reivilibre/quickpeep
1
0
Fork 0
Code Issues 11 Pull Requests Projects Releases 11 Wiki Activity

Enqueue favicons to be raked

Browse Source
This commit is contained in:
Olivier 'reivilibre' 2022-03-26 17:04:39 +00:00
parent d26b4271ce
commit 8714f0ec80
2 changed files with 21 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
quickpeep_raker/src/raking.rs
View File

@ -103,6 +103,7 @@ pub enum RakeIntent {
Page,
Feed,
SiteMap,
Icon,
}
impl From<ReferenceKind> for RakeIntent {

23
quickpeep_raker/src/raking/task.rs
View File

@ -1,8 +1,8 @@
use crate::raking::analysis::get_reduced_domain;
use crate::raking::references::references_from_urlrakes;
use crate::raking::{
get_robots_txt_for, robots_txt_url_for, PermanentFailure, PermanentFailureReason, RakeOutcome,
Raker, RedirectReason, RobotsTxt, TemporaryFailure, TemporaryFailureReason,
get_robots_txt_for, robots_txt_url_for, PermanentFailure, PermanentFailureReason, RakeIntent,
RakeOutcome, Raker, RedirectReason, RobotsTxt, TemporaryFailure, TemporaryFailureReason,
};
use crate::storage::records::{AllowedDomainRecord, UrlVisitedRecord, WeedDomainRecord};
use crate::storage::{RakerStore, RandomActiveDomainAcquisition};
@ -429,7 +429,12 @@ pub struct EventProcessor<'a> {
}
impl EventProcessor<'_> {
pub async fn process_page(&self, url: Url, datestamp: u16) -> anyhow::Result<()> {
pub async fn process_page(
&self,
url: Url,
page: &RakedPageEntry,
datestamp: u16,
) -> anyhow::Result<()> {
self.store
.as_ref()
.async_rw_txn(move |txn| {
@ -441,6 +446,18 @@ impl EventProcessor<'_> {
last_visited_days: datestamp,
},
)?;
// If there's a favicon to be tried, add it to the list...
let favicon_url_rel = if page.document.head.icon.is_empty() {
"/favicon.ico"
} else {
page.document.head.icon.as_str()
};
if let Ok(favicon_url) = url.join(favicon_url_rel) {
txn.enqueue_url(favicon_url.as_str(), None, RakeIntent::Icon)?;
}
txn.commit()?;
Ok(())
})