diff --git a/quickpeep_raker/src/raking.rs b/quickpeep_raker/src/raking.rs index bd6a293..18e5789 100644 --- a/quickpeep_raker/src/raking.rs +++ b/quickpeep_raker/src/raking.rs @@ -33,6 +33,7 @@ pub enum RakeOutcome { RakedPage(RakedPage), RakedFeed(Vec), RakedSitemap(Vec), + RakedIcon(()), Redirect { reason: RedirectReason, new_url: Url, @@ -252,7 +253,6 @@ impl Raker { let content_type = content_type .to_str() .context("Can't convert content-type to str")?; - eprintln!("CT {:?}", content_type); content_type.split(";").next().unwrap().trim().to_owned() } else { return Ok(RakeOutcome::TemporaryFailure(TemporaryFailure { @@ -299,6 +299,17 @@ impl Raker { } } + if intent == RakeIntent::Icon { + match rake_icon(&content, &content_type) { + Ok(icon) => { + return Ok(RakeOutcome::RakedIcon(icon)); + } + Err(error) => { + debug!("Failed to rake as icon: {:?}", error); + } + } + } + return Ok(RakeOutcome::PermanentFailure(PermanentFailure { reason: PermanentFailureReason::UnknownContentType(content_type.to_owned()), })); @@ -432,6 +443,10 @@ pub fn rake_sitemap(content: &[u8]) -> anyhow::Result> { Ok(urls) } +pub fn rake_icon(content: &[u8], content_type: &str) -> anyhow::Result<()> { + todo!() +} + pub fn robots_txt_url_for(url: &Url) -> anyhow::Result { url.join("/robots.txt") .context("Whilst resolving /robots.txt on URL") diff --git a/quickpeep_raker/src/raking/task.rs b/quickpeep_raker/src/raking/task.rs index 8083422..a1db9d3 100644 --- a/quickpeep_raker/src/raking/task.rs +++ b/quickpeep_raker/src/raking/task.rs @@ -296,7 +296,7 @@ impl TaskContext { RakeOutcome::RakedPage(page) => { self.submission .pages - .send((url.clone(), page.page_entry)) + .send((url.clone(), page.page_entry.clone())) .await .context("Page processor shut down; can't stream page!")?; @@ -307,7 +307,7 @@ impl TaskContext { .context("Reference processor shut down; can't stream references!")?; self.as_event_processor() - .process_page(url.clone(), today) + .process_page(url.clone(), page.page_entry, today) .await?; self.as_event_processor() .process_refs(url.clone(), page.referrer_entry, today) @@ -349,6 +349,10 @@ impl TaskContext { Ok(NextAction::Continue) } + RakeOutcome::RakedIcon(()) => { + // Store icon to icon store + todo!(); + } RakeOutcome::Redirect { reason, new_url } => { let refs = RakedReferrerEntry { references: [RakedReference { @@ -432,7 +436,7 @@ impl EventProcessor<'_> { pub async fn process_page( &self, url: Url, - page: &RakedPageEntry, + page: RakedPageEntry, datestamp: u16, ) -> anyhow::Result<()> { self.store