Add structure to support raking icons

quickpeep_raker/src/raking.rs

@@ -33,6 +33,7 @@ pub enum RakeOutcome {
     RakedPage(RakedPage),
     RakedFeed(Vec<UrlRaked>),
     RakedSitemap(Vec<UrlRaked>),
+    RakedIcon(()),
     Redirect {
         reason: RedirectReason,
         new_url: Url,
@@ -252,7 +253,6 @@ impl Raker {
             let content_type = content_type
                 .to_str()
                 .context("Can't convert content-type to str")?;
-            eprintln!("CT {:?}", content_type);
             content_type.split(";").next().unwrap().trim().to_owned()
         } else {
             return Ok(RakeOutcome::TemporaryFailure(TemporaryFailure {
@@ -299,6 +299,17 @@ impl Raker {
             }
         }
 
+        if intent == RakeIntent::Icon {
+            match rake_icon(&content, &content_type) {
+                Ok(icon) => {
+                    return Ok(RakeOutcome::RakedIcon(icon));
+                }
+                Err(error) => {
+                    debug!("Failed to rake as icon: {:?}", error);
+                }
+            }
+        }
+
         return Ok(RakeOutcome::PermanentFailure(PermanentFailure {
             reason: PermanentFailureReason::UnknownContentType(content_type.to_owned()),
         }));
@@ -432,6 +443,10 @@ pub fn rake_sitemap(content: &[u8]) -> anyhow::Result<Vec<UrlRaked>> {
     Ok(urls)
 }
 
+pub fn rake_icon(content: &[u8], content_type: &str) -> anyhow::Result<()> {
+    todo!()
+}
+
 pub fn robots_txt_url_for(url: &Url) -> anyhow::Result<Url> {
     url.join("/robots.txt")
         .context("Whilst resolving /robots.txt on URL")

quickpeep_raker/src/raking/task.rs

@@ -296,7 +296,7 @@ impl TaskContext {
             RakeOutcome::RakedPage(page) => {
                 self.submission
                     .pages
-                    .send((url.clone(), page.page_entry))
+                    .send((url.clone(), page.page_entry.clone()))
                     .await
                     .context("Page processor shut down; can't stream page!")?;
 
@@ -307,7 +307,7 @@ impl TaskContext {
                     .context("Reference processor shut down; can't stream references!")?;
 
                 self.as_event_processor()
-                    .process_page(url.clone(), today)
+                    .process_page(url.clone(), page.page_entry, today)
                     .await?;
                 self.as_event_processor()
                     .process_refs(url.clone(), page.referrer_entry, today)
@@ -349,6 +349,10 @@ impl TaskContext {
 
                 Ok(NextAction::Continue)
             }
+            RakeOutcome::RakedIcon(()) => {
+                // Store icon to icon store
+                todo!();
+            }
             RakeOutcome::Redirect { reason, new_url } => {
                 let refs = RakedReferrerEntry {
                     references: [RakedReference {
@@ -432,7 +436,7 @@ impl EventProcessor<'_> {
     pub async fn process_page(
         &self,
         url: Url,
-        page: &RakedPageEntry,
+        page: RakedPageEntry,
         datestamp: u16,
     ) -> anyhow::Result<()> {
         self.store