Allow passing permanent failures up as errors

quickpeep_raker/src/raking.rs

@@ -17,6 +17,8 @@ use reqwest::{Client, Response, Url};
 use serde::{Deserialize, Serialize};
 use sitemap::reader::SiteMapEntity;
 use std::collections::{HashMap, HashSet};
+use std::error::Error;
+use std::fmt::{Debug, Display, Formatter};
 use std::io::Cursor;
 use std::str::FromStr;
 use std::time::Duration;
@@ -125,8 +127,17 @@ pub enum PermanentFailureReason {
     DeniedToRobots,
     WrongLanguage(String),
     UnknownContentType(String),
+    ExceedsSizeLimit,
 }
 
+impl Display for PermanentFailure {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        Debug::fmt(&self, f)
+    }
+}
+
+impl Error for PermanentFailure {}
+
 #[derive(Copy, Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
 pub enum RakeIntent {
     Any,

quickpeep_raker/src/raking/task.rs

@@ -260,15 +260,21 @@ impl TaskContext {
 
             let rake_outcome = raked.unwrap_or_else(|err| {
                 warn!("Failed to rake {:?}: {:?}", url, err);
-                // Treat this as a temporary rejection (backoff).
+
-                RakeOutcome::TemporaryFailure(TemporaryFailure {
+                match err.downcast::<PermanentFailure>() {
-                    reason: TemporaryFailureReason::UnknownClientError(format!(
+                    Ok(permanent) => RakeOutcome::PermanentFailure(permanent),
-                        "Failed to rake {:?}: {:?}",
+                    Err(err) => {
-                        url, err
+                        // Treat this as a temporary rejection (backoff).
-                    )),
+                        RakeOutcome::TemporaryFailure(TemporaryFailure {
-                    // Back off for a day: this ought to be enough time for the operator to fix the problem... maybe?
+                            reason: TemporaryFailureReason::UnknownClientError(format!(
-                    backoff_sec: 86400,
+                                "Failed to rake {:?}: {:?}",
-                })
+                                url, err
+                            )),
+                            // Back off for a day: this ought to be enough time for the operator to fix the problem... maybe?
+                            backoff_sec: 86400,
+                        })
+                    }
+                }
             });
 
             match self.process_outcome(&url, rake_outcome).await? {