Allow passing permanent failures up as errors

quickpeep_raker/src/raking.rs

@@ -17,6 +17,8 @@ use reqwest::{Client, Response, Url};
 use serde::{Deserialize, Serialize};
 use sitemap::reader::SiteMapEntity;
 use std::collections::{HashMap, HashSet};
+use std::error::Error;
+use std::fmt::{Debug, Display, Formatter};
 use std::io::Cursor;
 use std::str::FromStr;
 use std::time::Duration;
@@ -125,8 +127,17 @@ pub enum PermanentFailureReason {
     DeniedToRobots,
     WrongLanguage(String),
     UnknownContentType(String),
+    ExceedsSizeLimit,
 }
 
+impl Display for PermanentFailure {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        Debug::fmt(&self, f)
+    }
+}
+
+impl Error for PermanentFailure {}
+
 #[derive(Copy, Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
 pub enum RakeIntent {
     Any,

quickpeep_raker/src/raking/task.rs

@@ -260,6 +260,10 @@ impl TaskContext {
 
             let rake_outcome = raked.unwrap_or_else(|err| {
                 warn!("Failed to rake {:?}: {:?}", url, err);
+
+                match err.downcast::<PermanentFailure>() {
+                    Ok(permanent) => RakeOutcome::PermanentFailure(permanent),
+                    Err(err) => {
                         // Treat this as a temporary rejection (backoff).
                         RakeOutcome::TemporaryFailure(TemporaryFailure {
                             reason: TemporaryFailureReason::UnknownClientError(format!(
@@ -269,6 +273,8 @@ impl TaskContext {
                             // Back off for a day: this ought to be enough time for the operator to fix the problem... maybe?
                             backoff_sec: 86400,
                         })
+                    }
+                }
             });
 
             match self.process_outcome(&url, rake_outcome).await? {