Accept forbidden robots.txt — if they forbid us from knowing about things, we will be cheeky

quickpeep_raker/src/raking.rs

@@ -555,8 +555,10 @@ pub async fn get_robots_txt_for(url: &Url, client: &Client) -> anyhow::Result<Op
 
     if !resp.status().is_success() {
         let code = resp.status().as_u16();
-        if code == 404 || code == 410 {
+        if code == 403 || code == 404 || code == 410 {
             // not found or gone? Assume there is intentionally no robots.txt file.
+            // If they deny us access to the robots file, then they deserve whatever they get and
+            // we proceed.
             return Ok(None);
         }