From 5719d0b402162db00689e92c1788f5dff63a9b58 Mon Sep 17 00:00:00 2001
From: Olivier 'reivilibre <olivier@librepush.net>
Date: Sun, 27 Mar 2022 22:43:16 +0100
Subject: [PATCH] Use the redirect-following client for robots.txt

---
 quickpeep_raker/src/raking/task.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/quickpeep_raker/src/raking/task.rs b/quickpeep_raker/src/raking/task.rs
index 391a341..82d8b58 100644
--- a/quickpeep_raker/src/raking/task.rs
+++ b/quickpeep_raker/src/raking/task.rs
@@ -126,7 +126,7 @@ impl TaskContext {
     }
 
     pub async fn get_robot_rules(&self, url_of_site: &Url) -> anyhow::Result<Option<Cylon>> {
-        let robots = get_robots_txt_for(url_of_site, &self.client).await?;
+        let robots = get_robots_txt_for(url_of_site, &self.redirect_following_client).await?;
         Ok(robots.map(|robots: RobotsTxt| robots.rules))
     }