Use the redirect-following client for robots.txt
This commit is contained in:
parent
ab25e0bb9a
commit
5719d0b402
|
@ -126,7 +126,7 @@ impl TaskContext {
|
|||
}
|
||||
|
||||
pub async fn get_robot_rules(&self, url_of_site: &Url) -> anyhow::Result<Option<Cylon>> {
|
||||
let robots = get_robots_txt_for(url_of_site, &self.client).await?;
|
||||
let robots = get_robots_txt_for(url_of_site, &self.redirect_following_client).await?;
|
||||
Ok(robots.map(|robots: RobotsTxt| robots.rules))
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in New Issue