reivilibre/quickpeep
1
0
Fork 0
Code Issues 11 Pull Requests Projects Releases 11 Wiki Activity

Use the redirect-following client for robots.txt

Browse Source
This commit is contained in:
Olivier 'reivilibre' 2022-03-27 22:43:16 +01:00
parent ab25e0bb9a
commit 5719d0b402
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
quickpeep_raker/src/raking/task.rs
View File

@ -126,7 +126,7 @@ impl TaskContext {
} }
pub async fn get_robot_rules(&self, url_of_site: &Url) -> anyhow::Result<Option<Cylon>> { pub async fn get_robot_rules(&self, url_of_site: &Url) -> anyhow::Result<Option<Cylon>> {
let robots = get_robots_txt_for(url_of_site, &self.client).await?; let robots = get_robots_txt_for(url_of_site, &self.redirect_following_client).await?;
Ok(robots.map(|robots: RobotsTxt| robots.rules)) Ok(robots.map(|robots: RobotsTxt| robots.rules))
} }