reivilibre
/
quickpeep
1
0
Fork 0
Code Issues 11 Pull Requests Projects Releases 11 Wiki Activity

Don't fall over on unknown schemes e.g. mailto:
ci/woodpecker/push/check Pipeline was successful Details
ci/woodpecker/push/manual Pipeline failed Details
ci/woodpecker/push/release Pipeline was successful Details

rei/rakerstore_postgres_overhaul
Olivier 'reivilibre' 2022-11-26 23:47:23 +00:00
parent c940900fab
commit 4bba2fc89b
1 changed files with 16 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
quickpeep_raker/src/raking/references.rs
View File

@ -6,6 +6,10 @@ use quickpeep_utils::dates::date_to_quickpeep_days;
use reqwest::Url;
use std::collections::BTreeSet;
/// Supported schemes.
/// References in all other schemes will be ignored.
pub const SUPPORTED_SCHEMES: [&'static str; 2] = ["http", "https"];
pub fn find_references(
doc: &Vec<DenseTree>,
feeds: &Vec<Url>,
@ -41,11 +45,18 @@ pub fn find_references(
} => {
if !nofollow {
if let Ok(full_url) = page_url.join(&href) {
refs.insert(RakedReference {
target: clean_url(&full_url).to_string(),
kind: ReferenceKind::Link,
last_mod: None,
});
if SUPPORTED_SCHEMES.contains(&full_url.scheme()) {
refs.insert(RakedReference {
target: clean_url(&full_url).to_string(),
kind: ReferenceKind::Link,
last_mod: None,
});
} else {
debug!(
"ignoring reference {:?}: not a supported scheme",
full_url.as_str()
)
}
} else {
debug!("Can't join {:?} + {:?} to get full URL", page_url, href);
}