reivilibre
/
quickpeep
1
0
Fork 0
Code Issues 11 Pull Requests Projects Releases 11 Wiki Activity

Push antifeatures as tags for now (a temporary solution)
continuous-integration/drone the build failed Details

This commit is contained in:
Olivier 'reivilibre' 2022-03-27 22:58:57 +01:00
parent 53db706cc0
commit 4231bec80a
1 changed files with 23 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
quickpeep_indexer/src/bin/qp-indexer.rs
View File

@ -15,7 +15,8 @@ use quickpeep_seed_parser::loader::{
find_seed_files, seed_loader, Seed, UrlOrUrlPattern, SEED_EXTENSION, find_seed_files, seed_loader, Seed, UrlOrUrlPattern, SEED_EXTENSION,
}; };
use quickpeep_structs::rake_entries::{ use quickpeep_structs::rake_entries::{
IconEntry, PackRecord, RakedPageEntry, SCHEMA_RAKED_ICONS, SCHEMA_RAKED_PAGES, AnalysisAntifeatures, IconEntry, PackRecord, RakedPageEntry, SCHEMA_RAKED_ICONS,
SCHEMA_RAKED_PAGES,
}; };
use quickpeep_utils::urls::get_reduced_domain; use quickpeep_utils::urls::get_reduced_domain;
use smartstring::alias::CompactString; use smartstring::alias::CompactString;
@ -112,6 +113,27 @@ pub fn handle_page_pack(
.map(|seed: &Seed| seed.tags.iter().map(|cs| cs.to_string()).collect()) .map(|seed: &Seed| seed.tags.iter().map(|cs| cs.to_string()).collect())
.unwrap_or_else(|| Vec::with_capacity(0)); .unwrap_or_else(|| Vec::with_capacity(0));
// TODO(temporary): add antifeatures as tags, for now.
let antifeatures = page_record.record.analysed_antifeatures;
if antifeatures.contains(AnalysisAntifeatures::CLOUDFLARE) {
tags.push("ZCloudflare".to_owned());
}
if antifeatures.contains(AnalysisAntifeatures::ADVERTS) {
tags.push("ZAdverts".to_owned());
}
if antifeatures.contains(AnalysisAntifeatures::ANNOYANCE) {
tags.push("ZAnnoyance".to_owned());
}
if antifeatures.contains(AnalysisAntifeatures::COOKIE_NAG) {
tags.push("ZCookieNags".to_owned());
}
if antifeatures.contains(AnalysisAntifeatures::PRIVACY) {
tags.push("ZTrackers".to_owned());
}
if antifeatures.contains(AnalysisAntifeatures::SOCIAL) {
tags.push("ZSocialTrackers".to_owned());
}
// TODO Store the actual structure of the document in the store? // TODO Store the actual structure of the document in the store?
let favicon_url_relative = document.head.effective_favicon_url(); let favicon_url_relative = document.head.effective_favicon_url();