Add metric for new enqueued URLs

This commit is contained in:
Olivier 'reivilibre' 2022-06-04 23:38:12 +01:00
parent aa08463499
commit d3600bfb73
2 changed files with 11 additions and 1 deletions

View File

@ -18,4 +18,10 @@ pub fn describe_raking_metrics() {
Unit::Bytes, Unit::Bytes,
"Number of content bytes raked." "Number of content bytes raked."
); );
describe_counter!(
"qprake_queue_new_url",
Unit::Count,
"Number of new URLs enqueued"
);
} }

View File

@ -10,6 +10,7 @@ use chrono::Utc;
use cylon::Cylon; use cylon::Cylon;
use log::{error, warn}; use log::{error, warn};
use lru::LruCache; use lru::LruCache;
use metrics::increment_counter;
use quickpeep_structs::rake_entries::{ use quickpeep_structs::rake_entries::{
IconEntry, RakedPageEntry, RakedReference, RakedReferrerEntry, ReferenceKind, IconEntry, RakedPageEntry, RakedReference, RakedReferrerEntry, ReferenceKind,
}; };
@ -537,11 +538,14 @@ impl EventProcessor<'_> {
.map(|record: AllowedDomainRecord| record.applies_to_url(&ref_url)) .map(|record: AllowedDomainRecord| record.applies_to_url(&ref_url))
.unwrap_or(false); .unwrap_or(false);
if allowed { if allowed {
txn.enqueue_url( let is_fresh = txn.enqueue_url(
&reference.target, &reference.target,
reference.last_mod, reference.last_mod,
reference.kind.into(), reference.kind.into(),
)?; )?;
if is_fresh {
increment_counter!("qprake_queue_new_url");
}
continue; continue;
} }