reivilibre
/
quickpeep
1
0
Fork 0
Code Issues 11 Pull Requests Projects Releases 11 Wiki Activity

Use the unified config in the raker

This commit is contained in:
Olivier 'reivilibre' 2022-04-05 22:17:27 +01:00
parent dd0097c1aa
commit 4fd2dc393e
7 changed files with 41 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.lock generated
View File

@ -3813,6 +3813,7 @@ dependencies = [
"rand 0.8.5",
"reivilibre_fork_cylon",
"reqwest",
"ron",
"serde",
"serde_bare",
"serde_json",
@ -3820,7 +3821,6 @@ dependencies = [
"sitemap",
"smartstring",
"tokio",
"toml",
"webp",
"zstd",
]

2
quickpeep_raker/Cargo.toml
View File

@ -27,7 +27,7 @@ serde = { version = "1.0.136", features = ["derive"] }
serde_bare = "0.5.0"
serde_json = "1.0.79"
toml = "0.5.8"
ron = "0.7.0"
bytesize = {version = "1.1.0", features = ["serde"]}
### Dates

6
quickpeep_raker/src/bin/qp-raker-db.rs
View File

@ -45,17 +45,17 @@ pub async fn main() -> anyhow::Result<()> {
.unwrap_or_else(|| PathBuf::from("qp_raker.toml"));
let config = config::RakerConfig::load(&config_path).context("Failed to load config")?;
if !config.workbench_dir.exists() {
if !config.raker.workbench_dir.exists() {
bail!(
"Workbench directory ({:?}) doesn't exist.",
config.workbench_dir
config.raker.workbench_dir
);
}
if !config.seed_dir.exists() {
bail!("Seed directory ({:?}) doesn't exist.", config.seed_dir);
}
let store = RakerStore::open(&config.workbench_dir.join("raker.mdbx"))?;
let store = RakerStore::open(&config.raker.workbench_dir.join("raker.mdbx"))?;
let txn = store.ro_txn()?;
match opts.table.as_ref() {

6
quickpeep_raker/src/bin/qp-raker-onhold.rs
View File

@ -36,14 +36,14 @@ pub async fn main() -> anyhow::Result<()> {
.unwrap_or_else(|| PathBuf::from("qp_raker.toml"));
let config = config::RakerConfig::load(&config_path).context("Failed to load config")?;
if !config.workbench_dir.exists() {
if !config.raker.workbench_dir.exists() {
bail!(
"Workbench directory ({:?}) doesn't exist.",
config.workbench_dir
config.raker.workbench_dir
);
}
let store = RakerStore::open(&config.workbench_dir.join("raker.mdbx"))?;
let store = RakerStore::open(&config.raker.workbench_dir.join("raker.mdbx"))?;
let is_urls = opts.urls;
let counts = tokio::task::spawn_blocking(move || -> anyhow::Result<Vec<_>> {

35
quickpeep_raker/src/bin/qp-raker.rs
View File

@ -70,13 +70,13 @@ pub async fn main() -> anyhow::Result<()> {
let config_path = opts
.config
.unwrap_or_else(|| PathBuf::from("qp_raker.toml"));
.unwrap_or_else(|| PathBuf::from("quickpeep.ron"));
let config = config::RakerConfig::load(&config_path).context("Failed to load config")?;
if !config.workbench_dir.exists() {
if !config.raker.workbench_dir.exists() {
bail!(
"Workbench directory ({:?}) doesn't exist.",
config.workbench_dir
config.raker.workbench_dir
);
}
@ -101,11 +101,11 @@ pub async fn main() -> anyhow::Result<()> {
.redirect(Policy::limited(5))
.build()?;
let store = RakerStore::open(&config.workbench_dir.join("raker.mdbx"))?;
let store = RakerStore::open(&config.raker.workbench_dir.join("raker.mdbx"))?;
let mut adblock_engines = Vec::new();
for (antifeature, name) in &ADBLOCK_FILTER_PATHS {
let path = PathBuf::from(config.data_dir.join(format!("{}.adblock", name)));
let path = PathBuf::from(config.raker.data_dir.join(format!("{}.adblock", name)));
if !path.exists() {
warn!("Missing adblock rules: {:?}.", path);
continue;
@ -119,7 +119,7 @@ pub async fn main() -> anyhow::Result<()> {
let mut antifeature_ip_set = IpSet::new();
let ips_file = File::open(config.data_dir.join("cf_ips.txt"))
let ips_file = File::open(config.raker.data_dir.join("cf_ips.txt"))
.await
.context("Failed to open CF IPs file")?;
antifeature_ip_set.add_all_from_file(ips_file).await?;
@ -129,15 +129,16 @@ pub async fn main() -> anyhow::Result<()> {
page_extraction: PageExtractionService::new(adblock_engines)?,
};
if let Some(addr) = config.metrics.prometheus {
if let Some(addr) = config.raker.metrics.prometheus {
PrometheusBuilder::new()
.with_http_listener(addr)
.install()?;
} else if config.metrics.bare_metrics {
} else if config.raker.metrics.bare_metrics {
warn!("BARE Metrics not supported yet, sorry.");
}
let metrics_enabled = config.metrics.prometheus.is_some() || config.metrics.bare_metrics;
let metrics_enabled =
config.raker.metrics.prometheus.is_some() || config.raker.metrics.bare_metrics;
if metrics_enabled {
metrics_process_promstyle::describe();
@ -158,8 +159,8 @@ pub async fn main() -> anyhow::Result<()> {
let mut emitters = Vec::with_capacity(3);
{
let emit_dir = config.emit_dir.clone();
let settings = config.pack_emitter.clone();
let emit_dir = config.raker.emit_dir.clone();
let settings = config.raker.pack_emitter.clone();
let stop = graceful_stop.clone();
let notify = graceful_stop_notify.clone();
emitters.push(
@ -181,8 +182,8 @@ pub async fn main() -> anyhow::Result<()> {
}
{
let emit_dir = config.emit_dir.clone();
let settings = config.pack_emitter.clone();
let emit_dir = config.raker.emit_dir.clone();
let settings = config.raker.pack_emitter.clone();
let stop = graceful_stop.clone();
let notify = graceful_stop_notify.clone();
emitters.push(
@ -204,8 +205,8 @@ pub async fn main() -> anyhow::Result<()> {
}
{
let emit_dir = config.emit_dir.clone();
let settings = config.pack_emitter.clone();
let emit_dir = config.raker.emit_dir.clone();
let settings = config.raker.pack_emitter.clone();
let stop = graceful_stop.clone();
let notify = graceful_stop_notify.clone();
emitters.push(
@ -227,8 +228,8 @@ pub async fn main() -> anyhow::Result<()> {
}
{
let emit_dir = config.emit_dir.clone();
let settings = config.pack_emitter.clone();
let emit_dir = config.raker.emit_dir.clone();
let settings = config.raker.pack_emitter.clone();
let stop = graceful_stop.clone();
let notify = graceful_stop_notify.clone();
emitters.push(

6
quickpeep_raker/src/bin/qp-seedrake.rs
View File

@ -44,17 +44,17 @@ pub async fn main() -> anyhow::Result<()> {
.unwrap_or_else(|| PathBuf::from("qp_raker.toml"));
let config = config::RakerConfig::load(&config_path).context("Failed to load config")?;
if !config.workbench_dir.exists() {
if !config.raker.workbench_dir.exists() {
bail!(
"Workbench directory ({:?}) doesn't exist.",
config.workbench_dir
config.raker.workbench_dir
);
}
if !config.seed_dir.exists() {
bail!("Seed directory ({:?}) doesn't exist.", config.seed_dir);
}
let store = RakerStore::open(&config.workbench_dir.join("raker.mdbx"))?;
let store = RakerStore::open(&config.raker.workbench_dir.join("raker.mdbx"))?;
import_seeds(store.clone(), &config).await?;

19
quickpeep_raker/src/config.rs
View File

@ -8,12 +8,17 @@ use std::path::{Path, PathBuf};
/// Config for a raker. All paths are relative to the config file if needed, but will be resolved
/// when loading.
pub struct RakerConfig {
/// Path to data files
pub data_dir: PathBuf,
/// Path to seeds
pub seed_dir: PathBuf,
pub raker: RakerOnlyConfig,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct RakerOnlyConfig {
/// Path to data files
pub data_dir: PathBuf,
/// Path to the raker's workbench (queue etc)
pub workbench_dir: PathBuf,
@ -31,12 +36,12 @@ impl RakerConfig {
pub fn load(path: &Path) -> anyhow::Result<RakerConfig> {
let config_dir = path.parent().context("Can't get parent of config file.")?;
let bytes = std::fs::read(path)?;
let mut raker_config: RakerConfig = toml::from_slice(&bytes)?;
let mut raker_config: RakerConfig = ron::de::from_bytes(&bytes)?;
raker_config.data_dir = config_dir.join(raker_config.data_dir);
raker_config.raker.data_dir = config_dir.join(raker_config.raker.data_dir);
raker_config.seed_dir = config_dir.join(raker_config.seed_dir);
raker_config.workbench_dir = config_dir.join(raker_config.workbench_dir);
raker_config.emit_dir = config_dir.join(raker_config.emit_dir);
raker_config.raker.workbench_dir = config_dir.join(raker_config.raker.workbench_dir);
raker_config.raker.emit_dir = config_dir.join(raker_config.raker.emit_dir);
Ok(raker_config)
}