diff --git a/.gitignore b/.gitignore index db0c65f..a084586 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,9 @@ quickpeep_static/dist quickpeep_static/node_modules quickpeep/testdb.sqlite qp_web.ron -dist -book -workbench -rakepacks \ No newline at end of file +/dist +/book +/workbench +/rakepacks +/index +qp_indexer.toml \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 7db7a21..67e1967 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3461,12 +3461,15 @@ name = "quickpeep_indexer" version = "0.1.0" dependencies = [ "anyhow", + "clap", + "colour", "env_logger", "log", "quickpeep_index", "quickpeep_structs", "serde", "serde_bare", + "serde_json", "tokio", "toml", ] diff --git a/quickpeep_index/src/backend/tantivy.rs b/quickpeep_index/src/backend/tantivy.rs index a6450c9..32470ad 100644 --- a/quickpeep_index/src/backend/tantivy.rs +++ b/quickpeep_index/src/backend/tantivy.rs @@ -58,6 +58,7 @@ impl TantivyBackend { }; let schema = schema_builder.build(); + std::fs::create_dir(&dir_path)?; let index = Index::create_in_dir(dir_path, schema)?; (index, fields) diff --git a/quickpeep_index/src/config.rs b/quickpeep_index/src/config.rs index 2a464ab..caafead 100644 --- a/quickpeep_index/src/config.rs +++ b/quickpeep_index/src/config.rs @@ -2,8 +2,11 @@ use serde::{Deserialize, Serialize}; use std::path::PathBuf; #[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(tag = "type")] pub enum BackendConfig { + #[serde(rename = "tantivy")] Tantivy(TantivyBackendConfig), + #[serde(rename = "meili")] Meili(MeiliBackendConfig), } diff --git a/quickpeep_indexer/Cargo.toml b/quickpeep_indexer/Cargo.toml index fd08553..4b26ba9 100644 --- a/quickpeep_indexer/Cargo.toml +++ b/quickpeep_indexer/Cargo.toml @@ -12,8 +12,10 @@ log = "0.4.16" env_logger = "0.9.0" serde = { version = "1.0.136", features = ["derive"] } serde_bare = "0.5.0" +serde_json = "1.0.79" toml = "0.5.8" - +clap = { version = "3.1.6", features = ["derive"] } +colour = "0.6.0" quickpeep_index = { path = "../quickpeep_index" } quickpeep_structs = { path = "../quickpeep_structs" } diff --git a/quickpeep_indexer/src/bin/qp-indexer.rs b/quickpeep_indexer/src/bin/qp-indexer.rs index 7b8f22d..0ce731d 100644 --- a/quickpeep_indexer/src/bin/qp-indexer.rs +++ b/quickpeep_indexer/src/bin/qp-indexer.rs @@ -1,6 +1,36 @@ -pub struct IndexerConfig {} +use anyhow::Context; +use clap::Parser; +use colour::{blue, yellow_ln}; +use env_logger::Env; + +use quickpeep_indexer::config::IndexerConfig; +use std::path::PathBuf; + +/// Seeds a raker's queue with URLs +#[derive(Clone, Debug, Parser)] +pub struct Opts { + #[clap(long = "config")] + config: Option, + + rakepacks: Vec, +} + +pub fn main() -> anyhow::Result<()> { + env_logger::Builder::from_env(Env::default().default_filter_or("info,qp_indexer=debug")).init(); + + let opts: Opts = Opts::parse(); + + let config_path = opts + .config + .unwrap_or_else(|| PathBuf::from("qp_indexer.toml")); + let config = IndexerConfig::load(&config_path).context("Failed to load config")?; + + let _indexer_backend = config.open_indexer_backend()?; + + for pack in opts.rakepacks { + blue!("Indexing: "); + yellow_ln!("{:?}", pack); + } -#[tokio::main] -pub async fn main() -> anyhow::Result<()> { Ok(()) } diff --git a/quickpeep_indexer/src/config.rs b/quickpeep_indexer/src/config.rs index 14e8771..717667a 100644 --- a/quickpeep_indexer/src/config.rs +++ b/quickpeep_indexer/src/config.rs @@ -1,4 +1,6 @@ use anyhow::Context; +use quickpeep_index::backend::tantivy::TantivyBackend; +use quickpeep_index::backend::Backend; use quickpeep_index::config::BackendConfig; use serde::{Deserialize, Serialize}; use std::path::{Path, PathBuf}; @@ -32,4 +34,15 @@ impl IndexerConfig { Ok(indexer_config) } + + pub fn open_indexer_backend(&self) -> anyhow::Result> { + match &self.backend { + BackendConfig::Tantivy(tantivy) => { + Ok(Box::new(TantivyBackend::open(&tantivy.index_dir)?)) + } + BackendConfig::Meili(_) => { + todo!() + } + } + } }