Prepare indexer

.gitignore

@@ -10,7 +10,9 @@ quickpeep_static/dist
 quickpeep_static/node_modules
 quickpeep/testdb.sqlite
 qp_web.ron
-dist
-book
-workbench
-rakepacks
+/dist
+/book
+/workbench
+/rakepacks
+/index
+qp_indexer.toml

Cargo.lock

@@ -3461,12 +3461,15 @@ name = "quickpeep_indexer"
 version = "0.1.0"
 dependencies = [
  "anyhow",
+ "clap",
+ "colour",
  "env_logger",
  "log",
  "quickpeep_index",
  "quickpeep_structs",
  "serde",
  "serde_bare",
+ "serde_json",
  "tokio",
  "toml",
 ]

quickpeep_index/src/backend/tantivy.rs

@@ -58,6 +58,7 @@ impl TantivyBackend {
             };
             let schema = schema_builder.build();
 
+            std::fs::create_dir(&dir_path)?;
             let index = Index::create_in_dir(dir_path, schema)?;
 
             (index, fields)

quickpeep_index/src/config.rs

@@ -2,8 +2,11 @@ use serde::{Deserialize, Serialize};
 use std::path::PathBuf;
 
 #[derive(Serialize, Deserialize, Debug, Clone)]
+#[serde(tag = "type")]
 pub enum BackendConfig {
+    #[serde(rename = "tantivy")]
     Tantivy(TantivyBackendConfig),
+    #[serde(rename = "meili")]
     Meili(MeiliBackendConfig),
 }
 

quickpeep_indexer/Cargo.toml

@@ -12,8 +12,10 @@ log = "0.4.16"
 env_logger = "0.9.0"
 serde = { version = "1.0.136", features = ["derive"] }
 serde_bare = "0.5.0"
+serde_json = "1.0.79"
 toml = "0.5.8"
-
+clap = { version = "3.1.6", features = ["derive"] }
+colour = "0.6.0"
 
 quickpeep_index = { path = "../quickpeep_index" }
 quickpeep_structs = { path = "../quickpeep_structs" }

quickpeep_indexer/src/bin/qp-indexer.rs

@@ -1,6 +1,36 @@
-pub struct IndexerConfig {}
+use anyhow::Context;
+use clap::Parser;
+use colour::{blue, yellow_ln};
+use env_logger::Env;
+
+use quickpeep_indexer::config::IndexerConfig;
+use std::path::PathBuf;
+
+/// Seeds a raker's queue with URLs
+#[derive(Clone, Debug, Parser)]
+pub struct Opts {
+    #[clap(long = "config")]
+    config: Option<PathBuf>,
+
+    rakepacks: Vec<PathBuf>,
+}
+
+pub fn main() -> anyhow::Result<()> {
+    env_logger::Builder::from_env(Env::default().default_filter_or("info,qp_indexer=debug")).init();
+
+    let opts: Opts = Opts::parse();
+
+    let config_path = opts
+        .config
+        .unwrap_or_else(|| PathBuf::from("qp_indexer.toml"));
+    let config = IndexerConfig::load(&config_path).context("Failed to load config")?;
+
+    let _indexer_backend = config.open_indexer_backend()?;
+
+    for pack in opts.rakepacks {
+        blue!("Indexing: ");
+        yellow_ln!("{:?}", pack);
+    }
 
-#[tokio::main]
-pub async fn main() -> anyhow::Result<()> {
     Ok(())
 }

quickpeep_indexer/src/config.rs

@@ -1,4 +1,6 @@
 use anyhow::Context;
+use quickpeep_index::backend::tantivy::TantivyBackend;
+use quickpeep_index::backend::Backend;
 use quickpeep_index::config::BackendConfig;
 use serde::{Deserialize, Serialize};
 use std::path::{Path, PathBuf};
@@ -32,4 +34,15 @@ impl IndexerConfig {
 
         Ok(indexer_config)
     }
+
+    pub fn open_indexer_backend(&self) -> anyhow::Result<Box<dyn Backend>> {
+        match &self.backend {
+            BackendConfig::Tantivy(tantivy) => {
+                Ok(Box::new(TantivyBackend::open(&tantivy.index_dir)?))
+            }
+            BackendConfig::Meili(_) => {
+                todo!()
+            }
+        }
+    }
 }