From 4f4c3e36d17787cb89c2ea0be8bfb377589165b2 Mon Sep 17 00:00:00 2001
From: Olivier 'reivilibre <olivier@librepush.net>
Date: Thu, 24 Mar 2022 23:19:32 +0000
Subject: [PATCH] Prepare indexer

---
 .gitignore                              | 10 ++++---
 Cargo.lock                              |  3 +++
 quickpeep_index/src/backend/tantivy.rs  |  1 +
 quickpeep_index/src/config.rs           |  3 +++
 quickpeep_indexer/Cargo.toml            |  4 ++-
 quickpeep_indexer/src/bin/qp-indexer.rs | 36 ++++++++++++++++++++++---
 quickpeep_indexer/src/config.rs         | 13 +++++++++
 7 files changed, 62 insertions(+), 8 deletions(-)

diff --git a/.gitignore b/.gitignore
index db0c65f..a084586 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,7 +10,9 @@ quickpeep_static/dist
 quickpeep_static/node_modules
 quickpeep/testdb.sqlite
 qp_web.ron
-dist
-book
-workbench
-rakepacks
\ No newline at end of file
+/dist
+/book
+/workbench
+/rakepacks
+/index
+qp_indexer.toml
\ No newline at end of file
diff --git a/Cargo.lock b/Cargo.lock
index 7db7a21..67e1967 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3461,12 +3461,15 @@ name = "quickpeep_indexer"
 version = "0.1.0"
 dependencies = [
  "anyhow",
+ "clap",
+ "colour",
  "env_logger",
  "log",
  "quickpeep_index",
  "quickpeep_structs",
  "serde",
  "serde_bare",
+ "serde_json",
  "tokio",
  "toml",
 ]
diff --git a/quickpeep_index/src/backend/tantivy.rs b/quickpeep_index/src/backend/tantivy.rs
index a6450c9..32470ad 100644
--- a/quickpeep_index/src/backend/tantivy.rs
+++ b/quickpeep_index/src/backend/tantivy.rs
@@ -58,6 +58,7 @@ impl TantivyBackend {
             };
             let schema = schema_builder.build();
 
+            std::fs::create_dir(&dir_path)?;
             let index = Index::create_in_dir(dir_path, schema)?;
 
             (index, fields)
diff --git a/quickpeep_index/src/config.rs b/quickpeep_index/src/config.rs
index 2a464ab..caafead 100644
--- a/quickpeep_index/src/config.rs
+++ b/quickpeep_index/src/config.rs
@@ -2,8 +2,11 @@ use serde::{Deserialize, Serialize};
 use std::path::PathBuf;
 
 #[derive(Serialize, Deserialize, Debug, Clone)]
+#[serde(tag = "type")]
 pub enum BackendConfig {
+    #[serde(rename = "tantivy")]
     Tantivy(TantivyBackendConfig),
+    #[serde(rename = "meili")]
     Meili(MeiliBackendConfig),
 }
 
diff --git a/quickpeep_indexer/Cargo.toml b/quickpeep_indexer/Cargo.toml
index fd08553..4b26ba9 100644
--- a/quickpeep_indexer/Cargo.toml
+++ b/quickpeep_indexer/Cargo.toml
@@ -12,8 +12,10 @@ log = "0.4.16"
 env_logger = "0.9.0"
 serde = { version = "1.0.136", features = ["derive"] }
 serde_bare = "0.5.0"
+serde_json = "1.0.79"
 toml = "0.5.8"
-
+clap = { version = "3.1.6", features = ["derive"] }
+colour = "0.6.0"
 
 quickpeep_index = { path = "../quickpeep_index" }
 quickpeep_structs = { path = "../quickpeep_structs" }
diff --git a/quickpeep_indexer/src/bin/qp-indexer.rs b/quickpeep_indexer/src/bin/qp-indexer.rs
index 7b8f22d..0ce731d 100644
--- a/quickpeep_indexer/src/bin/qp-indexer.rs
+++ b/quickpeep_indexer/src/bin/qp-indexer.rs
@@ -1,6 +1,36 @@
-pub struct IndexerConfig {}
+use anyhow::Context;
+use clap::Parser;
+use colour::{blue, yellow_ln};
+use env_logger::Env;
+
+use quickpeep_indexer::config::IndexerConfig;
+use std::path::PathBuf;
+
+/// Seeds a raker's queue with URLs
+#[derive(Clone, Debug, Parser)]
+pub struct Opts {
+    #[clap(long = "config")]
+    config: Option<PathBuf>,
+
+    rakepacks: Vec<PathBuf>,
+}
+
+pub fn main() -> anyhow::Result<()> {
+    env_logger::Builder::from_env(Env::default().default_filter_or("info,qp_indexer=debug")).init();
+
+    let opts: Opts = Opts::parse();
+
+    let config_path = opts
+        .config
+        .unwrap_or_else(|| PathBuf::from("qp_indexer.toml"));
+    let config = IndexerConfig::load(&config_path).context("Failed to load config")?;
+
+    let _indexer_backend = config.open_indexer_backend()?;
+
+    for pack in opts.rakepacks {
+        blue!("Indexing: ");
+        yellow_ln!("{:?}", pack);
+    }
 
-#[tokio::main]
-pub async fn main() -> anyhow::Result<()> {
     Ok(())
 }
diff --git a/quickpeep_indexer/src/config.rs b/quickpeep_indexer/src/config.rs
index 14e8771..717667a 100644
--- a/quickpeep_indexer/src/config.rs
+++ b/quickpeep_indexer/src/config.rs
@@ -1,4 +1,6 @@
 use anyhow::Context;
+use quickpeep_index::backend::tantivy::TantivyBackend;
+use quickpeep_index::backend::Backend;
 use quickpeep_index::config::BackendConfig;
 use serde::{Deserialize, Serialize};
 use std::path::{Path, PathBuf};
@@ -32,4 +34,15 @@ impl IndexerConfig {
 
         Ok(indexer_config)
     }
+
+    pub fn open_indexer_backend(&self) -> anyhow::Result<Box<dyn Backend>> {
+        match &self.backend {
+            BackendConfig::Tantivy(tantivy) => {
+                Ok(Box::new(TantivyBackend::open(&tantivy.index_dir)?))
+            }
+            BackendConfig::Meili(_) => {
+                todo!()
+            }
+        }
+    }
 }