Move the indexer around

Cargo.lock

@@ -3441,7 +3441,7 @@ dependencies = [
 ]
 
 [[package]]
-name = "quickpeep_indexer"
+name = "quickpeep_index"
 version = "0.1.0"
 dependencies = [
  "anyhow",
@@ -3456,6 +3456,21 @@ dependencies = [
  "toml",
 ]
 
+[[package]]
+name = "quickpeep_indexer"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "env_logger",
+ "log",
+ "quickpeep_index",
+ "quickpeep_structs",
+ "serde",
+ "serde_bare",
+ "tokio",
+ "toml",
+]
+
 [[package]]
 name = "quickpeep_moz_readability"
 version = "0.1.0"

Cargo.toml

@@ -1,6 +1,7 @@
 [workspace]
 members = [
     "quickpeep",
+    "quickpeep_index",
     "quickpeep_indexer",
     "quickpeep_raker",
     "quickpeep_densedoc",

quickpeep_index/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "quickpeep_index"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+tantivy = "0.17.0"
+meilisearch-sdk = "0.15.0"
+anyhow = "1.0.56"
+tokio = { version = "1.17.0", features = ["full"] }
+log = "0.4.16"
+env_logger = "0.9.0"
+serde = { version = "1.0.136", features = ["derive"] }
+serde_bare = "0.5.0"
+toml = "0.5.8"
+
+
+quickpeep_structs = { path = "../quickpeep_structs" }

quickpeep_index/src/backend/tantivy.rs

@@ -1,7 +1,7 @@
 use crate::backend::{Backend, BackendIndependentDocument};
 use std::path::Path;
 use tantivy::schema::{Schema, STORED, TEXT};
-use tantivy::{doc, Index, IndexWriter};
+use tantivy::{Index, IndexWriter};
 
 fn experiment_tantivy() -> anyhow::Result<()> {
     let mut schema_builder = Schema::builder();
@@ -15,7 +15,7 @@ fn experiment_tantivy() -> anyhow::Result<()> {
     // schema_builder.add_bytes_field()
     let schema = schema_builder.build();
     let index = tantivy::Index::create_in_dir(Path::new("/tmp/tindex"), schema)?;
-    let writer = index.writer(100 * 1024 * 1024)?;
+    let _writer = index.writer(100 * 1024 * 1024)?;
 
     Ok(())
 }
@@ -26,14 +26,14 @@ pub struct TantivyBackend {
 }
 
 impl Backend for TantivyBackend {
-    fn add_document(&mut self, document: BackendIndependentDocument) -> anyhow::Result<()> {
-        self.index_writer.add_document(doc! {
-            "title" => document.title,
-            "article" => document.article_body,
-            "nonarticle" => document.nonarticle_body,
-            "url" => document.url,
-            "tags" => document.tags
-        })?;
+    fn add_document(&mut self, _document: BackendIndependentDocument) -> anyhow::Result<()> {
+        // self.index_writer.add_document(doc! {
+        //     "title" => document.title,
+        //     "article" => document.article_body,
+        //     "nonarticle" => document.nonarticle_body,
+        //     "url" => document.url,
+        //     "tags" => document.tags
+        // })?;
         todo!()
     }
 

quickpeep_index/src/config.rs

@@ -0,0 +1,19 @@
+use serde::{Deserialize, Serialize};
+use std::path::PathBuf;
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub enum BackendConfig {
+    Tantivy(TantivyBackendConfig),
+    Meili(MeiliBackendConfig),
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct TantivyBackendConfig {
+    pub index_dir: PathBuf,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct MeiliBackendConfig {
+    pub url: String,
+    pub token: String,
+}

quickpeep_index/src/lib.rs

@@ -0,0 +1,3 @@
+pub mod config;
+
+pub mod backend;

quickpeep_indexer/Cargo.toml

@@ -6,8 +6,6 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-tantivy = "0.17.0"
-meilisearch-sdk = "0.15.0"
 anyhow = "1.0.56"
 tokio = { version = "1.17.0", features = ["full"] }
 log = "0.4.16"
@@ -17,4 +15,5 @@ serde_bare = "0.5.0"
 toml = "0.5.8"
 
 
+quickpeep_index = { path = "../quickpeep_index" }
 quickpeep_structs = { path = "../quickpeep_structs" }

quickpeep_indexer/src/config.rs

@@ -1,4 +1,5 @@
 use anyhow::Context;
+use quickpeep_index::config::BackendConfig;
 use serde::{Deserialize, Serialize};
 use std::path::{Path, PathBuf};
 
@@ -32,20 +33,3 @@ impl IndexerConfig {
         Ok(indexer_config)
     }
 }
-
-#[derive(Serialize, Deserialize, Debug, Clone)]
-pub enum BackendConfig {
-    Tantivy(TantivyBackendConfig),
-    Meili(MeiliBackendConfig),
-}
-
-#[derive(Serialize, Deserialize, Debug, Clone)]
-pub struct TantivyBackendConfig {
-    index_dir: PathBuf,
-}
-
-#[derive(Serialize, Deserialize, Debug, Clone)]
-pub struct MeiliBackendConfig {
-    url: String,
-    token: String,
-}

quickpeep_indexer/src/lib.rs

@@ -1,3 +1 @@
 pub mod config;
-
-pub mod backend;