diff --git a/Cargo.lock b/Cargo.lock index a0a61a3..7db7a21 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3441,7 +3441,7 @@ dependencies = [ ] [[package]] -name = "quickpeep_indexer" +name = "quickpeep_index" version = "0.1.0" dependencies = [ "anyhow", @@ -3456,6 +3456,21 @@ dependencies = [ "toml", ] +[[package]] +name = "quickpeep_indexer" +version = "0.1.0" +dependencies = [ + "anyhow", + "env_logger", + "log", + "quickpeep_index", + "quickpeep_structs", + "serde", + "serde_bare", + "tokio", + "toml", +] + [[package]] name = "quickpeep_moz_readability" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index daf2c51..815212b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] members = [ "quickpeep", + "quickpeep_index", "quickpeep_indexer", "quickpeep_raker", "quickpeep_densedoc", diff --git a/quickpeep_index/Cargo.toml b/quickpeep_index/Cargo.toml new file mode 100644 index 0000000..864d1d9 --- /dev/null +++ b/quickpeep_index/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "quickpeep_index" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +tantivy = "0.17.0" +meilisearch-sdk = "0.15.0" +anyhow = "1.0.56" +tokio = { version = "1.17.0", features = ["full"] } +log = "0.4.16" +env_logger = "0.9.0" +serde = { version = "1.0.136", features = ["derive"] } +serde_bare = "0.5.0" +toml = "0.5.8" + + +quickpeep_structs = { path = "../quickpeep_structs" } diff --git a/quickpeep_indexer/src/backend.rs b/quickpeep_index/src/backend.rs similarity index 100% rename from quickpeep_indexer/src/backend.rs rename to quickpeep_index/src/backend.rs diff --git a/quickpeep_indexer/src/backend/meili.rs b/quickpeep_index/src/backend/meili.rs similarity index 100% rename from quickpeep_indexer/src/backend/meili.rs rename to quickpeep_index/src/backend/meili.rs diff --git a/quickpeep_indexer/src/backend/tantivy.rs b/quickpeep_index/src/backend/tantivy.rs similarity index 69% rename from quickpeep_indexer/src/backend/tantivy.rs rename to quickpeep_index/src/backend/tantivy.rs index 3cd9b4c..d5370da 100644 --- a/quickpeep_indexer/src/backend/tantivy.rs +++ b/quickpeep_index/src/backend/tantivy.rs @@ -1,7 +1,7 @@ use crate::backend::{Backend, BackendIndependentDocument}; use std::path::Path; use tantivy::schema::{Schema, STORED, TEXT}; -use tantivy::{doc, Index, IndexWriter}; +use tantivy::{Index, IndexWriter}; fn experiment_tantivy() -> anyhow::Result<()> { let mut schema_builder = Schema::builder(); @@ -15,7 +15,7 @@ fn experiment_tantivy() -> anyhow::Result<()> { // schema_builder.add_bytes_field() let schema = schema_builder.build(); let index = tantivy::Index::create_in_dir(Path::new("/tmp/tindex"), schema)?; - let writer = index.writer(100 * 1024 * 1024)?; + let _writer = index.writer(100 * 1024 * 1024)?; Ok(()) } @@ -26,14 +26,14 @@ pub struct TantivyBackend { } impl Backend for TantivyBackend { - fn add_document(&mut self, document: BackendIndependentDocument) -> anyhow::Result<()> { - self.index_writer.add_document(doc! { - "title" => document.title, - "article" => document.article_body, - "nonarticle" => document.nonarticle_body, - "url" => document.url, - "tags" => document.tags - })?; + fn add_document(&mut self, _document: BackendIndependentDocument) -> anyhow::Result<()> { + // self.index_writer.add_document(doc! { + // "title" => document.title, + // "article" => document.article_body, + // "nonarticle" => document.nonarticle_body, + // "url" => document.url, + // "tags" => document.tags + // })?; todo!() } diff --git a/quickpeep_index/src/config.rs b/quickpeep_index/src/config.rs new file mode 100644 index 0000000..2a464ab --- /dev/null +++ b/quickpeep_index/src/config.rs @@ -0,0 +1,19 @@ +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub enum BackendConfig { + Tantivy(TantivyBackendConfig), + Meili(MeiliBackendConfig), +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct TantivyBackendConfig { + pub index_dir: PathBuf, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct MeiliBackendConfig { + pub url: String, + pub token: String, +} diff --git a/quickpeep_index/src/lib.rs b/quickpeep_index/src/lib.rs new file mode 100644 index 0000000..27b6d15 --- /dev/null +++ b/quickpeep_index/src/lib.rs @@ -0,0 +1,3 @@ +pub mod config; + +pub mod backend; diff --git a/quickpeep_indexer/Cargo.toml b/quickpeep_indexer/Cargo.toml index 6e535e2..fd08553 100644 --- a/quickpeep_indexer/Cargo.toml +++ b/quickpeep_indexer/Cargo.toml @@ -6,8 +6,6 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -tantivy = "0.17.0" -meilisearch-sdk = "0.15.0" anyhow = "1.0.56" tokio = { version = "1.17.0", features = ["full"] } log = "0.4.16" @@ -17,4 +15,5 @@ serde_bare = "0.5.0" toml = "0.5.8" +quickpeep_index = { path = "../quickpeep_index" } quickpeep_structs = { path = "../quickpeep_structs" } diff --git a/quickpeep_indexer/src/config.rs b/quickpeep_indexer/src/config.rs index 985ee9e..14e8771 100644 --- a/quickpeep_indexer/src/config.rs +++ b/quickpeep_indexer/src/config.rs @@ -1,4 +1,5 @@ use anyhow::Context; +use quickpeep_index::config::BackendConfig; use serde::{Deserialize, Serialize}; use std::path::{Path, PathBuf}; @@ -32,20 +33,3 @@ impl IndexerConfig { Ok(indexer_config) } } - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub enum BackendConfig { - Tantivy(TantivyBackendConfig), - Meili(MeiliBackendConfig), -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct TantivyBackendConfig { - index_dir: PathBuf, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct MeiliBackendConfig { - url: String, - token: String, -} diff --git a/quickpeep_indexer/src/lib.rs b/quickpeep_indexer/src/lib.rs index 27b6d15..ef68c36 100644 --- a/quickpeep_indexer/src/lib.rs +++ b/quickpeep_indexer/src/lib.rs @@ -1,3 +1 @@ pub mod config; - -pub mod backend;