diff --git a/Cargo.lock b/Cargo.lock index f07542e..6cff251 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -950,6 +950,20 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" +[[package]] +name = "fancy_mdbx" +version = "0.1.0" +dependencies = [ + "anyhow", + "libmdbx", + "log", + "ouroboros", + "serde", + "serde_bare", + "thiserror", + "zstd", +] + [[package]] name = "fastdivide" version = "0.4.0" @@ -3446,6 +3460,7 @@ version = "0.1.0" dependencies = [ "anyhow", "env_logger", + "fancy_mdbx", "log", "meilisearch-sdk", "quickpeep_structs", diff --git a/quickpeep_index/Cargo.toml b/quickpeep_index/Cargo.toml index 864d1d9..59d05fc 100644 --- a/quickpeep_index/Cargo.toml +++ b/quickpeep_index/Cargo.toml @@ -16,5 +16,6 @@ serde = { version = "1.0.136", features = ["derive"] } serde_bare = "0.5.0" toml = "0.5.8" +fancy_mdbx = { path = "../../../libraries/fancy_mdbx" } quickpeep_structs = { path = "../quickpeep_structs" } diff --git a/quickpeep_index/src/backend/tantivy.rs b/quickpeep_index/src/backend/tantivy.rs index 32470ad..fd21aaf 100644 --- a/quickpeep_index/src/backend/tantivy.rs +++ b/quickpeep_index/src/backend/tantivy.rs @@ -1,5 +1,9 @@ use crate::backend::{Backend, BackendIndependentDocument}; use anyhow::Context; +use fancy_mdbx::database::WrappedTable; +use fancy_mdbx::environment::Env; +use fancy_mdbx::wrapper::{CompressorWrapper, SerdeBareWrapper}; +use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::path::Path; use tantivy::schema::{Facet, Field, SchemaBuilder, STORED, TEXT}; @@ -17,6 +21,15 @@ pub struct TantivyBackend { index: Index, fields: Fields, index_writer: Option, + env: Env, + tables: StoreTables, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct DocumentStoreRow {} + +pub struct StoreTables { + pub documents: WrappedTable>>, } impl TantivyBackend { @@ -28,7 +41,7 @@ impl TantivyBackend { let dir_path = path.join("tantivy"); let (index, fields) = if dir_path.exists() { - let index = Index::open_in_dir(dir_path)?; + let index = Index::open_in_dir(&dir_path)?; let schema = index.schema(); let mut field_map: HashMap<_, _> = schema @@ -59,15 +72,29 @@ impl TantivyBackend { let schema = schema_builder.build(); std::fs::create_dir(&dir_path)?; - let index = Index::create_in_dir(dir_path, schema)?; + let index = Index::create_in_dir(&dir_path, schema)?; (index, fields) }; + // Open a libMDBX environment + let env = Env::open(&dir_path.join("store.mdbx"))?; + + let tables = StoreTables { + documents: env.open_wrapped_table( + Some("documents"), + (), + Default::default(), + CompressorWrapper::new()?, + )?, + }; + Ok(TantivyBackend { index, fields, index_writer: None, + env, + tables, }) } } diff --git a/quickpeep_indexer/src/bin/qp-indexer.rs b/quickpeep_indexer/src/bin/qp-indexer.rs index 08aa0a5..5e3989c 100644 --- a/quickpeep_indexer/src/bin/qp-indexer.rs +++ b/quickpeep_indexer/src/bin/qp-indexer.rs @@ -5,7 +5,7 @@ use env_logger::Env; use std::fs::File; use std::io::{BufRead, BufReader}; -use quickpeep_densedoc::{DenseTree}; +use quickpeep_densedoc::DenseTree; use quickpeep_index::backend::BackendIndependentDocument; use quickpeep_indexer::config::IndexerConfig; use quickpeep_structs::rake_entries::{PackRecord, RakedPageEntry, SCHEMA_RAKED_PAGES};