Open a fancy_mdbx database to store documents in for the Tantivy backend

Cargo.lock

@@ -950,6 +950,20 @@ version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed"
 
+[[package]]
+name = "fancy_mdbx"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "libmdbx",
+ "log",
+ "ouroboros",
+ "serde",
+ "serde_bare",
+ "thiserror",
+ "zstd",
+]
+
 [[package]]
 name = "fastdivide"
 version = "0.4.0"
@@ -3446,6 +3460,7 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "env_logger",
+ "fancy_mdbx",
  "log",
  "meilisearch-sdk",
  "quickpeep_structs",

quickpeep_index/Cargo.toml

@@ -16,5 +16,6 @@ serde = { version = "1.0.136", features = ["derive"] }
 serde_bare = "0.5.0"
 toml = "0.5.8"
 
+fancy_mdbx = { path = "../../../libraries/fancy_mdbx" }
 
 quickpeep_structs = { path = "../quickpeep_structs" }

quickpeep_index/src/backend/tantivy.rs

@@ -1,5 +1,9 @@
 use crate::backend::{Backend, BackendIndependentDocument};
 use anyhow::Context;
+use fancy_mdbx::database::WrappedTable;
+use fancy_mdbx::environment::Env;
+use fancy_mdbx::wrapper::{CompressorWrapper, SerdeBareWrapper};
+use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::path::Path;
 use tantivy::schema::{Facet, Field, SchemaBuilder, STORED, TEXT};
@@ -17,6 +21,15 @@ pub struct TantivyBackend {
     index: Index,
     fields: Fields,
     index_writer: Option<IndexWriter>,
+    env: Env,
+    tables: StoreTables,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct DocumentStoreRow {}
+
+pub struct StoreTables {
+    pub documents: WrappedTable<String, CompressorWrapper<SerdeBareWrapper<DocumentStoreRow>>>,
 }
 
 impl TantivyBackend {
@@ -28,7 +41,7 @@ impl TantivyBackend {
         let dir_path = path.join("tantivy");
 
         let (index, fields) = if dir_path.exists() {
-            let index = Index::open_in_dir(dir_path)?;
+            let index = Index::open_in_dir(&dir_path)?;
 
             let schema = index.schema();
             let mut field_map: HashMap<_, _> = schema
@@ -59,15 +72,29 @@ impl TantivyBackend {
             let schema = schema_builder.build();
 
             std::fs::create_dir(&dir_path)?;
-            let index = Index::create_in_dir(dir_path, schema)?;
+            let index = Index::create_in_dir(&dir_path, schema)?;
 
             (index, fields)
         };
 
+        // Open a libMDBX environment
+        let env = Env::open(&dir_path.join("store.mdbx"))?;
+
+        let tables = StoreTables {
+            documents: env.open_wrapped_table(
+                Some("documents"),
+                (),
+                Default::default(),
+                CompressorWrapper::new()?,
+            )?,
+        };
+
         Ok(TantivyBackend {
             index,
             fields,
             index_writer: None,
+            env,
+            tables,
         })
     }
 }

quickpeep_indexer/src/bin/qp-indexer.rs

@@ -5,7 +5,7 @@ use env_logger::Env;
 use std::fs::File;
 use std::io::{BufRead, BufReader};
 
-use quickpeep_densedoc::{DenseTree};
+use quickpeep_densedoc::DenseTree;
 use quickpeep_index::backend::BackendIndependentDocument;
 use quickpeep_indexer::config::IndexerConfig;
 use quickpeep_structs::rake_entries::{PackRecord, RakedPageEntry, SCHEMA_RAKED_PAGES};