From c01740113f90614e3cb3584d113776b48082910e Mon Sep 17 00:00:00 2001
From: Olivier 'reivilibre <olivier@librepush.net>
Date: Fri, 25 Mar 2022 19:17:34 +0000
Subject: [PATCH] Open a fancy_mdbx database to store documents in for the
Tantivy backend
---
Cargo.lock | 15 ++++++++++++
quickpeep_index/Cargo.toml | 1 +
quickpeep_index/src/backend/tantivy.rs | 31 +++++++++++++++++++++++--
quickpeep_indexer/src/bin/qp-indexer.rs | 2 +-
4 files changed, 46 insertions(+), 3 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index f07542e..6cff251 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -950,6 +950,20 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed"
+[[package]]
+name = "fancy_mdbx"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "libmdbx",
+ "log",
+ "ouroboros",
+ "serde",
+ "serde_bare",
+ "thiserror",
+ "zstd",
+]
+
[[package]]
name = "fastdivide"
version = "0.4.0"
@@ -3446,6 +3460,7 @@ version = "0.1.0"
dependencies = [
"anyhow",
"env_logger",
+ "fancy_mdbx",
"log",
"meilisearch-sdk",
"quickpeep_structs",
diff --git a/quickpeep_index/Cargo.toml b/quickpeep_index/Cargo.toml
index 864d1d9..59d05fc 100644
--- a/quickpeep_index/Cargo.toml
+++ b/quickpeep_index/Cargo.toml
@@ -16,5 +16,6 @@ serde = { version = "1.0.136", features = ["derive"] }
serde_bare = "0.5.0"
toml = "0.5.8"
+fancy_mdbx = { path = "../../../libraries/fancy_mdbx" }
quickpeep_structs = { path = "../quickpeep_structs" }
diff --git a/quickpeep_index/src/backend/tantivy.rs b/quickpeep_index/src/backend/tantivy.rs
index 32470ad..fd21aaf 100644
--- a/quickpeep_index/src/backend/tantivy.rs
+++ b/quickpeep_index/src/backend/tantivy.rs
@@ -1,5 +1,9 @@
use crate::backend::{Backend, BackendIndependentDocument};
use anyhow::Context;
+use fancy_mdbx::database::WrappedTable;
+use fancy_mdbx::environment::Env;
+use fancy_mdbx::wrapper::{CompressorWrapper, SerdeBareWrapper};
+use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::path::Path;
use tantivy::schema::{Facet, Field, SchemaBuilder, STORED, TEXT};
@@ -17,6 +21,15 @@ pub struct TantivyBackend {
index: Index,
fields: Fields,
index_writer: Option<IndexWriter>,
+ env: Env,
+ tables: StoreTables,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct DocumentStoreRow {}
+
+pub struct StoreTables {
+ pub documents: WrappedTable<String, CompressorWrapper<SerdeBareWrapper<DocumentStoreRow>>>,
}
impl TantivyBackend {
@@ -28,7 +41,7 @@ impl TantivyBackend {
let dir_path = path.join("tantivy");
let (index, fields) = if dir_path.exists() {
- let index = Index::open_in_dir(dir_path)?;
+ let index = Index::open_in_dir(&dir_path)?;
let schema = index.schema();
let mut field_map: HashMap<_, _> = schema
@@ -59,15 +72,29 @@ impl TantivyBackend {
let schema = schema_builder.build();
std::fs::create_dir(&dir_path)?;
- let index = Index::create_in_dir(dir_path, schema)?;
+ let index = Index::create_in_dir(&dir_path, schema)?;
(index, fields)
};
+ // Open a libMDBX environment
+ let env = Env::open(&dir_path.join("store.mdbx"))?;
+
+ let tables = StoreTables {
+ documents: env.open_wrapped_table(
+ Some("documents"),
+ (),
+ Default::default(),
+ CompressorWrapper::new()?,
+ )?,
+ };
+
Ok(TantivyBackend {
index,
fields,
index_writer: None,
+ env,
+ tables,
})
}
}
diff --git a/quickpeep_indexer/src/bin/qp-indexer.rs b/quickpeep_indexer/src/bin/qp-indexer.rs
index 08aa0a5..5e3989c 100644
--- a/quickpeep_indexer/src/bin/qp-indexer.rs
+++ b/quickpeep_indexer/src/bin/qp-indexer.rs
@@ -5,7 +5,7 @@ use env_logger::Env;
use std::fs::File;
use std::io::{BufRead, BufReader};
-use quickpeep_densedoc::{DenseTree};
+use quickpeep_densedoc::DenseTree;
use quickpeep_index::backend::BackendIndependentDocument;
use quickpeep_indexer::config::IndexerConfig;
use quickpeep_structs::rake_entries::{PackRecord, RakedPageEntry, SCHEMA_RAKED_PAGES};