reivilibre
/
fancy_mdbx
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Overhaul a bit to make it more ergonomic and use newer libMDBX

This commit is contained in:
Olivier 'reivilibre' 2023-05-08 20:58:30 +01:00
parent 2554674a47
commit b64bc5eb09
14 changed files with 302 additions and 317 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.envrc Normal file
View File

@ -0,0 +1,2 @@
use flake

3
.gitignore vendored
View File

@ -1,4 +1,5 @@
/target
Cargo.lock
.idea
.idea
.direnv

12
Cargo.toml
View File

@ -4,7 +4,7 @@ description = "Fancy wrapper for libMDBX"
repository = "https://bics.ga/reivilibre/fancy_mdbx.git"
authors = ["Olivier 'reivilibre'"]
license = "MIT OR Apache-2.0"
version = "0.1.1"
version = "0.2.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@ -16,16 +16,12 @@ log = "0.4.16"
thiserror = "1.0.30"
libmdbx = "0.1.1"
# 0.11.1+zstd.1.5.2
zstd = "0.11.1"
serde = "1.0.136"
serde_bare = "0.5.0"
libmdbx = "0.3.3"
ouroboros = "0.14.2"
byte_lamination = "0.1.1"
[dev-dependencies]
tempfile = "3.3.0"

58
flake.lock Normal file
View File

@ -0,0 +1,58 @@
{
"nodes": {
"nixpkgs": {
"locked": {
"lastModified": 1677075010,
"narHash": "sha256-X+UmR1AkdR//lPVcShmLy8p1n857IGf7y+cyCArp8bU=",
"path": "/nix/store/b1vy558z7lxph5mbg7n50b5njp393ia9-source",
"rev": "c95bf18beba4290af25c60cbaaceea1110d0f727",
"type": "path"
},
"original": {
"id": "nixpkgs",
"type": "indirect"
}
},
"root": {
"inputs": {
"nixpkgs": "nixpkgs",
"utils": "utils"
}
},
"systems": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
},
"utils": {
"inputs": {
"systems": "systems"
},
"locked": {
"lastModified": 1681202837,
"narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "cfacdce06f30d2b68473a46042957675eebb3401",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
}
},
"root": "root",
"version": 7
}

45
flake.nix Normal file
View File

@ -0,0 +1,45 @@
{
description = "fancy_mdbx";
inputs = {
utils.url = "github:numtide/flake-utils";
};
outputs = { self, nixpkgs, utils }:
utils.lib.eachDefaultSystem (system: let
pkgs = nixpkgs.legacyPackages."${system}";
in rec {
# `nix develop`
devShell = pkgs.mkShell {
nativeBuildInputs =
let
rust-toolchain = pkgs.symlinkJoin {
name = "rust-toolchain";
paths = with pkgs; [rustc cargo rustfmt rustPlatform.rustcSrc];
};
in
[
rust-toolchain
];
# Needed for bindgen when binding to mdbx
LIBCLANG_PATH="${pkgs.llvmPackages_latest.libclang.lib}/lib";
# Cargo culted:
# Add to rustc search path
RUSTFLAGS = (builtins.map (a: ''-L ${a}/lib'') [
]);
# Add to bindgen search path
BINDGEN_EXTRA_CLANG_ARGS =
# Includes with normal include path
(builtins.map (a: ''-I"${a}/include"'') [
# C standard library
pkgs.glibc.dev
])
# Includes with special directory paths
++ [
''-I"${pkgs.llvmPackages_latest.libclang.lib}/lib/clang/${pkgs.llvmPackages_latest.libclang.version}/include"''
];
};
});
}

58
src/database/raw.rs
View File

@ -1,58 +0,0 @@
use crate::environment::Txn;
use crate::wrapper::ZeroCopyByteWrapper;
use libmdbx::{Database, Environment, TransactionKind, WriteFlags, WriteMap, RW};
use ouroboros::self_referencing;
use std::borrow::Cow;
use std::marker::PhantomData;
use std::sync::Arc;
#[self_referencing(pub_extras)]
pub struct RawTable<K: ?Sized, V: ?Sized> {
pub mdbx_env: Arc<Environment<WriteMap>>,
#[borrows(mdbx_env)]
#[covariant]
pub mdbx_db: Database<'this>,
pub(crate) phantom_k: PhantomData<K>,
pub(crate) phantom_v: PhantomData<V>,
}
impl<K: ?Sized + ZeroCopyByteWrapper, V: ?Sized + ZeroCopyByteWrapper> RawTable<K, V> {
pub fn get<'txn, TK: TransactionKind>(
&'txn self,
txn: &'txn Txn<'txn, TK>,
k: impl AsRef<K>,
) -> anyhow::Result<Option<Cow<'txn, V>>> {
let bytes = txn
.mdbx_txn
.get::<Cow<'txn, [u8]>>(self.borrow_mdbx_db(), k.as_ref().as_byte_slice())?;
Ok(match bytes {
None => None,
Some(Cow::Owned(owned)) => Some(Cow::Owned(V::from_owned_bytes(owned)?)),
Some(Cow::Borrowed(borrowed)) => Some(Cow::Borrowed(V::from_byte_slice(borrowed)?)),
})
}
pub fn put<'txn>(
&self,
txn: &Txn<'txn, RW>,
k: impl AsRef<K>,
v: impl AsRef<V>,
) -> anyhow::Result<()> {
txn.mdbx_txn.put(
self.borrow_mdbx_db(),
k.as_ref().as_byte_slice(),
v.as_ref().as_byte_slice(),
WriteFlags::empty(),
)?;
Ok(())
}
/// Returns true if the key was present.
pub fn delete<'txn>(&self, txn: &Txn<'txn, RW>, k: impl AsRef<K>) -> anyhow::Result<bool> {
Ok(txn
.mdbx_txn
.del(self.borrow_mdbx_db(), k.as_ref().as_byte_slice(), None)?)
}
}

63
src/database/wrapped.rs
View File

@ -1,63 +0,0 @@
use crate::database::raw::RawTable;
use crate::environment::Txn;
use crate::wrapper::ByteWrapper;
use anyhow::Context;
use libmdbx::{TransactionKind, RW};
use std::borrow::Borrow;
pub struct WrappedTable<K, V> {
pub raw: RawTable<[u8], [u8]>,
pub k_wrapper: K,
pub v_wrapper: V,
}
impl<K: ByteWrapper, V: ByteWrapper> WrappedTable<K, V> {
pub fn get<'txn, TK: TransactionKind>(
&self,
txn: &Txn<'txn, TK>,
k: impl Borrow<K::Item>,
) -> anyhow::Result<Option<V::Item>> {
let k_bytes = self
.k_wrapper
.dump_to_db_bytes(k.borrow())
.context("whilst converting key to bytes")?;
self.raw
.get(txn, k_bytes)?
.map(|v_bytes| self.v_wrapper.load_from_db_bytes(&v_bytes))
.transpose()
}
pub fn put<'txn>(
&self,
txn: &Txn<'txn, RW>,
k: impl Borrow<K::Item>,
v: impl Borrow<V::Item>,
) -> anyhow::Result<()> {
let k_bytes = self
.k_wrapper
.dump_to_db_bytes(k.borrow())
.context("whilst converting key to bytes")?;
let v_bytes = self
.v_wrapper
.dump_to_db_bytes(v.borrow())
.context("whilst converting value to bytes")?;
self.raw.put(txn, k_bytes, v_bytes)?;
Ok(())
}
/// Returns true if the key was present.
pub fn delete<'txn>(
&self,
txn: &Txn<'txn, RW>,
k: impl Borrow<K::Item>,
) -> anyhow::Result<bool> {
let k_bytes = self
.k_wrapper
.dump_to_db_bytes(k.borrow())
.context("whilst converting key to bytes")?;
self.raw.delete(txn, k_bytes)
}
}

49
src/environment.rs
View File

@ -1,22 +1,20 @@
use crate::database::raw::{RawTable, RawTableTryBuilder};
use crate::database::wrapped::WrappedTable;
use crate::wrapper::{ByteWrapper, ZeroCopyByteWrapper};
use anyhow::{ensure, Context};
use libmdbx::{
DatabaseFlags, Environment, EnvironmentFlags, Geometry, Transaction, TransactionKind, WriteMap,
RO, RW,
Database, DatabaseFlags, Geometry, TableFlags, Transaction, TransactionKind, WriteMap, RO, RW,
};
use std::marker::PhantomData;
use crate::table::{raw::RawTableTryBuilder, RawTable, Table};
use byte_lamination::ByteLamination;
use std::path::Path;
use std::sync::Arc;
pub struct Env {
pub mdbx_env: Arc<Environment<WriteMap>>,
pub struct Db {
pub mdbx_env: Arc<Database<WriteMap>>,
}
impl Env {
pub fn open(path: &Path) -> anyhow::Result<Env> {
let mut flags = EnvironmentFlags::default();
impl Db {
pub fn open(path: &Path) -> anyhow::Result<Self> {
let mut flags = DatabaseFlags::default();
flags.no_sub_dir = true;
// TODO make the geometry more configurable.
@ -31,31 +29,27 @@ impl Env {
geom.shrink_threshold = Some(16 * 1024 * 1024);
// (Yes these numbers represent a large database).
let environment = Environment::new()
.set_max_dbs(256)
let environment = Database::new()
.set_max_tables(256)
.set_geometry(geom)
.set_flags(flags)
.open(path)?;
Ok(Env {
Ok(Db {
mdbx_env: Arc::new(environment),
})
}
pub fn open_raw_table<K: ZeroCopyByteWrapper + ?Sized, V: ZeroCopyByteWrapper + ?Sized>(
&self,
name: Option<&str>,
_flags: (),
) -> anyhow::Result<RawTable<K, V>> {
pub fn open_raw_table(&self, name: Option<&str>, _flags: ()) -> anyhow::Result<RawTable> {
Ok(RawTableTryBuilder {
mdbx_env: self.mdbx_env.clone(),
mdbx_db_builder: |mdbx_env: &Arc<Environment<WriteMap>>| {
mdbx_db_builder: |mdbx_env: &Arc<Database<WriteMap>>| {
let txn = mdbx_env
.begin_rw_txn()
.context("Can't start RW transaction")?;
// TODO database flags
let db = txn
.create_db(name, DatabaseFlags::empty())
.create_table(name, TableFlags::empty())
.context("Can't open database")?;
txn.prime_for_permaopen(db);
@ -67,25 +61,20 @@ impl Env {
Ok(mdbx_db)
},
phantom_k: PhantomData::default(),
phantom_v: PhantomData::default(),
}
.try_build()?)
}
pub fn open_wrapped_table<K: ByteWrapper, V: ByteWrapper>(
pub fn open_wrapped_table<K: for<'a> ByteLamination<'a>, V: for<'a> ByteLamination<'a>>(
&self,
name: Option<&str>,
flags: (),
k_wrapper: K,
v_wrapper: V,
) -> anyhow::Result<WrappedTable<K, V>> {
) -> anyhow::Result<Table<K, V>> {
let raw_table = self.open_raw_table(name, flags)?;
Ok(WrappedTable {
Ok(Table {
raw: raw_table,
k_wrapper,
v_wrapper,
_marker: Default::default(),
})
}

3
src/lib.rs
View File

@ -1,6 +1,5 @@
pub mod database;
pub mod environment;
pub mod wrapper;
pub mod table;
#[cfg(test)]
mod tests;

2
src/database.rs → src/table.rs
View File

@ -2,4 +2,4 @@ pub(crate) mod raw;
pub(crate) mod wrapped;
pub use raw::RawTable;
pub use wrapped::WrappedTable;
pub use wrapped::Table;

47
src/table/raw.rs Normal file
View File

@ -0,0 +1,47 @@
use libmdbx::{Database, Table, TransactionKind, WriteFlags, WriteMap, RW};
use ouroboros::self_referencing;
use std::borrow::Cow;
use crate::environment::Txn;
use std::sync::Arc;
#[self_referencing(pub_extras)]
pub struct RawTable {
pub mdbx_env: Arc<Database<WriteMap>>,
#[borrows(mdbx_env)]
#[covariant]
pub mdbx_db: Table<'this>,
}
impl RawTable {
pub fn get<'txn, TK: TransactionKind>(
&'txn self,
txn: &'txn Txn<'txn, TK>,
k: &[u8],
) -> anyhow::Result<Option<Cow<'txn, [u8]>>> {
txn.mdbx_txn
.get::<Cow<'txn, [u8]>>(self.borrow_mdbx_db(), k.as_ref())
.map_err(|e| e.into())
}
pub fn put<'txn>(
&self,
txn: &Txn<'txn, RW>,
k: impl AsRef<[u8]>,
v: impl AsRef<[u8]>,
) -> anyhow::Result<()> {
txn.mdbx_txn.put(
self.borrow_mdbx_db(),
k.as_ref(),
v.as_ref(),
WriteFlags::empty(),
)?;
Ok(())
}
/// Returns true if the key was present.
pub fn delete<'txn>(&self, txn: &Txn<'txn, RW>, k: impl AsRef<[u8]>) -> anyhow::Result<bool> {
Ok(txn.mdbx_txn.del(self.borrow_mdbx_db(), k.as_ref(), None)?)
}
}

114
src/table/wrapped.rs Normal file
View File

@ -0,0 +1,114 @@
use crate::table::raw::RawTable;
use crate::environment::Txn;
use anyhow::anyhow;
use byte_lamination::ByteLamination;
use libmdbx::{TransactionKind, RW};
use std::borrow::{Borrow, Cow};
use std::marker::PhantomData;
pub struct Table<K, V> {
pub raw: RawTable,
pub _marker: PhantomData<(K, V)>,
}
impl<K: for<'a> ByteLamination<'a>, V: for<'a> ByteLamination<'a>> Table<K, V> {
pub fn get<'txn, 'qk, QK: ByteLaminationHelper<'qk, K>, TK: TransactionKind>(
&self,
txn: &Txn<'txn, TK>,
k: QK,
) -> anyhow::Result<Option<V>> {
let k_bytes = k.as_cow_bytes_helper();
let val_opt = self.raw.get(txn, &k_bytes)?;
match val_opt {
Some(cow_bytes) => {
let value = V::try_from_bytes(cow_bytes)
.map_err(|e| anyhow!("failed to convert value bytes to value: {e:?}"))?;
Ok(Some(value))
}
None => Ok(None),
}
}
pub fn put<
'txn,
'qk,
'qv,
QK: ByteLaminationHelper<'qk, K>,
QV: ByteLaminationHelper<'qv, V>,
>(
&self,
txn: &Txn<'txn, RW>,
k: QK,
v: QV,
) -> anyhow::Result<()> {
let k_bytes = k.as_cow_bytes_helper();
let v_bytes = v.as_cow_bytes_helper();
self.raw.put(txn, &k_bytes, &v_bytes)?;
Ok(())
}
/// Returns true if the key was present.
pub fn delete<'txn>(
&self,
txn: &Txn<'txn, RW>,
k: &(impl Borrow<K> + ?Sized),
) -> anyhow::Result<bool> {
let k_bytes = k.borrow().as_cow_bytes();
self.raw.delete(txn, &k_bytes)
}
}
/// `T`: a ByteLamination that this type is equivalent to.
pub trait ByteLaminationHelper<'a, T: ByteLamination<'a>> {
fn as_cow_bytes_helper(&self) -> Cow<'_, [u8]>;
}
impl<'a, T: ByteLamination<'a>> ByteLaminationHelper<'a, T> for T {
fn as_cow_bytes_helper(&self) -> Cow<'_, [u8]> {
self.as_cow_bytes()
}
}
impl<'a, 'b, T: ByteLamination<'a>> ByteLaminationHelper<'a, T> for &'b T {
fn as_cow_bytes_helper(&self) -> Cow<'_, [u8]> {
self.as_cow_bytes()
}
}
// can't get rust happy with this...
// impl<'a, T: ?Sized + ToOwned> ByteLaminationHelper<'a, Cow<'a, T>> for &'a T
// where for<'b> Cow<'b, T>: ByteLamination<'b> {
// fn as_cow_bytes_helper(&self) -> Cow<'_, [u8]> {
// let x: Cow<'a, T> = Cow::Borrowed(self);
// let y: &'a Cow<'a, T> = &x;
// Cow::<'_, T>::as_cow_bytes(y)
// }
// }
impl<'a> ByteLaminationHelper<'a, Cow<'a, str>> for &'a str {
fn as_cow_bytes_helper(&self) -> Cow<'_, [u8]> {
Cow::Borrowed(self.as_bytes())
}
}
impl<'a> ByteLaminationHelper<'a, String> for &'a str {
fn as_cow_bytes_helper(&self) -> Cow<'_, [u8]> {
Cow::Borrowed(self.as_bytes())
}
}
impl<'a> ByteLaminationHelper<'a, Cow<'a, [u8]>> for &'a [u8] {
fn as_cow_bytes_helper(&self) -> Cow<'_, [u8]> {
Cow::Borrowed(&self)
}
}
impl<'a> ByteLaminationHelper<'a, Vec<u8>> for &'a [u8] {
fn as_cow_bytes_helper(&self) -> Cow<'_, [u8]> {
Cow::Borrowed(&self)
}
}

16
src/tests.rs
View File

@ -1,15 +1,15 @@
use crate::environment::Env;
use crate::environment::Db;
use tempfile::NamedTempFile;
fn temp_env() -> Env {
fn temp_env() -> Db {
let tempfile = NamedTempFile::new().unwrap();
Env::open(tempfile.path()).unwrap()
Db::open(tempfile.path()).unwrap()
}
#[test]
fn test_raw_bytes_basics() {
let env = temp_env();
let db = env.open_raw_table::<[u8], [u8]>(Some("rawt"), ()).unwrap();
let db = env.open_raw_table(Some("rawt"), ()).unwrap();
let k: [u8; 4] = [0, 1, 2, 3];
@ -30,9 +30,11 @@ fn test_raw_bytes_basics() {
}
#[test]
fn test_raw_str_basics() {
fn test_wrapped_str_basics() {
let env = temp_env();
let db = env.open_raw_table::<str, str>(Some("rawt"), ()).unwrap();
let db = env
.open_wrapped_table::<String, String>(Some("rawt"), ())
.unwrap();
{
let txn = env.ro_txn().unwrap();
@ -43,6 +45,6 @@ fn test_raw_str_basics() {
{
let txn = env.ro_txn().unwrap();
assert_eq!(db.get(&txn, "ooh").unwrap().unwrap().as_ref(), "cool!");
assert_eq!(db.get(&txn, "ooh").unwrap().unwrap().as_str(), "cool!");
}
}

147
src/wrapper.rs
View File

@ -1,147 +0,0 @@
use anyhow::anyhow;
use serde::de::DeserializeOwned;
use serde::Serialize;
use std::marker::PhantomData;
use std::sync::{Arc, Mutex};
use zstd::bulk::{Compressor, Decompressor};
pub trait ByteWrapper {
type Item;
fn load_from_db_bytes(&self, bytes: &[u8]) -> anyhow::Result<Self::Item>;
fn dump_to_db_bytes(&self, item: &Self::Item) -> anyhow::Result<Vec<u8>>;
}
pub struct SerdeBareWrapper<T> {
phantom: PhantomData<T>,
}
impl<T> Default for SerdeBareWrapper<T> {
fn default() -> Self {
SerdeBareWrapper {
phantom: Default::default(),
}
}
}
impl<T: Serialize + DeserializeOwned> ByteWrapper for SerdeBareWrapper<T> {
type Item = T;
fn load_from_db_bytes(&self, bytes: &[u8]) -> anyhow::Result<Self::Item> {
Ok(serde_bare::from_slice(bytes)?)
}
fn dump_to_db_bytes(&self, item: &Self::Item) -> anyhow::Result<Vec<u8>> {
Ok(serde_bare::to_vec(item)?)
}
}
pub struct CompressorWrapper<T> {
inner: T,
compressor: Mutex<Compressor<'static>>,
decompressor: Mutex<Decompressor<'static>>,
}
impl<T: ByteWrapper> ByteWrapper for CompressorWrapper<T> {
type Item = T::Item;
fn load_from_db_bytes(&self, bytes: &[u8]) -> anyhow::Result<Self::Item> {
let mut decompressor = self
.decompressor
.lock()
.map_err(|_| anyhow!("Can't lock decompressor"))?;
// TODO be more sensible about capacity. 8× should be good though.
let raw_bytes = decompressor.decompress(bytes, bytes.len() * 8)?;
Ok(self.inner.load_from_db_bytes(&raw_bytes)?)
}
fn dump_to_db_bytes(&self, item: &Self::Item) -> anyhow::Result<Vec<u8>> {
let raw_bytes = self.inner.dump_to_db_bytes(item)?;
let mut compressor = self
.compressor
.lock()
.map_err(|_| anyhow!("Can't lock compressor"))?;
Ok(compressor.compress(&raw_bytes)?)
}
}
impl<T> CompressorWrapper<T> {
pub fn new_with(inner: T) -> anyhow::Result<Self> {
let compressor = Compressor::new(13)?;
let decompressor = Decompressor::new()?;
Ok(CompressorWrapper {
inner,
compressor: Mutex::new(compressor),
decompressor: Mutex::new(decompressor),
})
}
pub fn new() -> anyhow::Result<Self>
where
T: Default,
{
let inner: T = Default::default();
Self::new_with(inner)
}
}
impl ByteWrapper for String {
type Item = String;
fn load_from_db_bytes(&self, bytes: &[u8]) -> anyhow::Result<Self::Item> {
Ok(std::str::from_utf8(bytes)?.to_string())
}
fn dump_to_db_bytes(&self, item: &Self::Item) -> anyhow::Result<Vec<u8>> {
Ok(item.as_bytes().to_vec())
}
}
impl<T: ByteWrapper> ByteWrapper for Arc<T> {
type Item = T::Item;
fn load_from_db_bytes(&self, bytes: &[u8]) -> anyhow::Result<Self::Item> {
(&self as &T).load_from_db_bytes(bytes)
}
fn dump_to_db_bytes(&self, item: &Self::Item) -> anyhow::Result<Vec<u8>> {
(&self as &T).dump_to_db_bytes(item)
}
}
pub trait ZeroCopyByteWrapper: ToOwned {
fn as_byte_slice(&self) -> &[u8];
fn from_byte_slice(bytes: &[u8]) -> anyhow::Result<&Self>;
fn from_owned_bytes(bytes: Vec<u8>) -> anyhow::Result<Self::Owned>;
}
impl ZeroCopyByteWrapper for [u8] {
fn as_byte_slice(&self) -> &[u8] {
self
}
fn from_byte_slice(bytes: &[u8]) -> anyhow::Result<&Self> {
Ok(bytes)
}
fn from_owned_bytes(bytes: Vec<u8>) -> anyhow::Result<Self::Owned> {
Ok(bytes)
}
}
impl ZeroCopyByteWrapper for str {
fn as_byte_slice(&self) -> &[u8] {
self.as_bytes()
}
fn from_byte_slice(bytes: &[u8]) -> anyhow::Result<&Self> {
Ok(std::str::from_utf8(bytes)?)
}
fn from_owned_bytes(bytes: Vec<u8>) -> anyhow::Result<Self::Owned> {
Ok(String::from_utf8(bytes)?)
}
}