diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..e3fecb3 --- /dev/null +++ b/.envrc @@ -0,0 +1,2 @@ +use flake + diff --git a/.gitignore b/.gitignore index 7a53b1c..0111d76 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /target Cargo.lock -.idea \ No newline at end of file +.idea +.direnv \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 1719205..9e4e870 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ description = "Fancy wrapper for libMDBX" repository = "https://bics.ga/reivilibre/fancy_mdbx.git" authors = ["Olivier 'reivilibre'"] license = "MIT OR Apache-2.0" -version = "0.1.1" +version = "0.2.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -16,16 +16,12 @@ log = "0.4.16" thiserror = "1.0.30" -libmdbx = "0.1.1" - -# 0.11.1+zstd.1.5.2 -zstd = "0.11.1" - -serde = "1.0.136" -serde_bare = "0.5.0" +libmdbx = "0.3.3" ouroboros = "0.14.2" +byte_lamination = "0.1.1" + [dev-dependencies] tempfile = "3.3.0" diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..7405617 --- /dev/null +++ b/flake.lock @@ -0,0 +1,58 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1677075010, + "narHash": "sha256-X+UmR1AkdR//lPVcShmLy8p1n857IGf7y+cyCArp8bU=", + "path": "/nix/store/b1vy558z7lxph5mbg7n50b5njp393ia9-source", + "rev": "c95bf18beba4290af25c60cbaaceea1110d0f727", + "type": "path" + }, + "original": { + "id": "nixpkgs", + "type": "indirect" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs", + "utils": "utils" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1681202837, + "narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "cfacdce06f30d2b68473a46042957675eebb3401", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..e6539b7 --- /dev/null +++ b/flake.nix @@ -0,0 +1,45 @@ +{ + description = "fancy_mdbx"; + + inputs = { + utils.url = "github:numtide/flake-utils"; + }; + + outputs = { self, nixpkgs, utils }: + utils.lib.eachDefaultSystem (system: let + pkgs = nixpkgs.legacyPackages."${system}"; + in rec { + # `nix develop` + devShell = pkgs.mkShell { + nativeBuildInputs = + let + rust-toolchain = pkgs.symlinkJoin { + name = "rust-toolchain"; + paths = with pkgs; [rustc cargo rustfmt rustPlatform.rustcSrc]; + }; + in + [ + rust-toolchain + ]; + + # Needed for bindgen when binding to mdbx + LIBCLANG_PATH="${pkgs.llvmPackages_latest.libclang.lib}/lib"; + + # Cargo culted: + # Add to rustc search path + RUSTFLAGS = (builtins.map (a: ''-L ${a}/lib'') [ + ]); + # Add to bindgen search path + BINDGEN_EXTRA_CLANG_ARGS = + # Includes with normal include path + (builtins.map (a: ''-I"${a}/include"'') [ + # C standard library + pkgs.glibc.dev + ]) + # Includes with special directory paths + ++ [ + ''-I"${pkgs.llvmPackages_latest.libclang.lib}/lib/clang/${pkgs.llvmPackages_latest.libclang.version}/include"'' + ]; + }; + }); +} diff --git a/src/database/raw.rs b/src/database/raw.rs deleted file mode 100644 index e4de09a..0000000 --- a/src/database/raw.rs +++ /dev/null @@ -1,58 +0,0 @@ -use crate::environment::Txn; -use crate::wrapper::ZeroCopyByteWrapper; -use libmdbx::{Database, Environment, TransactionKind, WriteFlags, WriteMap, RW}; -use ouroboros::self_referencing; -use std::borrow::Cow; -use std::marker::PhantomData; -use std::sync::Arc; - -#[self_referencing(pub_extras)] -pub struct RawTable { - pub mdbx_env: Arc>, - - #[borrows(mdbx_env)] - #[covariant] - pub mdbx_db: Database<'this>, - - pub(crate) phantom_k: PhantomData, - pub(crate) phantom_v: PhantomData, -} - -impl RawTable { - pub fn get<'txn, TK: TransactionKind>( - &'txn self, - txn: &'txn Txn<'txn, TK>, - k: impl AsRef, - ) -> anyhow::Result>> { - let bytes = txn - .mdbx_txn - .get::>(self.borrow_mdbx_db(), k.as_ref().as_byte_slice())?; - Ok(match bytes { - None => None, - Some(Cow::Owned(owned)) => Some(Cow::Owned(V::from_owned_bytes(owned)?)), - Some(Cow::Borrowed(borrowed)) => Some(Cow::Borrowed(V::from_byte_slice(borrowed)?)), - }) - } - - pub fn put<'txn>( - &self, - txn: &Txn<'txn, RW>, - k: impl AsRef, - v: impl AsRef, - ) -> anyhow::Result<()> { - txn.mdbx_txn.put( - self.borrow_mdbx_db(), - k.as_ref().as_byte_slice(), - v.as_ref().as_byte_slice(), - WriteFlags::empty(), - )?; - Ok(()) - } - - /// Returns true if the key was present. - pub fn delete<'txn>(&self, txn: &Txn<'txn, RW>, k: impl AsRef) -> anyhow::Result { - Ok(txn - .mdbx_txn - .del(self.borrow_mdbx_db(), k.as_ref().as_byte_slice(), None)?) - } -} diff --git a/src/database/wrapped.rs b/src/database/wrapped.rs deleted file mode 100644 index 5c38f25..0000000 --- a/src/database/wrapped.rs +++ /dev/null @@ -1,63 +0,0 @@ -use crate::database::raw::RawTable; -use crate::environment::Txn; -use crate::wrapper::ByteWrapper; -use anyhow::Context; -use libmdbx::{TransactionKind, RW}; -use std::borrow::Borrow; - -pub struct WrappedTable { - pub raw: RawTable<[u8], [u8]>, - pub k_wrapper: K, - pub v_wrapper: V, -} - -impl WrappedTable { - pub fn get<'txn, TK: TransactionKind>( - &self, - txn: &Txn<'txn, TK>, - k: impl Borrow, - ) -> anyhow::Result> { - let k_bytes = self - .k_wrapper - .dump_to_db_bytes(k.borrow()) - .context("whilst converting key to bytes")?; - - self.raw - .get(txn, k_bytes)? - .map(|v_bytes| self.v_wrapper.load_from_db_bytes(&v_bytes)) - .transpose() - } - - pub fn put<'txn>( - &self, - txn: &Txn<'txn, RW>, - k: impl Borrow, - v: impl Borrow, - ) -> anyhow::Result<()> { - let k_bytes = self - .k_wrapper - .dump_to_db_bytes(k.borrow()) - .context("whilst converting key to bytes")?; - let v_bytes = self - .v_wrapper - .dump_to_db_bytes(v.borrow()) - .context("whilst converting value to bytes")?; - - self.raw.put(txn, k_bytes, v_bytes)?; - Ok(()) - } - - /// Returns true if the key was present. - pub fn delete<'txn>( - &self, - txn: &Txn<'txn, RW>, - k: impl Borrow, - ) -> anyhow::Result { - let k_bytes = self - .k_wrapper - .dump_to_db_bytes(k.borrow()) - .context("whilst converting key to bytes")?; - - self.raw.delete(txn, k_bytes) - } -} diff --git a/src/environment.rs b/src/environment.rs index 8446c48..fb5e1e2 100644 --- a/src/environment.rs +++ b/src/environment.rs @@ -1,22 +1,20 @@ -use crate::database::raw::{RawTable, RawTableTryBuilder}; -use crate::database::wrapped::WrappedTable; -use crate::wrapper::{ByteWrapper, ZeroCopyByteWrapper}; use anyhow::{ensure, Context}; use libmdbx::{ - DatabaseFlags, Environment, EnvironmentFlags, Geometry, Transaction, TransactionKind, WriteMap, - RO, RW, + Database, DatabaseFlags, Geometry, TableFlags, Transaction, TransactionKind, WriteMap, RO, RW, }; -use std::marker::PhantomData; + +use crate::table::{raw::RawTableTryBuilder, RawTable, Table}; +use byte_lamination::ByteLamination; use std::path::Path; use std::sync::Arc; -pub struct Env { - pub mdbx_env: Arc>, +pub struct Db { + pub mdbx_env: Arc>, } -impl Env { - pub fn open(path: &Path) -> anyhow::Result { - let mut flags = EnvironmentFlags::default(); +impl Db { + pub fn open(path: &Path) -> anyhow::Result { + let mut flags = DatabaseFlags::default(); flags.no_sub_dir = true; // TODO make the geometry more configurable. @@ -31,31 +29,27 @@ impl Env { geom.shrink_threshold = Some(16 * 1024 * 1024); // (Yes these numbers represent a large database). - let environment = Environment::new() - .set_max_dbs(256) + let environment = Database::new() + .set_max_tables(256) .set_geometry(geom) .set_flags(flags) .open(path)?; - Ok(Env { + Ok(Db { mdbx_env: Arc::new(environment), }) } - pub fn open_raw_table( - &self, - name: Option<&str>, - _flags: (), - ) -> anyhow::Result> { + pub fn open_raw_table(&self, name: Option<&str>, _flags: ()) -> anyhow::Result { Ok(RawTableTryBuilder { mdbx_env: self.mdbx_env.clone(), - mdbx_db_builder: |mdbx_env: &Arc>| { + mdbx_db_builder: |mdbx_env: &Arc>| { let txn = mdbx_env .begin_rw_txn() .context("Can't start RW transaction")?; // TODO database flags let db = txn - .create_db(name, DatabaseFlags::empty()) + .create_table(name, TableFlags::empty()) .context("Can't open database")?; txn.prime_for_permaopen(db); @@ -67,25 +61,20 @@ impl Env { Ok(mdbx_db) }, - phantom_k: PhantomData::default(), - phantom_v: PhantomData::default(), } .try_build()?) } - pub fn open_wrapped_table( + pub fn open_wrapped_table ByteLamination<'a>, V: for<'a> ByteLamination<'a>>( &self, name: Option<&str>, flags: (), - k_wrapper: K, - v_wrapper: V, - ) -> anyhow::Result> { + ) -> anyhow::Result> { let raw_table = self.open_raw_table(name, flags)?; - Ok(WrappedTable { + Ok(Table { raw: raw_table, - k_wrapper, - v_wrapper, + _marker: Default::default(), }) } diff --git a/src/lib.rs b/src/lib.rs index 6d6cac8..14ae332 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,5 @@ -pub mod database; pub mod environment; -pub mod wrapper; +pub mod table; #[cfg(test)] mod tests; diff --git a/src/database.rs b/src/table.rs similarity index 68% rename from src/database.rs rename to src/table.rs index 577d4e6..b71e73b 100644 --- a/src/database.rs +++ b/src/table.rs @@ -2,4 +2,4 @@ pub(crate) mod raw; pub(crate) mod wrapped; pub use raw::RawTable; -pub use wrapped::WrappedTable; +pub use wrapped::Table; diff --git a/src/table/raw.rs b/src/table/raw.rs new file mode 100644 index 0000000..07454c5 --- /dev/null +++ b/src/table/raw.rs @@ -0,0 +1,47 @@ +use libmdbx::{Database, Table, TransactionKind, WriteFlags, WriteMap, RW}; +use ouroboros::self_referencing; +use std::borrow::Cow; + +use crate::environment::Txn; +use std::sync::Arc; + +#[self_referencing(pub_extras)] +pub struct RawTable { + pub mdbx_env: Arc>, + + #[borrows(mdbx_env)] + #[covariant] + pub mdbx_db: Table<'this>, +} + +impl RawTable { + pub fn get<'txn, TK: TransactionKind>( + &'txn self, + txn: &'txn Txn<'txn, TK>, + k: &[u8], + ) -> anyhow::Result>> { + txn.mdbx_txn + .get::>(self.borrow_mdbx_db(), k.as_ref()) + .map_err(|e| e.into()) + } + + pub fn put<'txn>( + &self, + txn: &Txn<'txn, RW>, + k: impl AsRef<[u8]>, + v: impl AsRef<[u8]>, + ) -> anyhow::Result<()> { + txn.mdbx_txn.put( + self.borrow_mdbx_db(), + k.as_ref(), + v.as_ref(), + WriteFlags::empty(), + )?; + Ok(()) + } + + /// Returns true if the key was present. + pub fn delete<'txn>(&self, txn: &Txn<'txn, RW>, k: impl AsRef<[u8]>) -> anyhow::Result { + Ok(txn.mdbx_txn.del(self.borrow_mdbx_db(), k.as_ref(), None)?) + } +} diff --git a/src/table/wrapped.rs b/src/table/wrapped.rs new file mode 100644 index 0000000..b26ff31 --- /dev/null +++ b/src/table/wrapped.rs @@ -0,0 +1,114 @@ +use crate::table::raw::RawTable; + +use crate::environment::Txn; +use anyhow::anyhow; +use byte_lamination::ByteLamination; +use libmdbx::{TransactionKind, RW}; +use std::borrow::{Borrow, Cow}; +use std::marker::PhantomData; + +pub struct Table { + pub raw: RawTable, + pub _marker: PhantomData<(K, V)>, +} + +impl ByteLamination<'a>, V: for<'a> ByteLamination<'a>> Table { + pub fn get<'txn, 'qk, QK: ByteLaminationHelper<'qk, K>, TK: TransactionKind>( + &self, + txn: &Txn<'txn, TK>, + k: QK, + ) -> anyhow::Result> { + let k_bytes = k.as_cow_bytes_helper(); + + let val_opt = self.raw.get(txn, &k_bytes)?; + + match val_opt { + Some(cow_bytes) => { + let value = V::try_from_bytes(cow_bytes) + .map_err(|e| anyhow!("failed to convert value bytes to value: {e:?}"))?; + Ok(Some(value)) + } + None => Ok(None), + } + } + + pub fn put< + 'txn, + 'qk, + 'qv, + QK: ByteLaminationHelper<'qk, K>, + QV: ByteLaminationHelper<'qv, V>, + >( + &self, + txn: &Txn<'txn, RW>, + k: QK, + v: QV, + ) -> anyhow::Result<()> { + let k_bytes = k.as_cow_bytes_helper(); + let v_bytes = v.as_cow_bytes_helper(); + self.raw.put(txn, &k_bytes, &v_bytes)?; + Ok(()) + } + + /// Returns true if the key was present. + pub fn delete<'txn>( + &self, + txn: &Txn<'txn, RW>, + k: &(impl Borrow + ?Sized), + ) -> anyhow::Result { + let k_bytes = k.borrow().as_cow_bytes(); + + self.raw.delete(txn, &k_bytes) + } +} + +/// `T`: a ByteLamination that this type is equivalent to. +pub trait ByteLaminationHelper<'a, T: ByteLamination<'a>> { + fn as_cow_bytes_helper(&self) -> Cow<'_, [u8]>; +} + +impl<'a, T: ByteLamination<'a>> ByteLaminationHelper<'a, T> for T { + fn as_cow_bytes_helper(&self) -> Cow<'_, [u8]> { + self.as_cow_bytes() + } +} + +impl<'a, 'b, T: ByteLamination<'a>> ByteLaminationHelper<'a, T> for &'b T { + fn as_cow_bytes_helper(&self) -> Cow<'_, [u8]> { + self.as_cow_bytes() + } +} + +// can't get rust happy with this... +// impl<'a, T: ?Sized + ToOwned> ByteLaminationHelper<'a, Cow<'a, T>> for &'a T +// where for<'b> Cow<'b, T>: ByteLamination<'b> { +// fn as_cow_bytes_helper(&self) -> Cow<'_, [u8]> { +// let x: Cow<'a, T> = Cow::Borrowed(self); +// let y: &'a Cow<'a, T> = &x; +// Cow::<'_, T>::as_cow_bytes(y) +// } +// } + +impl<'a> ByteLaminationHelper<'a, Cow<'a, str>> for &'a str { + fn as_cow_bytes_helper(&self) -> Cow<'_, [u8]> { + Cow::Borrowed(self.as_bytes()) + } +} + +impl<'a> ByteLaminationHelper<'a, String> for &'a str { + fn as_cow_bytes_helper(&self) -> Cow<'_, [u8]> { + Cow::Borrowed(self.as_bytes()) + } +} + +impl<'a> ByteLaminationHelper<'a, Cow<'a, [u8]>> for &'a [u8] { + fn as_cow_bytes_helper(&self) -> Cow<'_, [u8]> { + Cow::Borrowed(&self) + } +} + +impl<'a> ByteLaminationHelper<'a, Vec> for &'a [u8] { + fn as_cow_bytes_helper(&self) -> Cow<'_, [u8]> { + Cow::Borrowed(&self) + } +} diff --git a/src/tests.rs b/src/tests.rs index 30f0a5a..d9900de 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -1,15 +1,15 @@ -use crate::environment::Env; +use crate::environment::Db; use tempfile::NamedTempFile; -fn temp_env() -> Env { +fn temp_env() -> Db { let tempfile = NamedTempFile::new().unwrap(); - Env::open(tempfile.path()).unwrap() + Db::open(tempfile.path()).unwrap() } #[test] fn test_raw_bytes_basics() { let env = temp_env(); - let db = env.open_raw_table::<[u8], [u8]>(Some("rawt"), ()).unwrap(); + let db = env.open_raw_table(Some("rawt"), ()).unwrap(); let k: [u8; 4] = [0, 1, 2, 3]; @@ -30,9 +30,11 @@ fn test_raw_bytes_basics() { } #[test] -fn test_raw_str_basics() { +fn test_wrapped_str_basics() { let env = temp_env(); - let db = env.open_raw_table::(Some("rawt"), ()).unwrap(); + let db = env + .open_wrapped_table::(Some("rawt"), ()) + .unwrap(); { let txn = env.ro_txn().unwrap(); @@ -43,6 +45,6 @@ fn test_raw_str_basics() { { let txn = env.ro_txn().unwrap(); - assert_eq!(db.get(&txn, "ooh").unwrap().unwrap().as_ref(), "cool!"); + assert_eq!(db.get(&txn, "ooh").unwrap().unwrap().as_str(), "cool!"); } } diff --git a/src/wrapper.rs b/src/wrapper.rs deleted file mode 100644 index 9d98f39..0000000 --- a/src/wrapper.rs +++ /dev/null @@ -1,147 +0,0 @@ -use anyhow::anyhow; -use serde::de::DeserializeOwned; -use serde::Serialize; -use std::marker::PhantomData; -use std::sync::{Arc, Mutex}; -use zstd::bulk::{Compressor, Decompressor}; - -pub trait ByteWrapper { - type Item; - - fn load_from_db_bytes(&self, bytes: &[u8]) -> anyhow::Result; - - fn dump_to_db_bytes(&self, item: &Self::Item) -> anyhow::Result>; -} - -pub struct SerdeBareWrapper { - phantom: PhantomData, -} - -impl Default for SerdeBareWrapper { - fn default() -> Self { - SerdeBareWrapper { - phantom: Default::default(), - } - } -} - -impl ByteWrapper for SerdeBareWrapper { - type Item = T; - - fn load_from_db_bytes(&self, bytes: &[u8]) -> anyhow::Result { - Ok(serde_bare::from_slice(bytes)?) - } - - fn dump_to_db_bytes(&self, item: &Self::Item) -> anyhow::Result> { - Ok(serde_bare::to_vec(item)?) - } -} - -pub struct CompressorWrapper { - inner: T, - compressor: Mutex>, - decompressor: Mutex>, -} - -impl ByteWrapper for CompressorWrapper { - type Item = T::Item; - - fn load_from_db_bytes(&self, bytes: &[u8]) -> anyhow::Result { - let mut decompressor = self - .decompressor - .lock() - .map_err(|_| anyhow!("Can't lock decompressor"))?; - // TODO be more sensible about capacity. 8× should be good though. - let raw_bytes = decompressor.decompress(bytes, bytes.len() * 8)?; - Ok(self.inner.load_from_db_bytes(&raw_bytes)?) - } - - fn dump_to_db_bytes(&self, item: &Self::Item) -> anyhow::Result> { - let raw_bytes = self.inner.dump_to_db_bytes(item)?; - let mut compressor = self - .compressor - .lock() - .map_err(|_| anyhow!("Can't lock compressor"))?; - Ok(compressor.compress(&raw_bytes)?) - } -} - -impl CompressorWrapper { - pub fn new_with(inner: T) -> anyhow::Result { - let compressor = Compressor::new(13)?; - let decompressor = Decompressor::new()?; - - Ok(CompressorWrapper { - inner, - compressor: Mutex::new(compressor), - decompressor: Mutex::new(decompressor), - }) - } - - pub fn new() -> anyhow::Result - where - T: Default, - { - let inner: T = Default::default(); - - Self::new_with(inner) - } -} - -impl ByteWrapper for String { - type Item = String; - - fn load_from_db_bytes(&self, bytes: &[u8]) -> anyhow::Result { - Ok(std::str::from_utf8(bytes)?.to_string()) - } - - fn dump_to_db_bytes(&self, item: &Self::Item) -> anyhow::Result> { - Ok(item.as_bytes().to_vec()) - } -} - -impl ByteWrapper for Arc { - type Item = T::Item; - - fn load_from_db_bytes(&self, bytes: &[u8]) -> anyhow::Result { - (&self as &T).load_from_db_bytes(bytes) - } - - fn dump_to_db_bytes(&self, item: &Self::Item) -> anyhow::Result> { - (&self as &T).dump_to_db_bytes(item) - } -} - -pub trait ZeroCopyByteWrapper: ToOwned { - fn as_byte_slice(&self) -> &[u8]; - fn from_byte_slice(bytes: &[u8]) -> anyhow::Result<&Self>; - fn from_owned_bytes(bytes: Vec) -> anyhow::Result; -} - -impl ZeroCopyByteWrapper for [u8] { - fn as_byte_slice(&self) -> &[u8] { - self - } - - fn from_byte_slice(bytes: &[u8]) -> anyhow::Result<&Self> { - Ok(bytes) - } - - fn from_owned_bytes(bytes: Vec) -> anyhow::Result { - Ok(bytes) - } -} - -impl ZeroCopyByteWrapper for str { - fn as_byte_slice(&self) -> &[u8] { - self.as_bytes() - } - - fn from_byte_slice(bytes: &[u8]) -> anyhow::Result<&Self> { - Ok(std::str::from_utf8(bytes)?) - } - - fn from_owned_bytes(bytes: Vec) -> anyhow::Result { - Ok(String::from_utf8(bytes)?) - } -}