From dad85e6057e60d020c266aae773dbfd168ce679f Mon Sep 17 00:00:00 2001 From: Olivier 'reivilibre Date: Sat, 12 Jun 2021 10:11:05 +0100 Subject: [PATCH] Remove old legacy cruft --- yama/Cargo.lock => Cargo.lock | 490 ++++++++++++---------------------- Cargo.toml | 6 + datman/Cargo.toml | 9 + datman/src/main.rs | 3 + yama/src/xdef.rs | 345 ------------------------ yama/src/xpile.rs | 55 ---- yama/src/xpile/local_pile.rs | 413 ---------------------------- 7 files changed, 190 insertions(+), 1131 deletions(-) rename yama/Cargo.lock => Cargo.lock (69%) create mode 100644 Cargo.toml create mode 100644 datman/Cargo.toml create mode 100644 datman/src/main.rs delete mode 100644 yama/src/xdef.rs delete mode 100644 yama/src/xpile.rs delete mode 100644 yama/src/xpile/local_pile.rs diff --git a/yama/Cargo.lock b/Cargo.lock similarity index 69% rename from yama/Cargo.lock rename to Cargo.lock index 5d69dec..0096f5f 100644 --- a/yama/Cargo.lock +++ b/Cargo.lock @@ -1,39 +1,5 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -[[package]] -name = "aes-ctr" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2e5b0458ea3beae0d1d8c0f3946564f8e10f90646cf78c06b4351052058d1ee" -dependencies = [ - "aes-soft", - "aesni", - "ctr", - "stream-cipher", -] - -[[package]] -name = "aes-soft" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfd7e7ae3f9a1fb5c03b389fc6bb9a51400d0c13053f0dca698c832bfd893a0d" -dependencies = [ - "block-cipher-trait", - "byteorder", - "opaque-debug", -] - -[[package]] -name = "aesni" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f70a6b5f971e473091ab7cfb5ffac6cde81666c4556751d8d5620ead8abf100" -dependencies = [ - "block-cipher-trait", - "opaque-debug", - "stream-cipher", -] - [[package]] name = "ahash" version = "0.4.7" @@ -42,35 +8,30 @@ checksum = "739f4a8db6605981345c5654f3a85b056ce52f37a39d34da03f25bf2151ea16e" [[package]] name = "aho-corasick" -version = "0.7.10" +version = "0.7.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8716408b8bc624ed7f65d223ddb9ac2d044c0547b6fa4b0d554f3a9540496ada" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" dependencies = [ "memchr", ] [[package]] name = "anyhow" -version = "1.0.36" +version = "1.0.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68803225a7b13e47191bab76f2687382b60d259e8cf37f6e1893658b84bb9479" +checksum = "15af2628f6890fe2609a3b91bef4c83450512802e59489f9c1cb1fa5df064a61" [[package]] name = "atty" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1803c647a3ec87095e7ae7acfca019e98de5ec9a7d01343f611cf3152ed71a90" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ + "hermit-abi", "libc", "winapi", ] -[[package]] -name = "autocfg" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" - [[package]] name = "autocfg" version = "1.0.1" @@ -85,46 +46,27 @@ checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" [[package]] name = "blake" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "200c5e8f05f9bf644dfcfde5d273bca1514e4283fd542500e755bbf7cee70b83" +checksum = "f76cff23583935d01f1d259e546bf988450648a644da066de24717fdd015aa81" dependencies = [ "gcc", "libc", ] -[[package]] -name = "block-cipher-trait" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c924d49bd09e7c06003acda26cd9742e796e34282ec6c1189404dee0c1f4774" -dependencies = [ - "generic-array", -] - [[package]] name = "byteorder" -version = "1.3.2" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5" - -[[package]] -name = "c2-chacha" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "214238caa1bf3a496ec3392968969cab8549f96ff30652c9e56885329315f6bb" -dependencies = [ - "ppv-lite86", -] +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "cc" -version = "1.0.48" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f52a465a666ca3d838ebbf08b241383421412fe7ebb463527bba275526d89f76" +checksum = "4a72c244c1ff497a746a7e1fb3d14bd08420ecda70c8f25c7112f2781652d787" dependencies = [ "jobserver", - "num_cpus", ] [[package]] @@ -186,27 +128,19 @@ dependencies = [ [[package]] name = "console" -version = "0.11.3" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c0994e656bba7b922d8dd1245db90672ffb701e684e45be58f20719d69abc5a" +checksum = "3993e6445baa160675931ec041a5e03ca84b9c6e32a056150d3aa2bdda0a1f45" dependencies = [ "encode_unicode", "lazy_static", "libc", "regex", "terminal_size", - "termios", "unicode-width", "winapi", - "winapi-util", ] -[[package]] -name = "const_fn" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd51eab21ab4fd6a3bf889e2d0958c0a6e3a61ad04260325e919e652a2a62826" - [[package]] name = "crc32fast" version = "1.2.1" @@ -218,21 +152,22 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.4.0" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acec9a3b0b3559f15aee4f90746c4e5e293b701c0f7d3925d24e01645267b68c" +checksum = "b153fe7cbef478c567df0f972e02e6d736db11affe43dfc9c56a9374d1adfb87" dependencies = [ - "crossbeam-utils 0.7.0", + "crossbeam-utils 0.7.2", + "maybe-uninit", ] [[package]] name = "crossbeam-channel" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775" +checksum = "06ed27e177f16d65f0f0c22a213e17c696ace5dd64b14258b52f9417ccb52db4" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils 0.8.1", + "crossbeam-utils 0.8.5", ] [[package]] @@ -243,18 +178,17 @@ checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9" dependencies = [ "cfg-if 1.0.0", "crossbeam-epoch", - "crossbeam-utils 0.8.1", + "crossbeam-utils 0.8.5", ] [[package]] name = "crossbeam-epoch" -version = "0.9.1" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1aaa739f95311c2c7887a76863f500026092fb1dce0161dab577e559ef3569d" +checksum = "4ec02e091aa634e2c3ada4a392989e7c3116673ef0ac5b72232439094d73b7fd" dependencies = [ "cfg-if 1.0.0", - "const_fn", - "crossbeam-utils 0.8.1", + "crossbeam-utils 0.8.5", "lazy_static", "memoffset", "scopeguard", @@ -262,35 +196,28 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.7.0" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce446db02cdc3165b94ae73111e570793400d0794e46125cc4056c81cbb039f4" +checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" dependencies = [ - "autocfg 0.1.7", + "autocfg", "cfg-if 0.1.10", "lazy_static", ] [[package]] name = "crossbeam-utils" -version = "0.8.1" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02d96d1e189ef58269ebe5b97953da3274d83a93af647c2ddd6f9dab28cedb8d" +checksum = "d82cfc11ce7f2c3faef78d8a684447b40d503d9681acebed6cb728d45940c4db" dependencies = [ - "autocfg 1.0.1", "cfg-if 1.0.0", "lazy_static", ] [[package]] -name = "ctr" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "022cd691704491df67d25d006fe8eca083098253c4d43516c2206479c58c6736" -dependencies = [ - "block-cipher-trait", - "stream-cipher", -] +name = "datman" +version = "0.1.0" [[package]] name = "dirs-next" @@ -304,9 +231,9 @@ dependencies = [ [[package]] name = "dirs-sys-next" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99de365f605554ae33f115102a02057d4fc18b01f3284d6870be0938743cfe7d" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" dependencies = [ "libc", "redox_users", @@ -352,13 +279,9 @@ checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" [[package]] name = "fastcdc" -version = "1.0.2" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e10bacfbef04c5c9c89635a18ccb6de1488d96e65393cb9b545747da1fa3a0c" -dependencies = [ - "aes-ctr", - "byteorder", -] +checksum = "5afa29be46b12c8c380b997def8d1ac77c2665da93eb0a768fab0bf4db79333f" [[package]] name = "fs2" @@ -386,23 +309,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2" [[package]] -name = "generic-array" -version = "0.12.3" +name = "getrandom" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" dependencies = [ - "typenum", + "cfg-if 1.0.0", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", ] [[package]] name = "getrandom" -version = "0.1.13" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7db7ca94ed4cd01190ceee0d8a8052f08a247aa1b469a7f68c6a3b71afcf407" +checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "libc", - "wasi 0.7.0", + "wasi 0.10.0+wasi-snapshot-preview1", ] [[package]] @@ -431,18 +356,18 @@ dependencies = [ [[package]] name = "heck" -version = "0.3.1" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" dependencies = [ "unicode-segmentation", ] [[package]] name = "hermit-abi" -version = "0.1.3" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "307c3c9f937f38e3534b1d6447ecf090cafcc9744e4a6360e8b037b2cf5af120" +checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c" dependencies = [ "libc", ] @@ -458,11 +383,11 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.6.1" +version = "1.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb1fa934250de4de8aef298d81c729a7d33d8c239daa3a7575e6b92bfc7313b" +checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3" dependencies = [ - "autocfg 1.0.1", + "autocfg", "hashbrown", ] @@ -498,13 +423,11 @@ dependencies = [ [[package]] name = "jobserver" -version = "0.1.17" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2b1d42ef453b30b7387e113da1c83ab1605d90c5b4e0eb8e96d016ed3b8c160" +checksum = "972f5ae5d1cb9c6ae417789196c803205313edde988685da5e3aae0827b9e7fd" dependencies = [ - "getrandom", "libc", - "log", ] [[package]] @@ -515,9 +438,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.81" +version = "0.2.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1482821306169ec4d07f6aca392a4681f66c75c9918aa49641a2595db64053cb" +checksum = "12b8adadd720df158f4d70dfe7ccc6adb0472d7c55ca83445f6a5ab3e36f8fb6" [[package]] name = "libsodium-sys" @@ -540,63 +463,43 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "libssh2-sys" -version = "0.2.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fcd5a428a31cbbfe059812d74f4b6cd3b9b7426c2bdaec56993c5365da1c328" -dependencies = [ - "cc", - "libc", - "libz-sys", - "openssl-sys", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "libz-sys" -version = "1.0.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eb5e43362e38e2bca2fd5f5134c4d4564a23a5c28e9b95411652021a8675ebe" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "lock_api" -version = "0.4.2" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd96ffd135b2fd7b973ac026d28085defbe8983df057ced3eb4f2130b0831312" +checksum = "0382880606dff6d15c9476c416d18690b72742aa7b605bb6dd6ec9030fbf07eb" dependencies = [ "scopeguard", ] [[package]] name = "log" -version = "0.4.11" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", ] [[package]] -name = "memchr" -version = "2.3.4" +name = "maybe-uninit" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" +checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" + +[[package]] +name = "memchr" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc" [[package]] name = "memoffset" -version = "0.6.1" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87" +checksum = "59accc507f1338036a0477ef61afdae33cde60840f4dfe481319ce3ad116ddf9" dependencies = [ - "autocfg 1.0.1", + "autocfg", ] [[package]] @@ -630,7 +533,7 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" dependencies = [ - "autocfg 1.0.1", + "autocfg", "num-traits", ] @@ -640,14 +543,14 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" dependencies = [ - "autocfg 1.0.1", + "autocfg", ] [[package]] name = "num_cpus" -version = "1.11.1" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76dac5ed2a876980778b8b85f75a71b6cbf0db0b1232ee12f826bccb00d09d72" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" dependencies = [ "hermit-abi", "libc", @@ -659,25 +562,6 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a" -[[package]] -name = "opaque-debug" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" - -[[package]] -name = "openssl-sys" -version = "0.9.53" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "465d16ae7fc0e313318f7de5cecf57b2fbe7511fd213978b457e1c96ff46736f" -dependencies = [ - "autocfg 0.1.7", - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "os_str_bytes" version = "2.4.0" @@ -697,9 +581,9 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.8.1" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7c6d9b8427445284a09c55be860a15855ab580a417ccad9da88f5a06787ced0" +checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018" dependencies = [ "cfg-if 1.0.0", "instant", @@ -711,15 +595,15 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.17" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05da548ad6865900e60eaba7f589cc0783590a92e940c26953ff81ddbab2d677" +checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" [[package]] name = "ppv-lite86" -version = "0.2.6" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b" +checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" [[package]] name = "proc-macro-error" @@ -747,9 +631,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.7" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0319972dcae462681daf4da1adeeaa066e3ebd29c69be96c6abb1259d2ee2bcc" +checksum = "f0d8caf72986c1a598726adc988bb5984792ef84f5ee5aa50209145ee8077038" dependencies = [ "unicode-xid", ] @@ -762,20 +646,20 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quote" -version = "1.0.2" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "053a8c8bcc71fcce321828dc897a98ab9760bef03a4fc36693c231e5b3216cfe" +checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" dependencies = [ "proc-macro2", ] [[package]] name = "rand" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ae1b169243eaf61759b8475a998f0a385e42042370f3a7dbaf35246eacc8412" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" dependencies = [ - "getrandom", + "getrandom 0.1.16", "libc", "rand_chacha", "rand_core", @@ -784,11 +668,11 @@ dependencies = [ [[package]] name = "rand_chacha" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03a2a90da8c7523f554344f921aa97283eadf6ac484a6d2a7d0212fa7f8d6853" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" dependencies = [ - "c2-chacha", + "ppv-lite86", "rand_core", ] @@ -798,7 +682,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" dependencies = [ - "getrandom", + "getrandom 0.1.16", ] [[package]] @@ -812,11 +696,11 @@ dependencies = [ [[package]] name = "rayon" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674" +checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90" dependencies = [ - "autocfg 1.0.1", + "autocfg", "crossbeam-deque", "either", "rayon-core", @@ -824,50 +708,52 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.9.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a" +checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e" dependencies = [ - "crossbeam-channel 0.5.0", + "crossbeam-channel 0.5.1", "crossbeam-deque", - "crossbeam-utils 0.8.1", + "crossbeam-utils 0.8.5", "lazy_static", "num_cpus", ] [[package]] name = "redox_syscall" -version = "0.1.56" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" +checksum = "742739e41cd49414de871ea5e549afb7e2a3ac77b589bcbebe8c82fab37147fc" +dependencies = [ + "bitflags", +] [[package]] name = "redox_users" -version = "0.3.5" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de0737333e7a9502c789a36d7c7fa6092a49895d4faa31ca5df163857ded2e9d" +checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64" dependencies = [ - "getrandom", + "getrandom 0.2.3", "redox_syscall", ] [[package]] name = "regex" -version = "1.3.7" +version = "1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6020f034922e3194c711b82a627453881bc4682166cabb07134a10c26ba7692" +checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" dependencies = [ "aho-corasick", "memchr", "regex-syntax", - "thread_local", ] [[package]] name = "regex-syntax" -version = "0.6.17" +version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fe5bd57d1d7414c6b5ed48563a2c855d995ff777729dcd91c369ec7fea395ae" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" [[package]] name = "rusqlite" @@ -913,9 +799,9 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "serde" -version = "1.0.104" +version = "1.0.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "414115f25f818d7dfccec8ee535d76949ae78584fc4f79a6f45a904bf8ab4449" +checksum = "ec7505abeacaec74ae4778d9d9328fe5a5d04253220a85c4ee022239fc996d03" dependencies = [ "serde_derive", ] @@ -931,15 +817,21 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.104" +version = "1.0.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "128f9e303a5a29922045a830221b8f78ec74a5f544944f3d5984f8ec3895ef64" +checksum = "963a7dbc9895aeac7ac90e74f34a5d5261828f79df35cbed41e10189d3804d43" dependencies = [ "proc-macro2", "quote", "syn", ] +[[package]] +name = "shell-escape" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45bb67a18fa91266cc7807181f62f9178a6873bfad7dc788c42e6430db40184f" + [[package]] name = "sled" version = "0.34.6" @@ -948,7 +840,7 @@ checksum = "1d0132f3e393bcb7390c60bb45769498cf4550bcb7a21d7f95c02b69f6362cdc" dependencies = [ "crc32fast", "crossbeam-epoch", - "crossbeam-utils 0.8.1", + "crossbeam-utils 0.8.5", "fs2", "fxhash", "libc", @@ -958,9 +850,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.5.1" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae524f056d7d770e174287294f562e95044c68e88dec909a00d2094805db9d75" +checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" [[package]] name = "sodiumoxide" @@ -973,34 +865,20 @@ dependencies = [ "serde", ] -[[package]] -name = "ssh2" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "875fc74ffc41049e306a247f4191dd9c2ebfb7c5e0a6e4ddfdccacaba732e8aa" -dependencies = [ - "bitflags", - "libc", - "libssh2-sys", -] - [[package]] name = "sshish" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64e1ee53c6d5fad4125a5aec2006c51b2713af3fbba8b5ae73ea094c59498869" +checksum = "47cb7a759a4596d259849125fd9d58aceaea76fa521fed2540b06967b82827f9" dependencies = [ - "ssh2", + "shell-escape", ] [[package]] -name = "stream-cipher" -version = "0.3.2" +name = "static_assertions" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8131256a5896cabcf5eb04f4d6dacbe1aefda854b0d9896e09cb58829ec5638c" -dependencies = [ - "generic-array", -] +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "strsim" @@ -1010,9 +888,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "1.0.12" +version = "1.0.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddc157159e2a7df58cd67b1cace10b8ed256a404fb0070593f137d8ba6bef4de" +checksum = "f71489ff30030d2ae598524f61326b902466f72a0fb1a8564c001cc63425bcc7" dependencies = [ "proc-macro2", "quote", @@ -1027,32 +905,23 @@ checksum = "af547b166dd1ea4b472165569fc456cfb6818116f854690b0ff205e636523dab" [[package]] name = "termcolor" -version = "1.1.0" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb6bfa289a4d7c5766392812c0a1f4c1ba45afa1ad47803c11e1f407d846d75f" +checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" dependencies = [ "winapi-util", ] [[package]] name = "terminal_size" -version = "0.1.12" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8038f95fc7a6f351163f4b964af631bd26c9e828f7db085f2a84aca56f70d13b" +checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df" dependencies = [ "libc", "winapi", ] -[[package]] -name = "termios" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f0fcee7b24a25675de40d5bb4de6e41b0df07bc9856295e7e2b3a3600c400c2" -dependencies = [ - "libc", -] - [[package]] name = "textwrap" version = "0.12.1" @@ -1064,33 +933,24 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.21" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "318234ffa22e0920fe9a40d7b8369b5f649d490980cf7aadcf1eb91594869b42" +checksum = "fa6f76457f59514c7eeb4e59d891395fab0b2fd1d40723ae737d64153392e9c6" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.21" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cae2447b6282786c3493999f40a9be2a6ad20cb8bd268b0a0dbf5a065535c0ab" +checksum = "8a36768c0fbf1bb15eca10defa29526bda730a2376c2ab4393ccfa16fb1a318d" dependencies = [ "proc-macro2", "quote", "syn", ] -[[package]] -name = "thread_local" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" -dependencies = [ - "lazy_static", -] - [[package]] name = "time" version = "0.1.44" @@ -1104,28 +964,24 @@ dependencies = [ [[package]] name = "toml" -version = "0.5.5" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01d1404644c8b12b16bfcffa4322403a91a451584daaaa7c28d3152e6cbc98cf" +checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" dependencies = [ "serde", ] [[package]] name = "twox-hash" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bfd5b7557925ce778ff9b9ef90e3ade34c524b5ff10e239c69a42d546d2af56" +checksum = "04f8ab788026715fa63b31960869617cba39117e520eb415b0139543e325ab59" dependencies = [ + "cfg-if 0.1.10", "rand", + "static_assertions", ] -[[package]] -name = "typenum" -version = "1.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d2783fe2d6b8c1101136184eb41be8b1ad379e4657050b8aaff0c79ee7575f9" - [[package]] name = "unicode-segmentation" version = "1.7.1" @@ -1134,15 +990,15 @@ checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796" [[package]] name = "unicode-width" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479" +checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" [[package]] name = "unicode-xid" -version = "0.2.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" [[package]] name = "users" @@ -1161,21 +1017,21 @@ checksum = "936e4b492acfd135421d8dca4b1aa80a7bfc26e702ef3af710e0752684df5372" [[package]] name = "vcpkg" -version = "0.2.8" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fc439f2794e98976c88a2a2dafce96b930fe8010b0a256b3c2199a773933168" +checksum = "025ce40a007e1907e58d5bc1a594def78e5573bb0b1160bc389634e8f12e4faa" [[package]] name = "vec_map" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" [[package]] name = "version_check" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" +checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" [[package]] name = "void" @@ -1185,9 +1041,9 @@ checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" [[package]] name = "wasi" -version = "0.7.0" +version = "0.9.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b89c3ce4ce14bdc6fb6beaf9ec7928ca331de5df7e5ea278375642a2f478570d" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" [[package]] name = "wasi" @@ -1197,9 +1053,9 @@ checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" [[package]] name = "winapi" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" dependencies = [ "winapi-i686-pc-windows-gnu", "winapi-x86_64-pc-windows-gnu", @@ -1235,8 +1091,8 @@ dependencies = [ "byteorder", "chrono", "clap", - "crossbeam-channel 0.4.0", - "crossbeam-utils 0.8.1", + "crossbeam-channel 0.4.4", + "crossbeam-utils 0.8.5", "env_logger", "fastcdc", "glob", @@ -1263,18 +1119,18 @@ dependencies = [ [[package]] name = "zstd" -version = "0.6.0+zstd.1.4.8" +version = "0.6.1+zstd.1.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e44664feba7f2f1a9f300c1f6157f2d1bfc3c15c6f3cf4beabf3f5abe9c237" +checksum = "5de55e77f798f205d8561b8fe2ef57abfb6e0ff2abe7fd3c089e119cdb5631a3" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "3.0.0+zstd.1.4.8" +version = "3.0.1+zstd.1.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9447afcd795693ad59918c7bbffe42fdd6e467d708f3537e3dc14dc598c573f" +checksum = "1387cabcd938127b30ce78c4bf00b30387dddf704e3f0881dbc4ff62b5566f8c" dependencies = [ "libc", "zstd-sys", @@ -1282,12 +1138,10 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "1.4.19+zstd.1.4.8" +version = "1.4.20+zstd.1.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec24a9273d24437afb8e71b16f3d9a5d569193cccdb7896213b59f552f387674" +checksum = "ebd5b733d7cf2d9447e2c3e76a5589b4f5e5ae065c22a2bc0b023cbc331b6c8e" dependencies = [ "cc", - "glob", - "itertools", "libc", ] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..9e4a9b8 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,6 @@ +[workspace] +members = [ + "yama", + "datman" +] + diff --git a/datman/Cargo.toml b/datman/Cargo.toml new file mode 100644 index 0000000..25453f4 --- /dev/null +++ b/datman/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "datman" +version = "0.1.0" +authors = ["Olivier 'reivilibre' "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/datman/src/main.rs b/datman/src/main.rs new file mode 100644 index 0000000..e7a11a9 --- /dev/null +++ b/datman/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, world!"); +} diff --git a/yama/src/xdef.rs b/yama/src/xdef.rs deleted file mode 100644 index b17bd2f..0000000 --- a/yama/src/xdef.rs +++ /dev/null @@ -1,345 +0,0 @@ -use glob::{MatchOptions, Pattern}; -use serde::{Deserialize, Serialize}; -use std::collections::{BTreeMap, HashSet}; -use std::convert::TryFrom; -use std::fs::File; -use std::io::{BufRead, BufReader}; -use std::path::Path; - -pub type ChunkId = [u8; 32]; -pub type XXHash = u64; - -pub const XXH64_SEED: u64 = 424242; - -pub type YamaResult = Result>; -// yet unused: pub type YamaResultSend = Result>; - -//pub type YamaResultA = Result>; - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PointerData { - pub chunk_ref: RecursiveChunkRef, - pub parent_pointer: Option, - pub uid_lookup: BTreeMap, - pub gid_lookup: BTreeMap, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct RecursiveChunkRef { - pub chunk_id: ChunkId, - pub depth: u32, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct TreeNode { - #[serde(rename = "n")] - pub name: String, - //#[serde(flatten)] - #[serde(rename = "c")] - pub content: TreeNodeContent, -} - -impl TreeNode { - /// whether the metadata invalidates these two nodes being equal, thus requiring a backup - pub fn metadata_invalidates(&self, other: &TreeNode, check_name: bool) -> bool { - if check_name { - self.name != other.name || self.content.metadata_invalidates(&other.content) - } else { - self.content.metadata_invalidates(&other.content) - } - } - - /// Guarantees consistent visit order. - pub fn visit_mut( - &mut self, - visitor: &mut F, - path_prefix: &str, - skip_components: u32, - ) -> YamaResult<()> - where - F: FnMut(&mut Self, &str) -> YamaResult<()>, - { - let mut my_path_buf = String::new(); - my_path_buf.push_str(path_prefix); - - if skip_components == 0 { - if !my_path_buf.is_empty() { - my_path_buf.push('/'); - } - my_path_buf.push_str(&self.name); - } - - visitor(self, &my_path_buf)?; - - if let TreeNodeContent::Directory { children, .. } = &mut self.content { - for child in children.iter_mut() { - let new_skip = if skip_components > 0 { - skip_components - 1 - } else { - 0 - }; - child.visit_mut(visitor, &my_path_buf, new_skip)?; - } - } - Ok(()) - } -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(untagged)] -pub enum TreeNodeContent { - NormalFile { - // modification time in ms - #[serde(rename = "m")] - mtime: u64, - #[serde(flatten)] - #[serde(rename = "o")] - ownership: FilesystemOwnership, - #[serde(flatten)] - #[serde(rename = "p")] - permissions: FilesystemPermissions, - // TODO size: u64 or not - // can perhaps cache chunk-wise (but not sure.) - #[serde(rename = "c")] - content: RecursiveChunkRef, - }, - Directory { - #[serde(flatten)] - #[serde(rename = "o")] - ownership: FilesystemOwnership, - #[serde(flatten)] - #[serde(rename = "p")] - permissions: FilesystemPermissions, - #[serde(rename = "C")] - children: Vec, - }, - SymbolicLink { - #[serde(flatten)] - #[serde(rename = "o")] - ownership: FilesystemOwnership, - #[serde(rename = "t")] - target: String, - }, - // TODO is there any other kind of file we need to store? - Deleted, -} - -#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct FilesystemOwnership { - pub uid: u16, - pub gid: u16, -} - -#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct FilesystemPermissions { - pub mode: u32, -} - -impl TreeNodeContent { - pub fn metadata_invalidates(&self, other: &TreeNodeContent) -> bool { - match self { - TreeNodeContent::NormalFile { - mtime, - ownership, - permissions, - .. - } => { - if let TreeNodeContent::NormalFile { - mtime: other_mtime, - ownership: other_ownership, - permissions: other_permissions, - .. - } = other - { - mtime != other_mtime - || ownership != other_ownership - || permissions != other_permissions - } else { - true - } - } - TreeNodeContent::Directory { - ownership, - permissions, - children, - } => { - if let TreeNodeContent::Directory { - ownership: other_ownership, - permissions: other_permissions, - children: other_children, - } = other - { - if ownership != other_ownership || permissions != other_permissions { - return true; - } - children - .iter() - .zip(other_children.iter()) - .any(|(left, right)| left.metadata_invalidates(right, true)) - } else { - true - } - } - TreeNodeContent::SymbolicLink { ownership, target } => { - if let TreeNodeContent::SymbolicLink { - ownership: other_ownership, - target: other_target, - } = other - { - ownership != other_ownership || target != other_target - } else { - true - } - } - TreeNodeContent::Deleted => { - // unreachable - false - } - } - } -} - -pub struct Exclusions { - pub rules: Vec, -} - -impl Exclusions { - pub fn load(path: &Path) -> YamaResult { - let file = File::open(path)?; - let bufreader = BufReader::new(file); - let mut rules = Vec::new(); - for line in bufreader.lines() { - let line = line?; - let trim_line = line.trim(); - if !trim_line.is_empty() { - rules.push(ExclusionRule::try_from(trim_line)?); - } - } - - Ok(Exclusions { rules }) - } -} - -pub struct ExclusionRule { - pub glob: Pattern, - pub effect: Option, - pub negated: bool, -} - -impl TryFrom<&str> for ExclusionRule { - type Error = String; - - fn try_from(value: &str) -> Result { - let mut effect = None; - let mut negated = false; - let mut glob_str = value; - - let split: Vec<&str> = value.splitn(2, "?⇒").collect(); - if split.len() == 2 { - // this is a conditional rule - glob_str = split[0].trim(); - effect = Some(split[0].trim().to_owned()); - } - if glob_str.starts_with("!") { - negated = true; - glob_str = &glob_str[1..]; - } - Ok(ExclusionRule { - glob: Pattern::new(glob_str).map_err(|e| e.to_string())?, - effect, - negated, - }) - } -} - -impl Exclusions { - pub fn apply_to(&self, node: &mut TreeNode) -> YamaResult<()> { - self.apply_to_rec(node, "", &mut HashSet::new()) - } - - fn apply_to_rec( - &self, - node: &mut TreeNode, - path_rel: &str, - exclusions: &mut HashSet, - ) -> YamaResult<()> { - let match_options = MatchOptions { - case_sensitive: true, - require_literal_separator: true, - require_literal_leading_dot: false, - }; - - if let TreeNodeContent::Directory { - ref mut children, .. - } = node.content - { - let mut child_pathrel = String::new(); - for rule in self.rules.iter() { - for child in children.iter() { - child_pathrel.clear(); - child_pathrel.push_str(path_rel); - child_pathrel.push('/'); - child_pathrel.push_str(&child.name); - - if rule.glob.matches_with(&child_pathrel, match_options) { - if let Some(relative_effect) = &rule.effect { - let mut path_pieces: Vec<&str> = - child_pathrel.split("/").skip(1).collect(); - let relative_pieces = relative_effect.split("/"); - - for (idx, relpiece) in relative_pieces.enumerate() { - match relpiece { - "" => { - if idx == 0 { - // this is an absolute path - // doubt we will use this feature much :/ - path_pieces.clear(); - path_pieces.push(""); - } - } - "." => { /* nop */ } - ".." => { - if path_pieces.len() > 1 { - path_pieces.pop(); - } - } - other => { - path_pieces.push(other); - } - } - } - child_pathrel = path_pieces.join("/"); - } - - if rule.negated { - exclusions.remove(&child_pathrel); - } else { - exclusions.insert(child_pathrel.clone()); - } - } - } - } - - // filter out excluded children - children.retain(|child| { - child_pathrel.clear(); - child_pathrel.push_str(path_rel); - child_pathrel.push('/'); - child_pathrel.push_str(&child.name); - - !exclusions.contains(&child_pathrel) - }); - - for child in children.iter_mut() { - child_pathrel.clear(); - child_pathrel.push_str(path_rel); - child_pathrel.push('/'); - child_pathrel.push_str(&child.name); - - self.apply_to_rec(child, &child_pathrel, exclusions)?; - } - } - - Ok(()) - } -} diff --git a/yama/src/xpile.rs b/yama/src/xpile.rs deleted file mode 100644 index c2264af..0000000 --- a/yama/src/xpile.rs +++ /dev/null @@ -1,55 +0,0 @@ -use async_trait::async_trait; - -use crate::def::{ChunkId, PointerData, XXHash, YamaResult}; - -mod interface { - use std::borrow::Cow; - - use crate::def::{ChunkId, PointerData}; - - pub enum PileProcessorRequest<'a> { - GetDictionary, - GetChunk { - chunk_id: ChunkId, - }, - PutChunk { - chunk_id: ChunkId, - chunk_data: Cow<'a, [u8]>, - }, - /* DelChunk { - chunk_id: - }, */ - XXHashChunk { - chunk_id: ChunkId, - }, - GetPointer { - pointer_id: Cow<'a, str>, - }, - PutPointer { - pointer_id: Cow<'a, str>, - pointer_data: PointerData, - }, - // DelPointer, - ListPointers, - } -} - -/// should be Cloneable to get another handle to the xpile. -#[async_trait] -pub trait Pile: Send + Sync + Clone { - /// Returns another handle to the Pile. - async fn get_dictionary(&self) -> YamaResult>; - async fn get_chunk(&self, chunk_id: &ChunkId) -> YamaResult>>; - async fn put_chunk(&self, chunk_id: &ChunkId, data: &[u8], xxhash: &[u8]) -> YamaResult<()>; - async fn get_xxhash_chunk(&self, chunk_id: &ChunkId) -> YamaResult>>; - async fn get_pointer(&self, pointer_id: &str) -> YamaResult>; - async fn put_pointer(&self, pointer_id: &str, pointer_data: PointerData) -> YamaResult<()>; - async fn list_pointers(&self) -> YamaResult>; -} - -#[async_trait] -pub trait RawPile: Send + Sync + Clone { - async fn -} - -pub mod local_pile; diff --git a/yama/src/xpile/local_pile.rs b/yama/src/xpile/local_pile.rs deleted file mode 100644 index 142ff2b..0000000 --- a/yama/src/xpile/local_pile.rs +++ /dev/null @@ -1,413 +0,0 @@ -use clap::crate_version; -use log::{info, trace, warn}; -use std::fs::File; -use std::hash::Hasher; -use std::io::{Read, Write}; -use std::ops::Deref; -use std::path::{Path, PathBuf}; -use std::sync::mpsc::{RecvTimeoutError, SyncSender}; -use std::sync::{mpsc, Arc, Condvar, Mutex}; -use std::time::Duration; -use std::{mem, thread}; - -use lmdb::{Cursor, Database, DatabaseFlags, Environment, EnvironmentFlags, Transaction}; - -use async_trait::async_trait; - -use crate::def::{ChunkId, PointerData, XXHash, YamaResult, XXH64_SEED}; -use crate::xpile::Pile; -use serde::{Deserialize, Serialize}; -use std::thread::JoinHandle; - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct LocalPileManifest { - max_lmdb_size: usize, - yama_version: String, -} - -impl Default for LocalPileManifest { - fn default() -> Self { - LocalPileManifest { - max_lmdb_size: 128 * 1024 * 1024 * 1024, - yama_version: String::from(crate_version!()), - } - } -} - -#[derive(Clone)] -pub struct LocalPile { - directory: PathBuf, - dictionary: Arc>, - pub(crate) lmdb_env: Arc, - pub(crate) lmdb_chunk_database: Database, - pub(crate) lmdb_pointer_database: Database, - lmdb_writing_thread_queue: Option>, - lmdb_writing_thread_handle: Option>>, -} - -impl Drop for LocalPile { - fn drop(&mut self) { - let mut opt_handle: Option>> = None; - self.lmdb_writing_thread_queue = None; - mem::swap(&mut self.lmdb_writing_thread_handle, &mut opt_handle); - if let Some(arc_join_handle) = opt_handle { - if let Ok(join_handle) = Arc::try_unwrap(arc_join_handle) { - //eprintln!("joining LMDB..."); - if let Err(_) = join_handle.join() { - warn!("LMDB writer thread had an error when joining."); - } - //eprintln!("joined LMDB..."); - } - } - } -} - -impl LocalPile { - pub fn create(dir: &Path) -> YamaResult<()> { - let zstd_path = dir.join("zstd.dict"); - eprintln!("Please add a Zstd dictionary to: {:?}", zstd_path); - - let manifest_path = dir.join("yama.toml"); - let manifest = LocalPileManifest::default(); - { - let bytes = toml::to_vec(&manifest).expect("toml to_vec"); - let mut file = File::create(manifest_path).expect("Failed to open yama manifest."); - file.write_all(&bytes)?; - // file automatically closed - } - - let lmdb_env = Arc::new( - Environment::new() - .set_max_dbs(8) - .set_flags(EnvironmentFlags::NO_SUB_DIR | EnvironmentFlags::NO_TLS) - .set_map_size(manifest.max_lmdb_size) - .open(&dir.join("yama.lmdb")) - .expect("Failed to open LMDB env"), - ); - - let _lmdb_chunk_database = lmdb_env - .create_db(Some("yama_chunks"), DatabaseFlags::empty()) - .expect("Failed to open yama_chunks"); - let _lmdb_pointer_database = lmdb_env - .create_db(Some("yama_pointers"), DatabaseFlags::empty()) - .expect("Failed to open yama_pointers"); - - //let lmdb_writing_thread_queue = - // make_lmdb_writer_thread(lmdb_env.clone(), lmdb_chunk_database, lmdb_pointer_database); - - Ok(()) - } - pub fn open(dir: &Path) -> YamaResult { - let zstd_path = dir.join("zstd.dict"); - if !zstd_path.exists() { - return Err("No Zstd dictionary".into()); - } - - let manifest_path = dir.join("yama.toml"); - if !manifest_path.exists() { - return Err("No Yama manifest".into()); - } - - let manifest = { - let mut file = File::open(manifest_path).expect("Failed to open yama manifest."); - let mut buf = Vec::new(); - file.read_to_end(&mut buf).expect("Failed to read buf."); - toml::from_slice::(&buf)? - }; - - let dictionary = Arc::new({ - let mut file = File::open(zstd_path)?; - let mut out = Vec::new(); - file.read_to_end(&mut out)?; - out - }); - - let lmdb_env = Arc::new( - Environment::new() - .set_max_dbs(8) - .set_flags(EnvironmentFlags::NO_SUB_DIR | EnvironmentFlags::NO_TLS) - .set_map_size(manifest.max_lmdb_size) - .open(&dir.join("yama.lmdb")) - .expect("Failed to open LMDB env"), - ); - - let lmdb_chunk_database = lmdb_env - .open_db(Some("yama_chunks")) - .expect("Failed to open yama_chunks"); - let lmdb_pointer_database = lmdb_env - .open_db(Some("yama_pointers")) - .expect("Failed to open yama_pointers"); - - let (writing_queue, join_handle) = - make_lmdb_writer_thread(lmdb_env.clone(), lmdb_chunk_database, lmdb_pointer_database); - - Ok(LocalPile { - directory: dir.to_owned(), - dictionary, - lmdb_env, - lmdb_chunk_database, - lmdb_pointer_database, - lmdb_writing_thread_queue: Some(writing_queue), - lmdb_writing_thread_handle: Some(Arc::new(join_handle)), - }) - } -} - -#[async_trait] -impl Pile for LocalPile { - fn clone_pile_handle(&self) -> Box { - Box::new(self.clone()) - } - - async fn get_dictionary(&self) -> YamaResult> { - Ok(self.dictionary.to_vec()) - } - - async fn get_chunk(&self, chunk_id: &ChunkId) -> YamaResult>> { - //eprintln!("get_chunk {}", chunkid_to_hex(chunk_id)); - - let lmdb_env = self.lmdb_env.clone(); - let chunk_db = self.lmdb_chunk_database; - let chunk_id = chunk_id.clone(); - let res: Result>, String> = async_std::task::spawn_blocking(move || { - let txn = lmdb_env.begin_ro_txn().map_err(|e| e.to_string())?; - match txn.get(chunk_db, &chunk_id) { - Ok(value) => Ok(Some(value.to_owned())), - Err(lmdb::Error::NotFound) => Ok(None), - Err(other) => Err(other.to_string()), - } - }) - .await; - - res.map_err(|e| e.into()) - } - - async fn put_chunk(&self, chunk_id: &ChunkId, data: Vec) -> YamaResult<()> { - //eprintln!("put_chunk {}", chunkid_to_hex(chunk_id)); - // TODO check matches hash - let sender = self - .lmdb_writing_thread_queue - .as_ref() - .expect("invalid LP ref") - .clone(); - let key = chunk_id.to_vec(); - - //self.lmdb_writing_thread_queue.send(); - - let res: Result<(), String> = async_std::task::spawn_blocking(move || { - let complete_flag = BackgroundCompletedFlag::new(); - sender - .send(LmdbWriteOp { - destination: LmdbDest::Chunk, - key, - value: data, - completion: complete_flag.clone(), - }) - .map_err(|e| e.to_string())?; - complete_flag.wait(); - Ok(()) - }) - .await; - res.map_err(|e| e.into()) - } - - async fn xxhash_chunk(&self, chunk_id: &ChunkId) -> YamaResult> { - //eprintln!("xxhash_chunk {}", chunkid_to_hex(chunk_id)); - let lmdb_env = self.lmdb_env.clone(); - let chunk_db = self.lmdb_chunk_database; - let chunk_id = chunk_id.clone(); - let dictionary = self.dictionary.clone(); - - async_std::task::spawn_blocking(move || { - let txn = lmdb_env.begin_ro_txn().map_err(|e| e.to_string())?; - match txn.get(chunk_db, &chunk_id) { - Ok(value) => { - // todo we might like to cache the decompressor - let mut decompressor = - zstd::block::Decompressor::with_dict(dictionary.deref().clone()); - let decompressed = decompressor - .decompress(value, 128 * 1024 * 1024) - .map_err(|e| e.to_string() + " (lp xxhash decompress)")?; - let mut hasher = twox_hash::XxHash64::with_seed(XXH64_SEED); - hasher.write(&decompressed); - Ok(Some(hasher.finish())) - } - Err(lmdb::Error::NotFound) => Ok(None), - Err(other) => Err(other.to_string()), - } - }) - .await - .map_err(|e| e.into()) - } - - async fn get_pointer(&self, pointer_id: &str) -> YamaResult> { - //eprintln!("get_pointer {:?}", pointer_id); - let lmdb_env = self.lmdb_env.clone(); - let pointer_db = self.lmdb_pointer_database; - let pointer_id = pointer_id.to_owned(); - let res: Result, String> = - async_std::task::spawn/*_blocking*/(async move { - let txn = lmdb_env.begin_ro_txn().map_err(|e| e.to_string())?; - match txn.get(pointer_db, &pointer_id.as_bytes()) { - Ok(value) => { - let pointer_data: PointerData = serde_cbor::from_slice(value) - .map_err(|e| e.to_string())?; - Ok(Some(pointer_data)) - } - Err(lmdb::Error::NotFound) => Ok(None), - Err(other) => Err(other.to_string()) - } - }) - .await; - - res.map_err(|e| e.into()) - } - - async fn put_pointer(&self, pointer_id: &str, pointer_data: PointerData) -> YamaResult<()> { - //eprintln!("put_pointer {:?}", pointer_id); - let sender = self - .lmdb_writing_thread_queue - .as_ref() - .expect("invalid LP ref") - .clone(); - let key = pointer_id.as_bytes().to_vec(); - let value = serde_cbor::ser::to_vec_packed(&pointer_data)?; - - //self.lmdb_writing_thread_queue.send(); - - let res: Result<(), String> = async_std::task::spawn/*_blocking*/(async move { - let complete_flag = BackgroundCompletedFlag::new(); - sender.send(LmdbWriteOp { - destination: LmdbDest::Pointer, - key, - value, - completion: complete_flag.clone() - }).map_err(|e| e.to_string())?; - complete_flag.wait(); - Ok(()) - }) - .await; - res.map_err(|e| e.into()) - } - - async fn list_pointers(&self) -> YamaResult> { - //eprintln!("list_pointers"); - let lmdb_env = self.lmdb_env.clone(); - let pointer_db = self.lmdb_pointer_database; - - let res: Result, String> = async_std::task::spawn/*_blocking*/(async move { - let mut pointer_keys: Vec = Vec::new(); - let txn = lmdb_env.begin_ro_txn().map_err(|e| e.to_string())?; - let mut cur = txn.open_ro_cursor(pointer_db).map_err(|e| e.to_string())?; - for item in cur.iter_start() { - let (key, _value) = item.map_err(|e| e.to_string())?; - let key_str: String = String::from_utf8(key.to_vec()) - .map_err(|e| e.to_string())?; - pointer_keys.push(key_str); - } - - Ok(pointer_keys) - }) - .await; - - res.map_err(|e| e.into()) - } -} - -enum LmdbDest { - Pointer, - Chunk, -} - -#[derive(Clone, Debug)] -pub struct BackgroundCompletedFlag(Arc<(Mutex, Condvar)>); - -impl BackgroundCompletedFlag { - pub fn new() -> Self { - BackgroundCompletedFlag(Arc::new((Mutex::new(false), Condvar::new()))) - } - - pub fn wait(&self) { - let (mutex, condvar) = self.0.deref(); - let mut guard = mutex.lock().expect("poison"); - while !guard.deref() { - guard = condvar.wait(guard).expect("poison"); - } - } - - pub fn finish(&self) { - let (mutex, condvar) = self.0.deref(); - let mut guard = mutex.lock().expect("poison"); - *guard = true; - condvar.notify_all(); - } -} - -struct LmdbWriteOp { - destination: LmdbDest, - key: Vec, - value: Vec, - completion: BackgroundCompletedFlag, -} - -fn make_lmdb_writer_thread( - env: Arc, - chunk_db: Database, - pointer_db: Database, -) -> (SyncSender, JoinHandle<()>) { - let (send, recv) = mpsc::sync_channel(256); - let join_handle = thread::spawn(move || { - let mut pending = 0; - let mut txn = env - .begin_rw_txn() - .expect("Can't start LMDB writer transaction."); - loop { - match recv.recv_timeout(Duration::from_secs(60)) { - Ok(LmdbWriteOp { - destination, - key, - value, - completion, - }) => { - { - let mut cur = txn - .open_rw_cursor(match destination { - LmdbDest::Pointer => pointer_db, - LmdbDest::Chunk => chunk_db, - }) - .expect("Can't open LMDB r/w cursor."); - cur.put(&key, &value, Default::default()) - .expect("Unable to put kv pair"); - pending += 1; - } - - if pending > 128 { - trace!("committing to LMDB due to high pending count"); - txn.commit().expect("Failed LMDB commit"); - txn = env - .begin_rw_txn() - .expect("Can't start LMDB writer transaction."); - pending = 0; - } - - completion.finish(); - } - Err(err) => { - if pending > 0 { - trace!("committing to LMDB due to inactivity or disconnect"); - txn.commit().expect("Failed LMDB commit"); - txn = env - .begin_rw_txn() - .expect("Can't start LMDB writer transaction."); - pending = 0; - } - if err == RecvTimeoutError::Disconnected { - info!("LMDB disconnected"); - break; - } - } - } - } - }); - (send, join_handle) -}