From 6948ba3821a888187a26849ad3bf76dcb2755594 Mon Sep 17 00:00:00 2001 From: Olivier Date: Tue, 21 Apr 2020 12:11:17 +0100 Subject: [PATCH] Works for one-off and incremental store/extract --- .gitignore | 9 + Cargo.lock | 1230 +++++++++++++++++++++++++++++ Cargo.toml | 30 + README.md | 94 +++ example_zstd.dict | Bin 0 -> 112640 bytes src/chunking.rs | 251 ++++++ src/def.rs | 313 ++++++++ src/main.rs | 151 ++++ src/operations.rs | 327 ++++++++ src/operations/chunking_flow.rs | 339 ++++++++ src/operations/extraction_flow.rs | 223 ++++++ src/pile.rs | 53 ++ src/pile/local_pile.rs | 405 ++++++++++ src/tree.rs | 257 ++++++ src/util.rs | 0 15 files changed, 3682 insertions(+) create mode 100644 .gitignore create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 README.md create mode 100644 example_zstd.dict create mode 100644 src/chunking.rs create mode 100644 src/def.rs create mode 100644 src/main.rs create mode 100644 src/operations.rs create mode 100644 src/operations/chunking_flow.rs create mode 100644 src/operations/extraction_flow.rs create mode 100644 src/pile.rs create mode 100644 src/pile/local_pile.rs create mode 100644 src/tree.rs create mode 100644 src/util.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..84113ae --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +/target +**/*.rs.bk + +/.idea +/yama.iml +/.project +/.gdb_history + + diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..2cee585 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,1230 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "aes-ctr" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "aes-soft 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "aesni 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "ctr 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "stream-cipher 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "aes-soft" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "block-cipher-trait 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "opaque-debug 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "aesni" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "block-cipher-trait 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", + "opaque-debug 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "stream-cipher 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "aho-corasick" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "ansi_term" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "async-std" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "async-task 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "broadcaster 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-channel 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-deque 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-io 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-timer 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "kv-log-macro 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)", + "mio-uds 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)", + "once_cell 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "pin-project-lite 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "pin-utils 0.1.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)", + "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "async-task" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "async-trait" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.12 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "atty" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "autocfg" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "blake" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "gcc 0.3.55 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "block-cipher-trait" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "generic-array 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "broadcaster" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures-channel-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-core-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-sink-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-util-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "byteorder" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "c2-chacha" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "cc" +version = "1.0.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "jobserver 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "clap" +version = "2.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-width 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crossbeam" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-channel 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-deque 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-epoch 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-queue 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crossbeam-channel" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crossbeam-deque" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crossbeam-epoch 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "memoffset 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crossbeam-queue" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "crossbeam-utils" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "ctr" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "block-cipher-trait 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", + "stream-cipher 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "env_logger" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)", + "humantime 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.7 (registry+https://github.com/rust-lang/crates.io-index)", + "termcolor 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "fastcdc" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "aes-ctr 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "fuchsia-zircon" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "futures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures-channel 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-executor 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-io 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-sink 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-task 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-util 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "futures-channel" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-sink 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "futures-channel-preview" +version = "0.3.0-alpha.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures-core-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-sink-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "futures-core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "futures-core-preview" +version = "0.3.0-alpha.19" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "futures-executor" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-task 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-util 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "futures-io" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "futures-macro" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro-hack 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.12 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "futures-sink" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "futures-sink-preview" +version = "0.3.0-alpha.19" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "futures-task" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "futures-timer" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "futures-util" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures-channel 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-io 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-macro 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-sink 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-task 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "pin-utils 0.1.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-hack 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-nested 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "futures-util-preview" +version = "0.3.0-alpha.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures-core-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-sink-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)", + "pin-utils 0.1.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)", + "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "gcc" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "generic-array" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "typenum 1.11.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "getrandom" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "wasi 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "hermit-abi" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "humantime" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "quick-error 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "iovec" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "jobserver" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "kernel32-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "kv-log-macro" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "libc" +version = "0.2.66" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "libssh2-sys" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cc 1.0.48 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "libz-sys 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", + "openssl-sys 0.9.53 (registry+https://github.com/rust-lang/crates.io-index)", + "pkg-config 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)", + "vcpkg 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "libz-sys" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cc 1.0.48 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "pkg-config 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)", + "vcpkg 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "lmdb-rkv" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "lmdb-rkv-sys 0.9.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "lmdb-rkv-sys" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cc 1.0.48 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "pkg-config 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "lock_api" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "log" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "maybe-uninit" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "memchr" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "memoffset" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "mio" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", + "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "mio-uds" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "miow" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "net2" +version = "0.2.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "nix" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.48 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "num_cpus" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "hermit-abi 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "once_cell" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "opaque-debug" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "openssl-sys" +version = "0.9.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.48 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "pkg-config 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)", + "vcpkg 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "parking_lot" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lock_api 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot_core 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "parking_lot_core" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "pin-project-lite" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "pin-utils" +version = "0.1.0-alpha.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "pkg-config" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "ppv-lite86" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "proc-macro-hack" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.12 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "proc-macro-nested" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "proc-macro2" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "quote" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_chacha 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_hc 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_chacha" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "redox_syscall" +version = "0.1.56" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "regex" +version = "1.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "aho-corasick 0.7.10 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.6.17 (registry+https://github.com/rust-lang/crates.io-index)", + "thread_local 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "regex-syntax" +version = "0.6.17" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "scopeguard" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "serde" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "serde_derive 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "serde_cbor" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "serde_derive" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.12 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "slab" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "smallvec" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "ssh2" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "libssh2-sys 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "sshish" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ssh2 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "stream-cipher" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "generic-array 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "syn" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "termcolor" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi-util 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-width 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "thread_local" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "toml" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "twox-hash" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "typenum" +version = "1.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unicode-width" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unicode-xid" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "users" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "vcpkg" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "vec_map" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "wasi" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "ws2_32-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "yama" +version = "0.0.1" +dependencies = [ + "async-std 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "async-trait 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", + "blake 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", + "env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "fastcdc 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "glob 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lmdb-rkv 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "nix 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_cbor 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", + "sshish 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "toml 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", + "twox-hash 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", + "users 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", + "zstd 0.5.1+zstd.1.4.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "zstd" +version = "0.5.1+zstd.1.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "zstd-safe 2.0.3+zstd.1.4.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "zstd-safe" +version = "2.0.3+zstd.1.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "zstd-sys 1.4.15+zstd.1.4.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "zstd-sys" +version = "1.4.15+zstd.1.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cc 1.0.48 (registry+https://github.com/rust-lang/crates.io-index)", + "glob 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[metadata] +"checksum aes-ctr 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d2e5b0458ea3beae0d1d8c0f3946564f8e10f90646cf78c06b4351052058d1ee" +"checksum aes-soft 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "cfd7e7ae3f9a1fb5c03b389fc6bb9a51400d0c13053f0dca698c832bfd893a0d" +"checksum aesni 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2f70a6b5f971e473091ab7cfb5ffac6cde81666c4556751d8d5620ead8abf100" +"checksum aho-corasick 0.7.10 (registry+https://github.com/rust-lang/crates.io-index)" = "8716408b8bc624ed7f65d223ddb9ac2d044c0547b6fa4b0d554f3a9540496ada" +"checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +"checksum async-std 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0bf6039b315300e057d198b9d3ab92ee029e31c759b7f1afae538145e6f18a3e" +"checksum async-task 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d22dc86693d375d2733b536fd8914bea0fa93adf4b1e6bcbd9c7c500cb62d920" +"checksum async-trait 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)" = "c8df72488e87761e772f14ae0c2480396810e51b2c2ade912f97f0f7e5b95e3c" +"checksum atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)" = "1803c647a3ec87095e7ae7acfca019e98de5ec9a7d01343f611cf3152ed71a90" +"checksum autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" +"checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +"checksum blake 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "200c5e8f05f9bf644dfcfde5d273bca1514e4283fd542500e755bbf7cee70b83" +"checksum block-cipher-trait 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1c924d49bd09e7c06003acda26cd9742e796e34282ec6c1189404dee0c1f4774" +"checksum broadcaster 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "07a1446420a56f1030271649ba0da46d23239b3a68c73591cea5247f15a788a0" +"checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5" +"checksum c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "214238caa1bf3a496ec3392968969cab8549f96ff30652c9e56885329315f6bb" +"checksum cc 1.0.48 (registry+https://github.com/rust-lang/crates.io-index)" = "f52a465a666ca3d838ebbf08b241383421412fe7ebb463527bba275526d89f76" +"checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +"checksum clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5067f5bb2d80ef5d68b4c87db81601f0b75bca627bc2ef76b141d7b846a3c6d9" +"checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +"checksum crossbeam 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "69323bff1fb41c635347b8ead484a5ca6c3f11914d784170b158d8449ab07f8e" +"checksum crossbeam-channel 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "acec9a3b0b3559f15aee4f90746c4e5e293b701c0f7d3925d24e01645267b68c" +"checksum crossbeam-deque 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c3aa945d63861bfe624b55d153a39684da1e8c0bc8fba932f7ee3a3c16cea3ca" +"checksum crossbeam-epoch 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5064ebdbf05ce3cb95e45c8b086f72263f4166b29b97f6baff7ef7fe047b55ac" +"checksum crossbeam-queue 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c695eeca1e7173472a32221542ae469b3e9aac3a4fc81f7696bcad82029493db" +"checksum crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ce446db02cdc3165b94ae73111e570793400d0794e46125cc4056c81cbb039f4" +"checksum ctr 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "022cd691704491df67d25d006fe8eca083098253c4d43516c2206479c58c6736" +"checksum env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" +"checksum fastcdc 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6e10bacfbef04c5c9c89635a18ccb6de1488d96e65393cb9b545747da1fa3a0c" +"checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +"checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" +"checksum futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b6f16056ecbb57525ff698bb955162d0cd03bee84e6241c27ff75c08d8ca5987" +"checksum futures-channel 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fcae98ca17d102fd8a3603727b9259fcf7fa4239b603d2142926189bc8999b86" +"checksum futures-channel-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)" = "d5e5f4df964fa9c1c2f8bddeb5c3611631cacd93baf810fc8bb2fb4b495c263a" +"checksum futures-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "79564c427afefab1dfb3298535b21eda083ef7935b4f0ecbfcb121f0aec10866" +"checksum futures-core-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)" = "b35b6263fb1ef523c3056565fa67b1d16f0a8604ff12b11b08c25f28a734c60a" +"checksum futures-executor 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1e274736563f686a837a0568b478bdabfeaec2dca794b5649b04e2fe1627c231" +"checksum futures-io 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e676577d229e70952ab25f3945795ba5b16d63ca794ca9d2c860e5595d20b5ff" +"checksum futures-macro 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "52e7c56c15537adb4f76d0b7a76ad131cb4d2f4f32d3b0bcabcbe1c7c5e87764" +"checksum futures-sink 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "171be33efae63c2d59e6dbba34186fe0d6394fb378069a76dfd80fdcffd43c16" +"checksum futures-sink-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)" = "86f148ef6b69f75bb610d4f9a2336d4fc88c4b5b67129d1a340dd0fd362efeec" +"checksum futures-task 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0bae52d6b29cf440e298856fec3965ee6fa71b06aa7495178615953fd669e5f9" +"checksum futures-timer 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a1de7508b218029b0f01662ed8f61b1c964b3ae99d6f25462d0f55a595109df6" +"checksum futures-util 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c0d66274fb76985d3c62c886d1da7ac4c0903a8c9f754e8fe0f35a6a6cc39e76" +"checksum futures-util-preview 0.3.0-alpha.19 (registry+https://github.com/rust-lang/crates.io-index)" = "5ce968633c17e5f97936bd2797b6e38fb56cf16a7422319f7ec2e30d3c470e8d" +"checksum gcc 0.3.55 (registry+https://github.com/rust-lang/crates.io-index)" = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2" +"checksum generic-array 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec" +"checksum getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "e7db7ca94ed4cd01190ceee0d8a8052f08a247aa1b469a7f68c6a3b71afcf407" +"checksum glob 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" +"checksum hermit-abi 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "307c3c9f937f38e3534b1d6447ecf090cafcc9744e4a6360e8b037b2cf5af120" +"checksum humantime 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" +"checksum iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" +"checksum jobserver 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)" = "f2b1d42ef453b30b7387e113da1c83ab1605d90c5b4e0eb8e96d016ed3b8c160" +"checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +"checksum kv-log-macro 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8c54d9f465d530a752e6ebdc217e081a7a614b48cb200f6f0aee21ba6bc9aabb" +"checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +"checksum libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)" = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558" +"checksum libssh2-sys 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)" = "5fcd5a428a31cbbfe059812d74f4b6cd3b9b7426c2bdaec56993c5365da1c328" +"checksum libz-sys 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)" = "2eb5e43362e38e2bca2fd5f5134c4d4564a23a5c28e9b95411652021a8675ebe" +"checksum lmdb-rkv 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)" = "605061e5465304475be2041f19967a900175ea1b6d8f47fbab84a84fb8c48452" +"checksum lmdb-rkv-sys 0.9.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7982ba0460e939e26a52ee12c8075deab0ebd44ed21881f656841b70e021b7c8" +"checksum lock_api 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "79b2de95ecb4691949fea4716ca53cdbcfccb2c612e19644a8bad05edcf9f47b" +"checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" +"checksum maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" +"checksum memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "88579771288728879b57485cc7d6b07d648c9f0141eb955f8ab7f9d45394468e" +"checksum memoffset 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "75189eb85871ea5c2e2c15abbdd541185f63b408415e5051f5cac122d8c774b9" +"checksum mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)" = "302dec22bcf6bae6dfb69c647187f4b4d0fb6f535521f7bc022430ce8e12008f" +"checksum mio-uds 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)" = "966257a94e196b11bb43aca423754d87429960a768de9414f3691d6957abf125" +"checksum miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" +"checksum net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88" +"checksum nix 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)" = "50e4785f2c3b7589a0d0c1dd60285e1188adac4006e8abd6dd578e1567027363" +"checksum num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)" = "76dac5ed2a876980778b8b85f75a71b6cbf0db0b1232ee12f826bccb00d09d72" +"checksum once_cell 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "891f486f630e5c5a4916c7e16c4b24a53e78c860b646e9f8e005e4f16847bfed" +"checksum opaque-debug 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" +"checksum openssl-sys 0.9.53 (registry+https://github.com/rust-lang/crates.io-index)" = "465d16ae7fc0e313318f7de5cecf57b2fbe7511fd213978b457e1c96ff46736f" +"checksum parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252" +"checksum parking_lot_core 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b876b1b9e7ac6e1a74a6da34d25c42e17e8862aa409cbbbdcfc8d86c6f3bc62b" +"checksum pin-project-lite 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f0af6cbca0e6e3ce8692ee19fb8d734b641899e07b68eb73e9bbbd32f1703991" +"checksum pin-utils 0.1.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)" = "5894c618ce612a3fa23881b152b608bafb8c56cfc22f434a3ba3120b40f7b587" +"checksum pkg-config 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)" = "05da548ad6865900e60eaba7f589cc0783590a92e940c26953ff81ddbab2d677" +"checksum ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b" +"checksum proc-macro-hack 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)" = "ecd45702f76d6d3c75a80564378ae228a85f0b59d2f3ed43c91b4a69eb2ebfc5" +"checksum proc-macro-nested 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "369a6ed065f249a159e06c45752c780bda2fb53c995718f9e484d08daa9eb42e" +"checksum proc-macro2 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)" = "0319972dcae462681daf4da1adeeaa066e3ebd29c69be96c6abb1259d2ee2bcc" +"checksum quick-error 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +"checksum quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "053a8c8bcc71fcce321828dc897a98ab9760bef03a4fc36693c231e5b3216cfe" +"checksum rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3ae1b169243eaf61759b8475a998f0a385e42042370f3a7dbaf35246eacc8412" +"checksum rand_chacha 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "03a2a90da8c7523f554344f921aa97283eadf6ac484a6d2a7d0212fa7f8d6853" +"checksum rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +"checksum rand_hc 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +"checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" +"checksum regex 1.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a6020f034922e3194c711b82a627453881bc4682166cabb07134a10c26ba7692" +"checksum regex-syntax 0.6.17 (registry+https://github.com/rust-lang/crates.io-index)" = "7fe5bd57d1d7414c6b5ed48563a2c855d995ff777729dcd91c369ec7fea395ae" +"checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +"checksum scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b42e15e59b18a828bbf5c58ea01debb36b9b096346de35d941dcb89009f24a0d" +"checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +"checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +"checksum serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)" = "414115f25f818d7dfccec8ee535d76949ae78584fc4f79a6f45a904bf8ab4449" +"checksum serde_cbor 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b4ad7872ff6e6c2a9221f4c1abe681e7eefc56ca5b3e87196afbfc717d141dc8" +"checksum serde_derive 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)" = "128f9e303a5a29922045a830221b8f78ec74a5f544944f3d5984f8ec3895ef64" +"checksum slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" +"checksum smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)" = "f7b0758c52e15a8b5e3691eae6cc559f08eee9406e548a4477ba4e67770a82b6" +"checksum ssh2 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "875fc74ffc41049e306a247f4191dd9c2ebfb7c5e0a6e4ddfdccacaba732e8aa" +"checksum sshish 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "64e1ee53c6d5fad4125a5aec2006c51b2713af3fbba8b5ae73ea094c59498869" +"checksum stream-cipher 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "8131256a5896cabcf5eb04f4d6dacbe1aefda854b0d9896e09cb58829ec5638c" +"checksum strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" +"checksum syn 1.0.12 (registry+https://github.com/rust-lang/crates.io-index)" = "ddc157159e2a7df58cd67b1cace10b8ed256a404fb0070593f137d8ba6bef4de" +"checksum termcolor 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb6bfa289a4d7c5766392812c0a1f4c1ba45afa1ad47803c11e1f407d846d75f" +"checksum textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +"checksum thread_local 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" +"checksum toml 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "01d1404644c8b12b16bfcffa4322403a91a451584daaaa7c28d3152e6cbc98cf" +"checksum twox-hash 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3bfd5b7557925ce778ff9b9ef90e3ade34c524b5ff10e239c69a42d546d2af56" +"checksum typenum 1.11.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6d2783fe2d6b8c1101136184eb41be8b1ad379e4657050b8aaff0c79ee7575f9" +"checksum unicode-width 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479" +"checksum unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" +"checksum users 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c72f4267aea0c3ec6d07eaabea6ead7c5ddacfafc5e22bcf8d186706851fb4cf" +"checksum vcpkg 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "3fc439f2794e98976c88a2a2dafce96b930fe8010b0a256b3c2199a773933168" +"checksum vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" +"checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" +"checksum wasi 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b89c3ce4ce14bdc6fb6beaf9ec7928ca331de5df7e5ea278375642a2f478570d" +"checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" +"checksum winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" +"checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" +"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +"checksum winapi-util 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +"checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" +"checksum zstd 0.5.1+zstd.1.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "5c5d978b793ae64375b80baf652919b148f6a496ac8802922d9999f5a553194f" +"checksum zstd-safe 2.0.3+zstd.1.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "bee25eac9753cfedd48133fa1736cbd23b774e253d89badbeac7d12b23848d3f" +"checksum zstd-sys 1.4.15+zstd.1.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "89719b034dc22d240d5b407fb0a3fe6d29952c181cff9a9f95c0bd40b4f8f7d8" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..fca029c --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "yama" +version = "0.0.1" +authors = ["Ollie"] +edition = "2018" +description = "Deduplicated content pile repository manager" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +fastcdc = "1.0.2" +zstd = "0.5.1" +sshish = "0.1.0" +clap = "2.33.0" +lmdb-rkv = "0.12.3" +futures = "0.3.1" +async-std = { version = "1.4.0", features = ["unstable"] } +blake = "2.0.0" +twox-hash = "1.5.0" +async-trait = "0.1.22" +serde = { version = "1.0.104", features = ["derive"] } +#serde_derive = "1.0.104" +serde_cbor = "0.8.2" +users = "0.9.1" +crossbeam = "0.7.3" +toml = "0.5.5" +glob = "0.3.0" +nix = "0.17.0" +log = "0.4" +env_logger = "0.7.1" diff --git a/README.md b/README.md new file mode 100644 index 0000000..fcee8ed --- /dev/null +++ b/README.md @@ -0,0 +1,94 @@ +# 山 (yama): deduplicated heap repository + +note: this readme is not yet updated to reality… + +## Subcommands + +### `check`: Check repository for consistency + +Verifies the full repository satisfies the following consistency constraints: + +- all chunks have the correct hash +- all pointers have a valid structure, recursively + +Usage: `yama check [--gc]` + +The amount of space occupied and occupied by unused chunks is reported. + +If `--gc` is specified, unused chunks will be removed. + +### `lsp`: List tree pointers + +Usage: `yama lsp` + +### `rmp`: Remove tree pointers + +Usage: `yama rmp pointer/path [--force]` + +If `--force` is not specified and the pointer is depended upon by another, then deletion is aborted with an error. + +### `store`: Store tree into repository + +Usage: `yama store [--dry-run] [ssh://user@host]/path/to/dir pointer/path [--exclusions path/to/exclusions.txt] [--differential pointer/parent]` + +The pointer must not exist and it will be created. If `--differential` is specified with an existing parent pointer, then the diretory listing is specified as a differential list to the parent. +The intention of this is to reduce the size of the directory list. + +#### Exclusion lists + +Exclusion lists have pretty much the same format as `.gitignore`, one glob per line of files to not include, relative to the tree root. + +### `extract`: Extract file(s) from repository + +Usage: `yama extract [--dry-run] pointer/path[:path] [ssh://user@host]/path/to/local/dir[/]` + +If no path specified, extract root /. Trailing slash means that the file will be extracted as a child of the specified directory. + +### `remote`: Run operations on a remote repository + +Usage: `yama remote ssh://user@host/path/to/repo ` + +#### remote `store`: Store local tree into remote repository + +Usage is identical to `yama store` except store path must be local. + +#### remote `extract`: Extract remote repository into local tree + +Usage is identical to `yama extract` except target path must be local. + +### `slave`: Remote-controlled yama + +Communicates over stdin/stdout to perform specified operations. Used when a yama command involves SSH. + +## Repository Storage Details + +Pointers are stored in `pointers.lmdb` and chunks are stored in `chunks.lmdb`. +It is expected that exclusion files will be kept in the same directory with the repository, if they are to be used +on a recurring basis. + +Chunks are compressed with `zstd`. It must first be trained and a training dictionary placed in `repo root/zstd.dict`. +**This dictionary file must not be lost or altered after chunks have been made using it. Doing so will void the integrity of the entire repository.** + +Chunks are hashed with BLAKE256, and chunks will have their xxHash calculated before being deduplicated away. (Collision being detected will result in abortion of the backup. It is expected to never happen but nevertheless we may not be sure.) + +## Remote Protocol Details + +* Compression is performed on the host where the data resides. +* Only required chunks are compressed and diffused across the SSH connection. +* There needs to be some mechanism to offer, decline and accept chunks, without buffers overflowing and bringing hosts down. + + +## Processor Details + + +## Other notes + +`zstd --train FILEs -o zstd.dict` + +* Candidate size: `find ~/Programming -size -4k -size +64c -type f -exec grep -Iq . {} \; -printf "%s\n" | jq -s 'add'` +* Want to sample: + * `find ~/Programming -size -4k -size +64c -type f -exec grep -Iq . {} \; -exec cp {} -t /tmp/d/ \;` + * `du -sh` + * `find > file.list` + * `wc -l < file.list` → gives a № lines + * `shuf -n 4242 file.list | xargs -x zstd --train -o zstd.dict` for 4242 files. Chokes if it receives a filename with a space, just re-run until you get a working set. diff --git a/example_zstd.dict b/example_zstd.dict new file mode 100644 index 0000000000000000000000000000000000000000..72184b7a3f10a6bb4b878c536075e815057f02d1 GIT binary patch literal 112640 zcmcG%?{8dLn%}nuf*>knl0^ao`zFBcl4e*`lebvK-;}6nlACU zmRPK+sauppX=#ktb};d-fpwhCdIRsmo2+eYU?UrQXT3p^b%32kf*>$@{3h_L0Q<`D z^yEe2KOmp)^PGFDD5-mPHVL~$R-GTu`Ek!Z=lSt{o-_YDQ(vz4|HX~9pZ~kR@Y!Fu zJ@xPZZ?B4f@cqfdU;gC#zp(e&=KuMB|L=e62mku)$G`Ef(qI1P|Lnj1gTGim`K7s) z=%a7{FW2tP{mcL8|M>TQ^1c83U%mhLzyCk~U;pM$|Lyv8va((feA=1!o6k)u`mN3& zN(c2pa+Y-HKVQ4i(RrsaXmzbjGpaY6sSl!c*6a2Mbh48y6$%kQNB!Wc zXtxfnh1RPiK5D&8QvI%Z-$RS{S}%7rmUma^*Uks6cGkPlTd!FtG}`qvjWPls(~k`s zXDLe>U$U|G&pWklznS!-Lst*SB z#tF-*&ST}FCd`j!T9Y28DfLItNnS+Bb5$H%^b&jC{ciKT;Y-n$>KWhLjG=X;Zp6`s zmT=OiQ~c|63pt5&f()(g#8yjKuTl4`H5l*)MC&8%q$l0;Hp@MUn%xeh(n5NE7%HZ5 zv_DD*l032mSMgR(X*I1}~C?#noHdpyUpx&}=*?ImV9oL;ouVGHqRL828; zt@uR2Q(v^$iJ0Sh#QWT9^HxXC>-|>!kgdgDux8iZaa$O^45CYak!h+_Yq7MaCZbxK z8C>RhAEz-Nj|V5WBJyrqhCaUk{&@5{EXHon=hVwTskNTo3OOvGQkMJPa@09*w};i( zJJe*Se#VB_i9)&2{D>W*cR@cw-V>G0iR&#G(8FZl>6EJ-wK~nm_4avEY9>i`ZKpp@xcH-qJo-zR9l*mSn$EZ%1bdn-p83Euv2M1$j5i1(mC{ z(ly&mmh&yN)IEGQ!3NvUMi!bV6`zp9ixU;2_m$7@;?v@Iw7k3=ZCg&<>vsoTEuwhP z-5apG9mkD&yDgG74(}X|Zola4^x0eAY!(8&H`;xR^hrIdp7aM7c>@zs`~0}onTUGb zGx|@v^~piE+b$NaU2C$gmW!N%^HU3z>B@X{u{u*M-zi+X-AsCVZ99!tlJfo9ZQo1S z!j9{mR%&nGa*=PbQ|yrPoVQj_cDa}y4kj)ASvuM1chi*qYl?~@b?tVa_c!JB4W*~z zYCKhpl77G2UoLjK13xB7dC8vagdQh-)u6L+rJCm)4)XMhNHn`II_++q?Pj@nRBxwr zC{5~pZK!!dquV)JF0$d*+1E=tx;TEvi%d`2()wR6PFB31*KX?}aY|erbf}t*wclzr z*z>yQlZWT6cC*M@RRe)A2grG6&^n_hjC#2k#>(eO%CTBFKL?Y6oV2yU+Z=N`au!VRvLwa~HP~;DIb7J+jsM^Q@kG0a;s{vaQC+0$ zy&rN;M}rqSdejF_>TZKwoIN=^7Hm-&H3)# zn`TQ5Fk2Mn)C)Y`ga%Y*4}@=MiwN(Gvx`ZT_0h^`u6ml>fr#;m8G|9~MlViazSNVye8>bBe67n*P$Wz$x0x&(Q(un)!hVRx#yG!;)(stZ#y zGgH%ZbBps6#Y#nq(_(<9rWPynGt(2r*?5-3%Zb&c*F|13tF$;ZU8VH2eVd({pVhaS zD|xCEPgQ59=jX_q)3@sU?Bd)E-{$qLGBv+2voKLxDpnSXmlHdXHwx9MxtY1y`N~Xn zftpTcp*p)TS6xslJ(`Q@MkSu9&dgP&DKyZG&I_fC3+@=cDJo6+7`osBr^+5Ss{(mn{hyAD0i zw^SBWma${S(}!Axt)x#jN77Y_{REBx2EtY>Z;5*V#GTZ;(1fo7MeXn^@TDM3VCZ(JL^t zytxCtioc;N2Zzibm!o3LRmzV!3Uc))x`TyYv65FQODSx~m&x z*lc_gKVt`lEo`U5iMI#F2t{CYGDiltm9QVOZZr=%u{A}bVlW&KJlRRV`y!Hzu$vrl z2*9$g>xgzv4<<*Olum0-FXYH(G-W>gI*{ULsg@UFi*c3rIC_zD!@E;IXBY2Bck4X!&29&&{UT(oWh=l$`boC6x#(`| zn7zIot=3=F>wQ1|C|)i&eOP?Sz$v`eYsRbH-FbqHOC>Szt4pEhyed1$CLh+H*Y_Iz zR&THr8J9uHONLRnesa+}NnnMN<|LbP(r?sL9)ngNLJ+0L#tMb&X{Uag)ZUrZ;90cS z-c@O*E`I7=SqP+Ss#e=Gx}nG$jp}(`-&@;VUwOE(b-xD3@X>m0ZDntr-s$IR4{KUN zlWm?M=gK4+3r?y=S8pt=1!72aNEg@OjSf*nFn|5zyXUQbQW}fPqd8;a6GmQX>qxb4 zmsDz2s+F$v`*lePLvnGsjoe%gS!1lFY$sgnCrq}6k&jgpu8qUN+=Q=rFDV~77i{}dF*T~ zGQ?BK(>0m{SOr zQ{n}kqpZ$Vrxy6;9A%|4GbgSw<0$85^(}CemFlAE8Al19>Lrb%%xNx&zl4EZsEX#I zeXGn(^QG!NSy-4+zpCDqP_XJL4enfci%s?CJ1%*?_l&v;M56wg>sDpmbQp7o?MH#?(|&UjCJ;$}ieoQCyd zx-v7f$k=AQAB)q|XhD{j3xzn2YJTo6p{?sTC$(W+OmL?0-i1s8XnQHW3pMu+`Y6J5t~j7E-4l@qG_B8?u_5!zmAi|t!= zDIU)|YAfi{JV(;2aYe&?-qeDWZ=5iRu)@Te|tEX z7YZr8gSUA{Zgj@sX9Iw-M>Wwx{RpMEN9v;F8A5k~lb{8+eGZq??)u!N#;DuL#9@|C z4{b-G?RU^DWVO8$N$7{2NO@nvH1@mQGgJkdVIIPcJ%r6}_w>A%7T&S5^I$U-t_Fxs z{KUlIzMT`ONt#Jq9V*L2uNYUh-E_hPOa4xoxQ2E2tnRv7rbZRic z?a)dw)%M+2(GL#0hfA5tm`$=+xGZGy9Qp&eqgR7ouUaSsR%7S<@H?c<7+(q7$+WjR zC247wBIAOdc*mZOFGcit!2T^BHH!EJr&%a6S>ccAHh8yX62lag4bjM6bQnrS z_Lk@AT2pyW50w*}O>-&Q-tC^8wHDWt!E*Pcchbs>oODyTbiE91eGKU|(txS!bkEb8 z_J-C7-(R?X9kycU)7_2x5BBY)Kz@Ps>#;>dFEHY~=*xzJ@OTmJvFWs$bP|Aj9bC8v>!N`VI z8mY+8*m2@Uo6S2*7cu<{2GV`h;;XFDQx{am;<(KJyoO1;oac>wqwtMj4}IWroV4j| zSZS1(iC$l-RLDn>Z}uv-nr|=;JtTIv6R%vHP#QL~q>V6~WwKoyYSYdn=cT9;LDHx? zuIT#X|Q2B{9F{z`alpMFxeD9zNkE%Gw&~Ce!Eo{iWy-yG&B= zFo+Eiyj9SyU({BoQ-p3LpA>vh%++njOiSw5$Ij<;CR zv~Ow7hSO;eM+TTM5+mS=UdhvUWdGWY|dv^}M3<+aU~jfd-Z z0Yhvd<45a{*SGegy$35u`uh*of#~F?dGug=^DYAa_AY5#`@0*fkM_5B_oCHxkUOSm zxJTB?)~C_kjotONeKd?);S*#ww$?W8LVIO%BHG(oU)#`!^-tCx?rg5?emW7T?yZ04 z5rxQ%?yfvsxxc;_m8iM-=q_ry3CPfQd_8;x9NB6h4@9v2-`j5w;i}uh-Mw{Dvq}_{9icHYT{)#mYk$W5ad$&}2_0itO z);gu61=-zww6njly*1AGKW2J%H76@nzROteMgSW$Waesn_fwJ8Tx@J^>@%j1AFPwU ztEsgaThXNKF>`DC>c_AkZ8DYnBV&!W*6(j_++W{XTUU>^Md#y!1htaUX z+u?+|x2S#=|~119bbnyW4B~ zpYE(PfoP%&xAiBIig>x0bTHcA7Q2GAiyu>S*+frDJB{fbEEUY%i5cj{(_v1(^=Nl< z+4(0OI%(Rjr59=Ws1xt)ZPwQIKdGgl0PSJg%<`mrb=VRK{rY_ls~NUm@zleK6#tMt zI@Qp_Dp}S3x8FIR#(Qp2}FzBN+g90_iJNaz^iJs-OPql=VZ;^?A3| z^arVACI|8FYWmd&eD(Rh`FNf08GXEOmdKv2x2YA-;6ZoUw8IRsN2@3A$T`&Zt-OYn z&UxcToON4Er0}{ZQNQ?jr8p7Trs9JxucltQ(qp(hAu-p4Qrwdy1B+AygEly!4mP8O z$*EaVT+vj62s4)Gh~SivA5Ki6`|Yg@!vd)-_#Xgh~&^T?o7Cf_w>)z{w^@1 zoJzZ9cR}<=EyaNh{?0LM$cjVyEp{IOCP40Tf>OU2ADsYVTv~d4S5Q@qfS@YbZ?lgA z6`_2PIn8uuy#ea#@ZA*|!|(Kd_~oqL7=8(wm0{11r;JWX0w+T`hKU)RoPm{n zf=UFYx3s1sf5K2GoI1(j`SD2CIKv=9kJrDe%)IyGPpyXuJ4N&W$}1V+z$L29eOMpP z>+0;>Ta9adTz>+!W>SI^F-|5bGN}|ULRqDc{3RuOV8fcGbja&ZW}M z$@w8>y0ghj(rC<87Z;m{i%DalI_+cy^{P}<_81UVy=lT6atn`K#}C5Ha7!-cuaE?w zv$)$ni=S6wIO^v~b#}J0I9X59Nv-5j6=$P`xf!11=2t&^`F?xvReA6Ez5d;MHr@K@;{K!A&-NOR2hTR9KDy}bf7<^zj^mLIIwfdncZ1fH zy@6S(12y$6NNxHUl&Up0BsISGs$E??O+PMgE?j(e|KpWU-d|{bTKVMSUhDnV?2EJa z@9v)L-#&=F(%&?whhhkEH2fFV)TTdT-_R-xBn`bZ|px1QxEsPyRx-!l7*Q4kC3V0=E+46p>yk>?CeTXK}Og> zl-gJ)S7OP^&JLDe3azh79@{f9Zx03OrnC5g?#B8a&+(?DGncXQhE2d)k>DeOk}|9# zRFIhPaD8{}0sYEE9nnC1GNH+&z-VV>cYkB;5yD-x^Jo_-10m!)k2co#qxG#1w?B1Z zZ5LT?YjR`j-Yyc232?BCH?!xQ!ldZ=Ykje2*N?{w;(FFVZ6MpZZ5xqBmg)6RHum=S zB3q0UOl#S}f@;WKGO?qOiT2k&aRE}cY+u4*1-~l`7A~o)`Dk_HKEj~2Wi_M8NjrHn z8DD1;mX&NZil%xTBo?GU#J&|#$htV}VU6=lB9e@GkG3`tfTL3AE+>t@UO#o@0o-7! zY;kIJblWd$dH;P9Hl-xxx$UCt&WDm4^v6CK3++!t8s0--Ij~>=eC$VH(qTNsZXuWUY& zVqnVf6Tj$OWH{v{e2%_cfmu0WCtwr+VtC6lUjil>;c_BEamG#?h-EL{$eRE3mdUQd z%W507HHq#SKa3_O4L~{__-7RKBR_%I8?T8RUVhqJ5*u%>WskALDF$NqSt7!cVJu@K z%XUTMzR$F4Wis+eH{S6v5PL_?pCc&{`?Pxws~(^Mvr~ptHAJdx;1k7QgP*NdW+#fl z1V1$;jB9|Y!t+d3_+~cv3Q*U%s?f1h2pH3`67^R$d0}$>`&{@tAO0@d-*^f@?~Fha zplm4*zcIVGV9*QyZY^C-+y}Jky_#BBtS-`*%4}sC%fB}Rg5r(znRse)CNwn@SNR;^ z36=SU*##U0rl+b@pQV`h5PY#z*sm@nL76a*i}C_qYotJ!efW; zt}sAA(_+6tjfZvlfmDLrYd{^ZxNdr412JVfCv@sEhUd;gZ#Gr$wGe#_kG!(@`g&*jF}V3U#l-Z*&sIg_&6Ib*5QN0U9(lt1p5EIhZKGg68Mvh3By~ z3+le2GPYEw=2+4J5i|vepD$tWsm=)|BW$s8I2dVq zW`WVn+ScW>GHcCO=H@i~S!=#3?w=pvB-5a(||fu0&Tr zNES%J>`7Fv#Akq|J0=a-rz(Cb$7T1~GHh-57<%D7$nrfAvBA29FmHmoJhsbdciWb zgURO9LSqV;;rwh;tv06{o>gCSLHS;(m%wFxSfbtQwQI zsVx<5Yqy5&mPrv^-f}Ta^u8E#wEZagLrTcsTisuhvmTh#L2o8qI&9ZpC6oGMi%%Q1QHm5UMTE*%xOWsDp)fBVCg$14~(^6b6sN4wbYtgsn8$~U96SD1_3 zRUKTSjwM;0#6Ca!j)9@~Ii-3nDFWrDPCk|h>U}(fA#Hb>fASm2v*>ZR`NzNg!@rT7 zsz`3cDcvwb?~Pc;ofAUEeoRaC_LH$jI`$N%L|`rDZ)5h_7Z3o~hQ}9RL9mPc@Nd-9 zv58PXSQGMo_$wg|C%k@0dmumf3oD844wt%+{!m{$9j|hX<{|y@Ao-Ku7}jXx+LJW$ zI{J2s`XM7IuhWt2Q+*xgtMBqo3DwoT?(;wX-Kh7+znz|StxrMF4JEcNI)D6+JYC3~ zkosBu`Jep8AOE94KHPh_Ow~2u2M?Cc&Mdipwk10NXos<8w-=91p!s-czsH_xmU_ed zZqxI3EPu!ITj}opX69#d4W0fqdo(Pd@W9uw9rh2qFU<~iF4&p2*w*p6vn_>d+UAwM z6s?}Ok4?oLl@3uZxCLIm<=C@8dOh^Khw*BwSub1zy+R!TI7rs<;c^ou|G;=ldT_rCbOFaF?*AAIpYlJbM-btrKe zeep*>`laYK4$}4ZCECMlL_Ls8K66(5;-69L2VeaD7k~Jp?^Eyhl-;p4Rs7Lk{^Gy4 ztbhE)@BQdM@hpQ)soFm!?U%mz$58&>7k{WqFHidByrj(E{Ne{c`YTlbXZ#h_KZ>lp z8u|m8`aP0<5UI83M}PT8fBlQ!|IuHE${$!d9f$58Ny&Mwb$0ycAo=du$cci#0|#-^ zeSv+d&q~q^sISZ!y2f+an`T3^Fbr17Kh`+g;}cBfTV2dIJX^q9FAIfT`D7h*x}4CY zO`aq#4_vb#kRqb%5yk?SzPz13Vit(KPfEZ+Q}WjGV~IZ zsXe2q@GHb)I7l05TDWo44XrOl*Jozukc{F(WGJc81;Un-OouZKQ3mE|Qf?^<9Z@1! z$wYL0`iPhw*PG2IIyJ1;)O=+%t$PqWIyzDYjs4W#*>IC4p!Iik4O7t1)U!iCu@m@L z6YS#MT{ECp*0<(a-9V_In%k8l93kTFlAdySjBjT*!ZNUf;#Afwbn8!+CM!>ea9W*4 zg37S98)62`W$AQ+1<*-?gs2kyC^vtJ!i@YET@^{VMl(#}v7v_sD8MM54} z3+Ssv_G;TI6x;%X6JB1o*QK#$)-k@7)O~u7IZ?=&t@X`e7ilDvrVr(x`Fh01I+ zMb}lZwWwT?hiYzR^)_}k!N2is$1-BLkx(SK>j`52B#Q(?=b~{VB2dCXO zti>(CQinBAZcLuEFz$A!t56gkxo9c!#8C=_-{Eyu^*0H?bDbrV&3gZIX(`xI3^*Ii zdvf-?CD1He87?FouZM_erLkm^pVWmKq zV)JZTsgR!4jAnW-qXqu)cQiljZ-%x*{~~UB^z-%VI{&SWrcqnt1>?B2aTkNwpfRYe z)(4G~l_s3pZt~r`=-bWjOA${p!IR@(66=}>GRL`N2mmG#+~u3Owbf7e*Y}oWuX?4us-h!6%m|TQd_f7{a0FKM4YIYv z3k=r4S0?Y6bX5bupf&jw#fPYJmeft}%aikkTW`~~4uu6Sf;{FwL;2x8LVp@HR8Mv! zU4J60`qKgf(1+Srj>5+3F!);X-S%f^lU!Y6IS)T;iD{Jhngjm=IZ za971%t23$4Tj}TEy+1Fe?fPN(G-!TKyqk%lTiDT{&MwRnU17E|J4@h(g{lHfObFs2 z!y?%;bMsYP%_uZW*ojJIaUKmU&}%?Aa|;V|^V8MExy9L;$|8<&3RNLPeR1i@6V$Q` z(+l{ap~=OE5OojWo#YJHjN*jvN2t=#OV7;DEG{lWW_EsYruuZ^$=m|IaY<7jaBF9Xa(l4f?i-7n_=x$7qbETqNdE$YI!@$KkI!HBAXLMrW;V?Yx<0 z6)OaaSgg*^&&^b(iEDwfd2x!GRkX|t40DQUSeT!lo04|Ci@mpoCsE!iQ8*T6@JE!Z zWMy`4er|e>6Yj|jT6N}R8qM=;WoCY1VHTY;;FwyYcRp+wy*E^=Rr-v(;~Z1Jpt|#* zUNBC;C^63qv*xMjAeOY&>So=hsp>rX^vWV*g8z=6N)fAhKbnEVafQ`b^BuyMYAO; z$NUN;|?e(5MdJqq7)h(md<)4OE?OQHso=t zNCru%)qI&3vs`fGgHrA+f;=Z1WGLcC6kSN+Jaf#PRa1(yGr?I`o$?%URAVr#@^V)y z`>teptZ*f0EZ@neLi6Q)z(L1s@`^Mi)#d1PSqa09#izs)D~;jT{L#kdW=38QK+RlE zc>BRkAchekgxp6|QR~(h0|7dmX(VXE$M$q@d6+(juLTE{haSH{)T4ZFbdG>iQLH?Y zvE@nwCMoB&@$5^{qQFP(^E1vyJV~-b%*;EB%npm#@eg8bjhFct&OCa;;ri4E$Qc3C z$`N!(go2Z#4HJ^WO|{ZTW{2k4<*ps!VF3&KnjfeFL(8h$x34Kp@D`Xb)=E|Z%b^9( z3yi4kI&?=t#8wP=&xh5gS9&EUAw64PpE4UEHOW}99LdW?Q|SpY+rml7bdBaA#N)GE zRS{0IElJ{ERBh`x6Km$e! zMz^WWm;X_Zp?Nh_72+wHL;@K7xiyLJrt1wE>Hj}b#H@0}YQ->&@3oS4)43$$cnp~g zep(EWp4jc)9R<7Tv!5CIrW@U^M<*C5mWwX5e+8|t$-G^z-vNPjyW9&+1jZ|8(L&^( z($}^y%EA52Je}{?MbxeI4q<0=6dT$UcfcZ_4Ya9!g*S_)Fu;n+u~ZW zaGhPSbwpA7OCZBY(-3CkLCw{?k~i*dKjiEPaZkd_{tc9O-xxZyS?IgruE$A3rM;{U z&=WT0E#%<|-=DXSjwpi1K&R=ceDE|jCHD&iWI~&4E)GVGneZXK@w|))x5H8xmMtD= z>`_m4P4uvPTLEsM+y`xfeMmf4dceV!7B% zc|$FR%!Ufa@>`;RfDQAl%L;p&7LCz2xa;W4f_#8hmQqFXVO7gOi^aezoL^I=x@|>bIQdfB7_3eAkyJH09okD+L zRESQH7q%vjgoUwBc*w@mU^_srl=UT40CSEcDP~E#fDVxmiCT|W=j^5%YxVlv*l6Cc zVh>vqZ2+`Jwg}i2ED{s}9nX6fCtKRNosCT^Eaxv6lbsBB5>78U{D76=mrXWl2KJ)%%Ev7lklKEv-C%~)4H9k{L%+6DFClJC1Grtr?aeGWL5`}G5sqeT}fVDHdVegaD| zw7VIM`f2F{p{MMSvso>3FpAw%AO$%SqP8EY6Q4E5d}Gh zLbU;Yg=oAE@IaHA!JeQfw-)&uZ${9YK1NDsg~gQTfC*#yxUO(loVBjNA!Ckw5#*HIAsN_ z>VXY1J>Yl~GpyZM2vs}rY&Dz#%=L~Lqs&zMwBRfyxVctM8*Yq6s@T7%9b}D=^1)!N z6vkp((CP!7EJ+B+`A9rEV!KO(!m@ffjwo|A{uVq~2B#S`rWfPj^wH{;_e?iE64{y8 z1`TPu4KGL=f|22R8N0eC z?8{xc_?@cDvE++ zojkUm#SOX?BibXz2QSGRJ)oL*!fW{%k2i6sw`&2@wIPm#y3ipnNIwEjHWT*t+{><& zP%s}lV!mZPxGafzPT8mDo(;QXof_&{WWw{ZdFfV~(F`P?Q0A$>nfyjs_?7}nuW?o> z^*3sl@Qhd9IhIUgkS*)Y0FhGwJzIeI1$)%W(Hxbw1)e2ta`I8!XvYmZmci0;jFe6e zU6UVFndt2?udpTEUR&ab4IY_yO5YOSLM#F%b1C{3Yg}3`^qqpnteH(>+x+ov`^p_^ z`izq`rbhy729LG)tK+C|Y8}(Qydhh2banf+_e#YFY>aI#o7dV7*KDrC==c+!q8MwvqyCfYYzZ%{?G7i+tc|55AG`6bPH`goRv2gYuroT_ zr}A&GO##KG&1LwW>HNGTtYHbw*NE~nqX2uEjgvxbJl?tvh+&_ujvE~!@Ddv>{RXHgU`TaTysKJ zU~TJOmcR38@4=2-aja^k>T4Tr0M}&m$Ck|BlF{wFSCUczpJpPJ;P_-%39F@mYM2sZ zy_VPdjYbWPCZ2#bORYJmqO`lVR#W3ETFxT`R_<+V|5@v+>>Jf_C)$`||6w!hpZDM_ z)Q1ki`a-+4R{y)DEn-j9tYmFt_d6MdJ$6;~Xo|33gtp=nmRgR;m2t6MV-{!u&(x$d z*;G{gF{gdp!_2~TD0-kzJMk-QR&ClhwwU#hplP=6wR@Y}DYD=@LfEqrjs3 zT;bGErb2ZRq(xRf( zBd>fh@fJjf&pJv(>LBan@O0!>0mX2J{jmV|F`KLPFk}8hcG?v6$$CC4SDR4h0I`;I|?z))I2&cF6MBxd(2^&C)o-pwb>)1{?CpjCw3C={O zZ9d@u2<*;_Wl7W=mTO|2Avzh`vzH!z&nqee+IRc)VHUJiX91h%<#Kdz<8`~+?Om2`yk_9eJTQU-ZTS31hw418Ij6`-ICi)D zF`*;~dkH74Fn(i^j%Jmn5ke(N@>RsbB-Au9HCPC(ry=2>*^OR*-nsF*Rk^(J`fATE zEq&_p@*s@m>R`MfAFQcbLsP?vxJoW3V$;GeCimJkG&96oaUcr=D(OW+Uu_A}VWd_~ zmdmRGopE*mBn?B({w@322>96<6{&klk`(L~rCa>D9bru8!~6MlNy-ig*0sLEn6#Wt z%IiKO4KUI(>Vh13Hmqq}U1_B5x7bKVm^ja&3)1_-1$(?s7eu7d z!2yL7+OcuM-EfkI4-RZk%+gH!=k-(x?VCo$=&@_tjda8qKLD)584i7jke<;O08^~Y z3Y}q0K_EftLu^+>|s-pJ-xg<`05qu4Ifn4;pjAS$&zcS0L#1d5=! z6_<80b3qx8M@s_y^6LB9ons?a<8qeDoetKL%*D~lS@R*>T9)SuP??2%*ODnTTp7U{ zC-s_1-9dY-KhU~n@VaXXihO}#PSHzDTNY|~pA;fDYlZn4DTa^JFw458-7KdP55j0t z?_7yUP?FLn#r;g$l1D5k`INv1ot$^TzOo=tBTMp7r+qMXGAQ_`bQM-E_TyjI7)rj! zh-h`r&?)ehlA5ZwtKS9EWO6rjbopEC(hoYaQdTi#RJ}mYNLre8Ca-ykm#$Pyc|^VgfyR-M|Mq)m zH0?w&5MT6G6fUL6*d1|=m;$LVQwlFhysbz3_a+z2{pH$iiywJshiT>wpN!U!)Cpq~ z4bIV9z}obqXXjXEq|TR9N%eNwD?rnY+QHdMQZz0%&*5i;F~oVQ&plbs(sIuz$DFl5 zgo<~DAVj^)xBZSzjbfRAmXH9H5`2P`Z)di28N;6(vpFgaD;vtX@bfJjf8pypddHOv z$4oDeO1G@|T*UWDDJ@LFhtBC&NL?ZJAP>qO5iEcdorFmC53E^cPtF?t7c;Y6Z2(Zx z{)Vl?m?GN37T}6UPCgAlN6K)l`T13Iww-e*j|I*4<{=tmg(6KAtM~Qg_!AH08)xO8 zme~CW#gO!FMelJFVQ79!mWB43izqTMz-(J7KIDb)H%&rOX* z+ok%p<`j+gm&(Tt#S;ofW(OlQ$31K)|PTx&4 zrw?(f!1n0i zmgY}EC_oM?bF>+Q2b$8DEY^UrgDwCE3y6>7>;hApzu(rG>?s9xbdZ^|<6iqKq1~#ZOnc`%GaZaGoySkJeVB`f7A{HEOIz&DAJbjgD5M4G<1`cWjR4_`kD zmF7`8tVV0mL+cP3x}*)y@nW6Nx|L0zFZgWt!Kpsw`KlY=jNCxqbordR@KJQW8a-b{ z8nnE7WO0+5WxY^2E_|~uFZrSy@v0MROT`^jmIl_cm5>%OQ$&}?xi~vJWkm*e=hDxwoA|VT%;?jxd#^`L4o$CjJq206z z3e)oLXi}mfdz@(e^oUX_%@4+}n*~B0A!3PF)BSNswJ(lP+n|pY|4#0TUCZTn%dO&c zCGL|myM$MCGg{fl*`(go<|#)1Y@7=Dj4koyZhrgP5@riz2uX61B_!#xX&^!rfv>mH zqN%H-mXKhCOCwqeE>}Vs1`+q3wm$46eRew}z*DYt?M!m1GdFRWu1!IFY!gK4fH+DD zU2qivd9)D~g3eY0mmp&jvN56tN7I}$WGvQQ*PeI8WQqsY(^A;hQ34Kix}rcab>DMa z8Yp@&nkGow$o+0UHrk^Uj5H=)rb5c63HxnkSSbr-w^qB^Y}t^AF)}g+iGsRmbjVGTM@2s}`qu_@V-|8Hu zJ^6e9TO#DQ`S$?mYc+~X4j?t}9s&A`hphp#f%i-oADg8m$gC2e;pyeXYvajyOZ61K z)gIhJ)4u<$2i7U=zF=}bCk zmh+0-eS+36^yH3xv34|K@na*6At)eG4O{X#EHjT8?v!a{`+0L%UXLpbeada#y`(=t zs$@eOPPU$5KHo3IMnp$v11{}(As0(3TAD7H^rs$v`?X=VLTwEn@Fq9yx}5_T{R zf*V&b;VSTDZ*U?TX6PpGWGZ!<44dA(a}YY@4QbAaK3U0_H@Cx+fex;*>Fh z2J#fe-05AAW>x!pHcS-^BbIVX{6HiXJ34{V`|o?>w}o!Q^q(=Q>Xz}iSs$KQIW$|g zlfmiQ0D+24Da?D`4^&*Zo{nc^oGH$VUQtQG4DKfArBYxPY(8OveZWMVOV3k5y;_XL zlr>WRqbMZ!n%%dXO51Kf2I((8>e$YltCOVo1}yyA3gH`LmmO?(4wF;l^sA+O&WmV))G`nRNv^CCO+&lq*AIrlNW<6Wfa9Rc+TChli94oT!Y$4Dv|$(>X*$fB-AvEf z4X(*}jf_M#g?@Ft7qEZ>$stE12hWhyPF3fJ} zWi>6B6-lE54cFTF*ut%LQ!AV6`^3I{KSZ_kUBrKP7fPi*Tq^Ev;}W~)$0?@7<98(< zjwDWB6_HW&;vfY2UU9#B1MSmz{RZf4klQ`&$X=bL``}CM zxc=lZtP)q_ba-VQ^vQu&2)FTk@_1i)R!MAiDgPAbpJ?0mLb0*1d1(1UEmuW*^5}Jv zUgXVIi%!MHCcAnDDK^@g#OvA412ndgrv{{OHXQ+ei`KE;EX|FNK#D~IZVA44&)mIT z#stVK4n4Xu3#dnvQA{ro)YDc^+rlBSW&CPYI814OI5WUnsm-SKwm( z%ayh-c_r4}-gLVrqh_Q2~<*^`OPMgCxB z0Q6&N(t_!D!xtXmA-m<_;NSyUUtp2c4D7TA0Aww}Hgzzz9!qNLM;mu#8>ei4IR(MUD$e~ z=an?vhmXMDTVPYbuBvW#7P8V*$Ioi1MzgDoGAJGaFDAWd1xY9v#|+SEI8g*O({2yig+EDDqlL5+M^r4Xi1bEONjI zf`_F`&kU$53=cCbPS3E*r8-egGrq=Dec5Rk2X0Nbj#aY~S_i}1u6-GYx%4%HEkEh4 z%j(3_HPjdj7Z0CEcAkMmUbWDSk5nb^FK z_8})HV((cuLBA4KV{~|B7T@rP>p_}p=2BQSqFuU zR=t`sbcOKrr-e&BnpW!OmQ4YwM--;S=o!fwUW>icW+i!*H;a#CnQ}C|a%=e&HV3vl znn73iZI)}@9K`VTlKEdRjNYmbv^VGo;G=Iu+nk4@}(ks@y#~-1yDa=_yQakmxuNyY zFGXY>?OT~kzZgLII;t8hUwS^}LVVtu^mIh5YeIM&ArIuiPKdW<7u8*fmqKhXMU$EW zi-O~Zg(9Jxg270Ph^xsNIqcIqlC2m4S%%FFKTg^q;X#^4>dyectr(4A+t{*LgU{Io zzm}H9K>rnGO+BL9 zF6S2>xIq6PptzHv#qCP1nQH8|#YSSAoz9)(gmn=ZytP)JAh~ptaj@!EyJ<&Pd6o{- zaqbd6@5yu*Gm{&Yg-+>p5hYqLdSt06ymtD@DT)tNj7@Bsoo5LeG|!9(Ux~DbDoY+J zFSw2r031F9V0q>45TLwEMHPDm1}h^FJ<}z?AhI|==tisE;;TaB47$$@^EEi6U4Bgz z_c)>X{5Wa1e)u;)hmo|VXm_N2Rv$bE+eONTuA?=}f^b}XYqXx}Y_Ib-J;g7!pZv*h zpVkHZ652_OpPh?R9{S)N2WQ8(8OJi_1niSv=_kT>>HjkrFnl%fx03${Cof=<=#+ho zE-)wtd47!bl+pIUM4$m?zB0g>wAGN=s=sPI&xxq}^70eIA;vnu4D-np=NcFf0*_Js zo|hxW+GzrU3{FYIK-|@0p(q9x&M;Ph7+zz?;1$;hln&PIFahI87`y&kw<7DBSn1N> zHuGz`9_;V$aEZZ&@j{FRl;?BNHKJLIcEyV!aF~iijKD;rFT8$jWe5spy{a|Rns$d8 zd&4D09{j`v(7O-~nF@iO!BW2Ss^&?4tZzb@@`7*;H7En)6MgR%Ym16b>}1eB8%v9& zJpzq|D#BPhc76N{<97kCvgiw=r(RqRsdj?tfMlLG>*)Zb8ai=aM4ghkwJSs&d(DY_ zg_kfECSw3Xcg0DFbuk;QF$-cc-YAswK02f`k{l+bF>4!R8SkEh1rDfFN1jY5!tas9bfSZV-{VvxUr_o0Wf&zZvgMeic zl<5j4D#Iad@c_Zf2(Hshsb@n&(tTYMmqzs6J zgvp!W{cv79gL8Qa^~ut`AhY8iNawk|g-af3f)d{-jC>#Zb}3XHOycl;lQLSta{As6 zy`ID7Bp~E%Udu|xt_pFFs%$XXNTFxor4hWjBdkOjPEu@J8Q=cEgN&kMws5FYt8sCt6+#JcO>Z51%LwiH5V zA(F|bF|1ccgS8S!`-9+(GLWr_vT$J9)$TrAiFQAMDaAyImcXow=#<{$3boEf2+9UoT~Reyw=2mQ zd2rvT6HX3p>Pn7nY)6%;>6b5SC&<3My8OW;K{GkErKNfbNhOiFw1k?aeXeWiZjK~! zpum}8N3*}98QMF&3FzWMB`jZnGAODIm+$fvMmn;zgnifUy~1+2v@}3-QyP!YJL2*r zB-%EpdS_@6Y%`{QFTO{lf zyoECcaBUOBo+j*8(&hz2nsXkd1@gi$f2OQT4YYQJ1CkEyeHHMd0AL08m3u$lGxQ=Uq)m84u}DxA)=81fc3lH-!VEaht0-^ zpNtpN_(Etc1SHpNnLdtJp20HA(1a^vMjPUIJ4DW5?+!2A2z4JJccsdTD6O+6*kErA zhd(kR7%Cs?yDKrex2tK=!wlZgXD4^1!+D>=T-l)(URj{RI`!>bUmJmbj~wg{1$_W> z%HEKl#<1=(BWIk1#fw8-{g#H&e8j@p#?dA{FRNSohog}FwJZ85C!(>p=k@H|)B?6c zP_;s~x>K?~s0ipcA4K`)dxaef_`p3qakY@+g`{hljy|oQbi3~r0z*2gMp@mvoAIe} zK~j(qA;XIr%NMFy?m?Oe{oTXz8Dd7#FK=>H_SW7y;TII-oFHobWYR^^up-JgQF9A; zp-$OLI1<;j)sYrt(!70q5x+vAPXs&vVwqV)S6*d6u%7WwHH9{NmLpq6y*lHBBaA$(|ac-<9xRdB)Hpp3IrXKkzHQ~u7y1*tZMKk@cP0AAfCo?RE*;2 zPrvt{>2FLVVM|$>#7v`3cu3-!6mRCJ*6P(;03n}`qgn_Bh@binilX^nI!#{{pxW2! zJx!gnL_nUPnfqSAksCd&kDsBVj8HxP$|jzN2AsVAeyh9c>6-4=WzE~Txdv{~Wq>MQ zV2i*|WAQA*EINVWo~OQNRV-}_E-fFOOUV;ak^C^KQ1PBDf=i`gy^_1C`B=Ojb!hx` z(Qr<>t|q85b~ZlQh}XIHpod0gq9{fUSq3EqUtE2gBdkV*nc@=bHc@=le%-`iHxCfc zl5N26;}j!pR>sMS&3{0fybce;vQi+ogz?!vDvf>K0fer2n8W<{1c$M&m%}~OgWj|+ zc5IDcNx<3G4+APs&|+3bE*%`UC8{&_)Eb%F6Ke5w|Yl5QLMDpDWhw;bvlD9HZ-NW2{x`FvQW0ou4Eg-LF#tQl zQt+lCH=KA&wgIfFu}v~VWCoGP=rb}U)*|x1Suv%w(AJP9S}lP4h|`%L6<|`cCI(1& z^$vF``qpDKw1^Iw+Cv0kG5dJ3gzQ6UBTBUzUej91I{C|{6)dY!)0EJt0=Y^7qz)ds zU!Rn8yiAn99fuL@%-kq+OKXG7wyvCyeR|v8})MEAN2Hl+IE~gO;HZa8DLS zQMup?b&9G@|&5hE~_ka{%K_PsCYgVhVE*F8JDEaR4A>YCqaw@tu zlfVff1Q-FS;G&u~kUsYXX3rVTL`x1_3L``igAgb<6z(D1@XP}-P!fu9keN^(V4c`W zlD-|y3Laww;+^mzOp>;DUQTvc3C<7E^<@AT4OKrTdzyMh3fR~kn-6t@#m~B^8J^@X zrwz`$RP;@q|W#;InK8-9x^Cy+3gqYC^CU&$R%tw9hI>XJL-3tR)hY(2NzFlc=B?8h_?+|H=>*EE2 z9m#dH5hf=LVF9s({7;}WJ8YY-rqE7<+CXLOj6}B5cfZ;XtwyjK=F9a8I!E1sTMu== zqKo^Z;IJ4c=?+0s;}!fyEiI5Ob9(UQ`&;XZeGg8g4a{ zW@NR+ga2DgY`VboP2^+pZC1Pga z)@xU3%z*!-ajjhwg=|>aT^d-G*tM}-TWWqq2fNTqO9?c2hk)rwB&u@^Sz7U<@is-^{y#5%Y=<38|sb5=RWS-tGAG+yJBn(Dt zIdtyru{-xEjKCzcHR;R}h-7#OSA7X3#cC^8n3bTC?!7#C(a4NwMoW$vO z`|MrkXU6;d%$(2X+y!BQTKwh-@sE>mEFpa48@Et!2}5;evBEtNGgYo7=GNj-)`)u` zbU{OPVQPMUx>DupQb4eQGt%`9i}Pg9&QHy9a{?C!4{xF1hDENIn45zDSD~`c2|cUZ zFtQ63K-~fWCLHI~%)&H`RR!*f8x@?Rnwp!Q)osUI>!5pFiFV$|>Twlub&=~Y;QHpc zV40hY7gdesVesh3Y=z4!*j2gXltZ?fRpaIeb)`DHz=ajuSizX7R^h^nIl9Jm#WRas z#j!Z2i&|TV6?uEy>A;P~pqLpAx5dIF&dr3HTw}qt6^pbuJ;x2JfU99>^9H#DV`iS) zlet5ht0A~=dRo^_RHpe-)wQeCo?4il=h6wTj5s>3wT?p7fVoL9r%TE}*lG-xrjU0e zJOq>Th-H{q1)hk5s4F`qQejCXiUwNS1}4;r`u zTw>L%)WZ_;R5dM2NzUDK=nl*yX+)ruWl>KB^~Ih@4nZ(q{`z10@o)c~AOEet6Mgy1 zf9K2J`kRzE6G(+$6U9?~;ump^IJV5I$VY%|2jfHLPk!ZBzWmMqA^P%n{^pPW$q%A0 z|Mu_v`0xGg3>ISHm4@(@Q1mB%`5*r4Km5IsqFdz^2pg)~|l~ zTYv4xzx@3l|MKssPkU&>h*+q}lx&xE{jdJrpZt~IB>S5v6x!k{E&_TP=GrX`SS8N^ zyAVq%jl9;Ezxn$S|9<@6{9*LtU-=&6`SRC)|I1(h)#%IL{{1h1>mUB)*M4`{>ODO> zb3zF%y0=DABh^4385{|I;G50zHv@)8O*Nx3Nuj-HAmODn$NUhd;U$G(|kW6hx{ zzDo#9iJuU9)Qs>*x)Xq7?q8I0+tUbX&l#~xg?W4nM-ZabjK((`Dle)tQ;Rs1%%XMR z!vfcMU(sE#L>w<*&Lnh63*SGRQUVW?OaoDw<8#rHZny5hL0(hnu1-@=lyF;I)?7Yslde?R&MZw1(0*H?1)!6*PHaCy6Tr3ZIS={* z;$8`0*OArA@8u}da;lKcs=((myLT}-NYb?dHunHwn*;)=TZY$6f52+L(#oHH@9(SQ ziV;SK&2XTGhzwXqxr_Oe8ENHfu;sqW575r-8SIVe5M-csRKDu!PF4+nBtX zra;n@A-02@*cBIjjfw2wV`C61oh_&x{3=pSac9!k@}yWd?wryMH#sEI@cJxQ(+2Cv z91TRqGC_3$=7bkin`%~W{I!-+2#wG^wO3O4oK_G=sqY^Ma@Q2@(w-~XB7-P-fhL~= zkuPe_!w}&OKvW;^#S>+5b)^&WZ&s283p>oML_Sbtinx8=smCxuu?&LJvc(SpeRg>s ztmpUR5Yw@5$jLHx#CyxMm!IKzt@X}$q!`0G!GBd$f_JWGr+_7=1C>W zYS`DRMGBzf(spv6k|v;zo%Z@`4C{S+`2gvLC7~MiQKMPu1fhMosJBHfGHF?rQhQlk zlaRj^;(nC*XC)EZueF+ROAnHl&MjS*{gy5>t8f7dXDVnHv(<}uRJuOy+?g!X%#Kh% z3jABt_hQJ~5z=ThGzlK%XKfVWq*jaLvFAG-k1%_sFmX(@!i+uDvobCS+y~1N*pi`{ zhU#lp5xf#7erOv#i%P3pGu34&|!v%{xl^vBHZJ3uL z=mxKhC~G#4Abd#&6t^l&vAWZ}>Z?*RmJ)j0iKUOWm_|5Gd;j!eX0Hqn%*tio8U3|u zDj+W)!kLh-WfY!CZUm5iAIps0yl~~6ZXKD@)Aj#4AYm)ai7oJTFr$OSim)pECk)-x zi9AdfdZm;@h9_8Phr9fkMY$*iz1-2LPL5L~t10LQ!E6;oBxw&dHEP9*G!FC2bs>?r zuk-62r4{_q;?NLuvfC*gb)@@l9|aFN^^Z5O=37nZ7+b)IwlMG3vi63&Ih~*=uoY~U z%RT7R)wYj}mUNS5qKx+I>1pYn-OvDSqoTHJ+-uny)M}w+RCp+Z)#YV87mr+bSJU^= z4k@Z5qEKFUI0x1j73X>DPu^SaB&lj4PrZ)L3|5kxHrTj$LqK6MENpQogVIw$gV4E2 zm#A=iO~nOqbv-Pkw8@41GHF&?Y0|EFojg4DyRQXPkUr40KwL7dtIB8X)>XiPg3Kyr z$_H-W2uo-e*8zOZk@cR#Io?L}J^7$_f?^vLD|RmS(z83Nlp8-(Pi7zjfTfyv+^A)4wJMa!8Erft*S8xx3imW8RI0Es#6bd4Y!nbN!J7r{)oGxqRv8))b zrbd922OJtD#U(axKQp4rfZ{xs4m)#|gheK50WT#?J{P87mXy6zw|fz*Pu_brZN8{N z*t4CJi?r29Yw8sFQ zFaVA|`>qG7m)yeFaUQb?AnKy-=phXjOF7yTvQ+p9fb{`C!H768@}_tCU3>_njQ^^J zIZt?aof6_(X#H#GvNeJqR1e-PJkqt%_HCmm*Iy6-W{KcfiqoB)f}BPyQuHL63q9ye zEiw$A=XV;vW~^8g&oBoxi7SDx;Ya?ZoAg7N3CAjW=8Dehj`=-h+3Vht=dYip3SODl z4X$&dtB>ia$9=Hq%1Xp6!_)IbNz&Y&O{z9RK8bnJ?Xmb7E0s^!Lp`-FSi2H2)b@D> zo*GVZ-Rw=ANx(<~1+=n8!)mudq7>pY_isEhGV>_T84o2R%Z_h>X z@9LV_bfgTlVJb7f>D*EoyJKB&6ur)E1kpo+wIyay+A}>7h>T45>2#5uLhd9-plw0A zrG$%H5i0Ghqdg`k&twMBQ-XEIp+XPA_TJ2sr{$86TfKWKyEr$3(*z<^E;n@=bZ;?hDf`J(QSpX3*BO29I$@4o;1;X$kbY<_3V4RT)B3O5EQ9 zmsW}H^UAQ+9yT5#qA{7bk1gQ?@LrA6w08pc0go4VUU4()uvwbu3mc{$^l|uWc3Ff# zPeCNgq^>0CmX~#f+2r2V%Ff<{?S17KLx3&3-mX7Jwze|dC^? z8>MQK=FI_=5mrC$eUz|YRL(GPzts-!x$6O&_>y|~IRIHnvlbu*1P(^q*g8A*;KQ#C zGbT5@yX*H>9&PT|RvxeHuk6-v&E=9=9lr#81BTEfpz>v|Kg2y3-Kf*DhIHr+y-^DV zcMpSQW^>jHmcNL4dF>?(DIPM42pC}GS;%bd1}YeW)oE~CFr2H`G;DM`VIpq!! z_3)fNCZdfFV4o$yvslp3;XI^uE)I?EukS|(<^B{#Db_Fg0bI@zLph$ljW;sQpj=wc zKOJE0$%{psDpo-2CL$<==m?qctJfx@n>X`HH*X4p!tQDGECvhV;BB6=?YK|W8$r+M zT7E{YfQOT6z#m`4a@D&>zbAc3df?IRXWjGzl1S+)0Yl1BH`|;dZ;{vkShAIGPsq!J*x_hQZdou=Nq7or zCq9#<#{Q|eKh3g&^xtIXMIo$9|43Dw4?j>CGJ!b*yygDZqv*Z>IV`EG7v&}|;5rg^Y?s}BOkK?EW1!04OfFpW6CXrx3PR8Qwh_$j< zKX265odLE;QvB#R5wS@KP^`n}B}ccWl*1G$+|;r7b=!$tnNbfyX9w}#r+c+?opz`7 zfp2Qu2wr~Y{{kP#m?7T%9TjDv_*pI!f4hyJ(ttGO++$_>mJot_G{xOQhH=|(mnD8% zn05B*J8o_u&b|x->GJE9|EIcliH$4E*7VRGISCZdFyMhlJ_@NTLJ}i_5y5wgO7ST% zldnsL@==r=3X;J`ii}_}NJ%Ue8yBz#9{9oo54s!ZHqb!1hc?_Z&um;9=z%8}XrKWb zHsCujj2_s40iz9j;_qK;?{iLsC@Hfl4culb$@AFnbI#stul28g&FY_#056Lv7*msY zexD^v$RwPYh%<@#c{K9TvXg<^4!-k@e!LqidnucvmEkueo8}YR z`9sxc{$T(RnBdH0Qi?^OKMzG9ckiU$VqVJNC4tBVJ&0M;oVE^Zc9*4mAmg3rOU4&v z*9a6C>QL`UQPf}e=8B!i+*))jy7lLzzvM|{S#KufIvkw8a)%Xj-EvVp-2GNkud0-m z9l|>+6ntvt8jXh_g$GBUd`CSyE+6@wE*6icn0jkOX zJX(3aL9n{*?al30lW=c(C6$JFfY{dpZ^bk^%oES*}q-; z7yoMhKmBL_)nEMOf8YAQ|H=RMZ~Uh4|NN~#{Ey@RC>TdkuQ_jpD;M+@^6%=Tn%*#0 z4KY_F$qt#2a<#+3P=oZOCxx&GWsSteBgAkPd<25a{;$w`!#n|+vY$NF^|(by%XOL! zgi+*~(!G0p$xgc=fAfwe)O+_zMTB<{YkY%?DhX}pep$5?1OJ9*0!iyK&7DiZsyJzB zNoWC1^Lfw8o*?GNz@S~N5`mHlRBNHA=)D;#3i{a2sj`qS*d}x~@BPMCN3YitBml%t zJ9N=xlQ1)8_Me3`EkqwQiR_8LxSk71Y}eGJ!RGif7$QE*8WP~gMQsc{AB1a2ETW!W zMGhd|3v*Idqg`i5F75uj6qLVTMKF^-?DtOjaKmFoQnGM_X*;FCKs#lO2Xdp*rad(h zxc2jgy**A)WR#d5)p&kS(v<6dwK|z_f)JJhmhy$gNo6p^qb#=4Ix&B#DNd-tK^ogp zyQWuaxW)gqCcFv_?!0m(1i>_z(=y9}l-TUaRj{on_g_U>F>WZNw09}f_7hp`$8=^; zD{y^J%$%SeuT53A>`p&;>_dD}6%skvloY~pPQukiE5QH+7ehxTT7z~N6dPS9N}rD+c-HIVFO}WquPm*-fDk%hSLP)u}e`5^k8o}>HgTRkFbDTl+gi5 zoAmO}xg^5InM z`pkW1(fCq3YV-q_b61y#<>?m%s7PSY?^gTABX{w8=)b-m*$4kVvYV6-(YP3GnxhF` zN;cxV9=!T{QG}f*<=-a7f@sK)^Y&je{T5#IAIugh=H6Ec6!W+oRBv%C{Q>}F0oE20%0UPgy`&F)0`o{$?0#E1R8yL#WM3a=?B&1Cq10i`+ zHF!n+@vm!Yxb;Vc6Wz~RqIA5v16j{M%!>5*KIvcqNgn^Yo)m0^m}%H8qkCujhAxC$ zy!Bi2&1V(k!IkyXDB0pz$ftp5R!Ws0(E__%8;&n=Nev_NbhX*qn%|yZ-1tb+%o*BpbXZRZsg-^ondqB?i1ZE)|02lY;SYJYla0;u3#V{CeA zY=62oao7kAOQ>(xmJ`)1fq)7js{&6oHGFs# z@x{by5pKzf*glIVa0TLf$tp^p4p>TvncO(=n)`2#s#5EA=0qA=yu#eyIR|fKf4|Iw z;E`mGPFpJksAbi+4VzUskmklIblG1D_+(!2mUxFkS8NzFkQ4K5AcNYct%_o5>F4En zjy~ms-G({nznyc(S8MNz^yr&fR)Jh8w&=votWtff3;3f$apZpLO(6B2w zXQKp$?0ym#Aa}$bpQ=rc%}f<+LpO&-U5+CyY=14*vc!pdvr z)kk>rV3a4CpsB@KEl`hY1M_}_F8N+&-2yka{5`T4;t+aC7NUXM>+LQ=P@oez6}>3i zThpKG9WX`gz5~Gq%G1k~rsPQz@|N#Zqxt6K;!=#+fA@p*mOx*#OqBN1KH%%;+c$HU zc57WWWtig-AKQ8*r4Qyu(Gp=_bxaV+@w|g}gYBE*C@HbMzS0b^#$>8P4hM9Hw}b?~ z?Co4iVH7?#zDjsU`4GXGa)XCQnID(!#0Tgb@YEk0G0*fmR;m?ZP`x1Qed%{CpTIu6 zAV#E9-0khj(ulV~DDNejhbS;|(U=_IpHqb~qqxj-8QCbkEF!^Rl%ff;x+;OmHhxk4 zlFr!f0V_|6UTMsfzaRP1EtyYlF`4C$_aF1*L!HB&wSITl;#wK8gK_g@{R90R<{|Hx z&Gq&Xiyr#DKa`9wT2rYopWbXur9<(3wurg7a9#5Iycy^sWp9+VLiDj%Y=R8t7mVEW+43R zx8C$l4qE5(@ln8zq%SSuk73M%n z1A({CX8FEmjW2LP!pD#B@rt8Bh1kk@5f<(8Wk zT4^spJ>^H3kFa)WOP_0x-`DGI8S@{lhCg-vcS zL&~pFf8ME9w%`@LYhY~GW${fauRv*ru2M8E zG+D;8tExNiIv4v2$D)BXw{J98{eBOdy=>so&`c4x4}mnJvVX#V#vTw&3BBvPahN>d z(PEhl6)%wmIjM{&1P%dXlQe+r>Q1t?z4`6x(()29wFnry0~|?lW;dTVlkeuYx92yS zKO~zg2?UkNH>(><>eTZ0Tib#h5Pf!aeQRxX86ua}jm5R+OE6d_3*_9`Y$j`~z?3Pr zxoK5~!d92bw?ZN7%iD|3D1Clmb#1lz!)UUy+T2*)fqb3jip7?}>}YwF%o3$x+m$GC zJHX@%G4`NmBOtQbvLw#&<)9ndl?Na3;$ZEuJ7MwUT|VpuNf}lkJ|~p%($@7vm>X&f z=U;}G76Hj7b<1jTjxDA8h=duMB<8fHA`DfF+%Ifw>PzD391WZHBnTH2Cjq{MpSZAw zan?tQ&pNF(sfC`<_=`!Yi!@LF^IxbsS5~~q{fI^0a*dBLWG2`AujTjy0Z`2@O7NDOx6y@{_bk+Mc@O)W?Q{tpciK-&weBBH;B) zyG7TOFuSl1jO{4meU?>VzK2ZFRtu|3ti^4lEEQs~*SOILEk0r5*gav=uf@t;M!6GH z-^QJ;ew!d;eG$E<@IPQqY!{Q&s&r}Z!ZVRM1mZS4o>Te8heC<5172ql|8(A92 zMUjF;_{2J#bNXeR*;Zobq7Z7}PiXQjMTL}EQCV%u$~(<)#I|{7Lb0_^mK&0Oq}&<> z2d0H2j#VhO{>fs)xRO0~eW1N{H5hKE*kF4?7J1W;*r5c`7jG+#I+bY`VWy>$ry+uz zJu?*UwfjyI<%1M*!p8aga(Gw)Bn1%qG?*(qYC?omn`fh$GOkU*tL0VFfdx6+NDDaw z{>seGmjyGu2U*>S1`uCmhJb#UxnIe79-TQ6$5@xx0EM5NJHaXM8($nrt#(o}g}!=a zZ^|poM`+bQ+VZqK)ZOy4Q@Arx_9GWrVX?y0LbeLKFmF2Z4m;(Ln=V=4iOhKn+1}VR zn;4uJ$laGArRiju{Dh>7J;4)}B-%BlmryYARi;77VoKg2zVule?bS%pqnkMrCdcDh zZ{DJq_Z}h5sz-=y`q*>TM7M`@evVbWL+&y+5YWgnW8$G4E>ytnG2Nsv*#ckp8#G9j z8vpDB*Rd|Ud?1#WaMsw8@bUY2P@kdmP4k;|4|6_6N|#kNclR$I(SsAYaQFQ@yAZWh z!Uq$H!9d2iAJEUa!jFJVwE%}B*JRBRwXeX4T_U4`2QC;-##~`SL@TsuCU zE7;qcZ$$KYL%rUs;dzSJe1#@J?8{MIt%>!0*cz`o-uFSuwOsd5X))Hk5NY241S5)OAq_cCEV9|cTj9QjgU*jD=sa?hqY>5& zrJ$hmca!o1h-wA07Vx(0A3)^c)Afpewd@M}vj=6eN^T zNAc8&&L|E=XGJzMZ+J7X(u(4+$t)HNSZbqP)TbJ>Z`gCO?Fb>k8f1tf#X2Bk@Q0j* za2FpU=96RQmU*Zhs18X?6mwG{P>9v_z(l3Z0!*~ z8?&CHC$Nw!ad@10P;8(2WMdVv`AlIQnalF$MyE^w-3E`(rip>_+*H@E_-Mb41@72l zu<3%%X!0VRU%q32;7Db{L2D&U(HdY)h9a2!d=r-JG}jyT6+$m;FEl4{Je2Qf3CgB# zJf^`YUI*VKSpDuBj!DHV88#GT(V0*b0vJoeQYEfj9$^E{CIYDd>d9F)TB}c2H&_D4 z-oe#jKcmUo^2XEVGcKwol7qY`UA!Gm4^h%^dRk*HFb}{JxMEr>os6u~1~W9DS*n} zR>V*XGe-kGk6nc-JKhLRAFJ|0=*ma`!GL4=^S8{zows`Rs{y&@<+NgsmbbJu+|qOp z{@L_V!PsX8@pnWYseV8vdIwG8#KQuCW7;&g)@Up|*%K&jVh?W{m5?q?ZW3Yg!kY%t z*2aA3E;6YZ>+YE@P8vt%=&;dXw;l~YQuS1%v8fY zmJR-phtG%U9w!K<^GtG88vOupJ7j~42yc@Y6cJ!5`YrN7$*|u6-cj-zJbC(?(rl8z z4~0FVPCT`JCVoYNEVn<%p0Opw(yS4&zCOW%WWx8Rpt8j^>YBhl{gyk;9Fzv?zLc15 z&f8vEP`TI?RN6U3!aCUYru9Wmfb^@{?#l=*SOT@Ct8q(1G}ztH6x*dEgj1FEExb9@ zcz-B>oOaj?SMmkYWF4L39s3F1(r+;zI>s3l63XgaLMeVO2zf#Zm82li+?E1Gr&7go z7AhdLPwz}rzXykeKM6_9B2D0bJrgAV&`ppxYgrrkjI8Mza=y+77E`B7o7;sZ|0X*K&?DDZl zrS66R$v14IZAJ6V5N7n~d{vc3;6Y%Xd8emi#<~7GajI zB8WOj8tB&PFCdO;!lGbkhc85;lb=H|ekK-<$!yi>s}EA>AmT?WDjIf87r%WBJ7lLhY z#F`O~n6SgqOgw6^ZuqqZb4@s6h6+baO5uo!DIBpLg(Iy=6U2emY|LQmh?nNwVy$EKp;shnOpnpj zuu|4ogjh%LqvsYJ&y&7LXa7{TDkQs|g ztk;97(I14wpKaoYRRVJ*3L_=_5qX&(S5CUXd9e;3p8S+R;Me`X{KNI-U;ZIE{mVZb zcP_8RlkB{_ABzO;gKyeDS(QWl9V3M1Rxpj(pVDr@{Lu<85r?tl0>Y3JttkyRdhF@M zUc}>y6QOhV`HtSoCj+L+k^)!i32J5W^2yZgo$*h#>sF|?O9T)rH3oT#CcsjrCXn%x z_jqWtFC_GkB3`H^eryHcui?v#d@pTp&(&_3c@Eoww4xGv_Ai$O?v56=N~m_!jIp$O zuUvde?=*(!NDvk!lrlW|&Znfuuz~9}OA^ zBcCyo!snOEz^cnmNkl+%yTvq9kKYH$IN;wqRNyxGy zsv4v&TFk~vn|4%Ac+IhU9TCe+>-_qnqj+ZQWjXWq5n*UK^679305pE_WMmV5P?Gp7 zQLB$Z8)B9;A(Bj(4bw}dte7wzqNz^rx^|8Ict0st_KU9kkWK*oF+w>g%VOpH`dzqT zI&Ss-!%5UEHlrCs$^)P#OxX_}yndVGM|1#_-IF;_{8tk0herinANo6)CABM%-Vi(w zh_{|;kkkNv?j~)RgMkBJv^Z#AZ(v00jXhfwv>2EjJf=`If z#d-?E?d9>jG`c^qRON17bGpOsU+g}`A~ujO?ui$-gO(HJIfC*f`tLU2$9`$xhU>vu zgo?)(yq8RO{Qo5VWT0~@CHH`FOPYKV-Z!1|B4nxd+P|}TD<&{A6e&jLNQqxJ_fc2m zNTGCv+ZBrE@v?uwx3H7BL zsn1T{K|}Hoc*JQ>N)@zOJ?Tu6!W%kw+LZ z;=|Pyaic+ARUgNgLxLv~Jnq-7U%N;pAjEH=`3CK7U4Tm1aD3R>)kvM`2X z?XXOo@1fV7z$$2srfbAznI^!>RGkN%zVDxkFf&=3L=!eXCc1p0z3>j?-gjOc0?AE^ z9Hp8lwqTiM$Z5+&UGEX6gs2&>O-!hbG*k*SDRtc}l|`Q61K~R%jv(m{a3482WQs6< z5T+bwW-GJO3iApG0SzmTgI8UaBk@08;BN0c&YB*i24) zhh%BR;F;k_%CK|k9m>z&OFw)w#IO2Nn#vjh}r$vFWG*_;+G z>?$`x;W+f`cC6FO4mVlmbl3lw=l>V!qQ@XEoe+D4SaZi$-Pfgk{a%HQ>^<%{Kkd0W z;W~m2^MM@Ct)TvHDCwo@rcwo@c5Fg%eqm|;ht@b+mrK_FECe}9{|+qc;xW{+p8D)O z3g&P=Z`sB#0K>@k$OtPS|0|?vOQQcf;VYN2pFhS-m+b_>v7MXS#rb3rNlF znX6v0H!Z6@u%xOV-hBHVI&NL2GHbz5Z|SD{ZZRlVd*nv~JWH=2DeWJZb%{H5I$Fu&T}$B$geXrr<()&%H8RRXtt8|C=nxxzeZJxjpIXUC7Ko=OkHXh<2W?m6aI z(<;zFuhp)l-g8_FJjne9e3&kfN{lU;MMdaJF`)oL3=TMMW8tcdED!VhI=$7NboT^H zjThx$(}T|b=}?pyON}!S&Xi}i3LLRl6o6(75 zNmVmv1}8GaQ`#6mKp0K;6-1Hloq(-)9~!Nzq#7)v)iQfqt0fUvoFAvx#vWo|b>f*4 zkt6O|qo)vsx~oSyKz=z|5WX1bRTJF6_iQWilp%~E-ns1B`0Cg*$>J+~Aj=vB_w)z>F$`{7Tojpm0yDY?N&udJ{N(z60Yvs$_1nC;qW7%-g({C7go z;oX-vmQbF^(6~#b83*3;e5{C@n?X!`%EN{}G;*&Q!CA_{rW?00u<81T3~YI}G%+Id zW8~-P9}(V73sZ6tRsF{11kkMAeF5h}0Q&*4NM(CHgoz6Q$;{^{Vk7L0 z5;*D#KqoI)4mOdZD_t*2{le*ZQ#Qm*u)ei ziftWCMW{LGA5sx%z)?csUM?zPY4^CuM_0~)<4#fvK!^AvKAfWH`$Ai8uSTRLMXAa( zFt_-gghyw-C$hzagnGhuE6g)nJ_;4&3f!Tp<2Hx0_9qa#QeZ*)@BxOFbD_kJ&p!WKnntmBT>fcs@9o+>6ZT zw<{jm%WBPLRd@7J8JFaj+hAx>KD(nA2K1NSM&wgz$I{|*?h~vp?#>p<`^Y@*<%Gfa zk=Mw9D>s*$%D1aKt4+w!zFGcZXICD5=9m|jS~XiKv$onQt^#t{2&Kpt^1<~x)Ww3N zR0C_cZrQkTQx%&JR_q1ZmfX{eSAC5NALE5yJbs)@Va1eE@`x3bYOYv%@#5fp{pP;? zZG^w$H}_>qw4`&z7cXkbix+kNZ*UcsdGQAj$o>e9&sHpGw>Tl68Zo#@zt-|)T{s_* zaoTCNmW zKU|l%$)$M~^Hct=D{JxVtIz_DPIY1a>gcL(`beC9jX{-*o3I~1l)=#kmXxEX(pV>Xz2eW2ni!=mg_hy6vf zX=JbWu1r>*FfK(N42DZ)P~qX2L}h_x(9ZG3>Cy20@*J;uH|)xLpN(isxeq3~`#SZD z`AqqL0{TGE5or^JYe2mi*zsiUSpf^1z+Ns?w8AQ{IU-((bI@_)J0F`WLwY^|YMd2_ zA$-cL3^Q2i)X$&v56@0p|Fy62< zgFok~x&1@=q#-w<&wXhHfLEYEDbIU*+AQ%>Xzh)X;ac#yZC{%At6ma0B0x^`_U&7Q zbRqrB4Iwz^_NxI)k9)zGWqIBO6QMBR7KzN*jRSZcWeL<5a2wEOh_dK65-2Uu(o3Ud zB5?9!8kwSaNR@Fc$JQ~%`2=iyA#4r~xyi!(&gzaFZ-*Rn&6jpNS=`)MT5YawZpgDX zbJUF_%O5_S=ro|=j1ADZ>paqu(=QL1O8^fHU|l@}1T0o`^GBz72QutE-oJ%RxmZLM;#KDFToO15Td0Tj^P7JM#ea6Zl>reUbcx5jILhy^bRo< zr^T}SuOMLhO*V^SK{u~8L_yiTCG}bKw^T!h$W;`g$Oi!+5#zXDg}woJz2XB=mKsnh zisOs6Xzot8r3fHe#!XK}8)%6dL``^Jl1D2j zKr7M5NFoJ(UlhenL+{`zGSCLk@x^6@*o$eOE3^SIr{qH~FH{j1gd*<)5Gtxgde=u zXQolVPmiI}%`hN9Y(O1OOpX)(Ti}y%2QUJW94A2c82{U~N(1~zxQEq4=tbS0xEcLs z_Ps!$7=#kSP1NfAuJE^3nc#%VZ$?FbEi{jVUmQXr9OM{5x}h$@$PqCw*d(}An<0R? zFpY+GVLl(5P_%Ef&a_YHQ(%theH>tf6A92#hPTkc`t2j5m`4 z2GQLqY&xJdr^gwYugR@4DOx?b8QtK59dgB?JD5vAXl`=d`ieEDaWncXp4PCVj89T8 z=AxU?HNRY+tV~bQNb|}v`0dT;ddyLuK|_8s`e*h`ga*9{Tam@)fyao$!zS#;B{9gQ zI7*{hE%J8}Rx7}9kX%etBR6ggKANbA>bM``(1k+BDieJsNCjdui9es%>-Zo=kW4Nt zOkIg6b(Zm5usfz-OtBakO=hX|#T$>5Q$FUm9q~a)Mge_~)ER^m)t-t&iu{CI+)v>>7z7v4EP;tvDyB;YLXW0xwdOc4 z5q2E4)0!LwGVgYjtHM|WJ1Za3`8!yP+!y(fnM2|nxTIQLL&8%U=xa?8{&Y2rou^C2 zDdUc>KL{sZCdtT+aho-N*ji=+g{j^sIN91?;c|fD8P#GT!M9Ma0hD9L{1oF= za&;juN9J{8iF@mtOV8Jq#dmqH8p0;WjE^h|F*98tF0=7j7IrXrd8VqMuMb|Q)!|u5 zCHZ6y2lmPf+I(nTL~(AL%8*ZW1@dGum<9L7=&2Q=9Im4hc&bPT{U{1C0~1o23ryE% z)#S+%3O26Q8|JGL-D#T02lL`CTqi6e3z{TlU$G5TA@Sk-O!kxelf*(HG^P7c zyC}lhe5r>*dC?^hq7wg|R74l%6wMMK zpeGYq-Qj!s zoYalQPXJ9X*WjF-`OF@wHY6t(y?CKFmD?tA){XkK7B6oy;d&^t&gxg^;)GWaN|{I} zV|8`HJ22rLm7PxV>7{DV6?EQyK^q>u!ABU=i9uxT(U+lXPlCqXkDZ)*B^YyLx(qlueb=d; zjSq)80McNHlf+i9TI@@E;yOMk-m)ULjqt`96=c!1$;Wp$u|Va`F}^wOZBdhyrL&1Z zYl?)q%v=*1S@Qn)JqPN^Wm)*Kj0FxK4H1rW%|--^jUH6Y8p1Yf{>CIlOjFWX`XRcw zoHDK`q8VlQG-X`CSFEnA&9@*?YHqfcHox2W1uz!2_h=7nFW7qMt;y|5xMCPkn(Etx z_E}yy_x*t=TYG!$(+V3TJyBVbbfC4-k1Ay2ax|i;0>7cF!eJ|Ee3s!BL(3*AK9nrE zT~>N&_EeGC{3FaK05mxTorE{hmfCrftr$5twvn5_o#peuu#y{liEkn)_@@ z{LtD*X5*{4RAXxHhYAbw{>GVE;;^(@XwVrDzGaP;K50cG;P1{QUqa=)ckX~-X>1i0 z#Xc($qHm*}6vy1XeCQM&L{fc#ng+Pf-qmU1@djmKU{Vx^h-}D zRvR9gDi|K_!R7_>FXOUN0l^kRITC+ez(mqQBY^TZ5XiChR&XqdBrvug;g2bSMSO06 zjnV$NDoawoYqLgd&PTk78YL-U@^0cp=`#y+&`cwh_7%FKlN5G>4vGNws^oxh8Y3(k zx)*Py+B`P;hPvhrfOW>$0n~=5?90?WE+%OQ(jTEvq%YaA&T*PSvhL#FQb7&5ETs9L$Aj%U$CQ9>Np1`e-aW^$Hq)2!pqBC!px-?pH>WiJ7D+ibld}8|)dq@)a{n(rAp0 zCJU&V^yt^;6MQ^twQ{Y&PCmBm`OZANiht^lwd|K8BozTj=}uS-N=5iczkAx+-rN+^ ze)C!J{(U^*S?o?&cb+l1+9!n~k`SwRv@o!vzjt-c0E4^Ckq|K<7v~|+M{9{jjVCLi zgYWWgi+vK&>dp4k#fmFYAw2h%qIQ&q8c>N0a%uE7Siz2MG1uc}925b-)T`m6BsO~Y zlGobQWA832(JITi#p*VD?}KE=9rjG0P91u?ix@`altXiwHY`+TCCzT90)dtaM!_&k zwA1cuz{7FHWmOkXcD6wH8m?&Bvh>?TK{1`s`08Q z+JY~e+|`rKuO)D~Ly}rZhi!PwS?dP|{ANT%IQ~{KVEdhFzk33!)+NyaEEv=1E+G(1 zo&i7-d~!JQ4L7lX(^2h=UXFI}zaL;U!MVjtlLZb2r%^62<0!0x6|UyvbOHMkE{alY z%rb!lL$|~e0r5m0tYcnqje`i7@RsZzoC=I(V8x5skC(u@UJE(R7c>^G%JB6{ECe=x zdHA^IbD(8Pf2{_u(leF{N#Xu1y7$suPyCR;d=HbmmrNzv8|6!e1(_mBg?}&PK*ewj z5!AcVT@^p7CaGxZ7}5WOa22|hN|1O8gm`+z`=x;xYPgb5Fp-VD<6vr@am)dh0q}xr zpFm~F!8E|xLBR4$L4mXwU$T{~qeex)iSiM6I52I%<%YVEzNawz3B0h7xDav8fKUgl z4oY47XGPEexdUEz0c8ji-|&Ovk$FFmq^iPC$BqC2s)xOiRG%8^|d2a zznR7rVY)Fgvi5XqEw!yREs*LH^bN`B(fKK%I864jjd9gB>Sl1&t7XPkeR4qtf)7q} z_D_jxO>0xkCUZf`sVfw@A?*qU+8nZebV?JnAokU`4}=?o#4 zKb4#mRd00__PgHO?rD!j8+dTge>!+IcY*g#-Uev`)+i!shpQ4-hvR66TcEb=4)QXw z40wYXl;DyjTD?Fl7an35&V`+&q)}ePsIDgx)N!AeY^WrAqZ{cm$@$(%!+~jyWSz4Z7gS{CB=#4a?8Sfwb$v^pU z7kp5VWw9fvG-3C7&93zM+w;msE-XjCGJgPE4sBK?P>74mNK4#qKH2wYr{Q)yQ) zeD9)@0inXBVA1}6PG1INsna)1U zCsiy|`^OjH7x-K(;`F=e$tsjO^@>mr&Zrkjhm2A&U5|N=hwhIR)eGic85>NmJXn zpeD4XEijdFK$Eme9N&F<>62 z$FCh!;4rD$r_nE8TxCZU!u}L8`Z#?A1*|_vp1bzR=|cR;KEHTHe+yobWYt`VIql`> zl2dq9%{Xb@Dc1HHk^;4Q^W>3_)6$N!g2V zY@T&3@%U#GZyHUqC7zfFsRNyQ*tvN82KKyYH&)pkYgZd@35bJ61eXYbS93Rpw#&|B z)F&{L?MxrD{j!$>Jruehfx8Sk6&Bh1sv_7xwv7WR%@`*JEuacAC!E&XhKUT#?ds0A zPlG3st)I*g`AK5aqNiApPj=7&wT*fs0HZlNv*bb2eZRme)ME1$AO<8f}=^jSYobX~oDcf<@&*H8MF2ZK(S82`Y@G!QEBH^#lVHr@D%yQ=91 zOe-~fibe3nr^xmx-BpEM5xyQck(kb4sdFyMN!j+;d0C(@JQidZI;s-hmdABro7ZZf za$*{|=#<9}JDrJXwr}mjvChoF3|75!5i*w};nS_DfRoSwyv%~XG5#5-0YX?sHj;3X zjZIEWjElvrLFx60=^HbQf0jBoz~{Vni^SfK;JK|I*kiEg<8O}T_F(UN-SRYZx(9`hTnrJd`zK5eE#I`xm8M1kiP?7o;^ge)>Z~P4x1V0^ z%qObl-sh`@u>Sm~S6BLKwmwZie-B9O{zv;OH{Z|pHw>Uwt2QJZKE^9{HUx$LBm|y8 z{V4o>w?0vxc?r1!*mg-KilpOjg!F8}<1q3jKZ^-kLRMEZ_oL?HuzD%gqb6(o)n&=? z#nZwUPbMKIR1-F|xKw3-!(Qmz;znsEk1FNGsAms_H7IOsI@vlt|E{z5l!-;a`dDTx z<@o$*=^ZVRUcmSR(P&5Z?FZ$dn zVszOkT#SsTTKy$Lk!59QZ)=9pB`J{1n8T&){x5NKi`4JZoM!sGWQQHDpFh`-*QI`re;fFQF)xh>5K#yF&t$`AYn<1Ar@$Q z+z04>Mq5KDhj_~Y8SdEr&OpU!4inGfqZetpUN%$ZvGlY9;zZ6qd9BsgJKrb7`MJb` zo)%MyN2HfVX zQLkR??+-FWevsnm{G|7pB(O*g6HVrdK{H1HYarVLOq57I!{su&bge4PMVCp-A;~(_ zrUY%2^y?4KFVTYQB8#B}xv;Ub5Gr{5?7xN$`||3%qzf`^eL)Xfxf_T&t>b>DRjZ#J z|E`71+M3dl%pxOSROQQLP8LtMRyXH??ol9WW2~=h#Ocz-zEjj&v51x@=f|ORUj|Cs zi;FF#TO2CTigGQjZkvr0S5hlm7h0GOx104=^ZWaz)b&&I+WF!FN0JK|bM~ZK_@;-E z<)WWF>Gxh=w0|}7p!%zk0n^E^Mt+4|6$Np^0wEa3vMl&H!MISV=I=1$SQewrGc@J8 zg?vDlWD}id*WOzpUGh1N3&N`MkiyO#@m@>~=H|_ys$?+hiq$7uAgw5Ttcn7OLsI-6 zL`;^Zjtel#bc>sHwcF5Xue5_B@KLnp03HVujN9GEhF1k{WW^XgW2O;fI zI3j)4b!hIXV@?i>R0F1Nc-hYWPUY8FLFZvk|(%hx|79T zw~O5o)5WiBW3#xlNmo(}l2>6|*1m132n1Xq6`#rnBj9tro-ytdq`{v;^CI&$(Wb$( zYE}(0t7t;9uwa4O%_mXquWa{DL+lBl5T}tB1h0Zpn}>8EjZd6#2U0*5SE61JD=c=8 z?v%?}y1121GpYdT(FU3tOXZsfv8^{21=1SEBm=KotHNvXVWZsKrxMti+8^(0G!f;x zi;N>aaibW?+lcemOGR}+8%`@rp26 z%@;$o+V7)ly*o8l$8jzIq5!08_`yREpCj8lGao~Nv8xdyvW|uFF1(zbEx+yTp`XS& z!4m=lk7J=TgN%MAgW0$88BB7HPKl1f2Nv|nsMpV%Da?LA&Zzj z&SwTVU8*d0W-K0hPv;679T?ga-G*Qd5b;>R=mLuiKqR`Q)GRy~WtvO(1J*qt)C0~v z+)Hun;VbXZk1)RsU#msitXAU!UzxK-}lC>_4NVu)XZl6P$(h-1t|kCQ8xWZx=}HMruDu39pCrPs&(7T4NRBcM;mqNO7s$WBiS`q&Qu z3i|<=V?O|J><3_t{Xi}*;mEo>*D>{f$;;49&VSKSa7u*2a2xkrxUEQfctf&H9hUcaA2XuRby zXnd?C@EzCc*z(>RD1=!jQ*FT&2kl#3epB$uL1i}yRDO`dELCXb(E$){fefgRL&r7F z6f-kEMm}Z~iE^lXAm@S-4o@Zc!myo!2jDJH2X0Nm(uE!X?%=Trun72!-LA&MeH$C@ z{*?$R^ez((fLLOm)HFd2M`bhcltGS9Py=}LZ-#=WyLL3dTDAgXMky@x&}f15?E9R7 zxxv>wQ``qmR}Em)3dc2f&VhbAm)EG)zm&iHx-0>{xe?0jQD3(nORDWz=!QHR+^hzk zw}c@%1p9XIu;trecOelZ#PDWeBNdPl90sA`z7*Xo)(11E^@3KJ>-&N{9T>C$BNC$P&RR4}^+2;@MZhi-`u2d$fU9fH%tA_uRb! zWXgmp4xc5}0D#8b>L^T9IG8oS!{9Dg<5R{TLv`1n45Wp;D`M<3C%sx_s=ID!}UCHAZ!rc5V^|J>2XVpVnE=QiWbaKxhtw*6VtmxDDeZW3g>r!C4c0`JksG z(gx%_Ew+)yBW^Rt6tI|acqZ$RB8E)Nw;|Ugewnc9jjPtZ_Q}CjxUP99BtA4=Z=Zyw z@JWm{|Ms^^YXKlEHOb+45-HFOaVdyvF%HS(SmP$F#gOXnCL%2)v;(V#xxMK-kms9m znF7>eF*`v#5BpNM0SY&X8=aHz7p$r50+-m71*$h0$6^g~;XNj=F6sf@Z~&Wr-yg{c zAb4l>s>+Stj3BoOh>07&2se1tkl;b(Hi<-ZEg zAu`jCr!Ng;c9|{a8vqZ*i~1W1RA@Q0?F04G=6s1m1=?__1UNqKe<+5dJ#kqdD_Iof z3#OL;v0n{31Y$9;{WrH947=aq_sF)>4fOd9p}RCTR}k40y;=TorBHko?6=T*DTTClJ3$}j1C(x=41^a(h_8tUJfP1D1L-f=^6Ex& zdp3ClMn7;EvWX}p$IKUV(tGko?pKwgvX5qlXe9v71lqZTW5!s=@LJv5C&-rW79QIl zB3M3Vh{vH&*zbzN$2c?8acesT?cxWc8|lq5IXN{e7aBo|WZDUGB8^PO9S-Vj~H5ae8$CA{iy4 zUaB-xyn!o^@_+6=RJsskb*e&23Mgkm%le-v@48=&!ZE3=ox@dRvQL*obhDwx z|HK|ED4ZAlsWHu|e>FOvPwFu?fS>%-vJVYr1@+KR%A;p&Rmj}ZLCFvS2?6HmES!Bz zCdL{PLp0iC2CUXqJa}Gi?uji#I@}urK=smd0A(RFJ~8+@rQi!$_|B1)z%ZDZS1oW~ z(a;VGOEc$H_U=2TCMe&$Tr4pImTZ2U^+4QfpkcY@YlTz|7hi#-)5K~g_lYZ#Hjs2- z3YLl{m3T5jyX|+CEVV^-vpP$=wbA-$<7H46eJ0gtLV~mb!Q~C55V=Dj)X)A&CML)B z52o4^(}$hO&i=t3?Ar$ijs1hh*xtnSUZXQJ-5wi1Xn+YhC72w)p6u=Mm3DY)>^r)h z`;QZl<8Yu$j!zzn)#D320nns~C2->7V`G58V|Zbr230IqI|SB zUp!RnbZUft~GTG7&0^eBLjGm41{g#up;e8jYOJM4{ zFm~u+O)+__0KkiuC?ifyOUo{O>1neIq%k*PD?Q##X;W`4A*eTFr9M8Pdt`WjQ{iOu zZ8vUbO@kUolLS#rELN(ziej#_Zy=cjgO&{7rU4yDSgE>c-0p<5q8q~qi&j~RYLcgK zaLKFzixL2$u;h=gG?<64L;+zvVbJrY<2{%(r!~Gnemy$}pMZYsHG(7$6%Ba9~Eu*H!EEH1k>;y#81 z=_@&N6$GAU7t(pJ8zs?d+@9EgN7g70dJSueoaRCqkwhw_sZ?4%10pMg6sv2!BcDb( z7fwG#B5!I;vIeROt;#d@M$F&&^eF??N0Ufb*)PRrRCKz?7-i8$OuV#I_Jyv!|7cN+ zoF>h9T(92i8&u$g;X*MeuqIUO8qW#sDaY_&dU7& zQD8b0I~oD3*a<%{JCDH|xE2h#&YTt;rxV38*9x_<*(9R56Pae12nQLfAWchh_kV4p zEP@=_G4!H<`x1PHGd_KEzLEv0H@b7;H-G?ld~(j{Kd?2F*2cDo$Ux`})Odmvi&ri~ zT6{tfR8j71o5_Mf67)cid3F-kW{;-Bq^(f~QZ1%1`Ph2T@N#j`2G;PON^R#TNP9Dm z1FEvgFluwF*WQ&0J}QVDuUfDewY%WCaj*#|3)io#4cG%TQQ0TUP^w9MEt2{X6U51g zyWN3BMAxm`7$$7lz6qoZYK4gm8qiQs0`&00#{Y={wR3>)n8ct**>Xf^UZxAIHu})5 zH9TK0A(f5XbaRtz=Qt_UOC`$@P-sZjx$J_B_LSZ^pS!imjowqC-NC6G&&4rRIM$cA zzvXKT%TvhnRRRxGzq507_F?hWP_P2JZ_IC)rRsyteQcdHsm%E6tIhZ_M9h-Vu-ZEK z2R#UhU|s{TXwF4)GNpidkFdT&hm*CiOJW+~a?fMAOTQc-2)Ihi1Pum>$fVbao^vHx zShdZv*+H0oOoXgZI3IQdImUT4k5?N7!bK9lB$!~hz=YpcStBD`&o(!fTkFfs`K9^h ze2aSKJWotgf_%TYunL-OAEP*INd}gm!-By>(-fHk4_!Y8YwK9px4U8Tl#NY)|7Rij zc8L-5QeeJ@WBDe9UHkx{IGrLCQ44ks@aO9O418j}E|j2^JdBDYI>$%xiCLv6UYVeN zR=?^q^yv4KyETHx{C;YeC_A744Xr4(s*~{mu9Y<+kns>A2Zgb5@sNTS?!xR~Qg|x@ zBl!bCPK?Vd#jn)Ffj@@>*P0uv8#h4RdkR?a;z39ul{hI(5wC-AKiqY-yL8;T=M9jB zi!+(G^v4U3RZC}>wDiYIF-sO_UP_*;Y1Hjj{P9w5A!$NSPatDbcv)tf;>;A|^%{|n zIPrGnNt#A)qi5oxw=2)f6w5buCQ^7=M$3e<-uq}oOq{vhV2bcRoVX>8&_L3c4@Kpq z?=j@G*XWIpkvP3aKDKEnY8x>R;hb7HCX+YG+vGa=ej(XjfcS3-T{?}ph4_6X`6hXm zfFMeijX!>XGCtW%wvsM`|A*u^$zHOb93(*OlcVGq2I?P^*9m*1 z1oDeyJ^3vpTr9`QRcOW9Lh^JWSy@Q-7J!~TTS!(HlCRY)#qCzJ1{?9sLbAA!tS=-R z3(4j}vbB)>*#eec=nP28z2tYvj&iT0otmQYy76_gmTV-ezeuy23(50^6b-G!vLken|hzqKMS7LvYFiE&Y>mXdkSV5YRQsqmoItzFy@lcJKfITm%I zj;}cuKckN46BQ;_#yUR8M^z@OiR#!)71nV1|cIRmM-Fc~KBr_NN-+cEq8r_TjJgU|9={5+W4#Cg3K)$`6^mKi{ zm|pXea#z1?U&JKEm$q+1@vE&zP;lMS4oK-!NjKmof)L!4mc@}3;Yzr_v(TJHrai)C zUtYF9_Gj(j$9`GTJ9rV6CT)LAFvs4;M>aK!*d5Rjj~3VFcXnLjmb~^zv6LMnK_XEJ z;+MJIdbYW~e6Na&@fGo~Iv4KFRDDxJ(TP_K{u}4L8g@`Pzf1(MqdG(?M>xlLv;5%p zW>vjepGs2pBp+-T2Acoq>-lfzmo^t2H-cu`g8oSNq=-M~K|NgrbmV*K$EeCpI}T%o zbh*;n79TUd`%-(O@kYx%pp6c3gaSPuZ_WDv!dl**Z#7#h>rEDDMJc-XzBYUF<%{kY zH!r#`x_9sD|DY#oeLzx*)xg3Dvn@ z*~UL^_j5PxkcV$fydt~W9VCwkz1q~Mw&!x+e0Rf@iWkq;6l7gFY3Sxo^ zH~mG&RycUq^L(tGv62s@(6bMF9n(h3N<8M*7~R1kB8y$+zN6GF(DafVlSFrF}O$cmf5ZNs>QdH zXJ$NFO4++;nu60}FT4nj-qcg=4=vhybJXP^%>6kRq*T5CRtNNZ^@+YCRwZKsjmNuPlB z8{w*LInv|wVX+m-Ql|qFwjUs7fZ&5l#vsPLDXl9%L=LjFU)|Zk_B^$4?dxg=&Oe=~C%N)v1udPI=qHd?Ul1iNo{is#kjtJKs&%~^e z^e1ic8CDNd3gHS(f0?-{1p7yaK!7x!AGxljsW7LUh>s6`Xv3F5EJ&Pq9 zwFuc)4=v?~W2HAlAZ0pJ)lOVqBT@hD=mNT5d zDNjt*1%_KsOp30sa>8!w&$M2F-5ODY*U8_GxE>LqK;uA3ESaEYAhr-}K>UlSO7|f* z+}8b9%gtx4tyXhsZReF%Yhlf)EK)kWl9>Cdf^;IqS=dYjv<+Vz8QcXzuEfkh{8k7K zKO)|x4zmg<#b6{RJI~H$H-NjVF^{T#`w_aS*Yp&o@@#Def|4!H`|ze@h?TCC=cjl? zRCN9E&&{q@^AIev;Z!t*p#nbZ>_{^B`2}ik`NuWINb$3V@S0?e7=oe54unI_5VEA( zvt(iR$VgJnv!=ubhRFogefl1WuFCmD(U`i24`gCS$|>TSaTwuM|Tm zaS-ynd=B+xoHQrzwnvjaRzgTq@EB){WDI2*IE@U6R<&s)p+^DW`X>g!M=q7=tAD3o zt%Sw)b)y&3oGjF)*wkJ z7!izK{|Vq1f@W+UCOay|El18r?nt^DTjrT5?60ju0?Js71PIC1jv;f8fIwI^GJDS} zp=w`QWBCp(>u@`HtHcBR%DhZo4H<^4gXpOgBhSCcAO{!(g0MZ9EeP#Ok1WOQ_8S#K zOqUQ~2KqEO2Bp=Nr-L-N4jA_k^)sHTPV{sHX5Su65-A0=HKMfSNu=VFUcZHcS3u@e ztBXZ9qMNu=2X(HRx2_zU1 z38@oN$qzXd1&?yhUO*~q`z?hvTZqYVkm3{U9oP_gPb;j3gYIG!ooAfQdGD`2}_%*X11ISsTsp$9CAgLLU(tc6R@f!cW{ZW*}6cHxar zO%_tsvt62DU>lQ@Q;Ii}cOmdS$fS#3BEI?Aju`gc<=imLrLlENpfgwQCa{Y4IqOoM9=MsOaid-_)6X)QM zH3{ujZg%EO+b;f8G6XlqSX~EEA9cH^WVvu-oN6{dimU9%7BX8}il?1J1<)!QtFsO; zlOpUDSz{s(QM5*TdvB2KOp?yeR?*0817SCOk^q|*a+$R`*Mc?bW%xk}Sz*vOYi#c) zbIzz?+4){IG2DzUy0(4bd*H+79!c!C+p*kGR!FW+WZ>AHXSOw50ri;Pu#5&XR8>n<;5d{ zna53xDCUZfcPZ+nE7@^u>7r-ncht|zx0N}%_;@kyT0W_#pPha5cJD&Ed^UW2A`nG^ zBy1###mC#tjn>9|Ghbo#xQ0mpG^%GRYOmniq~c?!N&GMg8Gjm%!?Ho7UV4Ple+i<34`XtG^BO>90>#0!ffXq#EK^lWkU+_lvW zfRH?adO|b<0(Jr>P|vDlW$j-V1@YWKmUjGkb>^~^`HNv(c$=$KMcm3T>gG* zdwGW-l-tSb`qtX&@{;KzRyP*co-e^Ink+nT3fPsbtx_jtH#d!H^t<`(?fH$SMYCK< z)|a;zpK&_Bu)4O|{K1M?S#55pUzn_E9@uReX&OrCA7EiH3#VVU;L zFTfRKwQc;6VDVaAw{|Seug^bSwv5i2s+8W$8t)t%+hWPY2N%?!@U_U8I%vdIvUNkh1ytQ*T-kRpHTRcJojS+>BW4*2>d zLX(2x%5+KjwZ3wA-`B~Qp$=jz}SSi`}{f4Nhv0aS8;22QsKfz z3d<0f`@c?Mq_CQvofPigF&_uCv{m-SY?cp*Q98gz;zAR2zw8yUyqRql6D}cTj<5E_ z4_*BctYP;u{t*lu^yNcoti+R0VcYp;xBgQ4T>{SF)mm^GJ}ZeLqX6bb%f&=uUH|yU zzA8-bVsZicIK(S8YpONTv<_~%#Pb4^rQ*#bjWZQxX$ zO?S~qKlHW+4dF4yJeoz3bTyJYqIE=30iygtBDG+Af#><=!<_WPmnkMv;ft+ff>Y*0 z9MgQ2Ff?z;()_t;3w&pI(N(}x$h$s0()tR7q($a6{+HN)fkc63O51r1B7n!e-+{6d ztqhBRVzLwygG1cpT;FRGGXk4U64bOdF+It4i+x@6?g8(y`y#qpevHczp0?@wbR9Z< z_EYTGa460>!GD4wA4As|0o-83V}}L>{^aC1;%%VFf9VwWe+-$U1f_CqV*hZDFsO&) z(;fW!L8o-iA?|>@w(MFzKWX_1$Z*{m&h$A>$8o~OPUHN~P8m1iNjTE$H&s!dS9_-? zRXhN_D*2Cj?cwy+1YGK^)v`jMb(O>W#{UG{v(? zRy2m5i=!Us4>|T;TpPJ_HrbSGo&9|Vbmm|$FhN_fV+5(KTT(^{d6E5&xAn8=DujJr zg#lz0ARr;G*cgK-$^c(En3Y4T@>wGoi@d^HDeu@XiL(OOrFKRH*T+#;s`=@MFssDj zRbI;C049qje}K7CDez1izRL3$0&5U|MMfd7ly*L}Z;5NdU+LO<+Gc=N21lD0zDg)8 zE1qMauyj0t!Xle?PK5}@Qi4mJC@il|^sufLk~g|W$XmPiRwvuxSeM78!bK18|LLggO2X|4-sDk#0d_Q+3Lw{L<$p0N2FVJbj3@9-1*(k2nf@oc$ z!Bl%tEP|9^GG=co{4btk^@73zJYs4PtzUN^?q8j2uEqMhAO*p>cfr@7QxU)@-rZ!SL5!O!2HeozE{#tkXI z8h_I_m2s?s(>B5c=H|>buhIf{+3o8xy-9U+I38YgUn8`2G0W`lXpxnT@Un8uCAXwjGR%?=4?p|NMN# z2#XS6T8JqqP$3Oy1?D~{?sHqUv@8%$&K1n1e$ayb=A`phc&?lU+ccbWm=keDah0(= zHt^7DgAilt>&zLxD{lu6RB}CHt)NWG4Bm8OXbD!o zZ*4aP-&>#m{#N^Z{6PLR8~t!qG~Z3}vH029;Uxb$*z6w%8iT~kE-4Fo5us-P?`11=de2y zrswe^XXrEJ0TWMl2jlSx0g>8!0zzzw3z~@FwhOgncgT8jc9il>>>;)vSNd;`3a+gO zMmSBZBgGQBJu~4kE(x6IO~Y@g;I;1`_l&PM08$;68ar-XWlz{Q0E{OQ9+EA7!yOS=bYaTGD&Gr0+Ipb<<96$z|+P(C>W)-j8E#vr|l8CAxAulLp-?foxr=#i0= z0mx;hd&R@N=&bX#JyJ_7WAdPeXCcH7sSN zsGN6?iuO%d29kwO6db?3lk}WdJNbm$toh8wMXnIkKj^p|RK{osrIkX%%q#ac*V$Nt zJZP=TP}0wEpN&PN%<%HEY#Z~-poireUdbCl1sgN{@k-cOLPW(4=vgy`T-1UNskTfJ z8;cm!$ELchyl#D43gx9$DW*|jub`&>A?E&Obz>=2%puETO)y`HpzF#MRjD^>SYKlS zKSMD~Sb1hCAp9{Xj3`mH!8(Ose6G-g%`Kt0(^Erzq|w`QX!W_m;;H6v)M4Jzzy9;T zkAdPJ2qo!L+bDL6zyFmkshy@X^Qo@?*MI&`BK$3#1>@(Ti5oHJQXijxK^ZVEUJsz3 zf#u>YfcMkemy1QbXmMnUx%pVl&&9sSs%CI5*qW9ncM@n0;U=}MNgN{QaV=ac9{#q= z_m;Qd;k7N-E4SkL9l6iK`+;~}c6}jP+P*aQj*bX@0Fy*no6>, + unchunked: Vec, + threshold: usize, +} + +impl RecursiveChunker { + pub fn new(recurse_threshold: usize) -> Self { + RecursiveChunker { + deeper_chunker: None, + unchunked: Vec::new(), + threshold: recurse_threshold, + } + } + + /// finalise: true iff this is the last chunk (we will not reject a chunk which may have been + /// truncated) + fn do_chunking(&mut self, chunk_output: &mut F, finalise: bool) -> YamaResult> + where F: FnMut(&[u8]) -> YamaResult { + let fastcdc = FastCDC::new(&self.unchunked, FASTCDC_MIN, FASTCDC_AVG, FASTCDC_MAX); + let mut new_chunks: Vec = Vec::new(); + let mut consumed_until: Option = None; + //let deeper = self.deeper_chunker.as_mut().expect("Deeper chunker must be present"); + for chunk in fastcdc { + let is_final = chunk.offset + chunk.length == self.unchunked.len(); + if !is_final || finalise { + consumed_until = Some(chunk.offset + chunk.length); + let chunk_id = chunk_output(&self.unchunked[chunk.offset..chunk.offset + chunk.length])?; + new_chunks.extend_from_slice(&chunk_id); + } + } + + if let Some(consumed_until) = consumed_until { + if consumed_until > 0 { + self.unchunked.drain(0..consumed_until); + } + } + + Ok(new_chunks) + } + + pub fn write(&mut self, data: &[u8], chunk_output: &mut F) -> YamaResult<()> + where F: FnMut(&[u8]) -> YamaResult { + self.unchunked.extend_from_slice(&data); + + if self.unchunked.len() > self.threshold { + if self.deeper_chunker.is_none() { + // start chunking + self.deeper_chunker = Some(Box::new(RecursiveChunker::new(self.threshold))); + } + + let new_chunks = self.do_chunking(chunk_output, false)?; + + self.deeper_chunker.as_mut().unwrap() + .write(&new_chunks, chunk_output)?; + } + + Ok(()) + } + + pub fn finish(mut self, chunk_output: &mut F) -> YamaResult + where F: FnMut(&[u8]) -> YamaResult { + if self.deeper_chunker.is_some() { + // we are chunking so make this the last chunk + let new_chunks = self.do_chunking(chunk_output, true)?; + let mut deeper = self.deeper_chunker.unwrap(); + deeper.write(&new_chunks, chunk_output)?; + let mut rcr = deeper.finish(chunk_output)?; + // as there is a level of chunking, increase the depth + rcr.depth += 1; + Ok(rcr) + } else { + // no chunking, so depth=0 (raw) and just emit our unchunked data + let chunk_id = chunk_output(&self.unchunked)?; + Ok(RecursiveChunkRef { + chunk_id, + depth: 0, + }) + } + } +} + +pub fn chunkid_to_hex(chunkid: &ChunkId) -> String { + let mut s = String::new(); + for &byte in chunkid.iter() { + write!(&mut s, "{:02x}", byte).expect("Unable to write"); + } + s +} + +pub struct RecursiveExtractor { + deeper_extractor: Option>, + chunk_id_queue: Vec, + chunk_id_offset: usize, +} + +impl RecursiveExtractor { + + pub fn new(mut chunkref: RecursiveChunkRef) -> Self { + if chunkref.depth == 0 { + RecursiveExtractor { + deeper_extractor: None, + chunk_id_queue: chunkref.chunk_id.to_vec(), + chunk_id_offset: 0 + } + } else { + chunkref.depth -= 1; + RecursiveExtractor { + deeper_extractor: Some(Box::new(RecursiveExtractor::new(chunkref))), + chunk_id_queue: Vec::new(), + chunk_id_offset: 0 + } + } + } + + pub async fn read_next(&mut self, pile: &mut Box) -> YamaResult>> { + let mut ztd_decompressor = zstd::block::Decompressor::with_dict(pile.get_dictionary().await?); + self.read_next_int(pile, &mut ztd_decompressor).await + } + + // the 'a is crucially important + fn read_next_int_bf<'a>(&'a mut self, pile: &'a mut Box, ztd_decompressor: &'a mut Decompressor) -> BoxFuture<'a, YamaResult>>> { + async move { + self.read_next_int(pile, ztd_decompressor).await + }.boxed() + } + + async fn read_next_int(&mut self, pile: &mut Box, ztd_decompressor: &mut Decompressor) -> YamaResult>> { + let mut next_chunk_id: ChunkId = Default::default(); + let chunkid_len = next_chunk_id.len(); + if self.chunk_id_queue.len() - self.chunk_id_offset < chunkid_len { + if let Some(extractor) = &mut self.deeper_extractor { + self.chunk_id_queue.drain(0..self.chunk_id_offset); + self.chunk_id_offset = 0; + if let Some(more_chunk_ids) = extractor.read_next_int_bf(pile, ztd_decompressor).await? { + self.chunk_id_queue.extend(more_chunk_ids); + } else if self.chunk_id_queue.is_empty() { + return Ok(None); + } else { + return Err("Partial chunk ID left over, deeper dried out, this is dodgy :/".into()); + } + } else if self.chunk_id_queue.len() - self.chunk_id_offset == 0 { + return Ok(None); + } else { + return Err("Partial chunk ID left over, no deeper, this is dodgy :/".into()); + } + } + + if self.chunk_id_queue.len() - self.chunk_id_offset < chunkid_len { + return Err("Partial chunk ID left over, already tried refill, this is dodgy :/".into()); + } + + let cio = self.chunk_id_offset; + + next_chunk_id.copy_from_slice(&self.chunk_id_queue[cio..cio+chunkid_len]); + + self.chunk_id_offset += chunkid_len; + + let next_chunk = pile.get_chunk(&next_chunk_id).await?; + + if let Some(chunk) = next_chunk { + let chunk = ztd_decompressor.decompress(&chunk,4 * FASTCDC_MAX) + .map_err(|e| e.to_string())?; + Ok(Some(chunk)) + } else { + Err(format!("Invalid reference: no chunk {}", chunkid_to_hex(&next_chunk_id)).into()) + } + } +} + +/* can't figure it out +pub async fn unchunk_to_end(recursive_chunk_ref: RecursiveChunkRef, pile: &mut (dyn Pile + Send)) -> YamaResult> { + let mut buf = Vec::new(); + let mut pile_mutex = Arc::new(Mutex::new(pile)); + unchunk_to_end_internal(&recursive_chunk_ref.chunk_id, recursive_chunk_ref.depth, pile_mutex, &mut buf).await?; + Ok(buf) +} + +fn unchunk_to_end_internal(chunk_id: &ChunkId, depth: u32, pile_mutex: Arc>, buf: &mut Vec) -> BoxFuture<'static, Result<(), String>> { + async move { + let chunk = { + let pile = pile_mutex.lock().expect("Poisoned Pile"); + pile.get_chunk(chunk_id).await + .map_err(|e| e.to_string())? + .ok_or_else(|| format!("Invalid reference: no chunk {}", chunkid_to_hex(chunk_id)))? + }; + if depth == 0 { + buf.extend_from_slice(&chunk); + } else { + let mut sub_chunk_id: ChunkId = Default::default(); + for sub_chunk_id_slice in chunk.chunks(sub_chunk_id.len()) { + sub_chunk_id.clone_from_slice(sub_chunk_id_slice); + unchunk_to_end_internal(&sub_chunk_id, depth - 1, pile_mutex, buf).await?; + } + } + Ok(()) + }.boxed() +} +*/ + +pub async fn unchunk_to_end(recursive_chunk_ref: RecursiveChunkRef, pile: &mut dyn Pile) -> YamaResult> { + // can't do recursive async so we get creative + let mut prev_buf = Vec::new(); + let mut next_buf = Vec::new(); + next_buf.extend_from_slice(&recursive_chunk_ref.chunk_id); + + let mut ztd_decompressor = zstd::block::Decompressor::with_dict(pile.get_dictionary().await?); + + for _ in 0..recursive_chunk_ref.depth+1 { + mem::swap(&mut prev_buf, &mut next_buf); + next_buf.clear(); + + let mut sub_chunk_id: ChunkId = Default::default(); + for sub_chunk_id_slice in prev_buf.chunks(sub_chunk_id.len()) { + sub_chunk_id.clone_from_slice(sub_chunk_id_slice); + let chunk = pile.get_chunk(&sub_chunk_id).await + .map_err(|e| e.to_string())? + .ok_or_else(|| format!("Invalid reference: no chunk {}", chunkid_to_hex(&sub_chunk_id)))?; + let chunk = ztd_decompressor.decompress(&chunk, 4 * FASTCDC_MAX)?; + next_buf.extend_from_slice(&chunk); + } + } + + Ok(next_buf) +} + +#[inline] +pub fn calculate_chunkid(chunk: &[u8]) -> ChunkId { + let mut chunk_id: ChunkId = Default::default(); + blake::hash(256, &chunk, &mut chunk_id) + .expect("BLAKE problem"); + chunk_id +} \ No newline at end of file diff --git a/src/def.rs b/src/def.rs new file mode 100644 index 0000000..ea47119 --- /dev/null +++ b/src/def.rs @@ -0,0 +1,313 @@ +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, HashSet}; +use glob::{Pattern, MatchOptions}; +use std::convert::TryFrom; +use std::path::Path; +use std::fs::File; +use std::io::{BufReader, BufRead}; + +pub type ChunkId = [u8; 32]; +pub type XXHash = u64; + +pub const XXH64_SEED: u64 = 424242; + +pub type YamaResult = Result>; +// yet unused: pub type YamaResultSend = Result>; + +//pub type YamaResultA = Result>; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PointerData { + pub chunk_ref: RecursiveChunkRef, + pub parent_pointer: Option, + pub uid_lookup: BTreeMap, + pub gid_lookup: BTreeMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct RecursiveChunkRef { + pub chunk_id: ChunkId, + pub depth: u32, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct TreeNode { + #[serde(rename="n")] + pub name: String, + //#[serde(flatten)] + #[serde(rename="c")] + pub content: TreeNodeContent, +} + +impl TreeNode { + /// whether the metadata invalidates these two nodes being equal, thus requiring a backup + pub fn metadata_invalidates(&self, other: &TreeNode, check_name: bool) -> bool { + if check_name { + self.name != other.name || self.content.metadata_invalidates(&other.content) + } else { + self.content.metadata_invalidates(&other.content) + } + } + + /// Guarantees consistent visit order. + pub fn visit_mut(&mut self, visitor: &mut F, path_prefix: &str) -> YamaResult<()> + where F: FnMut(&mut Self, &str) -> YamaResult<()> { + let mut my_path_buf = String::new(); + my_path_buf.push_str(path_prefix); + if !my_path_buf.is_empty() { + my_path_buf.push('/'); + } + my_path_buf.push_str(&self.name); + visitor(self, &my_path_buf)?; + + + if let TreeNodeContent::Directory { children, .. } = &mut self.content { + for child in children.iter_mut() { + child.visit_mut(visitor, &my_path_buf)?; + } + } + Ok(()) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(untagged)] +pub enum TreeNodeContent { + NormalFile { + // modification time in ms + #[serde(rename="m")] + mtime: u64, + #[serde(flatten)] + #[serde(rename="o")] + ownership: FilesystemOwnership, + #[serde(flatten)] + #[serde(rename="p")] + permissions: FilesystemPermissions, + // TODO size: u64 or not + // can perhaps cache chunk-wise (but not sure.) + #[serde(rename="c")] + content: RecursiveChunkRef, + }, + Directory { + #[serde(flatten)] + #[serde(rename="o")] + ownership: FilesystemOwnership, + #[serde(flatten)] + #[serde(rename="p")] + permissions: FilesystemPermissions, + #[serde(rename="C")] + children: Vec, + }, + SymbolicLink { + #[serde(flatten)] + #[serde(rename="o")] + ownership: FilesystemOwnership, + #[serde(rename="t")] + target: String, + }, + // TODO is there any other kind of file we need to store? + Deleted, +} + +#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct FilesystemOwnership { + pub uid: u16, + pub gid: u16, +} + +#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct FilesystemPermissions { + pub mode: u32 +} + +impl TreeNodeContent { + pub fn metadata_invalidates(&self, other: &TreeNodeContent) -> bool { + match self { + TreeNodeContent::NormalFile { mtime, ownership, permissions, .. } => { + if let TreeNodeContent::NormalFile { + mtime: other_mtime, + ownership: other_ownership, + permissions: other_permissions, + .. + } = other { + mtime != other_mtime || ownership != other_ownership || permissions != other_permissions + } else { + true + } + } + TreeNodeContent::Directory { ownership, permissions, children } => { + if let TreeNodeContent::Directory { + ownership: other_ownership, + permissions: other_permissions, + children: other_children + } = other { + if ownership != other_ownership || permissions != other_permissions { + return true; + } + children.iter().zip(other_children.iter()).any(|(left, right)| { + left.metadata_invalidates(right, true) + }) + } else { + true + } + } + TreeNodeContent::SymbolicLink { ownership, target } => { + if let TreeNodeContent::SymbolicLink { + ownership: other_ownership, + target: other_target + } = other { + ownership != other_ownership || target != other_target + } else { + true + } + } + TreeNodeContent::Deleted => { + // unreachable + false + } + } + } +} + +pub struct Exclusions { + pub rules: Vec +} + +impl Exclusions { + pub fn load(path: &Path) -> YamaResult { + let file = File::open(path)?; + let bufreader = BufReader::new(file); + let mut rules = Vec::new(); + for line in bufreader.lines() { + let line = line?; + let trim_line = line.trim(); + if !trim_line.is_empty() { + rules.push(ExclusionRule::try_from(trim_line)?); + } + } + + Ok(Exclusions { + rules + }) + } +} + +pub struct ExclusionRule { + pub glob: Pattern, + pub effect: Option, + pub negated: bool, +} + +impl TryFrom<&str> for ExclusionRule { + type Error = String; + + fn try_from(value: &str) -> Result { + let mut effect = None; + let mut negated = false; + let mut glob_str = value; + + let split: Vec<&str> = value.splitn(2, "?⇒").collect(); + if split.len() == 2 { + // this is a conditional rule + glob_str = split[0].trim(); + effect = Some(split[0].trim().to_owned()); + + } + if glob_str.starts_with("!") { + negated = true; + glob_str = &glob_str[1..]; + } + Ok(ExclusionRule { + glob: Pattern::new(glob_str) + .map_err(|e| e.to_string())?, + effect, + negated, + }) + } +} + +impl Exclusions { + pub fn apply_to(&self, node: &mut TreeNode) -> YamaResult<()> { + self.apply_to_rec(node, "", &mut HashSet::new()) + } + + fn apply_to_rec(&self, node: &mut TreeNode, path_rel: &str, exclusions: &mut HashSet) -> YamaResult<()> { + let match_options = MatchOptions { + case_sensitive: true, + require_literal_separator: true, + require_literal_leading_dot: false, + }; + + if let TreeNodeContent::Directory { + ref mut children, .. + } = node.content { + let mut child_pathrel = String::new(); + for rule in self.rules.iter() { + for child in children.iter() { + child_pathrel.clear(); + child_pathrel.push_str(path_rel); + child_pathrel.push('/'); + child_pathrel.push_str(&child.name); + + if rule.glob.matches_with(&child_pathrel, match_options) { + if let Some(relative_effect) = &rule.effect { + let mut path_pieces: Vec<&str> = child_pathrel.split("/") + .skip(1).collect(); + let relative_pieces = relative_effect.split("/"); + + for (idx, relpiece) in relative_pieces.enumerate() { + match relpiece { + "" => { + if idx == 0 { + // this is an absolute path + // doubt we will use this feature much :/ + path_pieces.clear(); + path_pieces.push(""); + } + } + "." => {/* nop */}, + ".." => { + if path_pieces.len() > 1 { + path_pieces.pop(); + } + }, + other => { + path_pieces.push(other); + } + } + } + child_pathrel = path_pieces.join("/"); + } + + if rule.negated { + exclusions.remove(&child_pathrel); + } else { + exclusions.insert(child_pathrel.clone()); + } + } + } + } + + // filter out excluded children + children.retain(|child| { + child_pathrel.clear(); + child_pathrel.push_str(path_rel); + child_pathrel.push('/'); + child_pathrel.push_str(&child.name); + + !exclusions.contains(&child_pathrel) + }); + + for child in children.iter_mut() { + child_pathrel.clear(); + child_pathrel.push_str(path_rel); + child_pathrel.push('/'); + child_pathrel.push_str(&child.name); + + self.apply_to_rec(child, &child_pathrel, exclusions)?; + } + } + + Ok(()) + } +} \ No newline at end of file diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..6135afc --- /dev/null +++ b/src/main.rs @@ -0,0 +1,151 @@ +use clap::{crate_authors, crate_description, crate_version, App, Arg, SubCommand}; +use crate::operations::{store_local_local, list_local, parse_pointer_subpath_pair, extract_local_local, list_local_tree}; +use std::path::Path; +use crate::pile::local_pile::LocalPile; +use crate::def::Exclusions; +use std::process::exit; + +use log::{error}; + +mod def; +mod pile; +mod tree; +mod chunking; +mod operations; +mod util; + +fn main() { + env_logger::init(); + let matches = App::new("山") + .version(crate_version!()) + .author(crate_authors!()) + .about(crate_description!()) + /*.arg( + Arg::with_name("v") + .short("v") + .multiple(true) + .help("Sets the level of verbosity"), + )*/ + .subcommand( + SubCommand::with_name("init") + .about("creates a new pile in the current working directory") + ) + .subcommand( + SubCommand::with_name("check") + .about("checks the consistency of this repository") + .arg( + Arg::with_name("debug") + .short("v") + .help("print debug information verbosely"), + ) + .arg( + Arg::with_name("gc") + .long("gc") + .help("Remove unused chunks to free up space."), + ), + ) + .subcommand(SubCommand::with_name("lsp").about("List tree pointers")) + .subcommand( + SubCommand::with_name("lst") + .arg( + Arg::with_name("POINTER") + .required(true) + .index(1) + .help("Pointer, with optional :path for subtree") + ) + .about("List contents of (sub-)tree") + ) + .subcommand( + SubCommand::with_name("store") + .arg( + Arg::with_name("TREE") + .required(true) + .index(1) + .help("Path to tree / file(s) on filesystem") + ) + .arg( + Arg::with_name("POINTER") + .required(true) + .index(2) + .help("Pointerspec") + ) + .arg( + Arg::with_name("exclusions") + .short("x") + .long("exclusions") + .takes_value(true) + .help("Path to exclusions file") + ) + .arg( + Arg::with_name("differential") + .short("d") + .long("differential") + .takes_value(true) + .help("Pointerspec to make a differential backup against") + )) + .subcommand( + SubCommand::with_name("extract") + .arg( + Arg::with_name("POINTER") + .required(true) + .index(1) + .help("Pointer, with optional :path for subextraction") + ) + .arg( + Arg::with_name("TARGET") + .required(true) + .index(2) + .help("Path to target on filesystem, or subtarget for subextraction") + ) + ) + .get_matches(); + + if let Some(_submatches) = matches.subcommand_matches("init") { + if Path::new("yama.toml").exists() { + error!("Refusing to overwrite existing pile"); + exit(2); + } else { + LocalPile::create(Path::new(".")) + .expect("Failed to initialise pile."); + } + } else if let Some(submatches) = matches.subcommand_matches("store") { + let exclusions = submatches.value_of("exclusions") + .map(|path| { + Exclusions::load(Path::new(path)) + }) + .transpose() + .expect("Failed to load exclusions."); + store_local_local( + Path::new(submatches.value_of("TREE").unwrap()), + Path::new("."), // TODO + submatches.value_of("POINTER").unwrap(), + submatches.value_of("differential"), + exclusions + ).expect("Problem with store_local_local"); + } else if let Some(_submatches) = matches.subcommand_matches("lsp") { + list_local(Path::new(".")) // TODO + .expect("Problem with list_local"); + } else if let Some(submatches) = matches.subcommand_matches("extract") { + let (pointer, subpath_opt) = parse_pointer_subpath_pair(submatches.value_of("POINTER") + .expect("Pointer not specified.")); + extract_local_local( + Path::new("."), // TODO + pointer, + subpath_opt, + Path::new(submatches.value_of("TARGET") + .expect("No target specified.")) + ) + .expect("Extraction failed"); + } else if let Some(submatches) = matches.subcommand_matches("lst") { + let (pointer, subpath_opt) = parse_pointer_subpath_pair(submatches.value_of("POINTER") + .expect("Pointer not specified.")); + list_local_tree( + Path::new("."), // TODO do this + pointer, + subpath_opt + ) + .expect("List local tree failed"); + } else if let Some(_submatches) = matches.subcommand_matches("check") { + unimplemented!() + } +} diff --git a/src/operations.rs b/src/operations.rs new file mode 100644 index 0000000..c643f06 --- /dev/null +++ b/src/operations.rs @@ -0,0 +1,327 @@ +use crate::def::{YamaResult, Exclusions, PointerData, TreeNode, TreeNodeContent, RecursiveChunkRef}; +use std::path::Path; +use crate::pile::{local_pile::LocalPile, Pile}; +use futures::executor::block_on; +use crate::chunking::{unchunk_to_end, chunkid_to_hex}; +use crate::tree::{differentiate_node, create_uidgid_lookup_tables}; +use std::collections::BTreeMap; +use std::collections::btree_map::Entry; +use async_std::task; +use crate::operations::chunking_flow::ChunkingFlowConfiguration; +use std::fs::File; +use std::io::Write; +use crate::operations::extraction_flow::ExtractionFlowConfiguration; +use log::{info}; + +mod chunking_flow; +mod extraction_flow; + +pub fn store_local_local(tree: &Path, pile: &Path, pointer: &str, parent_pointer: Option<&str>, exclusions: Option) -> YamaResult<()> { + info!("Going to perform dir scan, might take a while …"); + let mut dir_scan = crate::tree::scan(tree)? + .ok_or("No scan performed; does the file exist")?; + info!("Dir scan completed."); + + if let Some(exclusions) = exclusions { + exclusions.apply_to(&mut dir_scan)?; + } + + let mut local_pile = LocalPile::open(pile)?; + + if let Some(parent_pointer) = parent_pointer { + let (_parent_pointer_data, parent_node) = load_integrated_pointer(parent_pointer, &mut local_pile)?; + + dir_scan = differentiate_node(dir_scan, &parent_node)?; + } + + // eprintln!("dir scan: {:#?}", dir_scan); + + let mut uids = BTreeMap::new(); + let mut gids = BTreeMap::new(); + + create_uidgid_lookup_tables(&dir_scan, &mut uids, &mut gids)?; + + // eprintln!("uids: {:#?}", uids); + // eprintln!("gids: {:#?}", gids); + + // now perform the actual chunking + let pointer_ref: RecursiveChunkRef = { + let tree = tree.to_owned(); + let local_pile = local_pile.clone_pile_handle(); + task::block_on(async { + ChunkingFlowConfiguration::default() + .run(tree, dir_scan, local_pile).await + .map_err(|e| e.to_string()) + })? + }; + + info!("chunking finished"); + + fn map_identity((k, v): (K, Option)) -> Option<(K, V)> { + match v { + None => None, + Some(v) => Some((k, v)), + } + } + + task::block_on(local_pile.put_pointer(pointer, PointerData { + chunk_ref: pointer_ref, + parent_pointer: parent_pointer.map(|s| s.to_owned()), + uid_lookup: uids.into_iter().filter_map(map_identity).collect(), + gid_lookup: gids.into_iter().filter_map(map_identity).collect(), + }))?; + + info!("Pointer updated. store operation COMPLETED."); + + Ok(()) +} + +pub fn extract_local_local(pile: &Path, pointer: &str, subpath_opt: Option<&str>, target: &Path) + -> YamaResult<()> { + let mut local_pile = LocalPile::open(pile)?; + + let (_pointer_data, mut node) = load_integrated_pointer(pointer, &mut local_pile)?; + if let Some(subpath) = subpath_opt { + node = get_subnode_of(node, subpath)?; + } + + // TODO(0.3 onwards) what about uid and gid lookup maps? + // probably want to offer the choice of id-based and name-based extraction, with fallbacks + // in both ways. + + task::block_on(async { + ExtractionFlowConfiguration::default() + .run(node, target.to_path_buf(), local_pile.clone_pile_handle()).await + .map_err(|e| e.to_string()) + })?; + + Ok(()) +} + +pub fn list_local_tree(pile: &Path, pointer: &str, subpath_opt: Option<&str>) + -> YamaResult<()> { + let mut local_pile = LocalPile::open(pile)?; + + let (_pointer_data, mut node) = load_integrated_pointer(pointer, &mut local_pile)?; + + if let Some(subpath) = subpath_opt { + node = get_subnode_of(node, subpath)?; + } + + print_node_recursive(&mut node)?; + + Ok(()) +} + +fn get_subnode_of(mut node: TreeNode, subpath: &str) -> YamaResult { + for component in subpath.split("/") { + if component == "" { + continue; + } + if let TreeNodeContent::Directory { + children, .. + } = node.content { + let subchild = children + .into_iter() + .filter(|node| node.name == component) + .next(); + node = subchild.ok_or("Subextraction path does not exist; check your subextraction path for errors.")?; + } else { + return Err("Cannot subextract directory; check your subextraction path for errors.".into()); + } + } + Ok(node) +} + +pub fn list_local(pile: &Path) -> YamaResult<()> { + let local_pile = LocalPile::open(pile)?; + + for pointer in task::block_on(local_pile.list_pointers())?.iter() { + println!("{}", pointer); + } + + Ok(()) +} + +fn print_node_recursive(tree_node: &mut TreeNode) -> YamaResult<()> { + tree_node.visit_mut(&mut |node, path| { + eprint!("{} ", path); + match &node.content { + TreeNodeContent::NormalFile { content, .. } => { + eprintln!("file @ {} (depth={})", + chunkid_to_hex(&content.chunk_id), content.depth); + } + TreeNodeContent::Directory { .. } => { + eprintln!("directory"); + } + TreeNodeContent::SymbolicLink { target, .. } => { + eprintln!("symlink → {}", target); + } + TreeNodeContent::Deleted => { + eprintln!("deleted"); + } + } + Ok(()) + }, "") +} + +fn load_integrated_pointer(pointer_path: &str, pile: &mut (dyn Pile + Send)) -> YamaResult<(PointerData, TreeNode)> { + // TODO refactor this, as it needs recursive integration logic, + // perhaps up to a specified integration depth (or maybe just make integration occur + // when dependents are about to be removed from the database). + // TODO Is this^ TODO still applicable? + + let result: Result<(PointerData, TreeNode), String> = block_on(async { + let pointer_data = pile.get_pointer(pointer_path).await + .map_err(|e| e.to_string() + " (lip)")? + .ok_or_else(|| "Invalid pointer reference: ".to_owned() + pointer_path)?; + + let node_bytes = unchunk_to_end( + pointer_data.chunk_ref.clone(), + pile, + ).await.map_err(|e| e.to_string() + " (lip:ute)")?; + + File::create("/tmp/lip_data").expect("create debug file") + .write_all(&node_bytes).expect("write debug file"); // TODO DEBUG + + let node: TreeNode = serde_cbor::de::from_slice(&node_bytes) + .map_err(|e| e.to_string() + " (lip:cbor)")?; + + Ok((pointer_data, node)) + }); + let (mut pointer_data, mut node) = result?; + + if let Some(parent_pointer) = pointer_data.parent_pointer.as_ref() { + let (parent_pointer_data, parent_node) = load_integrated_pointer(parent_pointer, pile)?; + integrate_node(&mut node, &parent_node); + + // integrate UID and GID maps + let mut uid_lookup = parent_pointer_data.uid_lookup; + let mut gid_lookup = parent_pointer_data.gid_lookup; + uid_lookup.extend(pointer_data.uid_lookup); + gid_lookup.extend(pointer_data.gid_lookup); + pointer_data.uid_lookup = uid_lookup; + pointer_data.gid_lookup = gid_lookup; + } + + Ok((pointer_data, node)) +} + +fn integrate_node(new: &mut TreeNode, old: &TreeNode) { + if let TreeNodeContent::Directory { children: old_children, .. } = &old.content { + if let TreeNodeContent::Directory { + children, .. + } = &mut new.content { + let mut map = BTreeMap::new(); + while !children.is_empty() { + let treenode = children.remove(children.len() - 1); + map.insert(treenode.name.clone(), treenode); + } + + for old_child in old_children { + match map.entry(old_child.name.clone()) { + Entry::Vacant(vac) => { + vac.insert(old_child.clone()); + } + Entry::Occupied(mut occ) => { + integrate_node(occ.get_mut(), old_child); + } + } + } + + for (_, child) in map.into_iter() { + children.push(child); + } + } + } +} + +pub fn parse_pointer_subpath_pair(pair: &str) -> (&str, Option<&str>) { + let splitted: Vec<&str> = pair.splitn(2, ":").collect(); + if splitted.len() == 2 { + (splitted[0], Some(splitted[1])) + } else { + assert_eq!(splitted.len(), 1); + (splitted[0], None) + } +} + + + +/* TODO TODO TODO clean up / remove this +fn perform_storage_chunking(populate_node: &mut TreeNode, pile: &mut (dyn Pile + Send)) -> YamaResult<()> { + let mut paths = Vec::new(); + + populate_node.visit_mut(&mut |tn, path| { + if let TreeNodeContent::NormalFile { .. } = tn.content { + paths.push(path.to_owned()); + } + Ok(()) + }, "")?; + + let (send, recv) = mpsc::sync_channel::<(ChunkId, Vec)>(256); + + let mut chunkrefs_for_paths = crossbeam::scope(|s| { + s.spawn(move |_| { + for (chunk_id, chunk_data) in recv.iter() { + task::block_on(async { + pile.put_chunk(&chunk_id, &chunk_data) + }); + } + }); + + let mut chunkrefs_for_paths: BTreeMap> = { + let send_mutex = Mutex::new(send); + + paths.par_iter() + .enumerate() + .map(|(idx, path)| { + let sender = { + send_mutex + .lock() + .expect("poisoned") + .clone() + }; + + // hmm how to handle tihs + // we don't want to compress chunks if we already have them + // so we can't compress them in here; have to do in background task. + + //chunk_file(path, sender)?; + unimplemented!(); + + (idx, Some(RecursiveChunkRef { + chunk_id: Default::default(), + depth: 0, + })) + }) + .collect::>>() + }; + + chunkrefs_for_paths + }).expect("crossbeam fail :S"); + + + let mut index = 0; + + populate_node.visit_mut(&mut |tn, path| { + if let TreeNodeContent::NormalFile { ref mut content, .. } = &mut tn.content { + assert_eq!(&paths[index], path); + + match chunkrefs_for_paths.remove(&index).expect("Chunkref not there??") { + Some(chunkref) => { + *content = chunkref; + } + None => { + tn.content = TreeNodeContent::Deleted; + } + } + + index += 1; + } + Ok(()) + }, "")?; + + Ok(()) +} +*/ \ No newline at end of file diff --git a/src/operations/chunking_flow.rs b/src/operations/chunking_flow.rs new file mode 100644 index 0000000..4a9d31e --- /dev/null +++ b/src/operations/chunking_flow.rs @@ -0,0 +1,339 @@ +use std::collections::BTreeMap; +use std::fs::File; +use std::hash::Hasher; +use std::io::{ErrorKind, Read}; +use std::ops::Deref; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; + +use async_std::sync::{Receiver, Sender}; +use async_std::task; + +use log::{trace, error}; + +use crate::chunking::{chunkid_to_hex, RecursiveChunker}; +use crate::chunking; +use crate::def::{ChunkId, RecursiveChunkRef, TreeNode, TreeNodeContent, XXH64_SEED, XXHash, YamaResult}; +use crate::pile::Pile; + +pub struct ChunkingFlowConfiguration { + pub max_chunkers: u32, + pub offerer_queue_size: usize, + pub max_offerers: u32, + pub compressor_queue_size: usize, + pub max_compressors: u32, + pub uploader_queue_size: usize, + pub max_uploaders: u32, + pub zstd_level: i32, +} + +impl Default for ChunkingFlowConfiguration { + fn default() -> Self { + ChunkingFlowConfiguration { + max_chunkers: 4, + offerer_queue_size: 256, + max_offerers: 32, + compressor_queue_size: 32, + max_compressors: 4, + uploader_queue_size: 64, + max_uploaders: 4, + zstd_level: 12, + /* pretty darn slow but I'm willing to accept that */ + } + } +} + +impl ChunkingFlowConfiguration { + pub async fn run(&self, tree_root: PathBuf, mut tree_node: TreeNode, pile: Box) + -> YamaResult { + unsafe fn extend_lifetime<'b, T>(r: &'b T) -> &'static T { + std::mem::transmute::<&'b T, &'static T>(r) + } + + let filepaths: Arc> = Arc::new(unpack_dirscan_filepaths(&mut tree_node)?); + + let inputs = unsafe { + let fp_static = extend_lifetime(&filepaths); + Arc::new(Mutex::new(fp_static.iter().enumerate())) + }; + let outputs = Arc::new(Mutex::new(BTreeMap::new())); + let (mut chunker_send, offerer_recv) = async_std::sync::channel::<(ChunkId, XXHash, Vec)>(self.offerer_queue_size); + let (offerer_send, compressor_recv) = async_std::sync::channel::<(ChunkId, Vec)>(self.compressor_queue_size); + let (compressor_send, uploader_recv) = async_std::sync::channel::<(ChunkId, Vec)>(self.uploader_queue_size); + let mut chunker_handlers = Vec::new(); + let mut other_handlers = Vec::new(); + + for _ in 0..self.max_chunkers { + let tree_root = tree_root.to_owned(); + let chunker_send = chunker_send.clone(); + let inputs = inputs.clone(); + let outputs = outputs.clone(); + chunker_handlers.push(task::spawn_blocking(move || chunker(tree_root, inputs, outputs, chunker_send))); + } + + for _ in 0..self.max_offerers { + let offerer_recv = offerer_recv.clone(); + let offerer_send = offerer_send.clone(); + let pile = pile.clone_pile_handle(); + other_handlers.push(task::spawn(offerer(offerer_recv, offerer_send, pile))); + } + + + let zstd_dictionary = Arc::new(pile.get_dictionary().await + .expect("Can't carry on without a Zstd dictionary.")); + + for _ in 0..self.max_compressors { + let compressor_recv = compressor_recv.clone(); + let compressor_send = compressor_send.clone(); + let zstd_dictionary = zstd_dictionary.clone(); + other_handlers.push(task::spawn(compressor(compressor_recv, compressor_send, zstd_dictionary, + self.zstd_level))); + } + + for _ in 0..self.max_uploaders { + let uploader_recv = uploader_recv.clone(); + let pile = pile.clone_pile_handle(); + other_handlers.push(task::spawn(uploader(uploader_recv, pile))); + } + + // we drop these so the receivers get properly hung up later + drop(compressor_send); + drop(offerer_send); + + let mut errors: Option = None; + + trace!("waiting for chunkers to complete"); + for handler in chunker_handlers { + if let Err(err_string) = handler.await { + match errors.as_mut() { + None => { + errors = Some(err_string); + } + Some(errors) => { + errors.push('\n'); + errors.push_str(&err_string); + } + } + } + } + trace!("chunkers complete; chunking tree"); + + let mut outputs = Arc::try_unwrap(outputs) + .map_err(|_| "(dangling Arcs)".to_owned())? + .into_inner() + .map_err(|e| e.to_string())?; + + repack_dirscan_filepaths(&mut tree_node, &filepaths, &mut outputs)?; + + let treenode_chunkref = { + let send_chunk = &mut |data: &[u8]| { send_chunk_implementation(&mut chunker_send, data) }; + let send_data = serde_cbor::ser::to_vec_packed(&tree_node)?; + + /* + File::create("/tmp/send_data").expect("create debug file") + .write_all(&send_data).expect("write debug file"); // TODO DEBUG + */ + + // TODO debug only + let check: TreeNode = serde_cbor::from_slice(&send_data).expect("Serialisation doesn't produce deserialisable result."); + assert_eq!(tree_node, check, "SER CHECK failed"); + drop(tree_node); + + let mut chunker = RecursiveChunker::new(chunking::SENSIBLE_THRESHOLD); + chunker.write(&send_data, send_chunk)?; + drop(send_data); + + chunker.finish(send_chunk)? + }; + + // we are done chunking, hang up + drop(chunker_send); + + trace!("chunked tree ({}); waiting for all workers to finish", chunkid_to_hex(&treenode_chunkref.chunk_id)); + + for handler in other_handlers { + if let Err(err_string) = handler.await { + match errors.as_mut() { + None => { + errors = Some(err_string); + } + Some(errors) => { + errors.push('\n'); + errors.push_str(&err_string); + } + } + } + } + trace!("workers complete"); + + match errors { + Some(errors) => { + error!("chunking flow had errors:\n{}\n-----", errors); + Err(errors.into()) + } + None => { + Ok(treenode_chunkref) + } + } + } +} + +fn send_chunk_implementation(chunk_send: &mut Sender<(ChunkId, XXHash, Vec)>, data: &[u8]) -> YamaResult { + let chunk_id = chunking::calculate_chunkid(data); + let xxhash = { + let mut hasher = twox_hash::XxHash64::with_seed(XXH64_SEED); + hasher.write(data); + hasher.finish() + }; + + task::block_on(chunk_send.send( + (chunk_id, xxhash, data.to_vec()) + )); + + Ok(chunk_id) +} + +fn chunker<'a, I>(tree_root: PathBuf, inputs: Arc>, outputs: Arc>>>, + mut chunk_sender: Sender<(ChunkId, XXHash, Vec)>) -> Result<(), String> + where I: Iterator { + // &_ works around https://github.com/rust-lang/rust/issues/58639 + let send_chunk = &mut |data: &_| send_chunk_implementation(&mut chunk_sender, data); + + loop { + // get new task + let (next_idx, next_path) = { + let mut inputs_iterator = inputs.lock() + .expect("Poisoned"); + if let Some((idx, next)) = inputs_iterator.next() { + trace!("chunker < {}:{}", idx, next); + let absolute_file_path = tree_root.parent() + .unwrap_or_else(|| &tree_root) + .to_owned().join(&Path::new(&next)); + (idx, absolute_file_path) + } else { + // Nothing left to do + trace!("chunker //"); + break; + } + }; + + // load the file and chunk it away! + match File::open(next_path) { + Ok(mut file) => { + let mut buf = vec![1; 8 * 1024 * 1024]; + let mut chunker = RecursiveChunker::new(chunking::SENSIBLE_THRESHOLD); + + loop { + let read = file.read(&mut buf) + .map_err(|e| e.to_string())?; + if read == 0 { + break; + } + chunker.write(&buf[0..read], send_chunk) + .map_err(|e| e.to_string())?; + } + + let chunkref = chunker.finish(send_chunk) + .map_err(|e| e.to_string())?; + + outputs.lock() + .expect("poison") + .insert(next_idx, Some(chunkref)); + } + Err(e) => { + let e_kind = e.kind(); + if e_kind == ErrorKind::PermissionDenied || e_kind == ErrorKind::NotFound { + // vanished / can't read. + error!("chunker E {:?}", e); + outputs.lock() + .expect("poison") + .insert(next_idx, None); + } + } + } + } + Ok(()) +} + +async fn offerer(recv: Receiver<(ChunkId, XXHash, Vec)>, send: Sender<(ChunkId, Vec)>, pile: Box) + -> Result<(), String> { + while let Some((chunk_id, xxhash, buf)) = recv.recv().await { + let upload_required = if let Some(hash) = pile.xxhash_chunk(&chunk_id).await + .map_err(|e| e.to_string())? { + if hash != xxhash { + error!("FATAL : HASH COLLISION!!! ChunkId {}", + chunkid_to_hex(&chunk_id)); + error!("XXHash differs; incoming={:016X}, stored={:016X}", + xxhash, hash); + return Err("HASH COLLISION".into()); + } + trace!("deduped {}", chunkid_to_hex(&chunk_id)); + false + } else { + // chunk not present + trace!("new {}", chunkid_to_hex(&chunk_id)); + true + }; + if upload_required { + // really windy way to do this, but otherwise complains that Error is not Send :/ + // because need to let the condition in if drop first. Ouch. + send.send((chunk_id, buf)).await; + } + } + Ok(()) +} + +// `async` for a blocking fn feels weird to me, but async_std detects its blocking and will do the +// right thing ... eventually / apparently. :S +async fn compressor(recv: Receiver<(ChunkId, Vec)>, send: Sender<(ChunkId, Vec)>, dictionary: Arc>, zstd_level: i32) -> Result<(), String> { + let mut zstd_compressor = zstd::block::Compressor::with_dict(dictionary.deref().clone()); + + while let Some((chunk_id, uncompressed)) = recv.recv().await { + let compressed = zstd_compressor.compress(&uncompressed, zstd_level) + .map_err(|e| e.to_string())?; + send.send((chunk_id, compressed)).await; + } + + Ok(()) +} + +async fn uploader(recv: Receiver<(ChunkId, Vec)>, pile: Box) -> Result<(), String> { + while let Some((chunk_id, compressed)) = recv.recv().await { + pile.put_chunk(&chunk_id, compressed).await + .map_err(|e| e.to_string())?; + } + + Ok(()) +} + +fn unpack_dirscan_filepaths(node: &mut TreeNode) -> YamaResult> { + let mut paths = Vec::new(); + node.visit_mut(&mut |tn, path| { + if let TreeNodeContent::NormalFile { .. } = tn.content { + paths.push(path.to_owned()); + } + Ok(()) + }, "")?; + Ok(paths) +} + +fn repack_dirscan_filepaths(node: &mut TreeNode, paths: &Vec, outputs: &mut BTreeMap>) -> YamaResult<()> { + let mut index = 0; + node.visit_mut(&mut |tn, path| { + if let TreeNodeContent::NormalFile { ref mut content, .. } = &mut tn.content { + assert_eq!(paths[index], path); + match outputs.remove(&index) + .ok_or("Serious issue; index not present in map")? { + None => { + tn.content = TreeNodeContent::Deleted + } + Some(chunkref) => { + *content = chunkref + } + } + index += 1; + } + Ok(()) + }, "")?; + Ok(()) +} diff --git a/src/operations/extraction_flow.rs b/src/operations/extraction_flow.rs new file mode 100644 index 0000000..1c5070d --- /dev/null +++ b/src/operations/extraction_flow.rs @@ -0,0 +1,223 @@ +use crate::def::{YamaResult, TreeNode, RecursiveChunkRef, FilesystemPermissions, FilesystemOwnership, TreeNodeContent}; +use std::path::PathBuf; +use crate::pile::Pile; +use async_std::sync::Receiver; +use async_std::task; +use std::fs; +use nix::unistd::{Gid, Uid, FchownatFlags}; +use std::os::unix::fs::PermissionsExt; +use async_std::fs::File as AsyncFile; +use async_std::prelude::*; +use crate::chunking::RecursiveExtractor; +use nix::sys::time::{TimeSpec, TimeValLike}; +use std::os::unix::io::{RawFd, AsRawFd}; +use log::{trace, error}; + +pub struct ExtractionFlowConfiguration { + pub workers: u32, + pub worker_queue_size: usize, +} + +impl Default for ExtractionFlowConfiguration { + fn default() -> Self { + ExtractionFlowConfiguration { + workers: 4, + worker_queue_size: 256, + } + } +} + +struct ExtractorInput { + pub path: PathBuf, + pub recursive_chunk_ref: RecursiveChunkRef, + pub permissions: FilesystemPermissions, + pub ownership: FilesystemOwnership, + pub mtime: u64, +} + +impl ExtractionFlowConfiguration { + pub async fn run(self, mut node: TreeNode, extract_to: PathBuf, pile: Box) -> YamaResult<()> { + let (worker_send, worker_recv) = async_std::sync::channel::(self.worker_queue_size); + let mut workers = Vec::new(); + + for _ in 0..self.workers { + let inputs = worker_recv.clone(); + let pile = pile.clone_pile_handle(); + workers.push(task::spawn(extractor(inputs, pile))) + } + + node.visit_mut(&mut |node, path| { + let final_path = extract_to.join(path); + match &node.content { + TreeNodeContent::NormalFile { content: chunk_ref, mtime, permissions, ownership } => { + trace!("nlf {}", path); + + task::block_on(worker_send.send(ExtractorInput { + path: final_path.to_owned(), + recursive_chunk_ref: chunk_ref.clone(), + permissions: permissions.clone(), + ownership: ownership.clone(), + mtime: *mtime, + })); + } + TreeNodeContent::Directory { ownership, permissions, .. } => { + fs::create_dir(&final_path)?; + // TODO what to do if chown not allowed? + let mut perms = fs::metadata(&final_path)?.permissions(); + perms.set_mode(permissions.mode); + fs::set_permissions(&final_path, perms)?; + nix::unistd::chown( + &final_path, + Some(Uid::from_raw(ownership.uid as u32)), + Some(Gid::from_raw(ownership.gid as u32)), + )?; + + trace!("dir {}", path); + } + TreeNodeContent::SymbolicLink { target, ownership } => { + std::os::unix::fs::symlink(target, &final_path)?; + // TODO what to do if chown not allowed? + // nix::unistd::chown falls over on symlinks because it follows them + nix::unistd::fchownat( + None, + &final_path, + Some(Uid::from_raw(ownership.uid as u32)), + Some(Gid::from_raw(ownership.gid as u32)), + FchownatFlags::NoFollowSymlink + )?; + trace!("sym {}", path); + } + TreeNodeContent::Deleted => { + // this should not happen as we should be using an integrated pointer + // might be cool for incremental restore but don't really care right now. + trace!("WARNING this should not happen: del {}", path); + } + } + Ok(()) + }, "") + .map_err(|e| e.to_string())?; + + drop(worker_send); + + let mut errors: Option = None; + + trace!("waiting for extractors to complete"); + for handler in workers { + if let Err(err_string) = handler.await { + match errors.as_mut() { + None => { + errors = Some(err_string); + } + Some(errors) => { + errors.push('\n'); + errors.push_str(&err_string); + } + } + } + } + + //unimplemented!() + Ok(()) + } +} + +async fn extractor(inputs: Receiver, mut pile: Box) -> Result<(), String> { + while let Some(msg) = inputs.recv().await { + let mut recursive_extractor = RecursiveExtractor::new(msg.recursive_chunk_ref.clone()); + //eprintln!("ex s> {:?} = {:?}", msg.path, &msg.recursive_chunk_ref); + let mut file = AsyncFile::create(&msg.path).await + .map_err(|e| e.to_string())?; + + // this weird `loop` needed to prevent Error types being usable across awaits... + loop { + let chunk_opt: Option> = { + match recursive_extractor.read_next(&mut pile).await.map_err(|e| e.to_string()) { + Ok(co) => co, + Err(error) => { + error!("Error extracting {:?}: {:?}, skipping", msg.path, error); + break; + }, + } + }; + if let Some(chunk) = chunk_opt { + file.write_all(&chunk).await + .map_err(|e| e.to_string())?; + } else { + break; + } + } + + let mut perms = fs::metadata(&msg.path) + .map_err(|e| e.to_string())?.permissions(); + perms.set_mode(msg.permissions.mode); + fs::set_permissions(&msg.path, perms) + .map_err(|e| e.to_string())?; + nix::unistd::chown( + &msg.path, + Some(Uid::from_raw(msg.ownership.uid as u32)), + Some(Gid::from_raw(msg.ownership.gid as u32)), + ).map_err(|e| e.to_string())?; + + // needed for mtime to not be set to now? + file.flush().await + .map_err(|e| e.to_string())?; + + // set correct mtime + let raw_fd: RawFd = file.as_raw_fd(); + let mtimespec = TimeSpec::milliseconds(msg.mtime as i64); + nix::sys::stat::futimens(raw_fd, &mtimespec, &mtimespec) + .map_err(|e| e.to_string())?; + + //eprintln!("ex f< {:?}", msg.path); + } + Ok(()) +} + +/* +async fn extractor(inputs: Receiver, pile: Box) -> Result<(), String> { + let mut chunkref_stack: Vec = Vec::new(); + while let Some(msg) = inputs.recv().await { + chunkref_stack.clear(); + chunkref_stack.push(msg.recursive_chunk_ref); + + let mut file = AsyncFile::create(&msg.path).await + .map_err(|e| e.to_string())?; + + while let Some(popped_ref) = chunkref_stack.pop() { + let chunk = pile.get_chunk(&popped_ref.chunk_id).await + .map_err(|e| e.to_string())? + .ok_or_else(|| format!("CORRUPTION: Missing chunk {} when extracting", chunkid_to_hex(&popped_ref.chunk_id)))?; + if popped_ref.depth == 0 { + // This chunk is part of the desired file + file.write_all(&chunk).await + .map_err(|e| e.to_string())?; + } else { + // This chunk is a chunklist + // TODO this is wrong because chunklists are chunked and not necessarily % = 0 the chunk ID size? + // TODO probably want to do something akin to the RecursiveChunker + // yes we could build the stack ourselves but it might be a nicer structure. + let chunk_id_size = popped_ref.chunk_id.len(); + if chunk.len() % chunk_id_size != 0 { + return Err(format!( + "CORRUPTION: Contents of chunk {} are not a multiple of the ChunkId size; this can't be a chunklist", + chunkid_to_hex(&popped_ref.chunk_id) + )); + } + chunkref_stack.extend(chunk.chunks(chunk_id_size) + .rev() + .map(|sub_chunk_id| { + let mut rcr = RecursiveChunkRef { + chunk_id: Default::default(), + depth: popped_ref.depth - 1, + }; + rcr.chunk_id.copy_from_slice(sub_chunk_id); + rcr + })); + } + } + + break; + } + Ok(()) +} +*/ \ No newline at end of file diff --git a/src/pile.rs b/src/pile.rs new file mode 100644 index 0000000..117ba18 --- /dev/null +++ b/src/pile.rs @@ -0,0 +1,53 @@ +use async_trait::async_trait; + +use crate::def::{ChunkId, PointerData, XXHash, YamaResult}; + +mod interface { + use std::borrow::Cow; + + use crate::def::{ChunkId, PointerData}; + + pub enum PileProcessorRequest<'a> { + GetDictionary, + GetChunk { + chunk_id: ChunkId, + }, + PutChunk { + chunk_id: ChunkId, + chunk_data: Cow<'a, [u8]>, + }, + /* DelChunk { + chunk_id: + }, */ + XXHashChunk { + chunk_id: ChunkId, + }, + GetPointer { + pointer_id: Cow<'a, str>, + }, + PutPointer { + pointer_id: Cow<'a, str>, + pointer_data: PointerData, + }, + // DelPointer, + ListPointers, + } +} + + +/// should be Cloneable to get another reference which uses the same pile. +#[async_trait] +pub trait Pile: Send + Sync { + /// Returns another handle to the Pile. + fn clone_pile_handle(&self) -> Box; + async fn get_dictionary(&self) -> YamaResult>; + async fn get_chunk(&self, chunk_id: &ChunkId) -> YamaResult>>; + async fn put_chunk(&self, chunk_id: &ChunkId, data: Vec) -> YamaResult<()>; + async fn xxhash_chunk(&self, chunk_id: &ChunkId) -> YamaResult>; + async fn get_pointer(&self, pointer_id: &str) -> YamaResult>; + async fn put_pointer(&self, pointer_id: &str, pointer_data: PointerData) -> YamaResult<()>; + async fn list_pointers(&self) -> YamaResult>; +} + +pub mod local_pile; + diff --git a/src/pile/local_pile.rs b/src/pile/local_pile.rs new file mode 100644 index 0000000..e0598f8 --- /dev/null +++ b/src/pile/local_pile.rs @@ -0,0 +1,405 @@ +use std::fs::File; +use std::hash::Hasher; +use std::io::{Read, Write}; +use std::ops::Deref; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Condvar, mpsc, Mutex}; +use std::sync::mpsc::{RecvTimeoutError, SyncSender}; +use std::{thread, mem}; +use std::time::Duration; +use clap::crate_version; +use log::{warn, trace, info}; + +use lmdb::{ + Cursor, Database, DatabaseFlags, Environment, EnvironmentFlags, Transaction, +}; + +use async_trait::async_trait; + +use crate::def::{ChunkId, PointerData, XXH64_SEED, XXHash, YamaResult}; +use crate::pile::Pile; +use serde::{Serialize, Deserialize}; +use std::thread::JoinHandle; + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct LocalPileManifest { + max_lmdb_size: usize, + yama_version: String +} + +impl Default for LocalPileManifest { + fn default() -> Self { + LocalPileManifest { + max_lmdb_size: 128 * 1024 * 1024 * 1024, + yama_version: String::from(crate_version!()) + } + } +} + +#[derive(Clone)] +pub struct LocalPile { + directory: PathBuf, + dictionary: Arc>, + lmdb_env: Arc, + lmdb_chunk_database: Database, + lmdb_pointer_database: Database, + lmdb_writing_thread_queue: Option>, + lmdb_writing_thread_handle: Option>> +} + +impl Drop for LocalPile { + fn drop(&mut self) { + let mut opt_handle: Option>> = None; + self.lmdb_writing_thread_queue = None; + mem::swap(&mut self.lmdb_writing_thread_handle, &mut opt_handle); + if let Some(arc_join_handle) = opt_handle { + if let Ok(join_handle) = Arc::try_unwrap(arc_join_handle) { + //eprintln!("joining LMDB..."); + if let Err(_) = join_handle.join() { + warn!("LMDB writer thread had an error when joining."); + } + //eprintln!("joined LMDB..."); + } + } + } +} + +impl LocalPile { + pub fn create(dir: &Path) -> YamaResult<()> { + let zstd_path = dir.join("zstd.dict"); + eprintln!("Please add a Zstd dictionary to: {:?}", zstd_path); + + let manifest_path = dir.join("yama.toml"); + let manifest = LocalPileManifest::default(); + { + let bytes = toml::to_vec(&manifest).expect("toml to_vec"); + let mut file = File::create(manifest_path).expect("Failed to open yama manifest."); + file.write_all(&bytes)?; + // file automatically closed + } + + let lmdb_env = Arc::new( + Environment::new() + .set_max_dbs(8) + .set_flags(EnvironmentFlags::NO_SUB_DIR | EnvironmentFlags::NO_TLS) + .set_map_size(manifest.max_lmdb_size) + .open(&dir.join("yama.lmdb")) + .expect("Failed to open LMDB env"), + ); + + let _lmdb_chunk_database = lmdb_env + .create_db(Some("yama_chunks"), DatabaseFlags::empty()) + .expect("Failed to open yama_chunks"); + let _lmdb_pointer_database = lmdb_env + .create_db(Some("yama_pointers"), DatabaseFlags::empty()) + .expect("Failed to open yama_pointers"); + + //let lmdb_writing_thread_queue = + // make_lmdb_writer_thread(lmdb_env.clone(), lmdb_chunk_database, lmdb_pointer_database); + + Ok(()) + } + pub fn open(dir: &Path) -> YamaResult { + let zstd_path = dir.join("zstd.dict"); + if !zstd_path.exists() { + return Err("No Zstd dictionary".into()); + } + + let manifest_path = dir.join("yama.toml"); + if !manifest_path.exists() { + return Err("No Yama manifest".into()); + } + + let manifest = { + let mut file = File::open(manifest_path).expect("Failed to open yama manifest."); + let mut buf = Vec::new(); + file.read_to_end(&mut buf).expect("Failed to read buf."); + toml::from_slice::(&buf)? + }; + + let dictionary = Arc::new({ + let mut file = File::open(zstd_path)?; + let mut out = Vec::new(); + file.read_to_end(&mut out)?; + out + }); + + let lmdb_env = Arc::new( + Environment::new() + .set_max_dbs(8) + .set_flags(EnvironmentFlags::NO_SUB_DIR | EnvironmentFlags::NO_TLS) + .set_map_size(manifest.max_lmdb_size) + .open(&dir.join("yama.lmdb")) + .expect("Failed to open LMDB env"), + ); + + let lmdb_chunk_database = lmdb_env + .open_db(Some("yama_chunks")) + .expect("Failed to open yama_chunks"); + let lmdb_pointer_database = lmdb_env + .open_db(Some("yama_pointers")) + .expect("Failed to open yama_pointers"); + + let (writing_queue, join_handle) = + make_lmdb_writer_thread(lmdb_env.clone(), lmdb_chunk_database, lmdb_pointer_database); + + Ok(LocalPile { + directory: dir.to_owned(), + dictionary, + lmdb_env, + lmdb_chunk_database, + lmdb_pointer_database, + lmdb_writing_thread_queue: Some(writing_queue), + lmdb_writing_thread_handle: Some(Arc::new(join_handle)) + }) + } +} + +#[async_trait] +impl Pile for LocalPile { + fn clone_pile_handle(&self) -> Box { + Box::new(self.clone()) + } + + async fn get_dictionary(&self) -> YamaResult> { + Ok(self.dictionary.to_vec()) + } + + async fn get_chunk(&self, chunk_id: &ChunkId) -> YamaResult>> { + //eprintln!("get_chunk {}", chunkid_to_hex(chunk_id)); + + let lmdb_env = self.lmdb_env.clone(); + let chunk_db = self.lmdb_chunk_database; + let chunk_id = chunk_id.clone(); + let res: Result>, String> = async_std::task::spawn_blocking(move || { + let txn = lmdb_env.begin_ro_txn().map_err(|e| e.to_string())?; + match txn.get(chunk_db, &chunk_id) { + Ok(value) => Ok(Some(value.to_owned())), + Err(lmdb::Error::NotFound) => Ok(None), + Err(other) => Err(other.to_string()) + } + }) + .await; + + res.map_err(|e| e.into()) + } + + async fn put_chunk(&self, chunk_id: &ChunkId, data: Vec) -> YamaResult<()> { + //eprintln!("put_chunk {}", chunkid_to_hex(chunk_id)); + // TODO check matches hash + let sender = self.lmdb_writing_thread_queue + .as_ref().expect("invalid LP ref").clone(); + let key = chunk_id.to_vec(); + + //self.lmdb_writing_thread_queue.send(); + + let res: Result<(), String> = async_std::task::spawn_blocking(move || { + let complete_flag = BackgroundCompletedFlag::new(); + sender.send(LmdbWriteOp { + destination: LmdbDest::Chunk, + key, + value: data, + completion: complete_flag.clone(), + }).map_err(|e| e.to_string())?; + complete_flag.wait(); + Ok(()) + }).await; + res.map_err(|e| e.into()) + } + + async fn xxhash_chunk(&self, chunk_id: &ChunkId) -> YamaResult> { + //eprintln!("xxhash_chunk {}", chunkid_to_hex(chunk_id)); + let lmdb_env = self.lmdb_env.clone(); + let chunk_db = self.lmdb_chunk_database; + let chunk_id = chunk_id.clone(); + let dictionary = self.dictionary.clone(); + + async_std::task::spawn_blocking(move || { + let txn = lmdb_env.begin_ro_txn().map_err(|e| e.to_string())?; + match txn.get(chunk_db, &chunk_id) { + Ok(value) => { + // todo we might like to cache the decompressor + let mut decompressor = zstd::block::Decompressor::with_dict(dictionary.deref().clone()); + let decompressed = decompressor.decompress(value, 128 * 1024 * 1024) + .map_err(|e| e.to_string() + " (lp xxhash decompress)")?; + let mut hasher = twox_hash::XxHash64::with_seed(XXH64_SEED); + hasher.write(&decompressed); + Ok(Some(hasher.finish())) + } + Err(lmdb::Error::NotFound) => Ok(None), + Err(other) => Err(other.to_string()) + } + }) + .await + .map_err(|e| e.into()) + } + + async fn get_pointer(&self, pointer_id: &str) -> YamaResult> { + //eprintln!("get_pointer {:?}", pointer_id); + let lmdb_env = self.lmdb_env.clone(); + let pointer_db = self.lmdb_pointer_database; + let pointer_id = pointer_id.to_owned(); + let res: Result, String> = + async_std::task::spawn/*_blocking*/(async move { + let txn = lmdb_env.begin_ro_txn().map_err(|e| e.to_string())?; + match txn.get(pointer_db, &pointer_id.as_bytes()) { + Ok(value) => { + let pointer_data: PointerData = serde_cbor::from_slice(value) + .map_err(|e| e.to_string())?; + Ok(Some(pointer_data)) + } + Err(lmdb::Error::NotFound) => Ok(None), + Err(other) => Err(other.to_string()) + } + }) + .await; + + res.map_err(|e| e.into()) + } + + async fn put_pointer(&self, pointer_id: &str, pointer_data: PointerData) -> YamaResult<()> { + //eprintln!("put_pointer {:?}", pointer_id); + let sender = self.lmdb_writing_thread_queue + .as_ref().expect("invalid LP ref").clone(); + let key = pointer_id.as_bytes().to_vec(); + let value = serde_cbor::ser::to_vec_packed(&pointer_data)?; + + //self.lmdb_writing_thread_queue.send(); + + let res: Result<(), String> = async_std::task::spawn/*_blocking*/(async move { + let complete_flag = BackgroundCompletedFlag::new(); + sender.send(LmdbWriteOp { + destination: LmdbDest::Pointer, + key, + value, + completion: complete_flag.clone() + }).map_err(|e| e.to_string())?; + complete_flag.wait(); + Ok(()) + }).await; + res.map_err(|e| e.into()) + } + + async fn list_pointers(&self) -> YamaResult> { + //eprintln!("list_pointers"); + let lmdb_env = self.lmdb_env.clone(); + let pointer_db = self.lmdb_pointer_database; + + let res: Result, String> = async_std::task::spawn/*_blocking*/(async move { + let mut pointer_keys: Vec = Vec::new(); + let txn = lmdb_env.begin_ro_txn().map_err(|e| e.to_string())?; + let mut cur = txn.open_ro_cursor(pointer_db).map_err(|e| e.to_string())?; + for item in cur.iter_start() { + let (key, _value) = item.map_err(|e| e.to_string())?; + let key_str: String = String::from_utf8(key.to_vec()) + .map_err(|e| e.to_string())?; + pointer_keys.push(key_str); + } + + Ok(pointer_keys) + }) + .await; + + res.map_err(|e| e.into()) + } +} + + + +enum LmdbDest { + Pointer, + Chunk, +} + +#[derive(Clone, Debug)] +pub struct BackgroundCompletedFlag(Arc<(Mutex, Condvar)>); + +impl BackgroundCompletedFlag { + pub fn new() -> Self { + BackgroundCompletedFlag(Arc::new((Mutex::new(false), Condvar::new()))) + } + + pub fn wait(&self) { + let (mutex, condvar) = self.0.deref(); + let mut guard = mutex.lock().expect("poison"); + while !guard.deref() { + guard = condvar.wait(guard).expect("poison"); + } + } + + pub fn finish(&self) { + let (mutex, condvar) = self.0.deref(); + let mut guard = mutex.lock().expect("poison"); + *guard = true; + condvar.notify_all(); + } +} + +struct LmdbWriteOp { + destination: LmdbDest, + key: Vec, + value: Vec, + completion: BackgroundCompletedFlag, +} + +fn make_lmdb_writer_thread( + env: Arc, + chunk_db: Database, + pointer_db: Database, +) -> (SyncSender, JoinHandle<()>) { + let (send, recv) = mpsc::sync_channel(256); + let join_handle = thread::spawn(move || { + let mut pending = 0; + let mut txn = env + .begin_rw_txn() + .expect("Can't start LMDB writer transaction."); + loop { + match recv.recv_timeout(Duration::from_secs(60)) { + Ok(LmdbWriteOp { + destination, + key, + value, + completion + }) => { + { + let mut cur = txn + .open_rw_cursor(match destination { + LmdbDest::Pointer => pointer_db, + LmdbDest::Chunk => chunk_db, + }) + .expect("Can't open LMDB r/w cursor."); + cur.put(&key, &value, Default::default()) + .expect("Unable to put kv pair"); + pending += 1; + } + + if pending > 128 { + trace!("committing to LMDB due to high pending count"); + txn.commit().expect("Failed LMDB commit"); + txn = env + .begin_rw_txn() + .expect("Can't start LMDB writer transaction."); + pending = 0; + } + + completion.finish(); + } + Err(err) => { + if pending > 0 { + trace!("committing to LMDB due to inactivity or disconnect"); + txn.commit().expect("Failed LMDB commit"); + txn = env + .begin_rw_txn() + .expect("Can't start LMDB writer transaction."); + pending = 0; + } + if err == RecvTimeoutError::Disconnected { + info!("LMDB disconnected"); + break; + } + } + } + } + }); + (send, join_handle) +} diff --git a/src/tree.rs b/src/tree.rs new file mode 100644 index 0000000..f44dfde --- /dev/null +++ b/src/tree.rs @@ -0,0 +1,257 @@ +use crate::def::{TreeNode, YamaResult, TreeNodeContent, RecursiveChunkRef, FilesystemOwnership, FilesystemPermissions}; +use std::path::Path; +use std::fs::{symlink_metadata, read_link, DirEntry, Metadata}; +use std::collections::BTreeMap; +use std::collections::btree_map::Entry; +use std::os::unix::fs::MetadataExt; +use std::io::ErrorKind; +use log::warn; + +pub fn mtime_msec(metadata: &Metadata) -> u64 { + (metadata.mtime() * 1000 + metadata.mtime_nsec() / 1_000_000) as u64 +} + +pub fn scan(path: &Path) -> YamaResult> { + let metadata_res = symlink_metadata(path); + if let Err(e) = &metadata_res { + match e.kind() { + ErrorKind::NotFound => { + warn!("vanished: {:?}", path); + return Ok(None); + }, + ErrorKind::PermissionDenied => { + warn!("permission denied: {:?}", path); + return Ok(None); + }, + _ => { /* nop */ } + } + } + let metadata = metadata_res?; + let filetype = metadata.file_type(); + + let name = path.file_name().ok_or("No filename, wat")? + .to_str().ok_or("Filename can't be to_str()d")?.to_owned(); + + let ownership = FilesystemOwnership { + uid: metadata.uid() as u16, + gid: metadata.gid() as u16 + }; + + let permissions = FilesystemPermissions { + mode: metadata.mode() + }; + + if filetype.is_file() { + // Leave an unpopulated file node. It's not my responsibility to chunk it right now. + Ok(Some(TreeNode { + name, + content: TreeNodeContent::NormalFile { + mtime: mtime_msec(&metadata), + ownership, + permissions, + content: RecursiveChunkRef { + chunk_id: [0; 32], + depth: 0, + }, + }, + })) + } else if filetype.is_dir() { + let mut children = Vec::new(); + let dir_read = path.read_dir(); + + if let Err(e) = &dir_read { + match e.kind() { + ErrorKind::NotFound => { + warn!("vanished/: {:?}", path); + return Ok(None); + }, + ErrorKind::PermissionDenied => { + warn!("permission denied/: {:?}", path); + return Ok(None); + }, + _ => { /* nop */ } + } + } + + for entry in dir_read? { + let entry: DirEntry = entry?; + let scanned = scan(&entry.path())?; + if let Some(scanned) = scanned { + children.push(scanned); + } + } + children.sort_by(|l, r| { + l.name.cmp(&r.name) + }); + + Ok(Some(TreeNode { + name, + content: TreeNodeContent::Directory { + ownership, + permissions, + children, + }, + })) + } else if filetype.is_symlink() { + let target = read_link(path)?.to_str() + .ok_or("target path cannot be to_str()d")?.to_owned(); + + Ok(Some(TreeNode { + name, + content: TreeNodeContent::SymbolicLink { + ownership, + target, + }, + })) + } else { + Ok(None) + } +} + +pub fn differentiate_node(new: TreeNode, old: &TreeNode) -> YamaResult { + match new.content { + TreeNodeContent::Directory { + children, + ownership, permissions + } => match &old.content { + TreeNodeContent::Directory { + children: old_children, .. + } => { + // reformat into map for convenience. + let mut children = { + let mut map = BTreeMap::new(); + for child in children.into_iter() { + map.insert(child.name.clone(), child); + } + map + }; + + for old_child in old_children.iter() { + match children.entry(old_child.name.clone()) { + Entry::Vacant(ve) => { + ve.insert(TreeNode { + name: old_child.name.clone(), + content: TreeNodeContent::Deleted, + }); + } + Entry::Occupied(occ) => { + if !occ.get().metadata_invalidates(old_child, false) { + occ.remove(); + } + // leave it as it is TODO check this + } + } + } + + // repack into list + let mut children_vec = Vec::new(); + for (_, subnode) in children.into_iter() { + children_vec.push(subnode); + } + Ok(TreeNode { + name: new.name, + content: TreeNodeContent::Directory { + children: children_vec, + ownership, + permissions + } + }) + } + _ => Ok(TreeNode { + name: new.name, + content: TreeNodeContent::Directory { + children, + ownership, + permissions, + }, + }) + }, + _ => Ok(new), + } + /* + if let (TreeNodeContent::Directory { + children: new_children, + ow + }, TreeNodeContent::Directory { + children: old_children + }) = (&mut new.content, &old.content) { + let mut new_child_map = BTreeMap::new(); + let mut old_child_map = HashMap::new(); + let mut all_names = BTreeSet::new(); + + for new_child in new_children.into_iter() { + new_child_map.insert(new_child.name.clone(), new_child); + all_names.insert(new_child.name.clone()); + } + for old_child in old_children.iter() { + old_child_map.insert(old_child.name.clone(), old_child); + all_names.insert(old_child.name.clone()); + } + + for name in all_names.into_iter() { + let old = old_child_map.get(&name); + let new = new_child_map.get(&name); + + match (old, new) { + (None, None) => { + Err("wat. None, None")?; + }, + (None, Some(_)) => { + // nop + }, + (Some(_), None) => { + new_child_map.insert(name.clone(), TreeNode { + name, + content: TreeNodeContent::Deleted + }) + }, + (Some(old), Some(new)) => { + + } + } + } + + } + */ +} + +pub fn create_uidgid_lookup_tables(node: &TreeNode, uids: &mut BTreeMap>, gids: &mut BTreeMap>) -> YamaResult<()> { + let ownership = match &node.content { + TreeNodeContent::NormalFile { ownership, .. } => Some(ownership), + TreeNodeContent::Directory { ownership, .. } => Some(ownership), + TreeNodeContent::SymbolicLink { ownership, .. } => Some(ownership), + TreeNodeContent::Deleted => None, + }; + if let Some(ownership) = ownership { + if !uids.contains_key(&ownership.uid) { + if let Some(user) = users::get_user_by_uid(ownership.uid.into()) { + uids.insert(ownership.uid, Some( + user.name().to_str() + .ok_or("uid leads to non-String name")? + .to_owned() + )); + } else { + uids.insert(ownership.uid, None); + } + } + if !gids.contains_key(&ownership.gid) { + if let Some(group) = users::get_group_by_gid(ownership.gid.into()) { + gids.insert(ownership.gid, Some( + group.name().to_str() + .ok_or("gid leads to non-String name")? + .to_owned() + )); + } else { + gids.insert(ownership.gid, None); + } + } + } + + if let TreeNodeContent::Directory { children, .. } = &node.content { + for child in children { + create_uidgid_lookup_tables(child, uids, gids)?; + } + } + + Ok(()) +} \ No newline at end of file diff --git a/src/util.rs b/src/util.rs new file mode 100644 index 0000000..e69de29