diff --git a/Cargo.lock b/Cargo.lock index 1818e11..cab7847 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -102,6 +102,20 @@ dependencies = [ "syn 2.0.13", ] +[[package]] +name = "attohttpc" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fcf00bc6d5abb29b5f97e3c61a90b6d3caa12f3faf897d4a3e3607c050a35a7" +dependencies = [ + "http", + "log", + "native-tls", + "serde", + "serde_json", + "url", +] + [[package]] name = "atty" version = "0.2.14" @@ -135,6 +149,32 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5b3469636cdf8543cceab175efca534471f36eee12fb8374aba00eb5e7e7f8a" +[[package]] +name = "aws-creds" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1be164cb32cc9dd28dc0f6a8576c164c74516137ddc62dfe9eef584c81ee9488" +dependencies = [ + "attohttpc", + "dirs", + "log", + "rust-ini", + "serde", + "serde-xml-rs", + "thiserror", + "time 0.3.20", + "url", +] + +[[package]] +name = "aws-region" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f92a8af5850d0ea0916ca3e015ab86951ded0bf4b70fd27896e81ae1dfb0af37" +dependencies = [ + "thiserror", +] + [[package]] name = "bare-metrics-core" version = "0.1.0" @@ -170,6 +210,12 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +[[package]] +name = "base64" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" + [[package]] name = "bitflags" version = "1.3.2" @@ -192,6 +238,21 @@ dependencies = [ "libc", ] +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" + [[package]] name = "byteorder" version = "1.4.3" @@ -278,7 +339,7 @@ dependencies = [ "libc", "num-integer", "num-traits", - "time", + "time 0.1.44", "winapi", ] @@ -389,6 +450,31 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27d614f23f34f7b5165a77dc1591f497e2518f9cec4b4f4b92bfc4dc6cf7a190" +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + +[[package]] +name = "cpufeatures" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "280a9f2d8b3a38871a3c8a46fb80db65e5e5ed97da80c4d08bf27fb63e35e181" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.3.2" @@ -474,6 +560,16 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "dashmap" version = "4.0.2" @@ -537,6 +633,17 @@ dependencies = [ "syn 1.0.96", ] +[[package]] +name = "digest" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + [[package]] name = "dirs" version = "4.0.0" @@ -578,6 +685,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "dlv-list" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257" + [[package]] name = "duplex" version = "0.11.0" @@ -605,6 +718,15 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" +[[package]] +name = "encoding_rs" +version = "0.8.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "env_logger" version = "0.7.1" @@ -687,6 +809,36 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +dependencies = [ + "percent-encoding", +] + [[package]] name = "fs-set-times" version = "0.15.0" @@ -708,18 +860,95 @@ dependencies = [ "winapi", ] +[[package]] +name = "futures" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +dependencies = [ + "futures-core", + "futures-sink", +] + [[package]] name = "futures-core" version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +[[package]] +name = "futures-executor" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" + +[[package]] +name = "futures-macro" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.13", +] + [[package]] name = "futures-sink" version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" +[[package]] +name = "futures-task" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" + +[[package]] +name = "futures-util" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + [[package]] name = "fxhash" version = "0.2.1" @@ -729,6 +958,16 @@ dependencies = [ "byteorder", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.7" @@ -746,6 +985,25 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" +[[package]] +name = "h2" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5be7b54589b581f624f566bf5d8eb2bab1db736c51528720b6bd36b96b55924d" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.9.1" @@ -761,6 +1019,15 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash 0.7.6", +] + [[package]] name = "hashlink" version = "0.6.0" @@ -776,7 +1043,7 @@ version = "7.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31672b7011be2c4f7456c4ddbcb40e7e9a4a9fad8efe49a6ebaf5f307d0109c0" dependencies = [ - "base64", + "base64 0.13.0", "byteorder", "crossbeam-channel", "flate2", @@ -805,6 +1072,21 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "hostname" version = "0.3.1" @@ -816,6 +1098,40 @@ dependencies = [ "winapi", ] +[[package]] +name = "http" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" + [[package]] name = "humansize" version = "1.1.1" @@ -831,6 +1147,53 @@ dependencies = [ "quick-error", ] +[[package]] +name = "hyper" +version = "0.14.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc5e554ff619822309ffd57d8734d77cd5ce6238bc956f037ea06c58238c9899" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "idna" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "indexmap" version = "1.8.2" @@ -948,6 +1311,15 @@ dependencies = [ "libc", ] +[[package]] +name = "js-sys" +version = "0.3.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -1019,12 +1391,29 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" +[[package]] +name = "maybe-async" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f1b8c13cb1f814b634a96b2c725449fe7ed464a7b8781de8688be5ffbd3f305" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.96", +] + [[package]] name = "maybe-owned" version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4facc753ae494aeb6e3c22f839b158aebd4f9270f55cd3c79906c45476c47ab4" +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + [[package]] name = "memchr" version = "2.5.0" @@ -1063,6 +1452,21 @@ dependencies = [ "syn 1.0.96", ] +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "minidom" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e9ce45d459e358790a285e7609ff5ae4cfab88b75f237e8838e62029dda397b" +dependencies = [ + "rxml", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -1090,6 +1494,24 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nix" version = "0.17.0" @@ -1272,6 +1694,60 @@ dependencies = [ "vec-strings", ] +[[package]] +name = "openssl" +version = "0.10.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d2f106ab837a24e03672c59b1239669a0596406ff657c3c0835b6b7f0f35a33" +dependencies = [ + "bitflags 1.3.2", + "cfg-if 1.0.0", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.13", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a20eace9dc2d82904039cb76dcf50fb1a0bba071cfd1629720b5d6f1ddba0fa" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "ordered-multimap" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccd746e37177e1711c20dd619a1620f34f5c8b569c53590a72dedd5344d8924a" +dependencies = [ + "dlv-list", + "hashbrown 0.12.3", +] + [[package]] name = "os_pipe" version = "1.0.1" @@ -1340,6 +1816,12 @@ dependencies = [ "windows-sys 0.36.1", ] +[[package]] +name = "percent-encoding" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" + [[package]] name = "pin-project" version = "1.0.12" @@ -1366,6 +1848,12 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "pkg-config" version = "0.3.25" @@ -1423,6 +1911,16 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quick-xml" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quote" version = "1.0.26" @@ -1541,6 +2039,45 @@ version = "0.6.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49b3de9ec5dc0a3417da371aab17d729997c15010e7fd24ff707773a33bddb64" +[[package]] +name = "reqwest" +version = "0.11.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b71749df584b7f4cac2c426c127a7c785a5106cc98f7a8feb044115f0fa254" +dependencies = [ + "base64 0.21.0", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-native-tls", + "tokio-util", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "winreg", +] + [[package]] name = "rusqlite" version = "0.24.2" @@ -1556,6 +2093,49 @@ dependencies = [ "smallvec", ] +[[package]] +name = "rust-ini" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6d5f2436026b4f6e79dc829837d467cc7e9a55ee40e750d716713540715a2df" +dependencies = [ + "cfg-if 1.0.0", + "ordered-multimap", +] + +[[package]] +name = "rust-s3" +version = "0.33.0-beta5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceb8ffcf553348777f08c4c0f3fb6f62091ab32ee0b64b6b2fcedd6756c46dc" +dependencies = [ + "async-trait", + "aws-creds", + "aws-region", + "base64 0.13.0", + "bytes", + "cfg-if 1.0.0", + "futures", + "hex", + "hmac", + "http", + "log", + "maybe-async", + "md5", + "minidom", + "percent-encoding", + "quick-xml", + "reqwest", + "serde", + "serde_derive", + "sha2", + "thiserror", + "time 0.3.20", + "tokio", + "tokio-stream", + "url", +] + [[package]] name = "rustix" version = "0.33.7" @@ -1613,6 +2193,25 @@ dependencies = [ "winapi", ] +[[package]] +name = "rxml" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a071866b8c681dc2cfffa77184adc32b57b0caad4e620b6292609703bceb804" +dependencies = [ + "bytes", + "pin-project-lite", + "rxml_validation", + "smartstring", + "tokio", +] + +[[package]] +name = "rxml_validation" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53bc79743f9a66c2fb1f951cd83735f275d46bfe466259fbc5897bb60a0d00ee" + [[package]] name = "ryu" version = "1.0.10" @@ -1628,12 +2227,44 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" +dependencies = [ + "windows-sys 0.42.0", +] + [[package]] name = "scopeguard" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "security-framework" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "serde" version = "1.0.137" @@ -1643,6 +2274,18 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-xml-rs" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65162e9059be2f6a3421ebbb4fef3e74b7d9e7c60c50a0e292c6239f19f1edfa" +dependencies = [ + "log", + "serde", + "thiserror", + "xml-rs", +] + [[package]] name = "serde_bare" version = "0.3.0" @@ -1683,6 +2326,29 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "digest", +] + [[package]] name = "shell-escape" version = "0.1.5" @@ -1725,12 +2391,30 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f054c6c1a6e95179d6f23ed974060dcefb2d9388bb7256900badad682c499de4" +[[package]] +name = "slab" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" +dependencies = [ + "autocfg", +] + [[package]] name = "smallvec" version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" +[[package]] +name = "smartstring" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e714dff2b33f2321fdcd475b71cec79781a692d846f37f415fb395a1d2bcd48e" +dependencies = [ + "static_assertions", +] + [[package]] name = "socket2" version = "0.4.9" @@ -1750,7 +2434,7 @@ dependencies = [ "io-extras", "io-lifetimes 0.5.3", "rustix 0.33.7", - "uuid", + "uuid 0.8.2", "winapi", ] @@ -1823,6 +2507,12 @@ dependencies = [ "syn 1.0.96", ] +[[package]] +name = "subtle" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" + [[package]] name = "syn" version = "1.0.96" @@ -1955,6 +2645,33 @@ dependencies = [ "winapi", ] +[[package]] +name = "time" +version = "0.3.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" +dependencies = [ + "itoa", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" + +[[package]] +name = "time-macros" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" +dependencies = [ + "time-core", +] + [[package]] name = "tiny-keccak" version = "2.0.2" @@ -1964,6 +2681,21 @@ dependencies = [ "crunchy", ] +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.27.0" @@ -2002,6 +2734,16 @@ dependencies = [ "syn 2.0.13", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-pipe" version = "0.2.12" @@ -2012,6 +2754,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.7" @@ -2023,6 +2776,7 @@ dependencies = [ "futures-sink", "pin-project-lite", "tokio", + "tracing", ] [[package]] @@ -2034,6 +2788,32 @@ dependencies = [ "serde", ] +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a400e31aa60b9d44a52a8ee0343b5b18566b03a8321e0d321f695cf56e940160" +dependencies = [ + "cfg-if 1.0.0", + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b7358be39f2f274f322d2aaed611acc57f382e8eb1e5b48cb9ae30933495ce7" +dependencies = [ + "once_cell", +] + [[package]] name = "triomphe" version = "0.1.8" @@ -2045,6 +2825,12 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "try-lock" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" + [[package]] name = "twox-hash" version = "1.6.3" @@ -2056,12 +2842,33 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "typenum" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" + +[[package]] +name = "unicode-bidi" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" + [[package]] name = "unicode-ident" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5bd2fe26506023ed7b5e1e315add59d6f584c621d037f9368fea9cfb988f368c" +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + [[package]] name = "unicode-segmentation" version = "1.9.0" @@ -2074,6 +2881,17 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" +[[package]] +name = "url" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + [[package]] name = "users" version = "0.9.1" @@ -2098,6 +2916,16 @@ dependencies = [ "getrandom", ] +[[package]] +name = "uuid" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1674845326ee10d37ca60470760d4288a6f80f304007d92e5c53bab78c9cfd79" +dependencies = [ + "getrandom", + "rand", +] + [[package]] name = "vcpkg" version = "0.2.15" @@ -2137,6 +2965,16 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + [[package]] name = "wasi" version = "0.10.0+wasi-snapshot-preview1" @@ -2149,6 +2987,95 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" +dependencies = [ + "cfg-if 1.0.0", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 1.0.96", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" +dependencies = [ + "cfg-if 1.0.0", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.96", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" + +[[package]] +name = "wasm-streams" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bbae3363c08332cadccd13b67db371814cd214c2524020932f0804b8cf7c078" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "winapi" version = "0.3.9" @@ -2193,6 +3120,21 @@ dependencies = [ "windows_x86_64_msvc 0.36.1", ] +[[package]] +name = "windows-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc 0.42.2", +] + [[package]] name = "windows-sys" version = "0.45.0" @@ -2289,6 +3231,15 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" +[[package]] +name = "winreg" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +dependencies = [ + "winapi", +] + [[package]] name = "winx" version = "0.31.0" @@ -2300,6 +3251,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "xml-rs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2d7d3948613f75c98fd9328cfdcc45acc4d360655289d0a7d4ec931392200a3" + [[package]] name = "yama" version = "0.7.0-alpha.1" @@ -2356,6 +3313,18 @@ dependencies = [ [[package]] name = "yama_wormfile_s3" version = "0.1.0" +dependencies = [ + "async-trait", + "bytes", + "rand", + "rust-s3", + "thiserror", + "tokio", + "tokio-stream", + "tokio-util", + "uuid 1.3.0", + "yama_wormfile", +] [[package]] name = "yama_wormfile_sftp" diff --git a/yama_wormfile/src/lib.rs b/yama_wormfile/src/lib.rs index d1ad2c1..db53e6e 100644 --- a/yama_wormfile/src/lib.rs +++ b/yama_wormfile/src/lib.rs @@ -17,6 +17,8 @@ pub trait WormFileProvider: Debug + Send + Sync { /// Tests whether the path is a directory. /// Does not fail if the path does not exist, even if the parent path doesn't exist — returns /// false in that case. + /// Beware! Some backends do not support the concept of a directory and will happily report + /// `true` for any path. async fn is_dir(&self, path: impl AsRef + Send) -> Result; /// Tests whether the path is a regular file. diff --git a/yama_wormfile_s3/Cargo.toml b/yama_wormfile_s3/Cargo.toml index 32780f5..be0df27 100644 --- a/yama_wormfile_s3/Cargo.toml +++ b/yama_wormfile_s3/Cargo.toml @@ -6,3 +6,16 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +yama_wormfile = { version = "0.1.0", path = "../yama_wormfile" } + +#ouroboros = "0.15.6" +rust-s3 = { version = "0.33.0-beta5", features = ["with-tokio"] } +async-trait = "0.1.68" +tokio = { version = "1.27.0", features = ["io-std"] } +rand = "0.8.5" +thiserror = "1.0.40" + +tokio-stream = "0.1.12" +tokio-util = "0.7.7" +bytes = "1.4.0" +uuid = { version = "1.3.0", features = ["fast-rng", "v4"] } \ No newline at end of file diff --git a/yama_wormfile_s3/src/lib.rs b/yama_wormfile_s3/src/lib.rs index e69de29..a7e45ed 100644 --- a/yama_wormfile_s3/src/lib.rs +++ b/yama_wormfile_s3/src/lib.rs @@ -0,0 +1,393 @@ +use async_trait::async_trait; +use s3::error::S3Error; +use s3::serde_types::HeadObjectResult; +use s3::Bucket; +use std::fmt::{Debug, Formatter}; +use std::io; +use std::io::{ErrorKind, SeekFrom}; +use std::ops::Range; +use std::pin::Pin; +use std::task::{ready, Context, Poll}; +use tokio::io::{duplex, AsyncRead, AsyncSeek, AsyncWrite, AsyncWriteExt, DuplexStream, ReadBuf}; +use tokio::task::JoinHandle; +use uuid::Uuid; +use yama_wormfile::paths::{WormPath, WormPathBuf}; +use yama_wormfile::{WormFileProvider, WormFileReader, WormFileWriter}; + +/// WormFileProvider that uses an S3 bucket, with a given path prefix. +#[derive(Debug)] +pub struct S3WormFilesystem { + /// The path prefix for all S3 objects. + path_prefix: String, + + bucket: Bucket, +} + +impl S3WormFilesystem { + pub fn new(bucket: Bucket, path_prefix: String) -> io::Result { + Ok(S3WormFilesystem { + path_prefix, + bucket, + }) + } + + pub fn resolve_real_path(&self, path: impl AsRef) -> String { + format!("{}{}", self.path_prefix, path.as_ref().as_str()) + } + + async fn head_object(&self, full_path: &str) -> Result, S3Error> { + let (head, status) = self.bucket.head_object(full_path).await?; + if status == 404 { + return Ok(None); + } + if status == 200 { + return Ok(Some(head)); + } + Err(S3Error::Http( + status, + format!("bad status for HEAD {full_path}"), + )) + } +} + +#[async_trait] +impl WormFileProvider for S3WormFilesystem { + type WormFileReader = S3WormReader; + type WormFileWriter = S3WormWriter; + type Error = S3Error; + + async fn is_dir(&self, _path: impl AsRef + Send) -> Result { + Ok(true) + } + + async fn is_regular_file( + &self, + path: impl AsRef + Send, + ) -> Result { + let full_path = self.resolve_real_path(path.as_ref()); + Ok(self.head_object(&full_path).await?.is_some()) + } + + async fn list( + &self, + path: impl AsRef + Send, + ) -> Result, Self::Error> { + let path = path.as_ref(); + let full_path = self.resolve_real_path(path); + let list = self + .bucket + .list(format!("{full_path}/"), Some("/".to_owned())) + .await?; + Ok(list + .into_iter() + .map(|lbr| lbr.contents) + .flatten() + .filter_map(|obj| { + // Strip the path prefix from items, plus convert to `WormPathBuf`s + obj.key + .strip_prefix(&self.path_prefix) + .map(|s| WormPathBuf::new(s.to_owned())) + .flatten() + }) + .collect()) + } + + async fn read( + &self, + path: impl AsRef + Send, + ) -> Result { + let path = path.as_ref(); + let full_path = self.resolve_real_path(path); + let head = self.head_object(&full_path).await?.ok_or_else(|| { + S3Error::Http(404, format!("can't read {path:?} as it does not exist.")) + })?; + + let length = head + .content_length + .ok_or_else(|| S3Error::Http(999, format!("No content-length from HEAD {path:?}")))?; + + Ok(S3WormReader { + bucket: self.bucket.clone(), + path: full_path, + reader: None, + offset: 0, + length: length as u64, + next_read_size_hint: None, + }) + } + + async fn write(&self) -> Result { + let (tx, mut rx) = duplex(8192); + // The rx half won't be doing any writing. + rx.shutdown().await?; + + // let (tx, rx) = tokio::sync::mpsc::channel::>(4); + // let mut reader = StreamReader::new(ReceiverStream::new(rx)); + + let uuid = Uuid::new_v4(); + let temp_path = format!("tmp/{}.writing", uuid); + let upload_to = self.resolve_real_path(WormPath::new(&temp_path).unwrap()); + + let bucket = self.bucket.clone(); + let join_handle: JoinHandle> = tokio::spawn(async move { + let resp_code = bucket.put_object_stream(&mut rx, upload_to).await?; + if resp_code != 200 { + eprintln!("non-200 resp code for put!") + } + Ok(resp_code) + }); + + Ok(S3WormWriter { + tx, + join_handle, + temp_path, + bucket: self.bucket.clone(), + path_prefix: self.path_prefix.clone(), + }) + } +} + +pub struct S3WormReader { + /// S3 bucket + bucket: Bucket, + /// S3-side path (absolute) + path: String, + + /// A stream for reading bytes from S3, if open. + reader: Option<(DuplexStream, Range)>, + /// Our current stream offset + offset: u64, + /// The length of the file + length: u64, + /// Optionally, a hint for the expected size of the next read operation. + /// Can be used to request the correct range from S3. + next_read_size_hint: Option, +} + +impl S3WormReader { + fn spawn_reader_for_range(&self, range: Range) -> DuplexStream { + let (mut tx, rx) = duplex(65536); + + let bucket = self.bucket.clone(); + let path = self.path.clone(); + + tokio::spawn(async move { + // TODO it says this is 'inclusive', but asserts start < end... + // TODO it'd be nice to have a `get_object_range_stream`: PR one to rust-s3? + // alternatively rebuild on top of reqwest and serde_xml_rs, since rust-s3 is + // a little bit dodgy around the corners... + if let Err(err) = bucket + .get_object_range_to_writer(path, range.start, Some(range.end), &mut tx) + .await + { + eprintln!("TODO error reading from S3 {err:?}"); + } + }); + + rx + } +} + +impl Debug for S3WormReader { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "S3WormReader({:?})", self.path) + } +} + +/// Default size of read request from S3. +const DEFAULT_READ_WINDOW_SIZE: u64 = 64 * 1024 * 1024; + +impl AsyncRead for S3WormReader { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + if self.reader.is_none() { + // Need to spawn a reader + if self.offset > self.length { + panic!("offset {} > length {}", self.offset, self.length); + } + + if self.offset == self.length { + // EOF + return Poll::Ready(Ok(())); + } + + let read_window_size = self + .next_read_size_hint + .take() + .unwrap_or(DEFAULT_READ_WINDOW_SIZE) + .min(self.length - self.offset); + + let read_range = self.offset..(self.offset + read_window_size); + + let rx = self.spawn_reader_for_range(read_range.clone()); + self.reader = Some((rx, read_range)); + } + + let (reader, read_range) = self.reader.as_mut().unwrap(); + let orig_remaining = buf.remaining(); + let read = ready!(Pin::new(reader).poll_read(cx, buf)); + let bytes_read = (buf.remaining() - orig_remaining) as u64; + + if bytes_read == 0 && read_range.start != read_range.end { + // Unexpected EOF + return Poll::Ready(Err(io::Error::new( + ErrorKind::UnexpectedEof, + "early EOF reading from S3", + ))); + } + + read_range.start += bytes_read; + if read_range.start >= read_range.end { + self.reader = None; + } + self.offset += bytes_read; + + Poll::Ready(read) + } +} + +impl AsyncSeek for S3WormReader { + fn start_seek(mut self: Pin<&mut Self>, position: SeekFrom) -> io::Result<()> { + let desired_offset = match position { + SeekFrom::Start(offset) => offset, + SeekFrom::End(end_offset) => { + let wanted = self.length as i64 + end_offset; + if wanted < 0 { + return Err(io::Error::new( + ErrorKind::InvalidInput, + "can't seek below zero (using relative end seek)", + )); + } + wanted as u64 + } + SeekFrom::Current(delta) => { + let wanted = self.offset as i64 + delta; + if wanted < 0 { + return Err(io::Error::new( + ErrorKind::InvalidInput, + "can't seek below zero (using relative current seek)", + )); + } + wanted as u64 + } + }; + + // Don't seek beyond the end. (That makes no sense for reading, but is technically allowed.) + let desired_offset = desired_offset.min(self.length); + self.offset = desired_offset; + + // Discard any readers + self.reader = None; + self.next_read_size_hint = None; + + // Return OK right away, the actual work will get done when we read. + Ok(()) + } + + fn poll_complete(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + // Return OK right away, the actual work gets done when we read. + Poll::Ready(Ok(self.offset)) + } +} + +impl WormFileReader for S3WormReader {} + +pub struct S3WormWriter { + tx: DuplexStream, + temp_path: String, + join_handle: JoinHandle>, + bucket: Bucket, + path_prefix: String, +} + +impl Debug for S3WormWriter { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "FileWormWriter({:?})", self.temp_path) + } +} + +impl AsyncWrite for S3WormWriter { + fn poll_write( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + Pin::new(&mut self.tx).poll_write(cx, buf) + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.tx).poll_flush(cx) + } + + fn poll_shutdown( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll> { + Pin::new(&mut self.tx).poll_shutdown(cx) + } +} + +#[async_trait] +impl WormFileWriter for S3WormWriter { + async fn finalise(mut self, target_path: &WormPath, replace: bool) -> io::Result<()> { + self.tx.shutdown().await?; + + let resp_code = self + .join_handle + .await? + .map_err(|e| io::Error::new(ErrorKind::Other, e))?; + if resp_code != 200 { + return Err(io::Error::new( + ErrorKind::Other, + "non-200 resp code for PUT", + )); + } + + let full_target_path = format!("{}{}", self.path_prefix, target_path.as_str()); + + if !replace { + let (_head, head_code) = self + .bucket + .head_object(&full_target_path) + .await + .map_err(|e| io::Error::new(ErrorKind::Other, e))?; + if head_code != 404 { + return Err(io::Error::new( + ErrorKind::Other, + "won't replace file; HEAD of target path not 404", + )); + } + } + + // S3 moves are done as a copy + delete + + let response_code = self + .bucket + .copy_object_internal(&self.temp_path, &full_target_path) + .await + .map_err(|e| io::Error::new(ErrorKind::Other, e))?; + if response_code != 200 { + return Err(io::Error::new( + ErrorKind::Other, + "non-200 response for copy object", + )); + } + + let response_code = self + .bucket + .delete_object(&self.temp_path) + .await + .map_err(|e| io::Error::new(ErrorKind::Other, e))?; + if response_code.status_code() != 200 { + return Err(io::Error::new( + ErrorKind::Other, + "non-200 response for delete object", + )); + } + + Ok(()) + } +}