diff --git a/bao/Cargo.lock b/bao/Cargo.lock deleted file mode 100644 index af300d2..0000000 --- a/bao/Cargo.lock +++ /dev/null @@ -1,1415 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 3 - -[[package]] -name = "abao" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2daa0989489b05a455a9707adbbbc17443edf7bbc902ce499cd3b84148d68a40" -dependencies = [ - "arrayref", - "arrayvec", - "blake3", - "futures", - "tokio", -] - -[[package]] -name = "anyhow" -version = "1.0.71" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" - -[[package]] -name = "arrayref" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" - -[[package]] -name = "arrayvec" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" - -[[package]] -name = "async-stream" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" -dependencies = [ - "async-stream-impl", - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "async-stream-impl" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.15", -] - -[[package]] -name = "async-trait" -version = "0.1.68" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.15", -] - -[[package]] -name = "atomic-counter" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62f447d68cfa5a9ab0c1c862a703da2a65b5ed1b7ce1153c9eb0169506d56019" - -[[package]] -name = "autocfg" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" - -[[package]] -name = "axum" -version = "0.6.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8175979259124331c1d7bf6586ee7e0da434155e4b2d48ec2c8386281d8df39" -dependencies = [ - "async-trait", - "axum-core", - "bitflags", - "bytes", - "futures-util", - "http", - "http-body", - "hyper", - "itoa", - "matchit", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "rustversion", - "serde", - "sync_wrapper", - "tower", - "tower-layer", - "tower-service", -] - -[[package]] -name = "axum-core" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" -dependencies = [ - "async-trait", - "bytes", - "futures-util", - "http", - "http-body", - "mime", - "rustversion", - "tower-layer", - "tower-service", -] - -[[package]] -name = "bao" -version = "0.1.0" -dependencies = [ - "abao", - "anyhow", - "async-stream", - "async-trait", - "atomic-counter", - "futures", - "gag", - "hyper", - "log", - "parking_lot", - "portpicker", - "prost", - "serde", - "thiserror", - "tokio", - "tokio-stream", - "tonic", - "tonic-build", - "tonic-health", - "tower", -] - -[[package]] -name = "base64" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" - -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "blake3" -version = "1.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ae2468a89544a466886840aa467a25b766499f4f04bf7d9fcd10ecee9fccef" -dependencies = [ - "arrayref", - "arrayvec", - "cc", - "cfg-if", - "constant_time_eq", - "digest", -] - -[[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" -dependencies = [ - "generic-array", -] - -[[package]] -name = "bytes" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" - -[[package]] -name = "cc" -version = "1.0.79" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "constant_time_eq" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13418e745008f7349ec7e449155f419a61b92b58a99cc3616942b926825ec76b" - -[[package]] -name = "crypto-common" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" -dependencies = [ - "generic-array", - "typenum", -] - -[[package]] -name = "digest" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" -dependencies = [ - "block-buffer", - "crypto-common", - "subtle", -] - -[[package]] -name = "either" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" - -[[package]] -name = "errno" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" -dependencies = [ - "errno-dragonfly", - "libc", - "windows-sys 0.48.0", -] - -[[package]] -name = "errno-dragonfly" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" -dependencies = [ - "cc", - "libc", -] - -[[package]] -name = "fastrand" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] - -[[package]] -name = "filedescriptor" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7199d965852c3bac31f779ef99cbb4537f80e952e2d6aa0ffeb30cce00f4f46e" -dependencies = [ - "libc", - "thiserror", - "winapi", -] - -[[package]] -name = "fixedbitset" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" - -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "futures" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-channel" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" -dependencies = [ - "futures-core", - "futures-sink", -] - -[[package]] -name = "futures-core" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" - -[[package]] -name = "futures-executor" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-io" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" - -[[package]] -name = "futures-macro" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.15", -] - -[[package]] -name = "futures-sink" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" - -[[package]] -name = "futures-task" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" - -[[package]] -name = "futures-util" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" -dependencies = [ - "futures-channel", - "futures-core", - "futures-io", - "futures-macro", - "futures-sink", - "futures-task", - "memchr", - "pin-project-lite", - "pin-utils", - "slab", -] - -[[package]] -name = "gag" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a713bee13966e9fbffdf7193af71d54a6b35a0bb34997cd6c9519ebeb5005972" -dependencies = [ - "filedescriptor", - "tempfile", -] - -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - -[[package]] -name = "getrandom" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" -dependencies = [ - "cfg-if", - "libc", - "wasi", -] - -[[package]] -name = "h2" -version = "0.3.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d357c7ae988e7d2182f7d7871d0b963962420b0678b0997ce7de72001aeab782" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http", - "indexmap", - "slab", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - -[[package]] -name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" - -[[package]] -name = "hermit-abi" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" -dependencies = [ - "libc", -] - -[[package]] -name = "hermit-abi" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" - -[[package]] -name = "http" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - -[[package]] -name = "http-body" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" -dependencies = [ - "bytes", - "http", - "pin-project-lite", -] - -[[package]] -name = "httparse" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" - -[[package]] -name = "httpdate" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" - -[[package]] -name = "hyper" -version = "0.14.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4" -dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "socket2", - "tokio", - "tower-service", - "tracing", - "want", -] - -[[package]] -name = "hyper-timeout" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" -dependencies = [ - "hyper", - "pin-project-lite", - "tokio", - "tokio-io-timeout", -] - -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown", -] - -[[package]] -name = "instant" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "io-lifetimes" -version = "1.0.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" -dependencies = [ - "hermit-abi 0.3.1", - "libc", - "windows-sys 0.48.0", -] - -[[package]] -name = "itertools" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" -dependencies = [ - "either", -] - -[[package]] -name = "itoa" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" - -[[package]] -name = "lazy_static" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" - -[[package]] -name = "libc" -version = "0.2.144" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1" - -[[package]] -name = "linux-raw-sys" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ece97ea872ece730aed82664c424eb4c8291e1ff2480247ccf7409044bc6479f" - -[[package]] -name = "lock_api" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" -dependencies = [ - "autocfg", - "scopeguard", -] - -[[package]] -name = "log" -version = "0.4.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "matchit" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" - -[[package]] -name = "memchr" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" - -[[package]] -name = "mime" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" - -[[package]] -name = "mio" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" -dependencies = [ - "libc", - "log", - "wasi", - "windows-sys 0.45.0", -] - -[[package]] -name = "multimap" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" - -[[package]] -name = "num_cpus" -version = "1.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" -dependencies = [ - "hermit-abi 0.2.6", - "libc", -] - -[[package]] -name = "once_cell" -version = "1.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" - -[[package]] -name = "parking_lot" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall 0.2.16", - "smallvec", - "windows-sys 0.45.0", -] - -[[package]] -name = "percent-encoding" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" - -[[package]] -name = "petgraph" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" -dependencies = [ - "fixedbitset", - "indexmap", -] - -[[package]] -name = "pin-project" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c95a7476719eab1e366eaf73d0260af3021184f18177925b07f54b30089ceead" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39407670928234ebc5e6e580247dd567ad73a3578460c5990f9503df207e8f07" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.15", -] - -[[package]] -name = "pin-project-lite" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" - -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - -[[package]] -name = "portpicker" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be97d76faf1bfab666e1375477b23fde79eccf0276e9b63b92a39d676a889ba9" -dependencies = [ - "rand", -] - -[[package]] -name = "ppv-lite86" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" - -[[package]] -name = "prettyplease" -version = "0.1.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" -dependencies = [ - "proc-macro2", - "syn 1.0.109", -] - -[[package]] -name = "proc-macro2" -version = "1.0.56" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "prost" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" -dependencies = [ - "bytes", - "prost-derive", -] - -[[package]] -name = "prost-build" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" -dependencies = [ - "bytes", - "heck", - "itertools", - "lazy_static", - "log", - "multimap", - "petgraph", - "prettyplease", - "prost", - "prost-types", - "regex", - "syn 1.0.109", - "tempfile", - "which", -] - -[[package]] -name = "prost-derive" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" -dependencies = [ - "anyhow", - "itertools", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "prost-types" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" -dependencies = [ - "prost", -] - -[[package]] -name = "quote" -version = "1.0.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f4f29d145265ec1c483c7c654450edde0bfe043d3938d6972630663356d9500" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha", - "rand_core", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom", -] - -[[package]] -name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags", -] - -[[package]] -name = "redox_syscall" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" -dependencies = [ - "bitflags", -] - -[[package]] -name = "regex" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af83e617f331cc6ae2da5443c602dfa5af81e517212d9d611a5b3ba1777b5370" -dependencies = [ - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c" - -[[package]] -name = "rustix" -version = "0.37.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" -dependencies = [ - "bitflags", - "errno", - "io-lifetimes", - "libc", - "linux-raw-sys", - "windows-sys 0.48.0", -] - -[[package]] -name = "rustversion" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" - -[[package]] -name = "scopeguard" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" - -[[package]] -name = "serde" -version = "1.0.163" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.163" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.15", -] - -[[package]] -name = "slab" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" -dependencies = [ - "autocfg", -] - -[[package]] -name = "smallvec" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" - -[[package]] -name = "socket2" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" -dependencies = [ - "libc", - "winapi", -] - -[[package]] -name = "subtle" -version = "2.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" - -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "2.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "sync_wrapper" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" - -[[package]] -name = "tempfile" -version = "3.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" -dependencies = [ - "cfg-if", - "fastrand", - "redox_syscall 0.3.5", - "rustix", - "windows-sys 0.45.0", -] - -[[package]] -name = "thiserror" -version = "1.0.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.15", -] - -[[package]] -name = "tokio" -version = "1.28.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aa32867d44e6f2ce3385e89dceb990188b8bb0fb25b0cf576647a6f98ac5105" -dependencies = [ - "autocfg", - "bytes", - "libc", - "mio", - "num_cpus", - "pin-project-lite", - "socket2", - "tokio-macros", - "windows-sys 0.48.0", -] - -[[package]] -name = "tokio-io-timeout" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" -dependencies = [ - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tokio-macros" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.15", -] - -[[package]] -name = "tokio-stream" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tokio-util" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "pin-project-lite", - "tokio", - "tracing", -] - -[[package]] -name = "tonic" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a" -dependencies = [ - "async-trait", - "axum", - "base64", - "bytes", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "hyper", - "hyper-timeout", - "percent-encoding", - "pin-project", - "prost", - "tokio", - "tokio-stream", - "tower", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tonic-build" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6fdaae4c2c638bb70fe42803a26fbd6fc6ac8c72f5c59f67ecc2a2dcabf4b07" -dependencies = [ - "prettyplease", - "proc-macro2", - "prost-build", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "tonic-health" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "080964d45894b90273d2b1dd755fdd114560db8636bb41cea615213c45043c4d" -dependencies = [ - "async-stream", - "prost", - "tokio", - "tokio-stream", - "tonic", -] - -[[package]] -name = "tower" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" -dependencies = [ - "futures-core", - "futures-util", - "indexmap", - "pin-project", - "pin-project-lite", - "rand", - "slab", - "tokio", - "tokio-util", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tower-layer" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" - -[[package]] -name = "tower-service" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" - -[[package]] -name = "tracing" -version = "0.1.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" -dependencies = [ - "cfg-if", - "log", - "pin-project-lite", - "tracing-attributes", - "tracing-core", -] - -[[package]] -name = "tracing-attributes" -version = "0.1.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.15", -] - -[[package]] -name = "tracing-core" -version = "0.1.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" -dependencies = [ - "once_cell", -] - -[[package]] -name = "try-lock" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" - -[[package]] -name = "typenum" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" - -[[package]] -name = "unicode-ident" -version = "1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" - -[[package]] -name = "version_check" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" - -[[package]] -name = "want" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" -dependencies = [ - "log", - "try-lock", -] - -[[package]] -name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" - -[[package]] -name = "which" -version = "4.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" -dependencies = [ - "either", - "libc", - "once_cell", -] - -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets 0.42.2", -] - -[[package]] -name = "windows-sys" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" -dependencies = [ - "windows-targets 0.48.0", -] - -[[package]] -name = "windows-targets" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - -[[package]] -name = "windows-targets" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" -dependencies = [ - "windows_aarch64_gnullvm 0.48.0", - "windows_aarch64_msvc 0.48.0", - "windows_i686_gnu 0.48.0", - "windows_i686_msvc 0.48.0", - "windows_x86_64_gnu 0.48.0", - "windows_x86_64_gnullvm 0.48.0", - "windows_x86_64_msvc 0.48.0", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" - -[[package]] -name = "windows_i686_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" - -[[package]] -name = "windows_i686_gnu" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" - -[[package]] -name = "windows_i686_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" - -[[package]] -name = "windows_i686_msvc" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" diff --git a/bao/Cargo.toml b/bao/Cargo.toml deleted file mode 100644 index fca5c02..0000000 --- a/bao/Cargo.toml +++ /dev/null @@ -1,28 +0,0 @@ -[package] -name = "bao" -version = "0.1.0" -edition = "2021" - -[dependencies] -abao = { version = "0.2.0", features = ["group_size_256k", "tokio_io"], default-features = false } -anyhow = "1.0.71" -async-stream = "0.3.5" -async-trait = "0.1.68" -atomic-counter = "1.0.1" -futures = "0.3.28" -gag = "1.0.0" -hyper = "0.14.26" -log = "0.4.17" -parking_lot = "0.12.1" -portpicker = "0.1.1" -prost = "0.11.9" -serde = { version = "1.0.163", features = ["derive"] } -thiserror = "1.0.40" -tokio = { version = "1.28.1", features = ["rt", "rt-multi-thread"] } -tokio-stream = "0.1.14" -tonic = "0.9.2" -tonic-health = "0.9.2" -tower = "0.4.13" - -[build-dependencies] -tonic-build = "0.9.2" diff --git a/bao/bao.go b/bao/bao.go index 899bc31..27fdfe9 100644 --- a/bao/bao.go +++ b/bao/bao.go @@ -1,165 +1,44 @@ package bao import ( + "bufio" _ "embed" - "github.com/hashicorp/go-plugin" "io" - "io/fs" - "log" - "os" - "os/exec" - "os/signal" - "syscall" + "lukechampine.com/blake3" ) -//go:generate protoc --proto_path=proto/ bao.proto --go_out=proto --go_opt=paths=source_relative --go-grpc_out=proto --go-grpc_opt=paths=source_relative +const ( + chunkSize = 1024 +) -//go:embed target/release/bao -var baoPlugin []byte -var baoInstance Bao +func ComputeTree(reader io.Reader, size int64) ([]byte, [32]byte, error) { + bufSize := baoOutboardSize(int(size)) + buf := bufferAt{buf: make([]byte, bufSize)} -type Bao interface { - Init() (uint32, error) - Write(id uint32, data []byte) error - Finalize(id uint32) ([]byte, error) - Destroy(id uint32) error - ComputeFile(path string) ([]byte, error) + hash, err := blake3.BaoEncode(&buf, bufio.NewReader(reader), size, true) + if err != nil { + return nil, [32]byte{}, err + } + + return buf.buf, hash, nil } -func init() { - baoExec, err := os.CreateTemp("", "lumeportal") - - _, err = baoExec.Write(baoPlugin) - if err != nil { - log.Fatalf("Error:", err.Error()) +func baoOutboardSize(dataLen int) int { + if dataLen == 0 { + return 8 } - - err = baoExec.Sync() - if err != nil { - log.Fatalf("Error:", err.Error()) - } - - err = baoExec.Chmod(fs.ModePerm) - if err != nil { - log.Fatalf("Error:", err.Error()) - } - - err = baoExec.Close() - if err != nil { - log.Fatalf("Error:", err.Error()) - } - pluginMap := map[string]plugin.Plugin{ - "bao": &BAOPlugin{}, - } - client := plugin.NewClient(&plugin.ClientConfig{ - HandshakeConfig: plugin.HandshakeConfig{ - ProtocolVersion: 1, - MagicCookieKey: "foo", - MagicCookieValue: "bar", - }, - Plugins: pluginMap, - Cmd: exec.Command("sh", "-c", baoExec.Name()), - AllowedProtocols: []plugin.Protocol{plugin.ProtocolGRPC}, - }) - - // Connect via RPC - rpcClient, err := client.Client() - if err != nil { - log.Fatalf("Error:", err.Error()) - } - - // Request the plugin - raw, err := rpcClient.Dispense("bao") - if err != nil { - log.Fatalf("Error:", err.Error()) - } - - baoInstance = raw.(Bao) - - signalCh := make(chan os.Signal, 1) - signal.Notify(signalCh, os.Interrupt, syscall.SIGTERM) - - go func() { - <-signalCh - err := os.Remove(baoExec.Name()) - if err != nil { - log.Fatalf("Error:", err.Error()) - } - }() - + chunks := (dataLen + chunkSize - 1) / chunkSize + cvs := 2*chunks - 2 // no I will not elaborate + return 8 + cvs*32 } -func ComputeTreeStreaming(reader io.Reader) ([]byte, error) { - instance, err := baoInstance.Init() - if err != nil { - return nil, err - } - - b := make([]byte, 512) - for { - n, err := reader.Read(b) - - if n > 0 { - err := write(instance, &b) - if err != nil { - return nil, err - } - } - - if err != nil { - var result []byte - if err == io.EOF { - result, err = finalize(instance) - if err == nil { - return result, nil - } - } - return nil, err - } - } +type bufferAt struct { + buf []byte } -func ComputeTreeFile(file *os.File) ([]byte, error) { - tree, err := baoInstance.ComputeFile(file.Name()) - if err != nil { - return nil, err +func (b *bufferAt) WriteAt(p []byte, off int64) (int, error) { + if copy(b.buf[off:], p) != len(p) { + panic("bad buffer size") } - - return tree, nil -} - -func write(instance uint32, bytes *[]byte) error { - err := baoInstance.Write(instance, *bytes) - if err != nil { - derr := destroy(instance) - if derr != nil { - return derr - } - return err - } - if err != nil { - derr := destroy(instance) - if derr != nil { - return derr - } - return err - } - - return nil -} - -func finalize(instance uint32) ([]byte, error) { - result, err := baoInstance.Finalize(instance) - if err != nil { - derr := destroy(instance) - if derr != nil { - return nil, derr - } - return nil, err - } - - return result, nil -} -func destroy(instance uint32) error { - return baoInstance.Destroy(instance) + return len(p), nil } diff --git a/bao/build.rs b/bao/build.rs deleted file mode 100644 index 6c0b711..0000000 --- a/bao/build.rs +++ /dev/null @@ -1,15 +0,0 @@ -fn main() -> Result<(), Box> { - tonic_build::configure() - .build_server(true) - .out_dir("src/proto") - .compile_well_known_types(true) - .include_file("mod.rs") - .type_attribute(".", "#[derive(serde::Deserialize)]") - .type_attribute(".", "#[derive(serde::Serialize)]") - .compile(&[ - "proto/grpc_stdio.proto", - "proto/bao.proto" - ], &["bao"]) - .unwrap(); - Ok(()) -} diff --git a/bao/client.go b/bao/client.go deleted file mode 100644 index b490f26..0000000 --- a/bao/client.go +++ /dev/null @@ -1,55 +0,0 @@ -package bao - -import ( - "context" - "git.lumeweb.com/LumeWeb/portal/bao/proto" - "github.com/golang/protobuf/ptypes/empty" - "github.com/golang/protobuf/ptypes/wrappers" -) - -// GRPCClient is an implementation of KV that talks over RPC. -type GRPCClient struct{ client proto.BaoClient } - -func (g *GRPCClient) Init() (uint32, error) { - init, err := g.client.Init(context.Background(), &empty.Empty{}) - if err != nil { - return 0, err - } - - return init.Value, nil -} - -func (g *GRPCClient) Write(id uint32, data []byte) error { - _, err := g.client.Write(context.Background(), &proto.WriteRequest{Id: id, Data: data}) - if err != nil { - return err - } - - return nil -} - -func (g *GRPCClient) Finalize(id uint32) ([]byte, error) { - tree, err := g.client.Finalize(context.Background(), &wrappers.UInt32Value{Value: id}) - if err != nil { - return nil, err - } - - return tree.Value, nil -} - -func (g *GRPCClient) Destroy(id uint32) error { - _, err := g.client.Destroy(context.Background(), &wrappers.UInt32Value{Value: id}) - if err != nil { - return err - } - - return nil -} -func (g *GRPCClient) ComputeFile(path string) ([]byte, error) { - tree, err := g.client.ComputeFile(context.Background(), &wrappers.StringValue{Value: path}) - if err != nil { - return nil, err - } - - return tree.Value, nil -} diff --git a/bao/plugin.go b/bao/plugin.go deleted file mode 100644 index e2f3419..0000000 --- a/bao/plugin.go +++ /dev/null @@ -1,21 +0,0 @@ -package bao - -import ( - "context" - "git.lumeweb.com/LumeWeb/portal/bao/proto" - "github.com/hashicorp/go-plugin" - "google.golang.org/grpc" -) - -type BAOPlugin struct { - plugin.Plugin - Impl Bao -} - -func (p *BAOPlugin) GRPCServer(broker *plugin.GRPCBroker, s *grpc.Server) error { - return nil -} - -func (b *BAOPlugin) GRPCClient(_ context.Context, broker *plugin.GRPCBroker, c *grpc.ClientConn) (interface{}, error) { - return &GRPCClient{client: proto.NewBaoClient(c)}, nil -} diff --git a/bao/proto/bao.proto b/bao/proto/bao.proto deleted file mode 100644 index 7c09ec3..0000000 --- a/bao/proto/bao.proto +++ /dev/null @@ -1,21 +0,0 @@ -syntax = "proto3"; - -import "google/protobuf/empty.proto"; -import "google/protobuf/wrappers.proto"; - -option go_package = "git.lumeweb.com/LumeWeb/portal/bao/proto"; - -package bao; - -message WriteRequest { - uint32 id = 1; - bytes data = 2; -} - -service bao { - rpc Init (google.protobuf.Empty) returns (google.protobuf.UInt32Value); - rpc Write(WriteRequest) returns (google.protobuf.Empty); - rpc Finalize (google.protobuf.UInt32Value) returns (google.protobuf.BytesValue); - rpc Destroy (google.protobuf.UInt32Value) returns (google.protobuf.Empty); - rpc ComputeFile (google.protobuf.StringValue) returns (google.protobuf.BytesValue); -} diff --git a/bao/proto/grpc_stdio.proto b/bao/proto/grpc_stdio.proto deleted file mode 100644 index fd50786..0000000 --- a/bao/proto/grpc_stdio.proto +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: MPL-2.0 - -syntax = "proto3"; -package plugin; -option go_package = "plugin"; - -import "google/protobuf/empty.proto"; - -// GRPCStdio is a service that is automatically run by the plugin process -// to stream any stdout/err data so that it can be mirrored on the plugin -// host side. -service GRPCStdio { - // StreamStdio returns a stream that contains all the stdout/stderr. - // This RPC endpoint must only be called ONCE. Once stdio data is consumed - // it is not sent again. - // - // Callers should connect early to prevent blocking on the plugin process. - rpc StreamStdio(google.protobuf.Empty) returns (stream StdioData); -} - -// StdioData is a single chunk of stdout or stderr data that is streamed -// from GRPCStdio. -message StdioData { - enum Channel { - INVALID = 0; - STDOUT = 1; - STDERR = 2; - } - - Channel channel = 1; - bytes data = 2; -} diff --git a/bao/src/grpc/error.rs b/bao/src/grpc/error.rs deleted file mode 100644 index 0c530be..0000000 --- a/bao/src/grpc/error.rs +++ /dev/null @@ -1,34 +0,0 @@ -use hyper::http::uri::InvalidUri; -use thiserror::Error as ThisError; -use tokio::sync::mpsc::error::SendError; -use tonic::transport::Error as TonicError; -use std::fmt::{Debug}; - -pub fn into_status(err: Error) -> tonic::Status { - tonic::Status::unknown(format!("{}", err)) -} - -#[derive(Debug, ThisError)] -pub enum Error { - #[error("Error with IO: {0}")] - Io(#[from] std::io::Error), - #[error("Error with tonic (gRPC) transport: {0}")] - TonicTransport(#[from] TonicError), - #[error("Error parsing string into a network address: {0}")] - AddrParser(#[from] std::net::AddrParseError), - #[error("Error sending on a mpsc channel: {0}")] - Send(String), - #[error("Invalid Uri: {0}")] - InvalidUri(#[from] InvalidUri), - #[error(transparent)] - Other(#[from] anyhow::Error), -} - -impl From> for Error { - fn from(_err: SendError) -> Self { - Self::Send(format!( - "unable to send {} on a mpsc channel", - std::any::type_name::() - )) - } -} diff --git a/bao/src/grpc/grpc_stdio.rs b/bao/src/grpc/grpc_stdio.rs deleted file mode 100644 index d0635ca..0000000 --- a/bao/src/grpc/grpc_stdio.rs +++ /dev/null @@ -1,98 +0,0 @@ -// Copied from: https://github.com/hashicorp/go-plugin/blob/master/grpc_controller.go -use anyhow::{Context, Result}; -use async_stream::stream; -use futures::stream::Stream; -use gag::BufferRedirect; -use std::io::Read; -use std::pin::Pin; -use tokio::time::{sleep, Duration}; -use tokio_stream::StreamExt; -use tonic::{async_trait, Request, Response, Status}; -use crate::proto::google::protobuf::Empty; -use crate::proto::plugin::grpc_stdio_server::{GrpcStdio, GrpcStdioServer}; -use crate::proto::plugin::stdio_data::Channel; -use crate::proto::plugin::{StdioData}; -use crate::grpc::error::into_status; - -const CONSOLE_POLL_SLEEP_MILLIS: u64 = 500; - -pub fn new_server() -> GrpcStdioServer { - GrpcStdioServer::new(GrpcStdioImpl {}) -} - -#[derive(Clone)] -pub struct GrpcStdioImpl {} - -impl GrpcStdioImpl { - fn new_combined_stream() -> Result<::StreamStdioStream, Status> { - log::trace!("new_inner_stream called. Asked for a stream of stdout and stderr"); - log::info!("Gagging stdout and stderr to a buffer for redirection to plugin's host.",); - - let stdoutbuf = BufferRedirect::stdout() - .context("Failed to create a BufferRedirec from stdout") - .map_err(|e| e.into()) - .map_err(into_status)?; - let stdout_stream = GrpcStdioImpl::new_stream("stdout", Channel::Stdout as i32, stdoutbuf); - - let stderrbuf = BufferRedirect::stderr() - .context("Failed to create a BufferRedirec from stderr") - .map_err(|e| e.into()) - .map_err(into_status)?; - let stderr_stream = GrpcStdioImpl::new_stream("stderr", Channel::Stderr as i32, stderrbuf); - - let merged_stream = stdout_stream.merge(stderr_stream); - - Ok(Box::pin(merged_stream)) - } - - fn new_stream( - stream_name: &'static str, - channel: i32, - mut redirected_buf: BufferRedirect, - ) -> impl Stream> { - stream! { - loop { - log::trace!("beginning next iteration of {} reading and streaming...", stream_name); - let mut readbuf = String::new(); - match redirected_buf.read_to_string(&mut readbuf) { - Ok(len) => match len{ - 0 => { - log::trace!("{} had zero bytes. Sleeping to avoid polling...", stream_name); - sleep(Duration::from_millis(CONSOLE_POLL_SLEEP_MILLIS)).await; - }, - _ => { - log::trace!("Sending {} {} bytes of data: {}", stream_name, len, readbuf); - yield Ok(StdioData{ - channel, - data: readbuf.into_bytes(), - }); - }, - }, - Err(e) => { - log::error!("Error reading {} data: {:?}", stream_name, e); - yield Err(Status::unknown(format!("Error reading from Stderr of plugin's process: {:?}", e))); - }, - } - } - } - } -} - -#[async_trait] -impl GrpcStdio for GrpcStdioImpl { - type StreamStdioStream = - Pin> + Send + 'static>>; - - async fn stream_stdio( - &self, - _req: Request, - ) -> Result, Status> { - log::trace!("stream_stdio called."); - - let s = GrpcStdioImpl::new_combined_stream()?; - - log::trace!("stream_stdio responding with a stream of StdioData.",); - - Ok(Response::new(s)) - } -} diff --git a/bao/src/grpc/mod.rs b/bao/src/grpc/mod.rs deleted file mode 100644 index 724f465..0000000 --- a/bao/src/grpc/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod grpc_stdio; -pub mod error; diff --git a/bao/src/main.rs b/bao/src/main.rs deleted file mode 100644 index ac162de..0000000 --- a/bao/src/main.rs +++ /dev/null @@ -1,151 +0,0 @@ -#![feature(async_fn_in_trait)] -#![allow(incomplete_features)] - -use io::Read; -use std::collections::hash_map::Entry; -use std::collections::HashMap; -use std::fs::{File}; -use std::io; -use std::io::{Cursor, Write}; -use std::sync::{Arc}; - -use abao::encode::Encoder; -use async_trait::async_trait; -use atomic_counter::{AtomicCounter, ConsistentCounter}; -use parking_lot::Mutex; -use tonic::{Request, Response, Status}; -use tonic::transport::Server; -use tonic_health::server::HealthReporter; - -use crate::proto::bao::bao_server::{Bao, BaoServer}; -use crate::proto::bao::WriteRequest; -use crate::proto::google::protobuf::{BytesValue, Empty, StringValue, UInt32Value}; -use crate::unique_port::UniquePort; - -mod proto; -mod unique_port; -mod grpc; - -async fn driver_service_status(mut reporter: HealthReporter) { - reporter.set_serving::>().await; -} - -#[tokio::main] -async fn main() -> Result<(), Box> { - let mut uport = UniquePort::default(); - let port = uport.get_unused_port().expect("No ports free"); - println!("{}", format!("1|1|tcp|127.0.0.1:{}|grpc", port)); - - let (mut health_reporter, health_service) = tonic_health::server::health_reporter(); - - health_reporter.set_serving::>().await; - - tokio::spawn(driver_service_status(health_reporter.clone())); - - let addr = format!("127.0.0.1:{}", port).parse().unwrap(); - let bao_service = BaoService::default(); - let server = BaoServer::new(bao_service); - Server::builder() - .add_service(health_service) - .add_service(server) - .add_service(grpc::grpc_stdio::new_server()) - .serve(addr) - .await?; - - Ok(()) -} - -#[derive(Debug, Default)] -pub struct BaoService { - requests: Arc>>>>>, - counter: ConsistentCounter, -} - -#[async_trait] -impl Bao for BaoService { - async fn init(&self, _request: Request) -> Result, Status> { - let next_id = self.counter.inc() as u32; - let tree = Vec::new(); - let cursor = Cursor::new(tree); - let encoder = Encoder::new_outboard(cursor); - - let mut req = self.requests.lock(); - req.insert(next_id, encoder); - - Ok(Response::new(UInt32Value { value: next_id })) - } - - async fn write(&self, request: Request) -> Result, Status> { - let r = request.into_inner(); - let mut req = self.requests.lock(); - if let Some(encoder) = req.get_mut(&r.id) { - encoder.write(&r.data)?; - } else { - return Err(Status::invalid_argument("invalid id")); - } - - Ok(Response::new(Empty::default())) - } - - async fn finalize( - &self, - request: Request, - ) -> Result, Status> { - let r = request.into_inner(); - let mut req = self.requests.lock(); - match req.entry(r.value) { - Entry::Occupied(mut entry) => { - let encoder = entry.get_mut(); - let ret = encoder.finalize().unwrap(); - let bytes = ret.as_bytes().to_vec(); - Ok(Response::new(BytesValue { value: bytes })) - } - Entry::Vacant(_) => { - Err(Status::invalid_argument("invalid id")) - } - } - } - - - async fn destroy(&self, request: Request) -> Result, Status> { - let r = request.into_inner(); - let mut req = self.requests.lock(); - if req.remove(&r.value).is_none() { - return Err(Status::invalid_argument("invalid id")); - } - - Ok(Response::new(Empty::default())) - } - - async fn compute_file(&self, request: Request) -> Result, Status> { - let r = request.into_inner(); - let tree = Vec::new(); - let cursor = Cursor::new(tree); - let mut encoder = Encoder::new_outboard(cursor); - let mut input = File::open(r.value)?; - - copy_reader_to_writer(&mut input, &mut encoder)?; - - let ret = encoder.finalize().unwrap(); - let bytes = ret.as_bytes().to_vec(); - Ok(Response::new(BytesValue { value: bytes })) - } -} -fn copy_reader_to_writer( - reader: &mut impl Read, - writer: &mut impl Write, -) -> io::Result { - // At least 16 KiB is necessary to use AVX-512 with BLAKE3. - let mut buf = [0; 65536]; - let mut written = 0; - loop { - let len = match reader.read(&mut buf) { - Ok(0) => return Ok(written), - Ok(len) => len, - Err(ref e) if e.kind() == io::ErrorKind::Interrupted => continue, - Err(e) => return Err(e), - }; - writer.write_all(&buf[..len])?; - written += len as u64; - } -} diff --git a/bao/src/unique_port.rs b/bao/src/unique_port.rs deleted file mode 100644 index 451031c..0000000 --- a/bao/src/unique_port.rs +++ /dev/null @@ -1,45 +0,0 @@ -use portpicker::Port; - -pub struct UniquePort { - vended_ports: Vec, -} - -impl UniquePort { - pub fn new() -> Self { - Self { - vended_ports: vec![], - } - } - - pub fn get_unused_port(&mut self) -> Option { - let mut counter = 0; - - loop { - counter += 1; - if counter > 1000 { - // no luck in 1000 tries? Give up! - return None; - } - - match portpicker::pick_unused_port() { - None => return None, - Some(p) => { - if self.vended_ports.contains(&p) { - log::trace!("Skipped port: {} because it is in the list of previously vended ports: {:?}", p, self.vended_ports); - continue; - } else { - log::trace!("Vending port: {}", p); - self.vended_ports.push(p); - return Some(p); - } - } - } - } - } -} - -impl Default for UniquePort { - fn default() -> Self { - Self::new() - } -} diff --git a/controller/files.go b/controller/files.go index 2d0b053..9246065 100644 --- a/controller/files.go +++ b/controller/files.go @@ -23,7 +23,7 @@ func (f *FilesController) PostUpload() { return } - upload, err := files.Upload(file, nil) + upload, err := files.Upload(file, meta.Size) if internalError(ctx, err) { return diff --git a/go.mod b/go.mod index c663f72..5f741e4 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( gorm.io/driver/mysql v1.5.0 gorm.io/driver/sqlite v1.5.0 gorm.io/gorm v1.25.0 - lukechampine.com/blake3 v1.1.7 + lukechampine.com/blake3 v1.2.0 lukechampine.com/frand v1.4.2 ) diff --git a/go.sum b/go.sum index d002cc1..61dbb46 100644 --- a/go.sum +++ b/go.sum @@ -1949,6 +1949,10 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= +lukechampine.com/blake3 v1.1.8-0.20230515191509-85fa20df84c1 h1:NTVGxhcZ8DAwAZnyd4Q0L1dpdnVU3B3PkplDgNd/Jek= +lukechampine.com/blake3 v1.1.8-0.20230515191509-85fa20df84c1/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= +lukechampine.com/blake3 v1.2.0 h1:xBP4eLyBtfzoNTBaeECFxxoLyPX7hD1xntcqc1xywQE= +lukechampine.com/blake3 v1.2.0/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= lukechampine.com/frand v1.4.2 h1:RzFIpOvkMXuPMBb9maa4ND4wjBn71E1Jpf8BzJHMaVw= lukechampine.com/frand v1.4.2/go.mod h1:4S/TM2ZgrKejMcKMbeLjISpJMO+/eZ1zu3vYX9dtj3s= lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= diff --git a/service/files/files.go b/service/files/files.go index 29ad208..d490c1d 100644 --- a/service/files/files.go +++ b/service/files/files.go @@ -1,7 +1,6 @@ package files import ( - "bufio" "context" "encoding/hex" "errors" @@ -13,8 +12,6 @@ import ( "git.lumeweb.com/LumeWeb/portal/shared" "github.com/go-resty/resty/v2" "io" - "lukechampine.com/blake3" - "os" ) var client *resty.Client @@ -26,40 +23,18 @@ func Init() { client.SetDisableWarn(true) } -func Upload(r io.ReadSeeker, file *os.File) (model.Upload, error) { +func Upload(r io.ReadSeeker, size int64) (model.Upload, error) { var upload model.Upload - if r == nil && file == nil { - return upload, errors.New("invalid upload mode") - } - - hasher := blake3.New(32, nil) - - var err error - - if r != nil { - _, err = io.Copy(hasher, r) - } else { - _, err = io.Copy(hasher, file) - } + tree, hashBytes, err := bao.ComputeTree(r, size) if err != nil { return upload, err } - hashBytes := hasher.Sum(nil) - hashHex := hex.EncodeToString(hashBytes[:]) - if err != nil { - return upload, err - } - - if r != nil { - _, err = r.Seek(0, io.SeekStart) - } else { - _, err = file.Seek(0, io.SeekStart) - } + _, err = r.Seek(0, io.SeekStart) if err != nil { return upload, err @@ -83,37 +58,11 @@ func Upload(r io.ReadSeeker, file *os.File) (model.Upload, error) { return upload, errors.New("file already exists in network, but missing in database") } - var tree []byte - - if r != nil { - tree, err = bao.ComputeTreeStreaming(bufio.NewReader(r)) - } else { - tree, err = bao.ComputeTreeFile(file) - } - if err != nil { return upload, err } - if r != nil { - _, err = r.Seek(0, io.SeekStart) - } else { - _, err = file.Seek(0, io.SeekStart) - } - - if err != nil { - return upload, err - } - - var body interface{} - - if r != nil { - body = r - } else { - body = file - } - - ret, err := client.R().SetBody(body).Put(fmt.Sprintf("/worker/objects/%s", hashHex)) + ret, err := client.R().SetBody(r).Put(fmt.Sprintf("/worker/objects/%s", hashHex)) if ret.StatusCode() != 200 { err = errors.New(string(ret.Body())) return upload, err diff --git a/tus/tus.go b/tus/tus.go index 76877a0..6a13cc2 100644 --- a/tus/tus.go +++ b/tus/tus.go @@ -14,8 +14,8 @@ import ( "github.com/tus/tusd/pkg/filestore" tusd "github.com/tus/tusd/pkg/handler" "github.com/tus/tusd/pkg/memorylocker" + "io" "log" - "os" ) const TUS_API_PATH = "/files/tus" @@ -136,7 +136,7 @@ func tusWorker(upload *tusd.Upload) error { return err } - _, err = files.Upload(nil, file.(*os.File)) + _, err = files.Upload(file.(io.ReadSeeker), info.Size) if err != nil { log.Print(err) return err