From 8d98f131d5b090e22c1343356e8d4787a0f0157d Mon Sep 17 00:00:00 2001 From: Derrick Hammer Date: Sat, 30 Mar 2024 13:59:47 -0400 Subject: [PATCH] refactor: implement native bao support --- api/s5/s5.go | 30 +- bao/bao.go | 160 +++---- bao/buf.gen.yaml | 12 - bao/buf.yaml | 1 - bao/plugin.go | 95 ----- bao/proto/bao.proto | 51 --- bao/rust/Cargo.lock | 990 ------------------------------------------- bao/rust/Cargo.toml | 16 - bao/rust/src/main.rs | 211 --------- go.mod | 6 +- go.sum | 4 +- protocols/s5/tus.go | 16 +- storage/storage.go | 20 +- 13 files changed, 98 insertions(+), 1514 deletions(-) delete mode 100644 bao/buf.gen.yaml delete mode 100644 bao/buf.yaml delete mode 100644 bao/plugin.go delete mode 100644 bao/proto/bao.proto delete mode 100644 bao/rust/Cargo.lock delete mode 100644 bao/rust/Cargo.toml delete mode 100644 bao/rust/src/main.rs diff --git a/api/s5/s5.go b/api/s5/s5.go index 703c16e..760cca4 100644 --- a/api/s5/s5.go +++ b/api/s5/s5.go @@ -473,7 +473,7 @@ func (rsnc readSeekNopCloser) Close() error { func (s *S5API) smallFileUpload(jc jape.Context) { user := middleware.GetUserFromContext(jc.Request.Context()) - file, err := s.prepareFileUpload(jc) + file, size, err := s.prepareFileUpload(jc) if err != nil { s.sendErrorResponse(jc, err) return @@ -485,7 +485,7 @@ func (s *S5API) smallFileUpload(jc jape.Context) { } }(file) - newUpload, err2 := s.storage.UploadObject(jc.Request.Context(), s5.GetStorageProtocol(s.protocol), file, nil, nil) + newUpload, err2 := s.storage.UploadObject(jc.Request.Context(), s5.GetStorageProtocol(s.protocol), file, size, nil, nil) if err2 != nil { s.sendErrorResponse(jc, NewS5Error(ErrKeyFileUploadFailed, err2)) @@ -525,33 +525,35 @@ func (s *S5API) smallFileUpload(jc jape.Context) { }) } -func (s *S5API) prepareFileUpload(jc jape.Context) (file io.ReadSeekCloser, s5Err *S5Error) { +func (s *S5API) prepareFileUpload(jc jape.Context) (file io.ReadSeekCloser, size uint64, s5Err *S5Error) { r := jc.Request contentType := r.Header.Get("Content-Type") // Handle multipart form data uploads if strings.HasPrefix(contentType, "multipart/form-data") { if err := r.ParseMultipartForm(int64(s.config.Config().Core.PostUploadLimit)); err != nil { - return nil, NewS5Error(ErrKeyFileUploadFailed, err) + return nil, size, NewS5Error(ErrKeyFileUploadFailed, err) } multipartFile, _, err := r.FormFile("file") if err != nil { - return nil, NewS5Error(ErrKeyFileUploadFailed, err) + return nil, size, NewS5Error(ErrKeyFileUploadFailed, err) } - return multipartFile, nil + return multipartFile, size, nil } // Handle raw body uploads data, err := io.ReadAll(r.Body) if err != nil { - return nil, NewS5Error(ErrKeyFileUploadFailed, err) + return nil, size, NewS5Error(ErrKeyFileUploadFailed, err) } buffer := readSeekNopCloser{bytes.NewReader(data)} - return buffer, nil + size = uint64(len(data)) + + return buffer, size, nil } func (s *S5API) accountRegisterChallenge(jc jape.Context) { @@ -1275,7 +1277,7 @@ func (s *S5API) pinEntity(ctx context.Context, userId uint, userIp string, cid * data = append(data, dataCont...) - proof, err := s.storage.HashObject(ctx, bytes.NewReader(data)) + proof, err := s.storage.HashObject(ctx, bytes.NewReader(data), uint64(len(data))) if err != nil { return nil, false } @@ -1444,7 +1446,7 @@ func (s *S5API) processMultipartFiles(r *http.Request) (map[string]*metadata.Upl } }(file) - upload, err := s.storage.UploadObject(r.Context(), s5.GetStorageProtocol(s.protocol), file, nil, nil) + upload, err := s.storage.UploadObject(r.Context(), s5.GetStorageProtocol(s.protocol), file, uint64(fileHeader.Size), nil, nil) if err != nil { return nil, NewS5Error(ErrKeyStorageOperationFailed, err) } @@ -1515,7 +1517,7 @@ func (s *S5API) uploadAppMetadata(appData *s5libmetadata.WebAppMetadata, r *http file := bytes.NewReader(appDataRaw) - upload, err := s.storage.UploadObject(r.Context(), s5.GetStorageProtocol(s.protocol), file, nil, nil) + upload, err := s.storage.UploadObject(r.Context(), s5.GetStorageProtocol(s.protocol), file, uint64(len(appDataRaw)), nil, nil) if err != nil { return "", NewS5Error(ErrKeyStorageOperationFailed, err) } @@ -2163,7 +2165,7 @@ func (s *S5API) pinImportCronJob(cid string, url string, proofUrl string, userId return err // Error logged in fetchAndProcess } - hash, err := s.storage.HashObject(ctx, bytes.NewReader(fileData)) + hash, err := s.storage.HashObject(ctx, bytes.NewReader(fileData), uint64(len(fileData))) if err != nil { s.logger.Error("error hashing object", zap.Error(err)) return err @@ -2178,7 +2180,7 @@ func (s *S5API) pinImportCronJob(cid string, url string, proofUrl string, userId return err } - upload, err := s.storage.UploadObject(ctx, s5.GetStorageProtocol(s.protocol), bytes.NewReader(fileData), nil, hash) + upload, err := s.storage.UploadObject(ctx, s5.GetStorageProtocol(s.protocol), bytes.NewReader(fileData), parsedCid.Size, nil, hash) if err != nil { return err } @@ -2255,7 +2257,7 @@ func (s *S5API) pinImportCronJob(cid string, url string, proofUrl string, userId return err } - upload, err := s.storage.UploadObject(ctx, s5.GetStorageProtocol(s.protocol), nil, &renter.MultiPartUploadParams{ + upload, err := s.storage.UploadObject(ctx, s5.GetStorageProtocol(s.protocol), nil, 0, &renter.MultiPartUploadParams{ ReaderFactory: func(start uint, end uint) (io.ReadCloser, error) { rangeHeader := "bytes=%d-" if end != 0 { diff --git a/bao/bao.go b/bao/bao.go index 2527b96..f75e773 100644 --- a/bao/bao.go +++ b/bao/bao.go @@ -1,36 +1,27 @@ package bao import ( - "bufio" "bytes" _ "embed" "errors" "io" - "math" - "os" - "os/exec" "time" "github.com/samber/lo" "go.uber.org/zap" - "github.com/docker/go-units" - "github.com/hashicorp/go-plugin" + "lukechampine.com/blake3/bao" ) -//go:generate buf generate -//go:generate bash -c "cd rust && cargo build -r" -//go:embed rust/target/release/rust -var pluginBin []byte - -var bao Bao -var client *plugin.Client - var _ io.ReadCloser = (*Verifier)(nil) +var _ io.WriterAt = (*proofWriter)(nil) var ErrVerifyFailed = errors.New("verification failed") +const groupLog = 8 +const groupChunks = 1 << groupLog + type Verifier struct { r io.ReadCloser proof Result @@ -41,6 +32,12 @@ type Verifier struct { verifyTime time.Duration } +type Result struct { + Hash []byte + Proof []byte + Length uint +} + func (v *Verifier) Read(p []byte) (int, error) { // Initial attempt to read from the buffer n, err := v.buffer.Read(p) @@ -52,7 +49,7 @@ func (v *Verifier) Read(p []byte) (int, error) { return n, err } - buf := make([]byte, VERIFY_CHUNK_SIZE) + buf := make([]byte, groupChunks) // Continue reading from the source and verifying until we have enough data or hit an error for v.buffer.Len() < len(p)-n { readStart := time.Now() @@ -68,7 +65,7 @@ func (v *Verifier) Read(p []byte) (int, error) { timeStart := time.Now() if bytesRead > 0 { - if status, err := bao.Verify(buf[:bytesRead], v.read, v.proof.Proof, v.proof.Hash); err != nil || !status { + if status := bao.VerifyChunk(buf[:bytesRead], v.proof.Proof, groupChunks, v.read, [32]byte(v.proof.Hash)); !status { return n, errors.Join(ErrVerifyFailed, err) } v.read += uint64(bytesRead) @@ -92,7 +89,7 @@ func (v *Verifier) Read(p []byte) (int, error) { v.logger.Debug("Read time", zap.Duration("average", averageReadTime)) } - averageVerifyTime := v.verifyTime / time.Duration(v.read/VERIFY_CHUNK_SIZE) + averageVerifyTime := v.verifyTime / time.Duration(v.read/groupChunks) v.logger.Debug("Verification time", zap.Duration("average", averageVerifyTime)) // Attempt to read the remainder of the data from the buffer @@ -103,94 +100,20 @@ func (v *Verifier) Read(p []byte) (int, error) { func (v *Verifier) Close() error { return v.r.Close() } +func Hash(r io.Reader, size uint64) (*Result, error) { + reader := newSizeReader(r) + writer := newProofWriter(int(size)) -func init() { - temp, err := os.CreateTemp(os.TempDir(), "bao") + hash, err := bao.Encode(writer, reader, int64(size), groupLog, true) if err != nil { - panic(err) + return nil, err } - err = temp.Chmod(1755) - if err != nil { - panic(err) - } - - _, err = io.Copy(temp, bytes.NewReader(pluginBin)) - if err != nil { - panic(err) - } - - err = temp.Close() - if err != nil { - panic(err) - } - - clientInst := plugin.NewClient(&plugin.ClientConfig{ - HandshakeConfig: plugin.HandshakeConfig{ - ProtocolVersion: 1, - }, - Plugins: plugin.PluginSet{ - "bao": &BaoPlugin{}, - }, - Cmd: exec.Command(temp.Name()), - AllowedProtocols: []plugin.Protocol{plugin.ProtocolGRPC}, - }) - - rpcClient, err := clientInst.Client() - if err != nil { - panic(err) - } - - pluginInst, err := rpcClient.Dispense("bao") - if err != nil { - panic(err) - } - - bao = pluginInst.(Bao) -} - -func Shutdown() { - client.Kill() -} - -func Hash(r io.Reader) (*Result, error) { - hasherId := bao.NewHasher() - initialSize := 4 * units.KiB - maxSize := 3.5 * units.MiB - bufSize := initialSize - - reader := bufio.NewReaderSize(r, bufSize) - var totalReadSize int - - buf := make([]byte, bufSize) - for { - - n, err := reader.Read(buf) - if err != nil { - if err == io.EOF { - break - } - return nil, err - } - totalReadSize += n - - if !bao.Hash(hasherId, buf[:n]) { - return nil, errors.New("hashing failed") - } - - // Adaptively adjust buffer size based on read patterns - if n == bufSize && float64(bufSize) < maxSize { - // If buffer was fully used, consider increasing buffer size - bufSize = int(math.Min(float64(bufSize*2), float64(maxSize))) // Double the buffer size, up to a maximum - buf = make([]byte, bufSize) // Apply new buffer size - reader = bufio.NewReaderSize(r, bufSize) // Apply new buffer size - } - } - - result := bao.Finish(hasherId) - result.Length = uint(totalReadSize) - - return &result, nil + return &Result{ + Hash: hash[:], + Proof: writer.buf, + Length: uint(size), + }, nil } func NewVerifier(r io.ReadCloser, proof Result, logger *zap.Logger) *Verifier { @@ -201,3 +124,38 @@ func NewVerifier(r io.ReadCloser, proof Result, logger *zap.Logger) *Verifier { logger: logger, } } + +type proofWriter struct { + buf []byte +} + +func (p proofWriter) WriteAt(b []byte, off int64) (n int, err error) { + if copy(p.buf[off:], b) != len(b) { + panic("bad buffer size") + } + return len(b), nil +} + +func newProofWriter(size int) *proofWriter { + return &proofWriter{ + buf: make([]byte, bao.EncodedSize(size, groupLog, true)), + } +} + +type sizeReader struct { + reader io.Reader + read int64 +} + +func (s sizeReader) Read(p []byte) (int, error) { + n, err := s.reader.Read(p) + s.read += int64(n) + return n, err +} + +func newSizeReader(r io.Reader) *sizeReader { + return &sizeReader{ + reader: r, + read: 0, + } +} diff --git a/bao/buf.gen.yaml b/bao/buf.gen.yaml deleted file mode 100644 index 6de0e3a..0000000 --- a/bao/buf.gen.yaml +++ /dev/null @@ -1,12 +0,0 @@ -version: v1 -managed: - enabled: true -plugins: - - plugin: buf.build/protocolbuffers/go - out: . - - plugin: buf.build/grpc/go:v1.3.0 - out: . - - plugin: buf.build/community/neoeinstein-prost:v0.2.3 - out: rust/src/proto - - plugin: buf.build/community/neoeinstein-tonic:v0.3.0 - out: rust/src/proto diff --git a/bao/buf.yaml b/bao/buf.yaml deleted file mode 100644 index c126332..0000000 --- a/bao/buf.yaml +++ /dev/null @@ -1 +0,0 @@ -version: v1 diff --git a/bao/plugin.go b/bao/plugin.go deleted file mode 100644 index 71b73ca..0000000 --- a/bao/plugin.go +++ /dev/null @@ -1,95 +0,0 @@ -package bao - -import ( - "context" - "errors" - - "github.com/docker/go-units" - - "git.lumeweb.com/LumeWeb/portal/bao/proto" - "github.com/google/uuid" - "github.com/hashicorp/go-plugin" - "google.golang.org/grpc" -) - -var _ Bao = (*BaoGRPC)(nil) - -const VERIFY_CHUNK_SIZE = 256 * units.KiB - -type Bao interface { - NewHasher() uuid.UUID - Hash(id uuid.UUID, data []byte) bool - Finish(id uuid.UUID) Result - Verify(data []byte, offset uint64, proof []byte, hash []byte) (bool, error) -} - -type BaoPlugin struct { - plugin.Plugin -} - -func (p *BaoPlugin) GRPCServer(broker *plugin.GRPCBroker, s *grpc.Server) error { - return nil -} - -func (p *BaoPlugin) GRPCClient(ctx context.Context, broker *plugin.GRPCBroker, c *grpc.ClientConn) (interface{}, error) { - return &BaoGRPC{client: proto.NewBaoClient(c)}, nil -} - -type Result struct { - Hash []byte - Proof []byte - Length uint -} -type BaoGRPC struct { - client proto.BaoClient -} - -func (b *BaoGRPC) NewHasher() uuid.UUID { - ret, err := b.client.NewHasher(context.Background(), &proto.NewHasherRequest{}) - - if err != nil { - panic(err) - } - - id, err := uuid.Parse(ret.Id) - - if err != nil { - panic(err) - } - - return id -} - -func (b *BaoGRPC) Hash(id uuid.UUID, data []byte) bool { - ret, err := b.client.Hash(context.Background(), &proto.HashRequest{Id: id.String(), Data: data}) - - if err != nil { - panic(err) - } - - return ret.Status -} - -func (b *BaoGRPC) Finish(id uuid.UUID) Result { - ret, err := b.client.Finish(context.Background(), &proto.FinishRequest{Id: id.String()}) - - if err != nil { - panic(err) - } - - return Result{Hash: ret.Hash, Proof: ret.Proof} -} - -func (b *BaoGRPC) Verify(data []byte, offset uint64, proof []byte, hash []byte) (bool, error) { - ret, err := b.client.Verify(context.Background(), &proto.VerifyRequest{Data: data, Offset: offset, Proof: proof, Hash: hash}) - - if err != nil { - return false, err - } - - if ret.Error != "" { - err = errors.New(ret.Error) - } - - return ret.Status, err -} diff --git a/bao/proto/bao.proto b/bao/proto/bao.proto deleted file mode 100644 index e2e01d0..0000000 --- a/bao/proto/bao.proto +++ /dev/null @@ -1,51 +0,0 @@ -syntax = "proto3"; -package bao; - -option go_package = "./proto"; - -// Define an empty message for the request as proto3 requires specific message types -message NewHasherRequest { -} - -// Define a message for the response that includes the bytes you mentioned -message NewHasherResponse { - string id = 1; -} - -message HashRequest { - string id = 1; - bytes data = 2; -} - -message HashResponse { - bool status = 1; -} - -message FinishRequest { - string id = 1; -} - -message FinishResponse { - bytes hash = 1; - bytes proof = 2; -} - -message VerifyRequest { - bytes data = 1; - uint64 offset = 2; - bytes proof = 3; - bytes hash = 4; -} - -message VerifyResponse { - bool status = 1; - string error = 2; -} - - -service Bao { - rpc NewHasher(NewHasherRequest) returns (NewHasherResponse); - rpc Hash(HashRequest) returns (HashResponse); - rpc Finish(FinishRequest) returns (FinishResponse); - rpc Verify(VerifyRequest) returns (VerifyResponse); -} diff --git a/bao/rust/Cargo.lock b/bao/rust/Cargo.lock deleted file mode 100644 index d5d88da..0000000 --- a/bao/rust/Cargo.lock +++ /dev/null @@ -1,990 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 3 - -[[package]] -name = "abao" -version = "0.2.0" -source = "git+https://github.com/LumeWeb/abao.git?branch=feature/inner_mut#3179db880ab071aea543ee95baae6d4b984af618" -dependencies = [ - "arrayref", - "arrayvec", - "blake3", - "futures", - "tokio", -] - -[[package]] -name = "addr2line" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" -dependencies = [ - "gimli", -] - -[[package]] -name = "adler" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" - -[[package]] -name = "anyhow" -version = "1.0.79" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" - -[[package]] -name = "arrayref" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" - -[[package]] -name = "arrayvec" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" - -[[package]] -name = "async-stream" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" -dependencies = [ - "async-stream-impl", - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "async-stream-impl" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "async-trait" -version = "0.1.77" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "autocfg" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" - -[[package]] -name = "axum" -version = "0.6.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" -dependencies = [ - "async-trait", - "axum-core", - "bitflags", - "bytes", - "futures-util", - "http", - "http-body", - "hyper", - "itoa", - "matchit", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "rustversion", - "serde", - "sync_wrapper", - "tower", - "tower-layer", - "tower-service", -] - -[[package]] -name = "axum-core" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" -dependencies = [ - "async-trait", - "bytes", - "futures-util", - "http", - "http-body", - "mime", - "rustversion", - "tower-layer", - "tower-service", -] - -[[package]] -name = "backtrace" -version = "0.3.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" -dependencies = [ - "addr2line", - "cc", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", -] - -[[package]] -name = "base64" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "blake3" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0231f06152bf547e9c2b5194f247cd97aacf6dcd8b15d8e5ec0663f64580da87" -dependencies = [ - "arrayref", - "arrayvec", - "cc", - "cfg-if", - "constant_time_eq", -] - -[[package]] -name = "bytes" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" - -[[package]] -name = "cc" -version = "1.0.83" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" -dependencies = [ - "libc", -] - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "constant_time_eq" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" - -[[package]] -name = "either" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" - -[[package]] -name = "equivalent" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" - -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "futures" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" -dependencies = [ - "futures-channel", - "futures-core", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-channel" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" -dependencies = [ - "futures-core", - "futures-sink", -] - -[[package]] -name = "futures-core" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" - -[[package]] -name = "futures-io" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" - -[[package]] -name = "futures-sink" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" - -[[package]] -name = "futures-task" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" - -[[package]] -name = "futures-util" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" -dependencies = [ - "futures-core", - "futures-sink", - "futures-task", - "pin-project-lite", - "pin-utils", -] - -[[package]] -name = "getrandom" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" -dependencies = [ - "cfg-if", - "libc", - "wasi", -] - -[[package]] -name = "gimli" -version = "0.28.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" - -[[package]] -name = "h2" -version = "0.3.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http", - "indexmap 2.2.2", - "slab", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - -[[package]] -name = "hashbrown" -version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" - -[[package]] -name = "hermit-abi" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0c62115964e08cb8039170eb33c1d0e2388a256930279edca206fff675f82c3" - -[[package]] -name = "http" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - -[[package]] -name = "http-body" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" -dependencies = [ - "bytes", - "http", - "pin-project-lite", -] - -[[package]] -name = "httparse" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" - -[[package]] -name = "httpdate" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" - -[[package]] -name = "hyper" -version = "0.14.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" -dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "socket2", - "tokio", - "tower-service", - "tracing", - "want", -] - -[[package]] -name = "hyper-timeout" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" -dependencies = [ - "hyper", - "pin-project-lite", - "tokio", - "tokio-io-timeout", -] - -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", -] - -[[package]] -name = "indexmap" -version = "2.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "824b2ae422412366ba479e8111fd301f7b5faece8149317bb81925979a53f520" -dependencies = [ - "equivalent", - "hashbrown 0.14.3", -] - -[[package]] -name = "itertools" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" -dependencies = [ - "either", -] - -[[package]] -name = "itoa" -version = "1.0.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" - -[[package]] -name = "libc" -version = "0.2.153" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" - -[[package]] -name = "matchit" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" - -[[package]] -name = "memchr" -version = "2.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" - -[[package]] -name = "mime" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" - -[[package]] -name = "miniz_oxide" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" -dependencies = [ - "adler", -] - -[[package]] -name = "mio" -version = "0.8.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" -dependencies = [ - "libc", - "wasi", - "windows-sys", -] - -[[package]] -name = "num_cpus" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" -dependencies = [ - "hermit-abi", - "libc", -] - -[[package]] -name = "object" -version = "0.32.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" -dependencies = [ - "memchr", -] - -[[package]] -name = "once_cell" -version = "1.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" - -[[package]] -name = "percent-encoding" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" - -[[package]] -name = "pin-project" -version = "1.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0302c4a0442c456bd56f841aee5c3bfd17967563f6fadc9ceb9f9c23cf3807e0" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "266c042b60c9c76b8d53061e52b2e0d1116abc57cefc8c5cd671619a56ac3690" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "pin-project-lite" -version = "0.2.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" - -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - -[[package]] -name = "portpicker" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be97d76faf1bfab666e1375477b23fde79eccf0276e9b63b92a39d676a889ba9" -dependencies = [ - "rand", -] - -[[package]] -name = "ppv-lite86" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" - -[[package]] -name = "proc-macro2" -version = "1.0.78" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "prost" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c289cda302b98a28d40c8b3b90498d6e526dd24ac2ecea73e4e491685b94a" -dependencies = [ - "bytes", - "prost-derive", -] - -[[package]] -name = "prost-derive" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efb6c9a1dd1def8e2124d17e83a20af56f1570d6c2d2bd9e266ccb768df3840e" -dependencies = [ - "anyhow", - "itertools", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "quote" -version = "1.0.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha", - "rand_core", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom", -] - -[[package]] -name = "rust" -version = "0.1.0" -dependencies = [ - "abao", - "anyhow", - "portpicker", - "prost", - "tokio", - "tonic", - "tonic-health", - "uuid", -] - -[[package]] -name = "rustc-demangle" -version = "0.1.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" - -[[package]] -name = "rustversion" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" - -[[package]] -name = "serde" -version = "1.0.196" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.196" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "signal-hook-registry" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" -dependencies = [ - "libc", -] - -[[package]] -name = "slab" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" -dependencies = [ - "autocfg", -] - -[[package]] -name = "socket2" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" -dependencies = [ - "libc", - "windows-sys", -] - -[[package]] -name = "syn" -version = "2.0.48" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "sync_wrapper" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" - -[[package]] -name = "tokio" -version = "1.36.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" -dependencies = [ - "backtrace", - "bytes", - "libc", - "mio", - "num_cpus", - "pin-project-lite", - "signal-hook-registry", - "socket2", - "tokio-macros", - "windows-sys", -] - -[[package]] -name = "tokio-io-timeout" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" -dependencies = [ - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tokio-macros" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "tokio-stream" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tokio-util" -version = "0.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "pin-project-lite", - "tokio", - "tracing", -] - -[[package]] -name = "tonic" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76c4eb7a4e9ef9d4763600161f12f5070b92a578e1b634db88a6887844c91a13" -dependencies = [ - "async-stream", - "async-trait", - "axum", - "base64", - "bytes", - "h2", - "http", - "http-body", - "hyper", - "hyper-timeout", - "percent-encoding", - "pin-project", - "prost", - "tokio", - "tokio-stream", - "tower", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tonic-health" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cef6e24bc96871001a7e48e820ab240b3de2201e59b517cf52835df2f1d2350" -dependencies = [ - "async-stream", - "prost", - "tokio", - "tokio-stream", - "tonic", -] - -[[package]] -name = "tower" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" -dependencies = [ - "futures-core", - "futures-util", - "indexmap 1.9.3", - "pin-project", - "pin-project-lite", - "rand", - "slab", - "tokio", - "tokio-util", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tower-layer" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" - -[[package]] -name = "tower-service" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" - -[[package]] -name = "tracing" -version = "0.1.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" -dependencies = [ - "pin-project-lite", - "tracing-attributes", - "tracing-core", -] - -[[package]] -name = "tracing-attributes" -version = "0.1.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "tracing-core" -version = "0.1.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" -dependencies = [ - "once_cell", -] - -[[package]] -name = "try-lock" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" - -[[package]] -name = "unicode-ident" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" - -[[package]] -name = "uuid" -version = "1.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f00cc9702ca12d3c81455259621e676d0f7251cec66a21e98fe2e9a37db93b2a" -dependencies = [ - "getrandom", -] - -[[package]] -name = "want" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" -dependencies = [ - "try-lock", -] - -[[package]] -name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" - -[[package]] -name = "windows-sys" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" -dependencies = [ - "windows-targets", -] - -[[package]] -name = "windows-targets" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" - -[[package]] -name = "windows_i686_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" - -[[package]] -name = "windows_i686_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" diff --git a/bao/rust/Cargo.toml b/bao/rust/Cargo.toml deleted file mode 100644 index 04d0e5c..0000000 --- a/bao/rust/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -name = "rust" -version = "0.1.0" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -tonic = "0.11.0" -prost = "0.12.3" -tonic-health = "0.11.0" -tokio = { version = "1.0.0", features = ["rt", "rt-multi-thread", "macros","signal"] } -portpicker = "0.1.1" -uuid = { version = "1.7.0", features = ["v4"] } -abao = { git = "https://github.com/LumeWeb/abao.git", branch = "feature/inner_mut", default-features = false, features = ["group_size_256k", "tokio", "tokio_io"] } -anyhow = "1.0.79" diff --git a/bao/rust/src/main.rs b/bao/rust/src/main.rs deleted file mode 100644 index 28fc56e..0000000 --- a/bao/rust/src/main.rs +++ /dev/null @@ -1,211 +0,0 @@ -use std::collections::HashMap; -use std::io::{Cursor, Read, Seek, SeekFrom, Write}; -use std::net::SocketAddr; -use std::sync::Arc; - -use abao::encode::Encoder; -use portpicker::pick_unused_port; -use tokio::sync::{Mutex, oneshot, RwLock}; -use tonic::{Request, Response, Status}; -use tonic::transport::Server; -use uuid::Uuid; -use bao::bao_server::Bao; -use tokio::signal::unix::{signal, SignalKind}; - -use crate::bao::{bao_server, FinishRequest, FinishResponse, HashRequest, HashResponse, NewHasherRequest, NewHasherResponse, VerifyRequest, VerifyResponse}; - -#[path = "proto/bao.rs"] -mod bao; - -struct GlobalState { - hashers: HashMap>>>>>, -} - - -pub struct BaoService { - state: Arc>, -} - -#[tonic::async_trait] -impl Bao for BaoService { - async fn new_hasher(&self, _request: Request) -> Result, Status> { - let encoder = Encoder::new_outboard(Cursor::new(Vec::new())); - let id = Uuid::new_v4(); - { - let mut state = self.state.write().await; - state.hashers.insert(id, Arc::new(Mutex::new(encoder))); - } - Ok(Response::new(NewHasherResponse { - id: id.to_string(), - })) - } - - async fn hash(&self, request: Request) -> Result, Status> { - let id = Uuid::parse_str(&request.get_ref().id).map_err(|_| Status::invalid_argument("invalid id"))?; - { - let state = self.state.read().await; - let encoder = state.hashers.get(&id).ok_or_else(|| Status::not_found("hasher not found"))?.clone(); - let mut encoder = encoder.lock().await; - encoder.write_all(&request.get_ref().data).map_err(|_| Status::internal("write failed"))?; - } - Ok(Response::new(HashResponse { - status: true, - })) - } - - async fn finish(&self, request: Request) -> Result, Status> { - let id = Uuid::parse_str(&request.get_ref().id).map_err(|_| Status::invalid_argument("invalid id"))?; - let (hash, proof) = { - let mut state = self.state.write().await; - let encoder = state.hashers.remove(&id).ok_or_else(|| Status::not_found("hasher not found"))?; - let encoder = Arc::try_unwrap(encoder).unwrap(); // Unwrap the Arc - let mut encoder = encoder.lock().await; - let hash = encoder.finalize()?.as_bytes().to_vec(); - let proof = encoder.inner_mut().get_ref().to_vec(); - (hash, proof) - }; - Ok(Response::new(FinishResponse { - hash, - proof, - })) - } - - async fn verify(&self, request: Request) -> Result, Status> { - let req = request.get_ref(); - let res = verify_internal( - req.data.clone(), - req.offset, - req.proof.clone(), - from_vec_to_array(req.hash.clone()), - ); - - if res.is_err() { - - Ok(Response::new(VerifyResponse { - status: false, - error: res.unwrap_err().to_string(), - - })) - } else { - Ok(Response::new(VerifyResponse { - status: true, - error: String::from(""), - })) - } - } -} - -fn verify_internal( - chunk_bytes: Vec, - offset: u64, - bao_outboard_bytes: Vec, - blake3_hash: [u8; 32], -) -> anyhow::Result { - let mut slice_stream = abao::encode::SliceExtractor::new_outboard( - FakeSeeker::new(&chunk_bytes[..]), - Cursor::new(&bao_outboard_bytes), - offset, - 262144, - ); - - let mut decode_stream = abao::decode::SliceDecoder::new( - &mut slice_stream, - &abao::Hash::from(blake3_hash), - offset, - 262144, - ); - let mut decoded = Vec::new(); - decode_stream.read_to_end(&mut decoded)?; - - Ok(1) -} - -fn from_vec_to_array(v: Vec) -> [T; N] { - core::convert::TryInto::try_into(v) - .unwrap_or_else(|v: Vec| panic!("Expected a Vec of length {} but it was {}", N, v.len())) -} - -struct FakeSeeker { - reader: R, - bytes_read: u64, -} - -impl FakeSeeker { - fn new(reader: R) -> Self { - Self { - reader, - bytes_read: 0, - } - } -} - -impl Read for FakeSeeker { - fn read(&mut self, buf: &mut [u8]) -> std::io::Result { - let n = self.reader.read(buf)?; - self.bytes_read += n as u64; - Ok(n) - } -} - -impl Seek for FakeSeeker { - fn seek(&mut self, _: SeekFrom) -> std::io::Result { - // Do nothing and return the current position. - Ok(self.bytes_read) - } -} - -impl BaoService { - fn new(state: Arc>) -> Self { - BaoService { state } - } -} - - -#[tokio::main] -async fn main() -> Result<(), Box> { - let (tx, rx) = oneshot::channel::<()>(); - let health_reporter = tonic_health::server::health_reporter(); - let port = match pick_unused_port() { - Some(p) => p, - None => { - return Err("Failed to pick an unused port".into()); - } - }; - - let addr: SocketAddr = format!("127.0.0.1:{}", port).parse()?; - - println!("1|1|tcp|127.0.0.1:{}|grpc", addr.port()); - - let global_state = Arc::new(RwLock::new(GlobalState { - hashers: HashMap::new(), - })); - - tokio::spawn(async move { - let mut term_signal = signal(SignalKind::terminate()).expect("Could not create signal handler"); - - // Wait for the terminate signal - term_signal.recv().await; - println!("Termination signal received, shutting down server..."); - - // Sending a signal through the channel to initiate shutdown. - // If the receiver is dropped, we don't care about the error. - let _ = tx.send(()); - }); - - let server = bao_server::BaoServer::new(BaoService::new(global_state.clone())) - .max_decoding_message_size(usize::MAX) - .max_encoding_message_size(usize::MAX); - - Server::builder() - .max_frame_size((1 << 24) - 1) - .add_service(server) - .add_service(health_reporter.1) - .serve_with_shutdown(addr, async { - // This future completes when the shutdown signal is received, - // allowing the server to shut down gracefully. - rx.await.ok(); - }) - .await?; - - Ok(()) -} diff --git a/go.mod b/go.mod index ed185ae..611f6c8 100644 --- a/go.mod +++ b/go.mod @@ -42,11 +42,9 @@ require ( go.uber.org/zap v1.26.0 go.uber.org/zap/exp v0.2.0 golang.org/x/crypto v0.21.0 - google.golang.org/grpc v1.62.0 - google.golang.org/protobuf v1.32.0 gorm.io/driver/mysql v1.5.4 gorm.io/gorm v1.25.7 - lukechampine.com/blake3 v1.2.1 + lukechampine.com/blake3 v1.2.2-0.20240329192137-af604d0fbc33 nhooyr.io/websocket v1.8.10 ) @@ -144,6 +142,8 @@ require ( golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.19.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect + google.golang.org/grpc v1.62.0 // indirect + google.golang.org/protobuf v1.32.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/frand v1.4.2 // indirect diff --git a/go.sum b/go.sum index ba1ca8f..16e2675 100644 --- a/go.sum +++ b/go.sum @@ -608,8 +608,8 @@ gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde/go.mod h1:hbnx/Oo0ChWMn1BIhpy gorm.io/gorm v1.25.7 h1:VsD6acwRjz2zFxGO50gPO6AkNs7KKnvfzUjHQhZDz/A= gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI= -lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= +lukechampine.com/blake3 v1.2.2-0.20240329192137-af604d0fbc33 h1:oQavF3w54yDoQQUBdlFCTcziojGobCnxU/SHLQHRNLM= +lukechampine.com/blake3 v1.2.2-0.20240329192137-af604d0fbc33/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= lukechampine.com/frand v1.4.2 h1:RzFIpOvkMXuPMBb9maa4ND4wjBn71E1Jpf8BzJHMaVw= lukechampine.com/frand v1.4.2/go.mod h1:4S/TM2ZgrKejMcKMbeLjISpJMO+/eZ1zu3vYX9dtj3s= nhooyr.io/websocket v1.8.10 h1:mv4p+MnGrLDcPlBoWsvPP7XCzTYMXP9F9eIGoKbgx7Q= diff --git a/protocols/s5/tus.go b/protocols/s5/tus.go index 164cb8c..c878c01 100644 --- a/protocols/s5/tus.go +++ b/protocols/s5/tus.go @@ -384,7 +384,13 @@ func (t *TusHandler) uploadTask(hash []byte) error { return err } - proof, err := t.storage.HashObject(ctx, reader) + info, err := tusUpload.GetInfo(ctx) + if err != nil { + t.logger.Error("Could not get tus info", zap.Error(err)) + return err + } + + proof, err := t.storage.HashObject(ctx, reader, uint64(info.Size)) if err != nil { t.logger.Error("Could not compute proof", zap.Error(err)) @@ -396,13 +402,7 @@ func (t *TusHandler) uploadTask(hash []byte) error { return err } - info, err := tusUpload.GetInfo(ctx) - if err != nil { - t.logger.Error("Could not get tus info", zap.Error(err)) - return err - } - - uploadMeta, err := t.storage.UploadObject(ctx, t.storageProtocol, nil, &renter.MultiPartUploadParams{ + uploadMeta, err := t.storage.UploadObject(ctx, t.storageProtocol, nil, 0, &renter.MultiPartUploadParams{ ReaderFactory: func(start uint, end uint) (io.ReadCloser, error) { rangeHeader := "bytes=%d-" if end != 0 { diff --git a/storage/storage.go b/storage/storage.go index 7cd88f6..7ec02ce 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -63,9 +63,9 @@ var Module = fx.Module("storage", ) type StorageService interface { - UploadObject(ctx context.Context, protocol StorageProtocol, data io.ReadSeeker, muParams *renter.MultiPartUploadParams, proof *bao.Result) (*metadata.UploadMetadata, error) - UploadObjectProof(ctx context.Context, protocol StorageProtocol, data io.ReadSeeker, proof *bao.Result) error - HashObject(ctx context.Context, data io.Reader) (*bao.Result, error) + UploadObject(ctx context.Context, protocol StorageProtocol, data io.ReadSeeker, size uint64, muParams *renter.MultiPartUploadParams, proof *bao.Result) (*metadata.UploadMetadata, error) + UploadObjectProof(ctx context.Context, protocol StorageProtocol, data io.ReadSeeker, proof *bao.Result, size uint64) error + HashObject(ctx context.Context, data io.Reader, size uint64) (*bao.Result, error) DownloadObject(ctx context.Context, protocol StorageProtocol, objectHash []byte, start int64) (io.ReadCloser, error) DownloadObjectProof(ctx context.Context, protocol StorageProtocol, objectHash []byte) (io.ReadCloser, error) DeleteObject(ctx context.Context, protocol StorageProtocol, objectHash []byte) error @@ -101,7 +101,7 @@ func NewStorageService(params StorageServiceParams) *StorageServiceDefault { } } -func (s StorageServiceDefault) UploadObject(ctx context.Context, protocol StorageProtocol, data io.ReadSeeker, muParams *renter.MultiPartUploadParams, proof *bao.Result) (*metadata.UploadMetadata, error) { +func (s StorageServiceDefault) UploadObject(ctx context.Context, protocol StorageProtocol, data io.ReadSeeker, size uint64, muParams *renter.MultiPartUploadParams, proof *bao.Result) (*metadata.UploadMetadata, error) { readers := make([]io.ReadCloser, 0) defer func() { for _, reader := range readers { @@ -148,7 +148,7 @@ func (s StorageServiceDefault) UploadObject(ctx context.Context, protocol Storag } if proof == nil { - hashResult, err := s.HashObject(ctx, reader) + hashResult, err := s.HashObject(ctx, reader, size) if err != nil { return nil, err } @@ -204,7 +204,7 @@ func (s StorageServiceDefault) UploadObject(ctx context.Context, protocol Storag filename := protocol.EncodeFileName(proof.Hash) - err = s.UploadObjectProof(ctx, protocol, nil, proof) + err = s.UploadObjectProof(ctx, protocol, nil, proof, size) if err != nil { return nil, err @@ -237,9 +237,9 @@ func (s StorageServiceDefault) UploadObject(ctx context.Context, protocol Storag return uploadMeta, nil } -func (s StorageServiceDefault) UploadObjectProof(ctx context.Context, protocol StorageProtocol, data io.ReadSeeker, proof *bao.Result) error { +func (s StorageServiceDefault) UploadObjectProof(ctx context.Context, protocol StorageProtocol, data io.ReadSeeker, proof *bao.Result, size uint64) error { if proof == nil { - hashResult, err := s.HashObject(ctx, data) + hashResult, err := s.HashObject(ctx, data, size) if err != nil { return err } @@ -258,8 +258,8 @@ func (s StorageServiceDefault) UploadObjectProof(ctx context.Context, protocol S return s.renter.UploadObject(ctx, bytes.NewReader(proof.Proof), protocolName, s.getProofPath(protocol, proof.Hash)) } -func (s StorageServiceDefault) HashObject(ctx context.Context, data io.Reader) (*bao.Result, error) { - result, err := bao.Hash(data) +func (s StorageServiceDefault) HashObject(ctx context.Context, data io.Reader, size uint64) (*bao.Result, error) { + result, err := bao.Hash(data, size) if err != nil { return nil, err