chore: remove usage of buf and rust in docker image

Dockerfile

@@ -1,5 +1,3 @@
-# Use the bufbuild/buf image for extracting the buf command binary
-FROM bufbuild/buf:latest as buf
 # Use the official Node.js image as the base image for building the api/account/portal
 FROM node:20-alpine as nodejs-builder
 
@@ -28,18 +26,10 @@ RUN apk add --no-cache git && git clone --recurse-submodules https://git.lumeweb
 
 # Copy the built dashboard from the nodejs-builder stage
 COPY --from=nodejs-builder /portal/api/account/app/build/client /portal/api/account/app/build/client
-# Copy buf from the buf stage
-COPY --from=buf /usr/local/bin/buf /usr/local/bin/buf
 
 # Install the necessary dependencies
 RUN apk add bash gcc curl musl-dev
 
-# Install the necessary dependencies
-RUN curl -sSf https://sh.rustup.rs | bash -s -- -y
-
-# Set the necessary environment variables
-ENV PATH="/root/.cargo/bin:${PATH}"
-
 ## Build the Go application
 RUN go mod download
 

api/s5/s5.go

@@ -473,7 +473,7 @@ func (rsnc readSeekNopCloser) Close() error {
 func (s *S5API) smallFileUpload(jc jape.Context) {
 	user := middleware.GetUserFromContext(jc.Request.Context())
 
-	file, err := s.prepareFileUpload(jc)
+	file, size, err := s.prepareFileUpload(jc)
 	if err != nil {
 		s.sendErrorResponse(jc, err)
 		return
@@ -485,7 +485,7 @@ func (s *S5API) smallFileUpload(jc jape.Context) {
 		}
 	}(file)
 
-	newUpload, err2 := s.storage.UploadObject(jc.Request.Context(), s5.GetStorageProtocol(s.protocol), file, nil, nil)
+	newUpload, err2 := s.storage.UploadObject(jc.Request.Context(), s5.GetStorageProtocol(s.protocol), file, size, nil, nil)
 
 	if err2 != nil {
 		s.sendErrorResponse(jc, NewS5Error(ErrKeyFileUploadFailed, err2))
@@ -525,33 +525,35 @@ func (s *S5API) smallFileUpload(jc jape.Context) {
 	})
 }
 
-func (s *S5API) prepareFileUpload(jc jape.Context) (file io.ReadSeekCloser, s5Err *S5Error) {
+func (s *S5API) prepareFileUpload(jc jape.Context) (file io.ReadSeekCloser, size uint64, s5Err *S5Error) {
 	r := jc.Request
 	contentType := r.Header.Get("Content-Type")
 
 	// Handle multipart form data uploads
 	if strings.HasPrefix(contentType, "multipart/form-data") {
 		if err := r.ParseMultipartForm(int64(s.config.Config().Core.PostUploadLimit)); err != nil {
-			return nil, NewS5Error(ErrKeyFileUploadFailed, err)
+			return nil, size, NewS5Error(ErrKeyFileUploadFailed, err)
 		}
 
 		multipartFile, _, err := r.FormFile("file")
 		if err != nil {
-			return nil, NewS5Error(ErrKeyFileUploadFailed, err)
+			return nil, size, NewS5Error(ErrKeyFileUploadFailed, err)
 		}
 
-		return multipartFile, nil
+		return multipartFile, size, nil
 	}
 
 	// Handle raw body uploads
 	data, err := io.ReadAll(r.Body)
 	if err != nil {
-		return nil, NewS5Error(ErrKeyFileUploadFailed, err)
+		return nil, size, NewS5Error(ErrKeyFileUploadFailed, err)
 	}
 
 	buffer := readSeekNopCloser{bytes.NewReader(data)}
 
-	return buffer, nil
+	size = uint64(len(data))
+
+	return buffer, size, nil
 }
 
 func (s *S5API) accountRegisterChallenge(jc jape.Context) {
@@ -1275,7 +1277,7 @@ func (s *S5API) pinEntity(ctx context.Context, userId uint, userIp string, cid *
 
 			data = append(data, dataCont...)
 
-			proof, err := s.storage.HashObject(ctx, bytes.NewReader(data))
+			proof, err := s.storage.HashObject(ctx, bytes.NewReader(data), uint64(len(data)))
 			if err != nil {
 				return nil, false
 			}
@@ -1444,7 +1446,7 @@ func (s *S5API) processMultipartFiles(r *http.Request) (map[string]*metadata.Upl
 				}
 			}(file)
 
-			upload, err := s.storage.UploadObject(r.Context(), s5.GetStorageProtocol(s.protocol), file, nil, nil)
+			upload, err := s.storage.UploadObject(r.Context(), s5.GetStorageProtocol(s.protocol), file, uint64(fileHeader.Size), nil, nil)
 			if err != nil {
 				return nil, NewS5Error(ErrKeyStorageOperationFailed, err)
 			}
@@ -1515,7 +1517,7 @@ func (s *S5API) uploadAppMetadata(appData *s5libmetadata.WebAppMetadata, r *http
 
 	file := bytes.NewReader(appDataRaw)
 
-	upload, err := s.storage.UploadObject(r.Context(), s5.GetStorageProtocol(s.protocol), file, nil, nil)
+	upload, err := s.storage.UploadObject(r.Context(), s5.GetStorageProtocol(s.protocol), file, uint64(len(appDataRaw)), nil, nil)
 	if err != nil {
 		return "", NewS5Error(ErrKeyStorageOperationFailed, err)
 	}
@@ -2163,7 +2165,7 @@ func (s *S5API) pinImportCronJob(cid string, url string, proofUrl string, userId
 			return err // Error logged in fetchAndProcess
 		}
 
-		hash, err := s.storage.HashObject(ctx, bytes.NewReader(fileData))
+		hash, err := s.storage.HashObject(ctx, bytes.NewReader(fileData), uint64(len(fileData)))
 		if err != nil {
 			s.logger.Error("error hashing object", zap.Error(err))
 			return err
@@ -2178,7 +2180,7 @@ func (s *S5API) pinImportCronJob(cid string, url string, proofUrl string, userId
 			return err
 		}
 
-		upload, err := s.storage.UploadObject(ctx, s5.GetStorageProtocol(s.protocol), bytes.NewReader(fileData), nil, hash)
+		upload, err := s.storage.UploadObject(ctx, s5.GetStorageProtocol(s.protocol), bytes.NewReader(fileData), parsedCid.Size, nil, hash)
 		if err != nil {
 			return err
 		}
@@ -2255,7 +2257,7 @@ func (s *S5API) pinImportCronJob(cid string, url string, proofUrl string, userId
 		return err
 	}
 
-	upload, err := s.storage.UploadObject(ctx, s5.GetStorageProtocol(s.protocol), nil, &renter.MultiPartUploadParams{
+	upload, err := s.storage.UploadObject(ctx, s5.GetStorageProtocol(s.protocol), nil, 0, &renter.MultiPartUploadParams{
 		ReaderFactory: func(start uint, end uint) (io.ReadCloser, error) {
 			rangeHeader := "bytes=%d-"
 			if end != 0 {

bao/bao.go

@@ -1,36 +1,27 @@
 package bao
 
 import (
-	"bufio"
 	"bytes"
 	_ "embed"
 	"errors"
 	"io"
-	"math"
-	"os"
-	"os/exec"
 	"time"
 
 	"github.com/samber/lo"
 
 	"go.uber.org/zap"
 
-	"github.com/docker/go-units"
+	"lukechampine.com/blake3/bao"
-	"github.com/hashicorp/go-plugin"
 )
 
-//go:generate buf generate
-//go:generate bash -c "cd rust && cargo build -r"
-//go:embed rust/target/release/rust
-var pluginBin []byte
-
-var bao Bao
-var client *plugin.Client
-
 var _ io.ReadCloser = (*Verifier)(nil)
+var _ io.WriterAt = (*proofWriter)(nil)
 
 var ErrVerifyFailed = errors.New("verification failed")
 
+const groupLog = 8
+const groupChunks = 1 << groupLog
+
 type Verifier struct {
 	r          io.ReadCloser
 	proof      Result
@@ -41,6 +32,12 @@ type Verifier struct {
 	verifyTime time.Duration
 }
 
+type Result struct {
+	Hash   []byte
+	Proof  []byte
+	Length uint
+}
+
 func (v *Verifier) Read(p []byte) (int, error) {
 	// Initial attempt to read from the buffer
 	n, err := v.buffer.Read(p)
@@ -52,7 +49,7 @@ func (v *Verifier) Read(p []byte) (int, error) {
 		return n, err
 	}
 
-	buf := make([]byte, VERIFY_CHUNK_SIZE)
+	buf := make([]byte, groupChunks)
 	// Continue reading from the source and verifying until we have enough data or hit an error
 	for v.buffer.Len() < len(p)-n {
 		readStart := time.Now()
@@ -68,7 +65,7 @@ func (v *Verifier) Read(p []byte) (int, error) {
 		timeStart := time.Now()
 
 		if bytesRead > 0 {
-			if status, err := bao.Verify(buf[:bytesRead], v.read, v.proof.Proof, v.proof.Hash); err != nil || !status {
+			if status := bao.VerifyChunk(buf[:bytesRead], v.proof.Proof, groupChunks, v.read, [32]byte(v.proof.Hash)); !status {
 				return n, errors.Join(ErrVerifyFailed, err)
 			}
 			v.read += uint64(bytesRead)
@@ -92,7 +89,7 @@ func (v *Verifier) Read(p []byte) (int, error) {
 		v.logger.Debug("Read time", zap.Duration("average", averageReadTime))
 	}
 
-	averageVerifyTime := v.verifyTime / time.Duration(v.read/VERIFY_CHUNK_SIZE)
+	averageVerifyTime := v.verifyTime / time.Duration(v.read/groupChunks)
 	v.logger.Debug("Verification time", zap.Duration("average", averageVerifyTime))
 
 	// Attempt to read the remainder of the data from the buffer
@@ -103,94 +100,20 @@ func (v *Verifier) Read(p []byte) (int, error) {
 func (v *Verifier) Close() error {
 	return v.r.Close()
 }
+func Hash(r io.Reader, size uint64) (*Result, error) {
+	reader := newSizeReader(r)
+	writer := newProofWriter(int(size))
 
-func init() {
+	hash, err := bao.Encode(writer, reader, int64(size), groupLog, true)
-	temp, err := os.CreateTemp(os.TempDir(), "bao")
 	if err != nil {
-		panic(err)
-	}
-
-	err = temp.Chmod(1755)
-	if err != nil {
-		panic(err)
-	}
-
-	_, err = io.Copy(temp, bytes.NewReader(pluginBin))
-	if err != nil {
-		panic(err)
-	}
-
-	err = temp.Close()
-	if err != nil {
-		panic(err)
-	}
-
-	clientInst := plugin.NewClient(&plugin.ClientConfig{
-		HandshakeConfig: plugin.HandshakeConfig{
-			ProtocolVersion: 1,
-		},
-		Plugins: plugin.PluginSet{
-			"bao": &BaoPlugin{},
-		},
-		Cmd:              exec.Command(temp.Name()),
-		AllowedProtocols: []plugin.Protocol{plugin.ProtocolGRPC},
-	})
-
-	rpcClient, err := clientInst.Client()
-	if err != nil {
-		panic(err)
-	}
-
-	pluginInst, err := rpcClient.Dispense("bao")
-	if err != nil {
-		panic(err)
-	}
-
-	bao = pluginInst.(Bao)
-}
-
-func Shutdown() {
-	client.Kill()
-}
-
-func Hash(r io.Reader) (*Result, error) {
-	hasherId := bao.NewHasher()
-	initialSize := 4 * units.KiB
-	maxSize := 3.5 * units.MiB
-	bufSize := initialSize
-
-	reader := bufio.NewReaderSize(r, bufSize)
-	var totalReadSize int
-
-	buf := make([]byte, bufSize)
-	for {
-
-		n, err := reader.Read(buf)
-		if err != nil {
-			if err == io.EOF {
-				break
-			}
 		return nil, err
 	}
-		totalReadSize += n
 
-		if !bao.Hash(hasherId, buf[:n]) {
+	return &Result{
-			return nil, errors.New("hashing failed")
+		Hash:   hash[:],
-		}
+		Proof:  writer.buf,
-
+		Length: uint(size),
-		// Adaptively adjust buffer size based on read patterns
+	}, nil
-		if n == bufSize && float64(bufSize) < maxSize {
-			// If buffer was fully used, consider increasing buffer size
-			bufSize = int(math.Min(float64(bufSize*2), float64(maxSize))) // Double the buffer size, up to a maximum
-			buf = make([]byte, bufSize)                                   // Apply new buffer size
-			reader = bufio.NewReaderSize(r, bufSize)                      // Apply new buffer size
-		}
-	}
-
-	result := bao.Finish(hasherId)
-	result.Length = uint(totalReadSize)
-
-	return &result, nil
 }
 
 func NewVerifier(r io.ReadCloser, proof Result, logger *zap.Logger) *Verifier {
@@ -201,3 +124,38 @@ func NewVerifier(r io.ReadCloser, proof Result, logger *zap.Logger) *Verifier {
 		logger: logger,
 	}
 }
+
+type proofWriter struct {
+	buf []byte
+}
+
+func (p proofWriter) WriteAt(b []byte, off int64) (n int, err error) {
+	if copy(p.buf[off:], b) != len(b) {
+		panic("bad buffer size")
+	}
+	return len(b), nil
+}
+
+func newProofWriter(size int) *proofWriter {
+	return &proofWriter{
+		buf: make([]byte, bao.EncodedSize(size, groupLog, true)),
+	}
+}
+
+type sizeReader struct {
+	reader io.Reader
+	read   int64
+}
+
+func (s sizeReader) Read(p []byte) (int, error) {
+	n, err := s.reader.Read(p)
+	s.read += int64(n)
+	return n, err
+}
+
+func newSizeReader(r io.Reader) *sizeReader {
+	return &sizeReader{
+		reader: r,
+		read:   0,
+	}
+}

bao/buf.gen.yaml

@@ -1,12 +0,0 @@
-version: v1
-managed:
-    enabled: true
-plugins:
-    - plugin: buf.build/protocolbuffers/go
-      out: .
-    - plugin: buf.build/grpc/go:v1.3.0
-      out: .
-    - plugin: buf.build/community/neoeinstein-prost:v0.2.3
-      out: rust/src/proto
-    - plugin: buf.build/community/neoeinstein-tonic:v0.3.0
-      out: rust/src/proto

bao/buf.yaml

@@ -1 +0,0 @@
-version: v1

bao/plugin.go

@@ -1,95 +0,0 @@
-package bao
-
-import (
-	"context"
-	"errors"
-
-	"github.com/docker/go-units"
-
-	"git.lumeweb.com/LumeWeb/portal/bao/proto"
-	"github.com/google/uuid"
-	"github.com/hashicorp/go-plugin"
-	"google.golang.org/grpc"
-)
-
-var _ Bao = (*BaoGRPC)(nil)
-
-const VERIFY_CHUNK_SIZE = 256 * units.KiB
-
-type Bao interface {
-	NewHasher() uuid.UUID
-	Hash(id uuid.UUID, data []byte) bool
-	Finish(id uuid.UUID) Result
-	Verify(data []byte, offset uint64, proof []byte, hash []byte) (bool, error)
-}
-
-type BaoPlugin struct {
-	plugin.Plugin
-}
-
-func (p *BaoPlugin) GRPCServer(broker *plugin.GRPCBroker, s *grpc.Server) error {
-	return nil
-}
-
-func (p *BaoPlugin) GRPCClient(ctx context.Context, broker *plugin.GRPCBroker, c *grpc.ClientConn) (interface{}, error) {
-	return &BaoGRPC{client: proto.NewBaoClient(c)}, nil
-}
-
-type Result struct {
-	Hash   []byte
-	Proof  []byte
-	Length uint
-}
-type BaoGRPC struct {
-	client proto.BaoClient
-}
-
-func (b *BaoGRPC) NewHasher() uuid.UUID {
-	ret, err := b.client.NewHasher(context.Background(), &proto.NewHasherRequest{})
-
-	if err != nil {
-		panic(err)
-	}
-
-	id, err := uuid.Parse(ret.Id)
-
-	if err != nil {
-		panic(err)
-	}
-
-	return id
-}
-
-func (b *BaoGRPC) Hash(id uuid.UUID, data []byte) bool {
-	ret, err := b.client.Hash(context.Background(), &proto.HashRequest{Id: id.String(), Data: data})
-
-	if err != nil {
-		panic(err)
-	}
-
-	return ret.Status
-}
-
-func (b *BaoGRPC) Finish(id uuid.UUID) Result {
-	ret, err := b.client.Finish(context.Background(), &proto.FinishRequest{Id: id.String()})
-
-	if err != nil {
-		panic(err)
-	}
-
-	return Result{Hash: ret.Hash, Proof: ret.Proof}
-}
-
-func (b *BaoGRPC) Verify(data []byte, offset uint64, proof []byte, hash []byte) (bool, error) {
-	ret, err := b.client.Verify(context.Background(), &proto.VerifyRequest{Data: data, Offset: offset, Proof: proof, Hash: hash})
-
-	if err != nil {
-		return false, err
-	}
-
-	if ret.Error != "" {
-		err = errors.New(ret.Error)
-	}
-
-	return ret.Status, err
-}

bao/proto/bao.proto

@@ -1,51 +0,0 @@
-syntax = "proto3";
-package bao;
-
-option go_package = "./proto";
-
-// Define an empty message for the request as proto3 requires specific message types
-message NewHasherRequest {
-}
-
-// Define a message for the response that includes the bytes you mentioned
-message NewHasherResponse {
-    string id = 1;
-}
-
-message HashRequest {
-    string id = 1;
-    bytes data = 2;
-}
-
-message HashResponse {
-    bool status = 1;
-}
-
-message FinishRequest {
-    string id = 1;
-}
-
-message FinishResponse {
-    bytes hash = 1;
-    bytes proof = 2;
-}
-
-message VerifyRequest {
-    bytes data = 1;
-    uint64 offset = 2;
-    bytes proof = 3;
-    bytes hash = 4;
-}
-
-message VerifyResponse {
-    bool status = 1;
-    string error = 2;
-}
-
-
-service Bao {
-    rpc NewHasher(NewHasherRequest) returns (NewHasherResponse);
-    rpc Hash(HashRequest) returns (HashResponse);
-    rpc Finish(FinishRequest) returns (FinishResponse);
-    rpc Verify(VerifyRequest) returns (VerifyResponse);
-}

bao/rust/Cargo.lock

@@ -1,990 +0,0 @@
-# This file is automatically @generated by Cargo.
-# It is not intended for manual editing.
-version = 3
-
-[[package]]
-name = "abao"
-version = "0.2.0"
-source = "git+https://github.com/LumeWeb/abao.git?branch=feature/inner_mut#3179db880ab071aea543ee95baae6d4b984af618"
-dependencies = [
- "arrayref",
- "arrayvec",
- "blake3",
- "futures",
- "tokio",
-]
-
-[[package]]
-name = "addr2line"
-version = "0.21.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb"
-dependencies = [
- "gimli",
-]
-
-[[package]]
-name = "adler"
-version = "1.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
-
-[[package]]
-name = "anyhow"
-version = "1.0.79"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca"
-
-[[package]]
-name = "arrayref"
-version = "0.3.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545"
-
-[[package]]
-name = "arrayvec"
-version = "0.7.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711"
-
-[[package]]
-name = "async-stream"
-version = "0.3.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51"
-dependencies = [
- "async-stream-impl",
- "futures-core",
- "pin-project-lite",
-]
-
-[[package]]
-name = "async-stream-impl"
-version = "0.3.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "async-trait"
-version = "0.1.77"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "autocfg"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
-
-[[package]]
-name = "axum"
-version = "0.6.20"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf"
-dependencies = [
- "async-trait",
- "axum-core",
- "bitflags",
- "bytes",
- "futures-util",
- "http",
- "http-body",
- "hyper",
- "itoa",
- "matchit",
- "memchr",
- "mime",
- "percent-encoding",
- "pin-project-lite",
- "rustversion",
- "serde",
- "sync_wrapper",
- "tower",
- "tower-layer",
- "tower-service",
-]
-
-[[package]]
-name = "axum-core"
-version = "0.3.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c"
-dependencies = [
- "async-trait",
- "bytes",
- "futures-util",
- "http",
- "http-body",
- "mime",
- "rustversion",
- "tower-layer",
- "tower-service",
-]
-
-[[package]]
-name = "backtrace"
-version = "0.3.69"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837"
-dependencies = [
- "addr2line",
- "cc",
- "cfg-if",
- "libc",
- "miniz_oxide",
- "object",
- "rustc-demangle",
-]
-
-[[package]]
-name = "base64"
-version = "0.21.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
-
-[[package]]
-name = "bitflags"
-version = "1.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
-
-[[package]]
-name = "blake3"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0231f06152bf547e9c2b5194f247cd97aacf6dcd8b15d8e5ec0663f64580da87"
-dependencies = [
- "arrayref",
- "arrayvec",
- "cc",
- "cfg-if",
- "constant_time_eq",
-]
-
-[[package]]
-name = "bytes"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223"
-
-[[package]]
-name = "cc"
-version = "1.0.83"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0"
-dependencies = [
- "libc",
-]
-
-[[package]]
-name = "cfg-if"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
-
-[[package]]
-name = "constant_time_eq"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2"
-
-[[package]]
-name = "either"
-version = "1.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07"
-
-[[package]]
-name = "equivalent"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
-
-[[package]]
-name = "fnv"
-version = "1.0.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
-
-[[package]]
-name = "futures"
-version = "0.3.30"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0"
-dependencies = [
- "futures-channel",
- "futures-core",
- "futures-io",
- "futures-sink",
- "futures-task",
- "futures-util",
-]
-
-[[package]]
-name = "futures-channel"
-version = "0.3.30"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78"
-dependencies = [
- "futures-core",
- "futures-sink",
-]
-
-[[package]]
-name = "futures-core"
-version = "0.3.30"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d"
-
-[[package]]
-name = "futures-io"
-version = "0.3.30"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1"
-
-[[package]]
-name = "futures-sink"
-version = "0.3.30"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5"
-
-[[package]]
-name = "futures-task"
-version = "0.3.30"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004"
-
-[[package]]
-name = "futures-util"
-version = "0.3.30"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48"
-dependencies = [
- "futures-core",
- "futures-sink",
- "futures-task",
- "pin-project-lite",
- "pin-utils",
-]
-
-[[package]]
-name = "getrandom"
-version = "0.2.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5"
-dependencies = [
- "cfg-if",
- "libc",
- "wasi",
-]
-
-[[package]]
-name = "gimli"
-version = "0.28.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253"
-
-[[package]]
-name = "h2"
-version = "0.3.24"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9"
-dependencies = [
- "bytes",
- "fnv",
- "futures-core",
- "futures-sink",
- "futures-util",
- "http",
- "indexmap 2.2.2",
- "slab",
- "tokio",
- "tokio-util",
- "tracing",
-]
-
-[[package]]
-name = "hashbrown"
-version = "0.12.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
-
-[[package]]
-name = "hashbrown"
-version = "0.14.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604"
-
-[[package]]
-name = "hermit-abi"
-version = "0.3.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d0c62115964e08cb8039170eb33c1d0e2388a256930279edca206fff675f82c3"
-
-[[package]]
-name = "http"
-version = "0.2.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb"
-dependencies = [
- "bytes",
- "fnv",
- "itoa",
-]
-
-[[package]]
-name = "http-body"
-version = "0.4.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2"
-dependencies = [
- "bytes",
- "http",
- "pin-project-lite",
-]
-
-[[package]]
-name = "httparse"
-version = "1.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904"
-
-[[package]]
-name = "httpdate"
-version = "1.0.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
-
-[[package]]
-name = "hyper"
-version = "0.14.28"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80"
-dependencies = [
- "bytes",
- "futures-channel",
- "futures-core",
- "futures-util",
- "h2",
- "http",
- "http-body",
- "httparse",
- "httpdate",
- "itoa",
- "pin-project-lite",
- "socket2",
- "tokio",
- "tower-service",
- "tracing",
- "want",
-]
-
-[[package]]
-name = "hyper-timeout"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1"
-dependencies = [
- "hyper",
- "pin-project-lite",
- "tokio",
- "tokio-io-timeout",
-]
-
-[[package]]
-name = "indexmap"
-version = "1.9.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
-dependencies = [
- "autocfg",
- "hashbrown 0.12.3",
-]
-
-[[package]]
-name = "indexmap"
-version = "2.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "824b2ae422412366ba479e8111fd301f7b5faece8149317bb81925979a53f520"
-dependencies = [
- "equivalent",
- "hashbrown 0.14.3",
-]
-
-[[package]]
-name = "itertools"
-version = "0.11.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57"
-dependencies = [
- "either",
-]
-
-[[package]]
-name = "itoa"
-version = "1.0.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c"
-
-[[package]]
-name = "libc"
-version = "0.2.153"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd"
-
-[[package]]
-name = "matchit"
-version = "0.7.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94"
-
-[[package]]
-name = "memchr"
-version = "2.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149"
-
-[[package]]
-name = "mime"
-version = "0.3.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
-
-[[package]]
-name = "miniz_oxide"
-version = "0.7.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7"
-dependencies = [
- "adler",
-]
-
-[[package]]
-name = "mio"
-version = "0.8.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09"
-dependencies = [
- "libc",
- "wasi",
- "windows-sys",
-]
-
-[[package]]
-name = "num_cpus"
-version = "1.16.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
-dependencies = [
- "hermit-abi",
- "libc",
-]
-
-[[package]]
-name = "object"
-version = "0.32.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441"
-dependencies = [
- "memchr",
-]
-
-[[package]]
-name = "once_cell"
-version = "1.19.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
-
-[[package]]
-name = "percent-encoding"
-version = "2.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
-
-[[package]]
-name = "pin-project"
-version = "1.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0302c4a0442c456bd56f841aee5c3bfd17967563f6fadc9ceb9f9c23cf3807e0"
-dependencies = [
- "pin-project-internal",
-]
-
-[[package]]
-name = "pin-project-internal"
-version = "1.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "266c042b60c9c76b8d53061e52b2e0d1116abc57cefc8c5cd671619a56ac3690"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "pin-project-lite"
-version = "0.2.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58"
-
-[[package]]
-name = "pin-utils"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
-
-[[package]]
-name = "portpicker"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be97d76faf1bfab666e1375477b23fde79eccf0276e9b63b92a39d676a889ba9"
-dependencies = [
- "rand",
-]
-
-[[package]]
-name = "ppv-lite86"
-version = "0.2.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
-
-[[package]]
-name = "proc-macro2"
-version = "1.0.78"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae"
-dependencies = [
- "unicode-ident",
-]
-
-[[package]]
-name = "prost"
-version = "0.12.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "146c289cda302b98a28d40c8b3b90498d6e526dd24ac2ecea73e4e491685b94a"
-dependencies = [
- "bytes",
- "prost-derive",
-]
-
-[[package]]
-name = "prost-derive"
-version = "0.12.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "efb6c9a1dd1def8e2124d17e83a20af56f1570d6c2d2bd9e266ccb768df3840e"
-dependencies = [
- "anyhow",
- "itertools",
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "quote"
-version = "1.0.35"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef"
-dependencies = [
- "proc-macro2",
-]
-
-[[package]]
-name = "rand"
-version = "0.8.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
-dependencies = [
- "libc",
- "rand_chacha",
- "rand_core",
-]
-
-[[package]]
-name = "rand_chacha"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
-dependencies = [
- "ppv-lite86",
- "rand_core",
-]
-
-[[package]]
-name = "rand_core"
-version = "0.6.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
-dependencies = [
- "getrandom",
-]
-
-[[package]]
-name = "rust"
-version = "0.1.0"
-dependencies = [
- "abao",
- "anyhow",
- "portpicker",
- "prost",
- "tokio",
- "tonic",
- "tonic-health",
- "uuid",
-]
-
-[[package]]
-name = "rustc-demangle"
-version = "0.1.23"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
-
-[[package]]
-name = "rustversion"
-version = "1.0.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4"
-
-[[package]]
-name = "serde"
-version = "1.0.196"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32"
-dependencies = [
- "serde_derive",
-]
-
-[[package]]
-name = "serde_derive"
-version = "1.0.196"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "signal-hook-registry"
-version = "1.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1"
-dependencies = [
- "libc",
-]
-
-[[package]]
-name = "slab"
-version = "0.4.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67"
-dependencies = [
- "autocfg",
-]
-
-[[package]]
-name = "socket2"
-version = "0.5.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9"
-dependencies = [
- "libc",
- "windows-sys",
-]
-
-[[package]]
-name = "syn"
-version = "2.0.48"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f"
-dependencies = [
- "proc-macro2",
- "quote",
- "unicode-ident",
-]
-
-[[package]]
-name = "sync_wrapper"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
-
-[[package]]
-name = "tokio"
-version = "1.36.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931"
-dependencies = [
- "backtrace",
- "bytes",
- "libc",
- "mio",
- "num_cpus",
- "pin-project-lite",
- "signal-hook-registry",
- "socket2",
- "tokio-macros",
- "windows-sys",
-]
-
-[[package]]
-name = "tokio-io-timeout"
-version = "1.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf"
-dependencies = [
- "pin-project-lite",
- "tokio",
-]
-
-[[package]]
-name = "tokio-macros"
-version = "2.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "tokio-stream"
-version = "0.1.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842"
-dependencies = [
- "futures-core",
- "pin-project-lite",
- "tokio",
-]
-
-[[package]]
-name = "tokio-util"
-version = "0.7.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15"
-dependencies = [
- "bytes",
- "futures-core",
- "futures-sink",
- "pin-project-lite",
- "tokio",
- "tracing",
-]
-
-[[package]]
-name = "tonic"
-version = "0.11.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76c4eb7a4e9ef9d4763600161f12f5070b92a578e1b634db88a6887844c91a13"
-dependencies = [
- "async-stream",
- "async-trait",
- "axum",
- "base64",
- "bytes",
- "h2",
- "http",
- "http-body",
- "hyper",
- "hyper-timeout",
- "percent-encoding",
- "pin-project",
- "prost",
- "tokio",
- "tokio-stream",
- "tower",
- "tower-layer",
- "tower-service",
- "tracing",
-]
-
-[[package]]
-name = "tonic-health"
-version = "0.11.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2cef6e24bc96871001a7e48e820ab240b3de2201e59b517cf52835df2f1d2350"
-dependencies = [
- "async-stream",
- "prost",
- "tokio",
- "tokio-stream",
- "tonic",
-]
-
-[[package]]
-name = "tower"
-version = "0.4.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c"
-dependencies = [
- "futures-core",
- "futures-util",
- "indexmap 1.9.3",
- "pin-project",
- "pin-project-lite",
- "rand",
- "slab",
- "tokio",
- "tokio-util",
- "tower-layer",
- "tower-service",
- "tracing",
-]
-
-[[package]]
-name = "tower-layer"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0"
-
-[[package]]
-name = "tower-service"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
-
-[[package]]
-name = "tracing"
-version = "0.1.40"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef"
-dependencies = [
- "pin-project-lite",
- "tracing-attributes",
- "tracing-core",
-]
-
-[[package]]
-name = "tracing-attributes"
-version = "0.1.27"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "tracing-core"
-version = "0.1.32"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54"
-dependencies = [
- "once_cell",
-]
-
-[[package]]
-name = "try-lock"
-version = "0.2.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
-
-[[package]]
-name = "unicode-ident"
-version = "1.0.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
-
-[[package]]
-name = "uuid"
-version = "1.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f00cc9702ca12d3c81455259621e676d0f7251cec66a21e98fe2e9a37db93b2a"
-dependencies = [
- "getrandom",
-]
-
-[[package]]
-name = "want"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"
-dependencies = [
- "try-lock",
-]
-
-[[package]]
-name = "wasi"
-version = "0.11.0+wasi-snapshot-preview1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
-
-[[package]]
-name = "windows-sys"
-version = "0.48.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
-dependencies = [
- "windows-targets",
-]
-
-[[package]]
-name = "windows-targets"
-version = "0.48.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"
-dependencies = [
- "windows_aarch64_gnullvm",
- "windows_aarch64_msvc",
- "windows_i686_gnu",
- "windows_i686_msvc",
- "windows_x86_64_gnu",
- "windows_x86_64_gnullvm",
- "windows_x86_64_msvc",
-]
-
-[[package]]
-name = "windows_aarch64_gnullvm"
-version = "0.48.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
-
-[[package]]
-name = "windows_aarch64_msvc"
-version = "0.48.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
-
-[[package]]
-name = "windows_i686_gnu"
-version = "0.48.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
-
-[[package]]
-name = "windows_i686_msvc"
-version = "0.48.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
-
-[[package]]
-name = "windows_x86_64_gnu"
-version = "0.48.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
-
-[[package]]
-name = "windows_x86_64_gnullvm"
-version = "0.48.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
-
-[[package]]
-name = "windows_x86_64_msvc"
-version = "0.48.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"

bao/rust/Cargo.toml

@@ -1,16 +0,0 @@
-[package]
-name = "rust"
-version = "0.1.0"
-edition = "2021"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-tonic = "0.11.0"
-prost = "0.12.3"
-tonic-health = "0.11.0"
-tokio = { version = "1.0.0", features = ["rt", "rt-multi-thread", "macros","signal"] }
-portpicker = "0.1.1"
-uuid = { version = "1.7.0", features = ["v4"] }
-abao = { git = "https://github.com/LumeWeb/abao.git", branch = "feature/inner_mut", default-features = false, features = ["group_size_256k", "tokio", "tokio_io"] }
-anyhow = "1.0.79"

bao/rust/src/main.rs

@@ -1,211 +0,0 @@
-use std::collections::HashMap;
-use std::io::{Cursor, Read, Seek, SeekFrom, Write};
-use std::net::SocketAddr;
-use std::sync::Arc;
-
-use abao::encode::Encoder;
-use portpicker::pick_unused_port;
-use tokio::sync::{Mutex, oneshot, RwLock};
-use tonic::{Request, Response, Status};
-use tonic::transport::Server;
-use uuid::Uuid;
-use bao::bao_server::Bao;
-use tokio::signal::unix::{signal, SignalKind};
-
-use crate::bao::{bao_server, FinishRequest, FinishResponse, HashRequest, HashResponse, NewHasherRequest, NewHasherResponse, VerifyRequest, VerifyResponse};
-
-#[path = "proto/bao.rs"]
-mod bao;
-
-struct GlobalState {
-    hashers: HashMap<Uuid, Arc<Mutex<Encoder<Cursor<Vec<u8>>>>>>,
-}
-
-
-pub struct BaoService {
-    state: Arc<RwLock<GlobalState>>,
-}
-
-#[tonic::async_trait]
-impl Bao for BaoService {
-    async fn new_hasher(&self, _request: Request<NewHasherRequest>) -> Result<Response<NewHasherResponse>, Status> {
-        let encoder = Encoder::new_outboard(Cursor::new(Vec::new()));
-        let id = Uuid::new_v4();
-        {
-            let mut state = self.state.write().await;
-            state.hashers.insert(id, Arc::new(Mutex::new(encoder)));
-        }
-        Ok(Response::new(NewHasherResponse {
-            id: id.to_string(),
-        }))
-    }
-
-    async fn hash(&self, request: Request<HashRequest>) -> Result<Response<HashResponse>, Status> {
-        let id = Uuid::parse_str(&request.get_ref().id).map_err(|_| Status::invalid_argument("invalid id"))?;
-        {
-            let state = self.state.read().await;
-            let encoder = state.hashers.get(&id).ok_or_else(|| Status::not_found("hasher not found"))?.clone();
-            let mut encoder = encoder.lock().await;
-            encoder.write_all(&request.get_ref().data).map_err(|_| Status::internal("write failed"))?;
-        }
-        Ok(Response::new(HashResponse {
-            status: true,
-        }))
-    }
-
-    async fn finish(&self, request: Request<FinishRequest>) -> Result<Response<FinishResponse>, Status> {
-        let id = Uuid::parse_str(&request.get_ref().id).map_err(|_| Status::invalid_argument("invalid id"))?;
-        let (hash, proof) = {
-            let mut state = self.state.write().await;
-            let encoder = state.hashers.remove(&id).ok_or_else(|| Status::not_found("hasher not found"))?;
-            let encoder = Arc::try_unwrap(encoder).unwrap(); // Unwrap the Arc
-            let mut encoder = encoder.lock().await;
-            let hash = encoder.finalize()?.as_bytes().to_vec();
-            let proof = encoder.inner_mut().get_ref().to_vec();
-            (hash, proof)
-        };
-        Ok(Response::new(FinishResponse {
-            hash,
-            proof,
-        }))
-    }
-
-    async fn verify(&self, request: Request<VerifyRequest>) -> Result<Response<VerifyResponse>, Status> {
-        let req = request.get_ref();
-        let res = verify_internal(
-            req.data.clone(),
-            req.offset,
-            req.proof.clone(),
-            from_vec_to_array(req.hash.clone()),
-        );
-
-        if res.is_err() {
-
-            Ok(Response::new(VerifyResponse {
-                status: false,
-                error: res.unwrap_err().to_string(),
-
-            }))
-        } else {
-            Ok(Response::new(VerifyResponse {
-                status: true,
-                error: String::from(""),
-            }))
-        }
-    }
-}
-
-fn verify_internal(
-    chunk_bytes: Vec<u8>,
-    offset: u64,
-    bao_outboard_bytes: Vec<u8>,
-    blake3_hash: [u8; 32],
-) -> anyhow::Result<u8> {
-    let mut slice_stream = abao::encode::SliceExtractor::new_outboard(
-        FakeSeeker::new(&chunk_bytes[..]),
-        Cursor::new(&bao_outboard_bytes),
-        offset,
-        262144,
-    );
-
-    let mut decode_stream = abao::decode::SliceDecoder::new(
-        &mut slice_stream,
-        &abao::Hash::from(blake3_hash),
-        offset,
-        262144,
-    );
-    let mut decoded = Vec::new();
-    decode_stream.read_to_end(&mut decoded)?;
-
-    Ok(1)
-}
-
-fn from_vec_to_array<T, const N: usize>(v: Vec<T>) -> [T; N] {
-    core::convert::TryInto::try_into(v)
-        .unwrap_or_else(|v: Vec<T>| panic!("Expected a Vec of length {} but it was {}", N, v.len()))
-}
-
-struct FakeSeeker<R: Read> {
-    reader: R,
-    bytes_read: u64,
-}
-
-impl<R: Read> FakeSeeker<R> {
-    fn new(reader: R) -> Self {
-        Self {
-            reader,
-            bytes_read: 0,
-        }
-    }
-}
-
-impl<R: Read> Read for FakeSeeker<R> {
-    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
-        let n = self.reader.read(buf)?;
-        self.bytes_read += n as u64;
-        Ok(n)
-    }
-}
-
-impl<R: Read> Seek for FakeSeeker<R> {
-    fn seek(&mut self, _: SeekFrom) -> std::io::Result<u64> {
-        // Do nothing and return the current position.
-        Ok(self.bytes_read)
-    }
-}
-
-impl BaoService {
-    fn new(state: Arc<RwLock<GlobalState>>) -> Self {
-        BaoService { state }
-    }
-}
-
-
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
-    let (tx, rx) = oneshot::channel::<()>();
-    let health_reporter = tonic_health::server::health_reporter();
-    let port = match pick_unused_port() {
-        Some(p) => p,
-        None => {
-            return Err("Failed to pick an unused port".into());
-        }
-    };
-
-    let addr: SocketAddr = format!("127.0.0.1:{}", port).parse()?;
-
-    println!("1|1|tcp|127.0.0.1:{}|grpc", addr.port());
-
-    let global_state = Arc::new(RwLock::new(GlobalState {
-        hashers: HashMap::new(),
-    }));
-
-    tokio::spawn(async move {
-        let mut term_signal = signal(SignalKind::terminate()).expect("Could not create signal handler");
-
-        // Wait for the terminate signal
-        term_signal.recv().await;
-        println!("Termination signal received, shutting down server...");
-
-        // Sending a signal through the channel to initiate shutdown.
-        // If the receiver is dropped, we don't care about the error.
-        let _ = tx.send(());
-    });
-
-    let server = bao_server::BaoServer::new(BaoService::new(global_state.clone()))
-        .max_decoding_message_size(usize::MAX)
-        .max_encoding_message_size(usize::MAX);
-
-    Server::builder()
-        .max_frame_size((1 << 24) - 1)
-        .add_service(server)
-        .add_service(health_reporter.1)
-        .serve_with_shutdown(addr, async {
-            // This future completes when the shutdown signal is received,
-            // allowing the server to shut down gracefully.
-            rx.await.ok();
-        })
-        .await?;
-
-    Ok(())
-}

go.mod

@@ -42,11 +42,9 @@ require (
 	go.uber.org/zap v1.26.0
 	go.uber.org/zap/exp v0.2.0
 	golang.org/x/crypto v0.21.0
-	google.golang.org/grpc v1.62.0
-	google.golang.org/protobuf v1.32.0
 	gorm.io/driver/mysql v1.5.4
 	gorm.io/gorm v1.25.7
-	lukechampine.com/blake3 v1.2.1
+	lukechampine.com/blake3 v1.2.2-0.20240329192137-af604d0fbc33
 	nhooyr.io/websocket v1.8.10
 )
 
@@ -144,6 +142,8 @@ require (
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/tools v0.19.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect
+	google.golang.org/grpc v1.62.0 // indirect
+	google.golang.org/protobuf v1.32.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	lukechampine.com/frand v1.4.2 // indirect

go.sum

@@ -608,8 +608,8 @@ gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde/go.mod h1:hbnx/Oo0ChWMn1BIhpy
 gorm.io/gorm v1.25.7 h1:VsD6acwRjz2zFxGO50gPO6AkNs7KKnvfzUjHQhZDz/A=
 gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI=
+lukechampine.com/blake3 v1.2.2-0.20240329192137-af604d0fbc33 h1:oQavF3w54yDoQQUBdlFCTcziojGobCnxU/SHLQHRNLM=
-lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k=
+lukechampine.com/blake3 v1.2.2-0.20240329192137-af604d0fbc33/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k=
 lukechampine.com/frand v1.4.2 h1:RzFIpOvkMXuPMBb9maa4ND4wjBn71E1Jpf8BzJHMaVw=
 lukechampine.com/frand v1.4.2/go.mod h1:4S/TM2ZgrKejMcKMbeLjISpJMO+/eZ1zu3vYX9dtj3s=
 nhooyr.io/websocket v1.8.10 h1:mv4p+MnGrLDcPlBoWsvPP7XCzTYMXP9F9eIGoKbgx7Q=

protocols/s5/tus.go

@@ -384,7 +384,13 @@ func (t *TusHandler) uploadTask(hash []byte) error {
 		return err
 	}
 
-	proof, err := t.storage.HashObject(ctx, reader)
+	info, err := tusUpload.GetInfo(ctx)
+	if err != nil {
+		t.logger.Error("Could not get tus info", zap.Error(err))
+		return err
+	}
+
+	proof, err := t.storage.HashObject(ctx, reader, uint64(info.Size))
 
 	if err != nil {
 		t.logger.Error("Could not compute proof", zap.Error(err))
@@ -396,13 +402,7 @@ func (t *TusHandler) uploadTask(hash []byte) error {
 		return err
 	}
 
-	info, err := tusUpload.GetInfo(ctx)
+	uploadMeta, err := t.storage.UploadObject(ctx, t.storageProtocol, nil, 0, &renter.MultiPartUploadParams{
-	if err != nil {
-		t.logger.Error("Could not get tus info", zap.Error(err))
-		return err
-	}
-
-	uploadMeta, err := t.storage.UploadObject(ctx, t.storageProtocol, nil, &renter.MultiPartUploadParams{
 		ReaderFactory: func(start uint, end uint) (io.ReadCloser, error) {
 			rangeHeader := "bytes=%d-"
 			if end != 0 {

storage/storage.go

@@ -63,9 +63,9 @@ var Module = fx.Module("storage",
 )
 
 type StorageService interface {
-	UploadObject(ctx context.Context, protocol StorageProtocol, data io.ReadSeeker, muParams *renter.MultiPartUploadParams, proof *bao.Result) (*metadata.UploadMetadata, error)
+	UploadObject(ctx context.Context, protocol StorageProtocol, data io.ReadSeeker, size uint64, muParams *renter.MultiPartUploadParams, proof *bao.Result) (*metadata.UploadMetadata, error)
-	UploadObjectProof(ctx context.Context, protocol StorageProtocol, data io.ReadSeeker, proof *bao.Result) error
+	UploadObjectProof(ctx context.Context, protocol StorageProtocol, data io.ReadSeeker, proof *bao.Result, size uint64) error
-	HashObject(ctx context.Context, data io.Reader) (*bao.Result, error)
+	HashObject(ctx context.Context, data io.Reader, size uint64) (*bao.Result, error)
 	DownloadObject(ctx context.Context, protocol StorageProtocol, objectHash []byte, start int64) (io.ReadCloser, error)
 	DownloadObjectProof(ctx context.Context, protocol StorageProtocol, objectHash []byte) (io.ReadCloser, error)
 	DeleteObject(ctx context.Context, protocol StorageProtocol, objectHash []byte) error
@@ -101,7 +101,7 @@ func NewStorageService(params StorageServiceParams) *StorageServiceDefault {
 	}
 }
 
-func (s StorageServiceDefault) UploadObject(ctx context.Context, protocol StorageProtocol, data io.ReadSeeker, muParams *renter.MultiPartUploadParams, proof *bao.Result) (*metadata.UploadMetadata, error) {
+func (s StorageServiceDefault) UploadObject(ctx context.Context, protocol StorageProtocol, data io.ReadSeeker, size uint64, muParams *renter.MultiPartUploadParams, proof *bao.Result) (*metadata.UploadMetadata, error) {
 	readers := make([]io.ReadCloser, 0)
 	defer func() {
 		for _, reader := range readers {
@@ -148,7 +148,7 @@ func (s StorageServiceDefault) UploadObject(ctx context.Context, protocol Storag
 	}
 
 	if proof == nil {
-		hashResult, err := s.HashObject(ctx, reader)
+		hashResult, err := s.HashObject(ctx, reader, size)
 		if err != nil {
 			return nil, err
 		}
@@ -204,7 +204,7 @@ func (s StorageServiceDefault) UploadObject(ctx context.Context, protocol Storag
 
 	filename := protocol.EncodeFileName(proof.Hash)
 
-	err = s.UploadObjectProof(ctx, protocol, nil, proof)
+	err = s.UploadObjectProof(ctx, protocol, nil, proof, size)
 
 	if err != nil {
 		return nil, err
@@ -237,9 +237,9 @@ func (s StorageServiceDefault) UploadObject(ctx context.Context, protocol Storag
 	return uploadMeta, nil
 }
 
-func (s StorageServiceDefault) UploadObjectProof(ctx context.Context, protocol StorageProtocol, data io.ReadSeeker, proof *bao.Result) error {
+func (s StorageServiceDefault) UploadObjectProof(ctx context.Context, protocol StorageProtocol, data io.ReadSeeker, proof *bao.Result, size uint64) error {
 	if proof == nil {
-		hashResult, err := s.HashObject(ctx, data)
+		hashResult, err := s.HashObject(ctx, data, size)
 		if err != nil {
 			return err
 		}
@@ -258,8 +258,8 @@ func (s StorageServiceDefault) UploadObjectProof(ctx context.Context, protocol S
 	return s.renter.UploadObject(ctx, bytes.NewReader(proof.Proof), protocolName, s.getProofPath(protocol, proof.Hash))
 }
 
-func (s StorageServiceDefault) HashObject(ctx context.Context, data io.Reader) (*bao.Result, error) {
+func (s StorageServiceDefault) HashObject(ctx context.Context, data io.Reader, size uint64) (*bao.Result, error) {
-	result, err := bao.Hash(data)
+	result, err := bao.Hash(data, size)
 
 	if err != nil {
 		return nil, err