Compare commits

..

2 Commits

14 changed files with 98 additions and 1524 deletions

View File

@ -1,5 +1,3 @@
# Use the bufbuild/buf image for extracting the buf command binary
FROM bufbuild/buf:latest as buf
# Use the official Node.js image as the base image for building the api/account/portal
FROM node:20-alpine as nodejs-builder
@ -28,18 +26,10 @@ RUN apk add --no-cache git && git clone --recurse-submodules https://git.lumeweb
# Copy the built dashboard from the nodejs-builder stage
COPY --from=nodejs-builder /portal/api/account/app/build/client /portal/api/account/app/build/client
# Copy buf from the buf stage
COPY --from=buf /usr/local/bin/buf /usr/local/bin/buf
# Install the necessary dependencies
RUN apk add bash gcc curl musl-dev
# Install the necessary dependencies
RUN curl -sSf https://sh.rustup.rs | bash -s -- -y
# Set the necessary environment variables
ENV PATH="/root/.cargo/bin:${PATH}"
## Build the Go application
RUN go mod download

View File

@ -473,7 +473,7 @@ func (rsnc readSeekNopCloser) Close() error {
func (s *S5API) smallFileUpload(jc jape.Context) {
user := middleware.GetUserFromContext(jc.Request.Context())
file, err := s.prepareFileUpload(jc)
file, size, err := s.prepareFileUpload(jc)
if err != nil {
s.sendErrorResponse(jc, err)
return
@ -485,7 +485,7 @@ func (s *S5API) smallFileUpload(jc jape.Context) {
}
}(file)
newUpload, err2 := s.storage.UploadObject(jc.Request.Context(), s5.GetStorageProtocol(s.protocol), file, nil, nil)
newUpload, err2 := s.storage.UploadObject(jc.Request.Context(), s5.GetStorageProtocol(s.protocol), file, size, nil, nil)
if err2 != nil {
s.sendErrorResponse(jc, NewS5Error(ErrKeyFileUploadFailed, err2))
@ -525,33 +525,35 @@ func (s *S5API) smallFileUpload(jc jape.Context) {
})
}
func (s *S5API) prepareFileUpload(jc jape.Context) (file io.ReadSeekCloser, s5Err *S5Error) {
func (s *S5API) prepareFileUpload(jc jape.Context) (file io.ReadSeekCloser, size uint64, s5Err *S5Error) {
r := jc.Request
contentType := r.Header.Get("Content-Type")
// Handle multipart form data uploads
if strings.HasPrefix(contentType, "multipart/form-data") {
if err := r.ParseMultipartForm(int64(s.config.Config().Core.PostUploadLimit)); err != nil {
return nil, NewS5Error(ErrKeyFileUploadFailed, err)
return nil, size, NewS5Error(ErrKeyFileUploadFailed, err)
}
multipartFile, _, err := r.FormFile("file")
if err != nil {
return nil, NewS5Error(ErrKeyFileUploadFailed, err)
return nil, size, NewS5Error(ErrKeyFileUploadFailed, err)
}
return multipartFile, nil
return multipartFile, size, nil
}
// Handle raw body uploads
data, err := io.ReadAll(r.Body)
if err != nil {
return nil, NewS5Error(ErrKeyFileUploadFailed, err)
return nil, size, NewS5Error(ErrKeyFileUploadFailed, err)
}
buffer := readSeekNopCloser{bytes.NewReader(data)}
return buffer, nil
size = uint64(len(data))
return buffer, size, nil
}
func (s *S5API) accountRegisterChallenge(jc jape.Context) {
@ -1275,7 +1277,7 @@ func (s *S5API) pinEntity(ctx context.Context, userId uint, userIp string, cid *
data = append(data, dataCont...)
proof, err := s.storage.HashObject(ctx, bytes.NewReader(data))
proof, err := s.storage.HashObject(ctx, bytes.NewReader(data), uint64(len(data)))
if err != nil {
return nil, false
}
@ -1444,7 +1446,7 @@ func (s *S5API) processMultipartFiles(r *http.Request) (map[string]*metadata.Upl
}
}(file)
upload, err := s.storage.UploadObject(r.Context(), s5.GetStorageProtocol(s.protocol), file, nil, nil)
upload, err := s.storage.UploadObject(r.Context(), s5.GetStorageProtocol(s.protocol), file, uint64(fileHeader.Size), nil, nil)
if err != nil {
return nil, NewS5Error(ErrKeyStorageOperationFailed, err)
}
@ -1515,7 +1517,7 @@ func (s *S5API) uploadAppMetadata(appData *s5libmetadata.WebAppMetadata, r *http
file := bytes.NewReader(appDataRaw)
upload, err := s.storage.UploadObject(r.Context(), s5.GetStorageProtocol(s.protocol), file, nil, nil)
upload, err := s.storage.UploadObject(r.Context(), s5.GetStorageProtocol(s.protocol), file, uint64(len(appDataRaw)), nil, nil)
if err != nil {
return "", NewS5Error(ErrKeyStorageOperationFailed, err)
}
@ -2163,7 +2165,7 @@ func (s *S5API) pinImportCronJob(cid string, url string, proofUrl string, userId
return err // Error logged in fetchAndProcess
}
hash, err := s.storage.HashObject(ctx, bytes.NewReader(fileData))
hash, err := s.storage.HashObject(ctx, bytes.NewReader(fileData), uint64(len(fileData)))
if err != nil {
s.logger.Error("error hashing object", zap.Error(err))
return err
@ -2178,7 +2180,7 @@ func (s *S5API) pinImportCronJob(cid string, url string, proofUrl string, userId
return err
}
upload, err := s.storage.UploadObject(ctx, s5.GetStorageProtocol(s.protocol), bytes.NewReader(fileData), nil, hash)
upload, err := s.storage.UploadObject(ctx, s5.GetStorageProtocol(s.protocol), bytes.NewReader(fileData), parsedCid.Size, nil, hash)
if err != nil {
return err
}
@ -2255,7 +2257,7 @@ func (s *S5API) pinImportCronJob(cid string, url string, proofUrl string, userId
return err
}
upload, err := s.storage.UploadObject(ctx, s5.GetStorageProtocol(s.protocol), nil, &renter.MultiPartUploadParams{
upload, err := s.storage.UploadObject(ctx, s5.GetStorageProtocol(s.protocol), nil, 0, &renter.MultiPartUploadParams{
ReaderFactory: func(start uint, end uint) (io.ReadCloser, error) {
rangeHeader := "bytes=%d-"
if end != 0 {

View File

@ -1,36 +1,27 @@
package bao
import (
"bufio"
"bytes"
_ "embed"
"errors"
"io"
"math"
"os"
"os/exec"
"time"
"github.com/samber/lo"
"go.uber.org/zap"
"github.com/docker/go-units"
"github.com/hashicorp/go-plugin"
"lukechampine.com/blake3/bao"
)
//go:generate buf generate
//go:generate bash -c "cd rust && cargo build -r"
//go:embed rust/target/release/rust
var pluginBin []byte
var bao Bao
var client *plugin.Client
var _ io.ReadCloser = (*Verifier)(nil)
var _ io.WriterAt = (*proofWriter)(nil)
var ErrVerifyFailed = errors.New("verification failed")
const groupLog = 8
const groupChunks = 1 << groupLog
type Verifier struct {
r io.ReadCloser
proof Result
@ -41,6 +32,12 @@ type Verifier struct {
verifyTime time.Duration
}
type Result struct {
Hash []byte
Proof []byte
Length uint
}
func (v *Verifier) Read(p []byte) (int, error) {
// Initial attempt to read from the buffer
n, err := v.buffer.Read(p)
@ -52,7 +49,7 @@ func (v *Verifier) Read(p []byte) (int, error) {
return n, err
}
buf := make([]byte, VERIFY_CHUNK_SIZE)
buf := make([]byte, groupChunks)
// Continue reading from the source and verifying until we have enough data or hit an error
for v.buffer.Len() < len(p)-n {
readStart := time.Now()
@ -68,7 +65,7 @@ func (v *Verifier) Read(p []byte) (int, error) {
timeStart := time.Now()
if bytesRead > 0 {
if status, err := bao.Verify(buf[:bytesRead], v.read, v.proof.Proof, v.proof.Hash); err != nil || !status {
if status := bao.VerifyChunk(buf[:bytesRead], v.proof.Proof, groupChunks, v.read, [32]byte(v.proof.Hash)); !status {
return n, errors.Join(ErrVerifyFailed, err)
}
v.read += uint64(bytesRead)
@ -92,7 +89,7 @@ func (v *Verifier) Read(p []byte) (int, error) {
v.logger.Debug("Read time", zap.Duration("average", averageReadTime))
}
averageVerifyTime := v.verifyTime / time.Duration(v.read/VERIFY_CHUNK_SIZE)
averageVerifyTime := v.verifyTime / time.Duration(v.read/groupChunks)
v.logger.Debug("Verification time", zap.Duration("average", averageVerifyTime))
// Attempt to read the remainder of the data from the buffer
@ -103,94 +100,20 @@ func (v *Verifier) Read(p []byte) (int, error) {
func (v *Verifier) Close() error {
return v.r.Close()
}
func Hash(r io.Reader, size uint64) (*Result, error) {
reader := newSizeReader(r)
writer := newProofWriter(int(size))
func init() {
temp, err := os.CreateTemp(os.TempDir(), "bao")
hash, err := bao.Encode(writer, reader, int64(size), groupLog, true)
if err != nil {
panic(err)
return nil, err
}
err = temp.Chmod(1755)
if err != nil {
panic(err)
}
_, err = io.Copy(temp, bytes.NewReader(pluginBin))
if err != nil {
panic(err)
}
err = temp.Close()
if err != nil {
panic(err)
}
clientInst := plugin.NewClient(&plugin.ClientConfig{
HandshakeConfig: plugin.HandshakeConfig{
ProtocolVersion: 1,
},
Plugins: plugin.PluginSet{
"bao": &BaoPlugin{},
},
Cmd: exec.Command(temp.Name()),
AllowedProtocols: []plugin.Protocol{plugin.ProtocolGRPC},
})
rpcClient, err := clientInst.Client()
if err != nil {
panic(err)
}
pluginInst, err := rpcClient.Dispense("bao")
if err != nil {
panic(err)
}
bao = pluginInst.(Bao)
}
func Shutdown() {
client.Kill()
}
func Hash(r io.Reader) (*Result, error) {
hasherId := bao.NewHasher()
initialSize := 4 * units.KiB
maxSize := 3.5 * units.MiB
bufSize := initialSize
reader := bufio.NewReaderSize(r, bufSize)
var totalReadSize int
buf := make([]byte, bufSize)
for {
n, err := reader.Read(buf)
if err != nil {
if err == io.EOF {
break
}
return nil, err
}
totalReadSize += n
if !bao.Hash(hasherId, buf[:n]) {
return nil, errors.New("hashing failed")
}
// Adaptively adjust buffer size based on read patterns
if n == bufSize && float64(bufSize) < maxSize {
// If buffer was fully used, consider increasing buffer size
bufSize = int(math.Min(float64(bufSize*2), float64(maxSize))) // Double the buffer size, up to a maximum
buf = make([]byte, bufSize) // Apply new buffer size
reader = bufio.NewReaderSize(r, bufSize) // Apply new buffer size
}
}
result := bao.Finish(hasherId)
result.Length = uint(totalReadSize)
return &result, nil
return &Result{
Hash: hash[:],
Proof: writer.buf,
Length: uint(size),
}, nil
}
func NewVerifier(r io.ReadCloser, proof Result, logger *zap.Logger) *Verifier {
@ -201,3 +124,38 @@ func NewVerifier(r io.ReadCloser, proof Result, logger *zap.Logger) *Verifier {
logger: logger,
}
}
type proofWriter struct {
buf []byte
}
func (p proofWriter) WriteAt(b []byte, off int64) (n int, err error) {
if copy(p.buf[off:], b) != len(b) {
panic("bad buffer size")
}
return len(b), nil
}
func newProofWriter(size int) *proofWriter {
return &proofWriter{
buf: make([]byte, bao.EncodedSize(size, groupLog, true)),
}
}
type sizeReader struct {
reader io.Reader
read int64
}
func (s sizeReader) Read(p []byte) (int, error) {
n, err := s.reader.Read(p)
s.read += int64(n)
return n, err
}
func newSizeReader(r io.Reader) *sizeReader {
return &sizeReader{
reader: r,
read: 0,
}
}

View File

@ -1,12 +0,0 @@
version: v1
managed:
enabled: true
plugins:
- plugin: buf.build/protocolbuffers/go
out: .
- plugin: buf.build/grpc/go:v1.3.0
out: .
- plugin: buf.build/community/neoeinstein-prost:v0.2.3
out: rust/src/proto
- plugin: buf.build/community/neoeinstein-tonic:v0.3.0
out: rust/src/proto

View File

@ -1 +0,0 @@
version: v1

View File

@ -1,95 +0,0 @@
package bao
import (
"context"
"errors"
"github.com/docker/go-units"
"git.lumeweb.com/LumeWeb/portal/bao/proto"
"github.com/google/uuid"
"github.com/hashicorp/go-plugin"
"google.golang.org/grpc"
)
var _ Bao = (*BaoGRPC)(nil)
const VERIFY_CHUNK_SIZE = 256 * units.KiB
type Bao interface {
NewHasher() uuid.UUID
Hash(id uuid.UUID, data []byte) bool
Finish(id uuid.UUID) Result
Verify(data []byte, offset uint64, proof []byte, hash []byte) (bool, error)
}
type BaoPlugin struct {
plugin.Plugin
}
func (p *BaoPlugin) GRPCServer(broker *plugin.GRPCBroker, s *grpc.Server) error {
return nil
}
func (p *BaoPlugin) GRPCClient(ctx context.Context, broker *plugin.GRPCBroker, c *grpc.ClientConn) (interface{}, error) {
return &BaoGRPC{client: proto.NewBaoClient(c)}, nil
}
type Result struct {
Hash []byte
Proof []byte
Length uint
}
type BaoGRPC struct {
client proto.BaoClient
}
func (b *BaoGRPC) NewHasher() uuid.UUID {
ret, err := b.client.NewHasher(context.Background(), &proto.NewHasherRequest{})
if err != nil {
panic(err)
}
id, err := uuid.Parse(ret.Id)
if err != nil {
panic(err)
}
return id
}
func (b *BaoGRPC) Hash(id uuid.UUID, data []byte) bool {
ret, err := b.client.Hash(context.Background(), &proto.HashRequest{Id: id.String(), Data: data})
if err != nil {
panic(err)
}
return ret.Status
}
func (b *BaoGRPC) Finish(id uuid.UUID) Result {
ret, err := b.client.Finish(context.Background(), &proto.FinishRequest{Id: id.String()})
if err != nil {
panic(err)
}
return Result{Hash: ret.Hash, Proof: ret.Proof}
}
func (b *BaoGRPC) Verify(data []byte, offset uint64, proof []byte, hash []byte) (bool, error) {
ret, err := b.client.Verify(context.Background(), &proto.VerifyRequest{Data: data, Offset: offset, Proof: proof, Hash: hash})
if err != nil {
return false, err
}
if ret.Error != "" {
err = errors.New(ret.Error)
}
return ret.Status, err
}

View File

@ -1,51 +0,0 @@
syntax = "proto3";
package bao;
option go_package = "./proto";
// Define an empty message for the request as proto3 requires specific message types
message NewHasherRequest {
}
// Define a message for the response that includes the bytes you mentioned
message NewHasherResponse {
string id = 1;
}
message HashRequest {
string id = 1;
bytes data = 2;
}
message HashResponse {
bool status = 1;
}
message FinishRequest {
string id = 1;
}
message FinishResponse {
bytes hash = 1;
bytes proof = 2;
}
message VerifyRequest {
bytes data = 1;
uint64 offset = 2;
bytes proof = 3;
bytes hash = 4;
}
message VerifyResponse {
bool status = 1;
string error = 2;
}
service Bao {
rpc NewHasher(NewHasherRequest) returns (NewHasherResponse);
rpc Hash(HashRequest) returns (HashResponse);
rpc Finish(FinishRequest) returns (FinishResponse);
rpc Verify(VerifyRequest) returns (VerifyResponse);
}

990
bao/rust/Cargo.lock generated
View File

@ -1,990 +0,0 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 3
[[package]]
name = "abao"
version = "0.2.0"
source = "git+https://github.com/LumeWeb/abao.git?branch=feature/inner_mut#3179db880ab071aea543ee95baae6d4b984af618"
dependencies = [
"arrayref",
"arrayvec",
"blake3",
"futures",
"tokio",
]
[[package]]
name = "addr2line"
version = "0.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb"
dependencies = [
"gimli",
]
[[package]]
name = "adler"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
[[package]]
name = "anyhow"
version = "1.0.79"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca"
[[package]]
name = "arrayref"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545"
[[package]]
name = "arrayvec"
version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711"
[[package]]
name = "async-stream"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51"
dependencies = [
"async-stream-impl",
"futures-core",
"pin-project-lite",
]
[[package]]
name = "async-stream-impl"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "async-trait"
version = "0.1.77"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "autocfg"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]]
name = "axum"
version = "0.6.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf"
dependencies = [
"async-trait",
"axum-core",
"bitflags",
"bytes",
"futures-util",
"http",
"http-body",
"hyper",
"itoa",
"matchit",
"memchr",
"mime",
"percent-encoding",
"pin-project-lite",
"rustversion",
"serde",
"sync_wrapper",
"tower",
"tower-layer",
"tower-service",
]
[[package]]
name = "axum-core"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c"
dependencies = [
"async-trait",
"bytes",
"futures-util",
"http",
"http-body",
"mime",
"rustversion",
"tower-layer",
"tower-service",
]
[[package]]
name = "backtrace"
version = "0.3.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837"
dependencies = [
"addr2line",
"cc",
"cfg-if",
"libc",
"miniz_oxide",
"object",
"rustc-demangle",
]
[[package]]
name = "base64"
version = "0.21.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
[[package]]
name = "bitflags"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "blake3"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0231f06152bf547e9c2b5194f247cd97aacf6dcd8b15d8e5ec0663f64580da87"
dependencies = [
"arrayref",
"arrayvec",
"cc",
"cfg-if",
"constant_time_eq",
]
[[package]]
name = "bytes"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223"
[[package]]
name = "cc"
version = "1.0.83"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0"
dependencies = [
"libc",
]
[[package]]
name = "cfg-if"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "constant_time_eq"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2"
[[package]]
name = "either"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07"
[[package]]
name = "equivalent"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
[[package]]
name = "fnv"
version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "futures"
version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0"
dependencies = [
"futures-channel",
"futures-core",
"futures-io",
"futures-sink",
"futures-task",
"futures-util",
]
[[package]]
name = "futures-channel"
version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78"
dependencies = [
"futures-core",
"futures-sink",
]
[[package]]
name = "futures-core"
version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d"
[[package]]
name = "futures-io"
version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1"
[[package]]
name = "futures-sink"
version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5"
[[package]]
name = "futures-task"
version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004"
[[package]]
name = "futures-util"
version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48"
dependencies = [
"futures-core",
"futures-sink",
"futures-task",
"pin-project-lite",
"pin-utils",
]
[[package]]
name = "getrandom"
version = "0.2.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5"
dependencies = [
"cfg-if",
"libc",
"wasi",
]
[[package]]
name = "gimli"
version = "0.28.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253"
[[package]]
name = "h2"
version = "0.3.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9"
dependencies = [
"bytes",
"fnv",
"futures-core",
"futures-sink",
"futures-util",
"http",
"indexmap 2.2.2",
"slab",
"tokio",
"tokio-util",
"tracing",
]
[[package]]
name = "hashbrown"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
[[package]]
name = "hashbrown"
version = "0.14.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604"
[[package]]
name = "hermit-abi"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0c62115964e08cb8039170eb33c1d0e2388a256930279edca206fff675f82c3"
[[package]]
name = "http"
version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb"
dependencies = [
"bytes",
"fnv",
"itoa",
]
[[package]]
name = "http-body"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2"
dependencies = [
"bytes",
"http",
"pin-project-lite",
]
[[package]]
name = "httparse"
version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904"
[[package]]
name = "httpdate"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
[[package]]
name = "hyper"
version = "0.14.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80"
dependencies = [
"bytes",
"futures-channel",
"futures-core",
"futures-util",
"h2",
"http",
"http-body",
"httparse",
"httpdate",
"itoa",
"pin-project-lite",
"socket2",
"tokio",
"tower-service",
"tracing",
"want",
]
[[package]]
name = "hyper-timeout"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1"
dependencies = [
"hyper",
"pin-project-lite",
"tokio",
"tokio-io-timeout",
]
[[package]]
name = "indexmap"
version = "1.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
dependencies = [
"autocfg",
"hashbrown 0.12.3",
]
[[package]]
name = "indexmap"
version = "2.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "824b2ae422412366ba479e8111fd301f7b5faece8149317bb81925979a53f520"
dependencies = [
"equivalent",
"hashbrown 0.14.3",
]
[[package]]
name = "itertools"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57"
dependencies = [
"either",
]
[[package]]
name = "itoa"
version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c"
[[package]]
name = "libc"
version = "0.2.153"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd"
[[package]]
name = "matchit"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94"
[[package]]
name = "memchr"
version = "2.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149"
[[package]]
name = "mime"
version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
[[package]]
name = "miniz_oxide"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7"
dependencies = [
"adler",
]
[[package]]
name = "mio"
version = "0.8.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09"
dependencies = [
"libc",
"wasi",
"windows-sys",
]
[[package]]
name = "num_cpus"
version = "1.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
dependencies = [
"hermit-abi",
"libc",
]
[[package]]
name = "object"
version = "0.32.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441"
dependencies = [
"memchr",
]
[[package]]
name = "once_cell"
version = "1.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
[[package]]
name = "percent-encoding"
version = "2.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
[[package]]
name = "pin-project"
version = "1.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0302c4a0442c456bd56f841aee5c3bfd17967563f6fadc9ceb9f9c23cf3807e0"
dependencies = [
"pin-project-internal",
]
[[package]]
name = "pin-project-internal"
version = "1.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "266c042b60c9c76b8d53061e52b2e0d1116abc57cefc8c5cd671619a56ac3690"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "pin-project-lite"
version = "0.2.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58"
[[package]]
name = "pin-utils"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
[[package]]
name = "portpicker"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be97d76faf1bfab666e1375477b23fde79eccf0276e9b63b92a39d676a889ba9"
dependencies = [
"rand",
]
[[package]]
name = "ppv-lite86"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
[[package]]
name = "proc-macro2"
version = "1.0.78"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae"
dependencies = [
"unicode-ident",
]
[[package]]
name = "prost"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "146c289cda302b98a28d40c8b3b90498d6e526dd24ac2ecea73e4e491685b94a"
dependencies = [
"bytes",
"prost-derive",
]
[[package]]
name = "prost-derive"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "efb6c9a1dd1def8e2124d17e83a20af56f1570d6c2d2bd9e266ccb768df3840e"
dependencies = [
"anyhow",
"itertools",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "quote"
version = "1.0.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef"
dependencies = [
"proc-macro2",
]
[[package]]
name = "rand"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
dependencies = [
"libc",
"rand_chacha",
"rand_core",
]
[[package]]
name = "rand_chacha"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
dependencies = [
"ppv-lite86",
"rand_core",
]
[[package]]
name = "rand_core"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
dependencies = [
"getrandom",
]
[[package]]
name = "rust"
version = "0.1.0"
dependencies = [
"abao",
"anyhow",
"portpicker",
"prost",
"tokio",
"tonic",
"tonic-health",
"uuid",
]
[[package]]
name = "rustc-demangle"
version = "0.1.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
[[package]]
name = "rustversion"
version = "1.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4"
[[package]]
name = "serde"
version = "1.0.196"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.196"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "signal-hook-registry"
version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1"
dependencies = [
"libc",
]
[[package]]
name = "slab"
version = "0.4.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67"
dependencies = [
"autocfg",
]
[[package]]
name = "socket2"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9"
dependencies = [
"libc",
"windows-sys",
]
[[package]]
name = "syn"
version = "2.0.48"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "sync_wrapper"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
[[package]]
name = "tokio"
version = "1.36.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931"
dependencies = [
"backtrace",
"bytes",
"libc",
"mio",
"num_cpus",
"pin-project-lite",
"signal-hook-registry",
"socket2",
"tokio-macros",
"windows-sys",
]
[[package]]
name = "tokio-io-timeout"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf"
dependencies = [
"pin-project-lite",
"tokio",
]
[[package]]
name = "tokio-macros"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "tokio-stream"
version = "0.1.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842"
dependencies = [
"futures-core",
"pin-project-lite",
"tokio",
]
[[package]]
name = "tokio-util"
version = "0.7.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15"
dependencies = [
"bytes",
"futures-core",
"futures-sink",
"pin-project-lite",
"tokio",
"tracing",
]
[[package]]
name = "tonic"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76c4eb7a4e9ef9d4763600161f12f5070b92a578e1b634db88a6887844c91a13"
dependencies = [
"async-stream",
"async-trait",
"axum",
"base64",
"bytes",
"h2",
"http",
"http-body",
"hyper",
"hyper-timeout",
"percent-encoding",
"pin-project",
"prost",
"tokio",
"tokio-stream",
"tower",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
name = "tonic-health"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2cef6e24bc96871001a7e48e820ab240b3de2201e59b517cf52835df2f1d2350"
dependencies = [
"async-stream",
"prost",
"tokio",
"tokio-stream",
"tonic",
]
[[package]]
name = "tower"
version = "0.4.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c"
dependencies = [
"futures-core",
"futures-util",
"indexmap 1.9.3",
"pin-project",
"pin-project-lite",
"rand",
"slab",
"tokio",
"tokio-util",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
name = "tower-layer"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0"
[[package]]
name = "tower-service"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
[[package]]
name = "tracing"
version = "0.1.40"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef"
dependencies = [
"pin-project-lite",
"tracing-attributes",
"tracing-core",
]
[[package]]
name = "tracing-attributes"
version = "0.1.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "tracing-core"
version = "0.1.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54"
dependencies = [
"once_cell",
]
[[package]]
name = "try-lock"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
[[package]]
name = "unicode-ident"
version = "1.0.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
[[package]]
name = "uuid"
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f00cc9702ca12d3c81455259621e676d0f7251cec66a21e98fe2e9a37db93b2a"
dependencies = [
"getrandom",
]
[[package]]
name = "want"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"
dependencies = [
"try-lock",
]
[[package]]
name = "wasi"
version = "0.11.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "windows-sys"
version = "0.48.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
dependencies = [
"windows-targets",
]
[[package]]
name = "windows-targets"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"
dependencies = [
"windows_aarch64_gnullvm",
"windows_aarch64_msvc",
"windows_i686_gnu",
"windows_i686_msvc",
"windows_x86_64_gnu",
"windows_x86_64_gnullvm",
"windows_x86_64_msvc",
]
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
[[package]]
name = "windows_aarch64_msvc"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
[[package]]
name = "windows_i686_gnu"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
[[package]]
name = "windows_i686_msvc"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
[[package]]
name = "windows_x86_64_gnu"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
[[package]]
name = "windows_x86_64_msvc"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"

View File

@ -1,16 +0,0 @@
[package]
name = "rust"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
tonic = "0.11.0"
prost = "0.12.3"
tonic-health = "0.11.0"
tokio = { version = "1.0.0", features = ["rt", "rt-multi-thread", "macros","signal"] }
portpicker = "0.1.1"
uuid = { version = "1.7.0", features = ["v4"] }
abao = { git = "https://github.com/LumeWeb/abao.git", branch = "feature/inner_mut", default-features = false, features = ["group_size_256k", "tokio", "tokio_io"] }
anyhow = "1.0.79"

View File

@ -1,211 +0,0 @@
use std::collections::HashMap;
use std::io::{Cursor, Read, Seek, SeekFrom, Write};
use std::net::SocketAddr;
use std::sync::Arc;
use abao::encode::Encoder;
use portpicker::pick_unused_port;
use tokio::sync::{Mutex, oneshot, RwLock};
use tonic::{Request, Response, Status};
use tonic::transport::Server;
use uuid::Uuid;
use bao::bao_server::Bao;
use tokio::signal::unix::{signal, SignalKind};
use crate::bao::{bao_server, FinishRequest, FinishResponse, HashRequest, HashResponse, NewHasherRequest, NewHasherResponse, VerifyRequest, VerifyResponse};
#[path = "proto/bao.rs"]
mod bao;
struct GlobalState {
hashers: HashMap<Uuid, Arc<Mutex<Encoder<Cursor<Vec<u8>>>>>>,
}
pub struct BaoService {
state: Arc<RwLock<GlobalState>>,
}
#[tonic::async_trait]
impl Bao for BaoService {
async fn new_hasher(&self, _request: Request<NewHasherRequest>) -> Result<Response<NewHasherResponse>, Status> {
let encoder = Encoder::new_outboard(Cursor::new(Vec::new()));
let id = Uuid::new_v4();
{
let mut state = self.state.write().await;
state.hashers.insert(id, Arc::new(Mutex::new(encoder)));
}
Ok(Response::new(NewHasherResponse {
id: id.to_string(),
}))
}
async fn hash(&self, request: Request<HashRequest>) -> Result<Response<HashResponse>, Status> {
let id = Uuid::parse_str(&request.get_ref().id).map_err(|_| Status::invalid_argument("invalid id"))?;
{
let state = self.state.read().await;
let encoder = state.hashers.get(&id).ok_or_else(|| Status::not_found("hasher not found"))?.clone();
let mut encoder = encoder.lock().await;
encoder.write_all(&request.get_ref().data).map_err(|_| Status::internal("write failed"))?;
}
Ok(Response::new(HashResponse {
status: true,
}))
}
async fn finish(&self, request: Request<FinishRequest>) -> Result<Response<FinishResponse>, Status> {
let id = Uuid::parse_str(&request.get_ref().id).map_err(|_| Status::invalid_argument("invalid id"))?;
let (hash, proof) = {
let mut state = self.state.write().await;
let encoder = state.hashers.remove(&id).ok_or_else(|| Status::not_found("hasher not found"))?;
let encoder = Arc::try_unwrap(encoder).unwrap(); // Unwrap the Arc
let mut encoder = encoder.lock().await;
let hash = encoder.finalize()?.as_bytes().to_vec();
let proof = encoder.inner_mut().get_ref().to_vec();
(hash, proof)
};
Ok(Response::new(FinishResponse {
hash,
proof,
}))
}
async fn verify(&self, request: Request<VerifyRequest>) -> Result<Response<VerifyResponse>, Status> {
let req = request.get_ref();
let res = verify_internal(
req.data.clone(),
req.offset,
req.proof.clone(),
from_vec_to_array(req.hash.clone()),
);
if res.is_err() {
Ok(Response::new(VerifyResponse {
status: false,
error: res.unwrap_err().to_string(),
}))
} else {
Ok(Response::new(VerifyResponse {
status: true,
error: String::from(""),
}))
}
}
}
fn verify_internal(
chunk_bytes: Vec<u8>,
offset: u64,
bao_outboard_bytes: Vec<u8>,
blake3_hash: [u8; 32],
) -> anyhow::Result<u8> {
let mut slice_stream = abao::encode::SliceExtractor::new_outboard(
FakeSeeker::new(&chunk_bytes[..]),
Cursor::new(&bao_outboard_bytes),
offset,
262144,
);
let mut decode_stream = abao::decode::SliceDecoder::new(
&mut slice_stream,
&abao::Hash::from(blake3_hash),
offset,
262144,
);
let mut decoded = Vec::new();
decode_stream.read_to_end(&mut decoded)?;
Ok(1)
}
fn from_vec_to_array<T, const N: usize>(v: Vec<T>) -> [T; N] {
core::convert::TryInto::try_into(v)
.unwrap_or_else(|v: Vec<T>| panic!("Expected a Vec of length {} but it was {}", N, v.len()))
}
struct FakeSeeker<R: Read> {
reader: R,
bytes_read: u64,
}
impl<R: Read> FakeSeeker<R> {
fn new(reader: R) -> Self {
Self {
reader,
bytes_read: 0,
}
}
}
impl<R: Read> Read for FakeSeeker<R> {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
let n = self.reader.read(buf)?;
self.bytes_read += n as u64;
Ok(n)
}
}
impl<R: Read> Seek for FakeSeeker<R> {
fn seek(&mut self, _: SeekFrom) -> std::io::Result<u64> {
// Do nothing and return the current position.
Ok(self.bytes_read)
}
}
impl BaoService {
fn new(state: Arc<RwLock<GlobalState>>) -> Self {
BaoService { state }
}
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let (tx, rx) = oneshot::channel::<()>();
let health_reporter = tonic_health::server::health_reporter();
let port = match pick_unused_port() {
Some(p) => p,
None => {
return Err("Failed to pick an unused port".into());
}
};
let addr: SocketAddr = format!("127.0.0.1:{}", port).parse()?;
println!("1|1|tcp|127.0.0.1:{}|grpc", addr.port());
let global_state = Arc::new(RwLock::new(GlobalState {
hashers: HashMap::new(),
}));
tokio::spawn(async move {
let mut term_signal = signal(SignalKind::terminate()).expect("Could not create signal handler");
// Wait for the terminate signal
term_signal.recv().await;
println!("Termination signal received, shutting down server...");
// Sending a signal through the channel to initiate shutdown.
// If the receiver is dropped, we don't care about the error.
let _ = tx.send(());
});
let server = bao_server::BaoServer::new(BaoService::new(global_state.clone()))
.max_decoding_message_size(usize::MAX)
.max_encoding_message_size(usize::MAX);
Server::builder()
.max_frame_size((1 << 24) - 1)
.add_service(server)
.add_service(health_reporter.1)
.serve_with_shutdown(addr, async {
// This future completes when the shutdown signal is received,
// allowing the server to shut down gracefully.
rx.await.ok();
})
.await?;
Ok(())
}

6
go.mod
View File

@ -42,11 +42,9 @@ require (
go.uber.org/zap v1.26.0
go.uber.org/zap/exp v0.2.0
golang.org/x/crypto v0.21.0
google.golang.org/grpc v1.62.0
google.golang.org/protobuf v1.32.0
gorm.io/driver/mysql v1.5.4
gorm.io/gorm v1.25.7
lukechampine.com/blake3 v1.2.1
lukechampine.com/blake3 v1.2.2-0.20240329192137-af604d0fbc33
nhooyr.io/websocket v1.8.10
)
@ -144,6 +142,8 @@ require (
golang.org/x/text v0.14.0 // indirect
golang.org/x/tools v0.19.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect
google.golang.org/grpc v1.62.0 // indirect
google.golang.org/protobuf v1.32.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
lukechampine.com/frand v1.4.2 // indirect

4
go.sum
View File

@ -608,8 +608,8 @@ gorm.io/gorm v1.25.7-0.20240204074919-46816ad31dde/go.mod h1:hbnx/Oo0ChWMn1BIhpy
gorm.io/gorm v1.25.7 h1:VsD6acwRjz2zFxGO50gPO6AkNs7KKnvfzUjHQhZDz/A=
gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI=
lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k=
lukechampine.com/blake3 v1.2.2-0.20240329192137-af604d0fbc33 h1:oQavF3w54yDoQQUBdlFCTcziojGobCnxU/SHLQHRNLM=
lukechampine.com/blake3 v1.2.2-0.20240329192137-af604d0fbc33/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k=
lukechampine.com/frand v1.4.2 h1:RzFIpOvkMXuPMBb9maa4ND4wjBn71E1Jpf8BzJHMaVw=
lukechampine.com/frand v1.4.2/go.mod h1:4S/TM2ZgrKejMcKMbeLjISpJMO+/eZ1zu3vYX9dtj3s=
nhooyr.io/websocket v1.8.10 h1:mv4p+MnGrLDcPlBoWsvPP7XCzTYMXP9F9eIGoKbgx7Q=

View File

@ -384,7 +384,13 @@ func (t *TusHandler) uploadTask(hash []byte) error {
return err
}
proof, err := t.storage.HashObject(ctx, reader)
info, err := tusUpload.GetInfo(ctx)
if err != nil {
t.logger.Error("Could not get tus info", zap.Error(err))
return err
}
proof, err := t.storage.HashObject(ctx, reader, uint64(info.Size))
if err != nil {
t.logger.Error("Could not compute proof", zap.Error(err))
@ -396,13 +402,7 @@ func (t *TusHandler) uploadTask(hash []byte) error {
return err
}
info, err := tusUpload.GetInfo(ctx)
if err != nil {
t.logger.Error("Could not get tus info", zap.Error(err))
return err
}
uploadMeta, err := t.storage.UploadObject(ctx, t.storageProtocol, nil, &renter.MultiPartUploadParams{
uploadMeta, err := t.storage.UploadObject(ctx, t.storageProtocol, nil, 0, &renter.MultiPartUploadParams{
ReaderFactory: func(start uint, end uint) (io.ReadCloser, error) {
rangeHeader := "bytes=%d-"
if end != 0 {

View File

@ -63,9 +63,9 @@ var Module = fx.Module("storage",
)
type StorageService interface {
UploadObject(ctx context.Context, protocol StorageProtocol, data io.ReadSeeker, muParams *renter.MultiPartUploadParams, proof *bao.Result) (*metadata.UploadMetadata, error)
UploadObjectProof(ctx context.Context, protocol StorageProtocol, data io.ReadSeeker, proof *bao.Result) error
HashObject(ctx context.Context, data io.Reader) (*bao.Result, error)
UploadObject(ctx context.Context, protocol StorageProtocol, data io.ReadSeeker, size uint64, muParams *renter.MultiPartUploadParams, proof *bao.Result) (*metadata.UploadMetadata, error)
UploadObjectProof(ctx context.Context, protocol StorageProtocol, data io.ReadSeeker, proof *bao.Result, size uint64) error
HashObject(ctx context.Context, data io.Reader, size uint64) (*bao.Result, error)
DownloadObject(ctx context.Context, protocol StorageProtocol, objectHash []byte, start int64) (io.ReadCloser, error)
DownloadObjectProof(ctx context.Context, protocol StorageProtocol, objectHash []byte) (io.ReadCloser, error)
DeleteObject(ctx context.Context, protocol StorageProtocol, objectHash []byte) error
@ -101,7 +101,7 @@ func NewStorageService(params StorageServiceParams) *StorageServiceDefault {
}
}
func (s StorageServiceDefault) UploadObject(ctx context.Context, protocol StorageProtocol, data io.ReadSeeker, muParams *renter.MultiPartUploadParams, proof *bao.Result) (*metadata.UploadMetadata, error) {
func (s StorageServiceDefault) UploadObject(ctx context.Context, protocol StorageProtocol, data io.ReadSeeker, size uint64, muParams *renter.MultiPartUploadParams, proof *bao.Result) (*metadata.UploadMetadata, error) {
readers := make([]io.ReadCloser, 0)
defer func() {
for _, reader := range readers {
@ -148,7 +148,7 @@ func (s StorageServiceDefault) UploadObject(ctx context.Context, protocol Storag
}
if proof == nil {
hashResult, err := s.HashObject(ctx, reader)
hashResult, err := s.HashObject(ctx, reader, size)
if err != nil {
return nil, err
}
@ -204,7 +204,7 @@ func (s StorageServiceDefault) UploadObject(ctx context.Context, protocol Storag
filename := protocol.EncodeFileName(proof.Hash)
err = s.UploadObjectProof(ctx, protocol, nil, proof)
err = s.UploadObjectProof(ctx, protocol, nil, proof, size)
if err != nil {
return nil, err
@ -237,9 +237,9 @@ func (s StorageServiceDefault) UploadObject(ctx context.Context, protocol Storag
return uploadMeta, nil
}
func (s StorageServiceDefault) UploadObjectProof(ctx context.Context, protocol StorageProtocol, data io.ReadSeeker, proof *bao.Result) error {
func (s StorageServiceDefault) UploadObjectProof(ctx context.Context, protocol StorageProtocol, data io.ReadSeeker, proof *bao.Result, size uint64) error {
if proof == nil {
hashResult, err := s.HashObject(ctx, data)
hashResult, err := s.HashObject(ctx, data, size)
if err != nil {
return err
}
@ -258,8 +258,8 @@ func (s StorageServiceDefault) UploadObjectProof(ctx context.Context, protocol S
return s.renter.UploadObject(ctx, bytes.NewReader(proof.Proof), protocolName, s.getProofPath(protocol, proof.Hash))
}
func (s StorageServiceDefault) HashObject(ctx context.Context, data io.Reader) (*bao.Result, error) {
result, err := bao.Hash(data)
func (s StorageServiceDefault) HashObject(ctx context.Context, data io.Reader, size uint64) (*bao.Result, error) {
result, err := bao.Hash(data, size)
if err != nil {
return nil, err