From 88caa9bebef253311fe4783fe3d4d38f6b6e284a Mon Sep 17 00:00:00 2001 From: dapplion Date: Tue, 1 Dec 2020 09:01:45 +0000 Subject: [PATCH 1/5] Add benchmark results --- README.md | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 8b31c57..f1ade5d 100644 --- a/README.md +++ b/README.md @@ -85,14 +85,34 @@ try { The API is identical for all implementations. +## Benchmarks + +- `blst`: [src/blst](src/blst) +- `herumi`: [src/herumi](src/herumi) +- `noble`: [noble-bls12-381](https://github.com/paulmillr/noble-bls12-381) + +``` +blst verify: 502.72 ops/sec (100 runs) +blst verifyAgg: 489.60 ops/sec (100 runs) +blst aggPubkey: 8326.6 ops/sec (100 runs) +blst aggSigs: 6968.3 ops/sec (100 runs) +herumi verify: 53.792 ops/sec (100 runs) +herumi verifyAgg: 52.897 ops/sec (100 runs) +herumi aggPubkey: 3020.1 ops/sec (100 runs) +herumi aggSigs: 1151.2 ops/sec (100 runs) +noble verify: 13.868 ops/sec (10 runs) +noble verifyAgg: 11.241 ops/sec (10 runs) +noble aggPubkey: 47.309 ops/sec (10 runs) +``` + ## Spec versioning | Version | Bls spec hash-to-curve version | -| ------- | :--------------: | -| 5.x.x | draft #9 | -| 2.x.x | draft #7 | -| 1.x.x | draft #6 | -| 0.3.x | initial version | +| ------- | :----------------------------: | +| 5.x.x | draft #9 | +| 2.x.x | draft #7 | +| 1.x.x | draft #6 | +| 0.3.x | initial version | > [spec](https://github.com/ethereum/eth2.0-specs/blob/v1.0.0/specs/phase0/beacon-chain.md#bls-signatures) From a207c23f3e1f21d4ccf9ebec52c48869db029ddb Mon Sep 17 00:00:00 2001 From: dapplion Date: Tue, 1 Dec 2020 09:32:15 +0000 Subject: [PATCH 2/5] Link to CI run --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index f1ade5d..2b68d12 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,8 @@ noble verifyAgg: 11.241 ops/sec (10 runs) noble aggPubkey: 47.309 ops/sec (10 runs) ``` +Results from CI run https://github.com/ChainSafe/bls/runs/1478915060 + ## Spec versioning | Version | Bls spec hash-to-curve version | From e356bd1b377315750cb411814eda03f3bc558f5c Mon Sep 17 00:00:00 2001 From: dapplion Date: Wed, 2 Dec 2020 20:45:29 +0000 Subject: [PATCH 3/5] Present results as a table --- README.md | 23 ++++++++++------------- benchmark/index.ts | 9 ++++++--- benchmark/noble.ts | 12 +++++------- benchmark/params.ts | 3 +++ 4 files changed, 24 insertions(+), 23 deletions(-) create mode 100644 benchmark/params.ts diff --git a/README.md b/README.md index 2b68d12..a380685 100644 --- a/README.md +++ b/README.md @@ -91,19 +91,16 @@ The API is identical for all implementations. - `herumi`: [src/herumi](src/herumi) - `noble`: [noble-bls12-381](https://github.com/paulmillr/noble-bls12-381) -``` -blst verify: 502.72 ops/sec (100 runs) -blst verifyAgg: 489.60 ops/sec (100 runs) -blst aggPubkey: 8326.6 ops/sec (100 runs) -blst aggSigs: 6968.3 ops/sec (100 runs) -herumi verify: 53.792 ops/sec (100 runs) -herumi verifyAgg: 52.897 ops/sec (100 runs) -herumi aggPubkey: 3020.1 ops/sec (100 runs) -herumi aggSigs: 1151.2 ops/sec (100 runs) -noble verify: 13.868 ops/sec (10 runs) -noble verifyAgg: 11.241 ops/sec (10 runs) -noble aggPubkey: 47.309 ops/sec (10 runs) -``` +Results are in `ops/sec`. `blst` and `herumi` performed 100 runs each, `noble` 10 runs. + +| Function - `ops/sec` | `blst` | `herumi` | `noble`\* | +| ------------------------- | :----: | :------: | :-------: | +| `verify` | 502.72 | 53.792 | 13.868 | +| `verifyAggregate` (30) | 489.60 | 52.897 | 11.241 | +| `aggregate` (pubkeys, 30) | 8326.6 | 3020.1 | 47.309 | +| `aggregate` (sigs, 30) | 6968.3 | 1151.2 | - | + +\*`noble` methods include serialization and deserialization to bytes, which may impact the `aggregate` benchmark Results from CI run https://github.com/ChainSafe/bls/runs/1478915060 diff --git a/benchmark/index.ts b/benchmark/index.ts index 72b523d..6696642 100644 --- a/benchmark/index.ts +++ b/benchmark/index.ts @@ -2,8 +2,7 @@ import {runBenchmark} from "./runner"; import {runForAllImplementations} from "../test/switch"; import {PublicKey, Signature} from "../src/interface"; import {range, randomMessage} from "../test/util"; - -const aggCount = 30; +import {aggCount, runs} from "./params"; (async function () { await runForAllImplementations(async (bls, implementation) => { @@ -25,12 +24,13 @@ const aggCount = 30; testRunner: ({pk, msg, sig}) => { return sig.verify(pk, msg); }, + runs, }); // Fast aggregate await runBenchmark<{pks: PublicKey[]; msg: Uint8Array; sig: Signature}, boolean>({ - id: `${implementation} verifyAggregate`, + id: `${implementation} verifyAggregate (${aggCount})`, prepareTest: () => { const msg = randomMessage(); @@ -52,6 +52,7 @@ const aggCount = 30; testRunner: ({pks, msg, sig}) => { return sig.verifyAggregate(pks, msg); }, + runs, }); // Aggregate pubkeys @@ -67,6 +68,7 @@ const aggCount = 30; testRunner: (pks) => { bls.PublicKey.aggregate(pks); }, + runs, }); // Aggregate sigs @@ -87,6 +89,7 @@ const aggCount = 30; testRunner: (sigs) => { bls.Signature.aggregate(sigs); }, + runs, }); }); })(); diff --git a/benchmark/noble.ts b/benchmark/noble.ts index 23f598f..9980ae7 100644 --- a/benchmark/noble.ts +++ b/benchmark/noble.ts @@ -2,9 +2,7 @@ import {runBenchmark} from "./runner"; import {range, randomMessage} from "../test/util"; import {generateRandomSecretKey} from "@chainsafe/bls-keygen"; import * as noble from "noble-bls12-381"; - -const aggCount = 30; -const nobleRuns = 10; +import {aggCount, runsNoble} from "./params"; (async function () { // verify @@ -26,13 +24,13 @@ const nobleRuns = 10; testRunner: async ({pk, msg, sig}) => { return await noble.verify(sig, msg, pk); }, - runs: nobleRuns, + runs: runsNoble, }); // Fast aggregate await runBenchmark<{pks: Uint8Array[]; msg: Uint8Array; sig: Uint8Array}, boolean>({ - id: `noble verifyAggregate`, + id: `noble verifyAggregate (${aggCount})`, prepareTest: async () => { const msg = randomMessage(); @@ -57,7 +55,7 @@ const nobleRuns = 10; const pk = noble.aggregatePublicKeys(pks); return await noble.verify(sig, msg, pk); }, - runs: nobleRuns, + runs: runsNoble, }); // Aggregate pubkeys @@ -73,6 +71,6 @@ const nobleRuns = 10; testRunner: async (pks) => { noble.aggregatePublicKeys(pks); }, - runs: nobleRuns, + runs: runsNoble, }); })(); diff --git a/benchmark/params.ts b/benchmark/params.ts new file mode 100644 index 0000000..fa78e69 --- /dev/null +++ b/benchmark/params.ts @@ -0,0 +1,3 @@ +export const aggCount = 30; +export const runs = 100; +export const runsNoble = 10; From 324285c38a9b717d8395bcada33760c86402ddcb Mon Sep 17 00:00:00 2001 From: dapplion Date: Wed, 2 Dec 2020 20:54:46 +0000 Subject: [PATCH 4/5] Run benchmarks for verifyMultiple --- benchmark/index.ts | 29 +++++++++++++++++++++++++++++ benchmark/noble.ts | 31 +++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/benchmark/index.ts b/benchmark/index.ts index 6696642..e936fd3 100644 --- a/benchmark/index.ts +++ b/benchmark/index.ts @@ -55,6 +55,35 @@ import {aggCount, runs} from "./params"; runs, }); + // Verify multiple + + await runBenchmark<{pks: PublicKey[]; msgs: Uint8Array[]; sig: Signature}, boolean>({ + id: `${implementation} verifyMultiple (${aggCount})`, + + prepareTest: () => { + const dataArr = range(aggCount).map(() => { + const sk = bls.SecretKey.fromKeygen(); + const pk = sk.toPublicKey(); + const msg = randomMessage(); + const sig = sk.sign(msg); + return {pk, msg, sig}; + }); + + const pks = dataArr.map((data) => data.pk); + const msgs = dataArr.map((data) => data.msg); + const sig = bls.Signature.aggregate(dataArr.map((data) => data.sig)); + + return { + input: {pks, msgs, sig}, + resultCheck: (valid) => valid === true, + }; + }, + testRunner: ({pks, msgs, sig}) => { + return sig.verifyMultiple(pks, msgs); + }, + runs, + }); + // Aggregate pubkeys await runBenchmark({ diff --git a/benchmark/noble.ts b/benchmark/noble.ts index 9980ae7..3d70d10 100644 --- a/benchmark/noble.ts +++ b/benchmark/noble.ts @@ -58,6 +58,37 @@ import {aggCount, runsNoble} from "./params"; runs: runsNoble, }); + // Verify multiple + + await runBenchmark<{pks: Uint8Array[]; msgs: Uint8Array[]; sig: Uint8Array}, boolean>({ + id: `noble verifyMultiple (${aggCount})`, + + prepareTest: async () => { + const dataArr = await Promise.all( + range(aggCount).map(async () => { + const sk = generateRandomSecretKey(); + const pk = noble.getPublicKey(sk); + const msg = randomMessage(); + const sig = await noble.sign(msg, sk); + return {pk, msg, sig}; + }) + ); + + const pks = dataArr.map((data) => data.pk); + const msgs = dataArr.map((data) => data.msg); + const sig = noble.aggregateSignatures(dataArr.map((data) => data.sig)); + + return { + input: {pks, msgs, sig}, + resultCheck: (valid: boolean) => valid === true, + }; + }, + testRunner: async ({pks, msgs, sig}) => { + return await noble.verifyBatch(msgs, pks, sig); + }, + runs: runsNoble, + }); + // Aggregate pubkeys await runBenchmark({ From b324213292b60e1c0605aad4fb23d6a0c5aa0305 Mon Sep 17 00:00:00 2001 From: dapplion Date: Wed, 2 Dec 2020 21:10:34 +0000 Subject: [PATCH 5/5] Update benchmark results from CI run --- README.md | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index a380685..1230c22 100644 --- a/README.md +++ b/README.md @@ -91,18 +91,20 @@ The API is identical for all implementations. - `herumi`: [src/herumi](src/herumi) - `noble`: [noble-bls12-381](https://github.com/paulmillr/noble-bls12-381) -Results are in `ops/sec`. `blst` and `herumi` performed 100 runs each, `noble` 10 runs. +Results are in `ops/sec (x times slower)`, where `x times slower` = times slower than fastest implementation (`blst`). -| Function - `ops/sec` | `blst` | `herumi` | `noble`\* | -| ------------------------- | :----: | :------: | :-------: | -| `verify` | 502.72 | 53.792 | 13.868 | -| `verifyAggregate` (30) | 489.60 | 52.897 | 11.241 | -| `aggregate` (pubkeys, 30) | 8326.6 | 3020.1 | 47.309 | -| `aggregate` (sigs, 30) | 6968.3 | 1151.2 | - | +| Function - `ops/sec` | `blst` | `herumi` | `noble`\* | +| ------------------------- | :----: | :----------: | :-----------: | +| `verify` | 443.75 | 46.658 (x9) | 12.355 (x36) | +| `verifyAggregate` (30) | 438.68 | 46.615 (x9) | 9.8803 (x44) | +| `verifyMultiple` (30) | 35.138 | 3.4332 (x10) | 0.9217 (x38) | +| `aggregate` (pubkeys, 30) | 15761 | 2603.5 (x6) | 42.956 (x366) | +| `aggregate` (sigs, 30) | 6587.8 | 1018.7 (x6) | - | -\*`noble` methods include serialization and deserialization to bytes, which may impact the `aggregate` benchmark +\*`noble` methods include serialization and deserialization to bytes, which may impact all benchmarks specially `aggregate`. +\*\* `blst` and `herumi` performed 100 runs each, `noble` 10 runs. -Results from CI run https://github.com/ChainSafe/bls/runs/1478915060 +Results from CI run https://github.com/ChainSafe/bls/runs/1488856560?check_suite_focus=true#step:12:13 ## Spec versioning