Merge pull request #57 from ChainSafe/dapplion/benchmarks

Add benchmark results
This commit is contained in:
Lion - dapplion 2020-12-02 23:18:28 +00:00 committed by GitHub
commit e8d81fef96
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 100 additions and 15 deletions

View File

@ -85,14 +85,35 @@ try {
The API is identical for all implementations.
## Benchmarks
- `blst`: [src/blst](src/blst)
- `herumi`: [src/herumi](src/herumi)
- `noble`: [noble-bls12-381](https://github.com/paulmillr/noble-bls12-381)
Results are in `ops/sec (x times slower)`, where `x times slower` = times slower than fastest implementation (`blst`).
| Function - `ops/sec` | `blst` | `herumi` | `noble`\* |
| ------------------------- | :----: | :----------: | :-----------: |
| `verify` | 443.75 | 46.658 (x9) | 12.355 (x36) |
| `verifyAggregate` (30) | 438.68 | 46.615 (x9) | 9.8803 (x44) |
| `verifyMultiple` (30) | 35.138 | 3.4332 (x10) | 0.9217 (x38) |
| `aggregate` (pubkeys, 30) | 15761 | 2603.5 (x6) | 42.956 (x366) |
| `aggregate` (sigs, 30) | 6587.8 | 1018.7 (x6) | - |
\*`noble` methods include serialization and deserialization to bytes, which may impact all benchmarks specially `aggregate`.
\*\* `blst` and `herumi` performed 100 runs each, `noble` 10 runs.
Results from CI run https://github.com/ChainSafe/bls/runs/1488856560?check_suite_focus=true#step:12:13
## Spec versioning
| Version | Bls spec hash-to-curve version |
| ------- | :--------------: |
| 5.x.x | draft #9 |
| 2.x.x | draft #7 |
| 1.x.x | draft #6 |
| 0.3.x | initial version |
| ------- | :----------------------------: |
| 5.x.x | draft #9 |
| 2.x.x | draft #7 |
| 1.x.x | draft #6 |
| 0.3.x | initial version |
> [spec](https://github.com/ethereum/eth2.0-specs/blob/v1.0.0/specs/phase0/beacon-chain.md#bls-signatures)

View File

@ -2,8 +2,7 @@ import {runBenchmark} from "./runner";
import {runForAllImplementations} from "../test/switch";
import {PublicKey, Signature} from "../src/interface";
import {range, randomMessage} from "../test/util";
const aggCount = 30;
import {aggCount, runs} from "./params";
(async function () {
await runForAllImplementations(async (bls, implementation) => {
@ -25,12 +24,13 @@ const aggCount = 30;
testRunner: ({pk, msg, sig}) => {
return sig.verify(pk, msg);
},
runs,
});
// Fast aggregate
await runBenchmark<{pks: PublicKey[]; msg: Uint8Array; sig: Signature}, boolean>({
id: `${implementation} verifyAggregate`,
id: `${implementation} verifyAggregate (${aggCount})`,
prepareTest: () => {
const msg = randomMessage();
@ -52,6 +52,36 @@ const aggCount = 30;
testRunner: ({pks, msg, sig}) => {
return sig.verifyAggregate(pks, msg);
},
runs,
});
// Verify multiple
await runBenchmark<{pks: PublicKey[]; msgs: Uint8Array[]; sig: Signature}, boolean>({
id: `${implementation} verifyMultiple (${aggCount})`,
prepareTest: () => {
const dataArr = range(aggCount).map(() => {
const sk = bls.SecretKey.fromKeygen();
const pk = sk.toPublicKey();
const msg = randomMessage();
const sig = sk.sign(msg);
return {pk, msg, sig};
});
const pks = dataArr.map((data) => data.pk);
const msgs = dataArr.map((data) => data.msg);
const sig = bls.Signature.aggregate(dataArr.map((data) => data.sig));
return {
input: {pks, msgs, sig},
resultCheck: (valid) => valid === true,
};
},
testRunner: ({pks, msgs, sig}) => {
return sig.verifyMultiple(pks, msgs);
},
runs,
});
// Aggregate pubkeys
@ -67,6 +97,7 @@ const aggCount = 30;
testRunner: (pks) => {
bls.PublicKey.aggregate(pks);
},
runs,
});
// Aggregate sigs
@ -87,6 +118,7 @@ const aggCount = 30;
testRunner: (sigs) => {
bls.Signature.aggregate(sigs);
},
runs,
});
});
})();

View File

@ -2,9 +2,7 @@ import {runBenchmark} from "./runner";
import {range, randomMessage} from "../test/util";
import {generateRandomSecretKey} from "@chainsafe/bls-keygen";
import * as noble from "noble-bls12-381";
const aggCount = 30;
const nobleRuns = 10;
import {aggCount, runsNoble} from "./params";
(async function () {
// verify
@ -26,13 +24,13 @@ const nobleRuns = 10;
testRunner: async ({pk, msg, sig}) => {
return await noble.verify(sig, msg, pk);
},
runs: nobleRuns,
runs: runsNoble,
});
// Fast aggregate
await runBenchmark<{pks: Uint8Array[]; msg: Uint8Array; sig: Uint8Array}, boolean>({
id: `noble verifyAggregate`,
id: `noble verifyAggregate (${aggCount})`,
prepareTest: async () => {
const msg = randomMessage();
@ -57,7 +55,38 @@ const nobleRuns = 10;
const pk = noble.aggregatePublicKeys(pks);
return await noble.verify(sig, msg, pk);
},
runs: nobleRuns,
runs: runsNoble,
});
// Verify multiple
await runBenchmark<{pks: Uint8Array[]; msgs: Uint8Array[]; sig: Uint8Array}, boolean>({
id: `noble verifyMultiple (${aggCount})`,
prepareTest: async () => {
const dataArr = await Promise.all(
range(aggCount).map(async () => {
const sk = generateRandomSecretKey();
const pk = noble.getPublicKey(sk);
const msg = randomMessage();
const sig = await noble.sign(msg, sk);
return {pk, msg, sig};
})
);
const pks = dataArr.map((data) => data.pk);
const msgs = dataArr.map((data) => data.msg);
const sig = noble.aggregateSignatures(dataArr.map((data) => data.sig));
return {
input: {pks, msgs, sig},
resultCheck: (valid: boolean) => valid === true,
};
},
testRunner: async ({pks, msgs, sig}) => {
return await noble.verifyBatch(msgs, pks, sig);
},
runs: runsNoble,
});
// Aggregate pubkeys
@ -73,6 +102,6 @@ const nobleRuns = 10;
testRunner: async (pks) => {
noble.aggregatePublicKeys(pks);
},
runs: nobleRuns,
runs: runsNoble,
});
})();

3
benchmark/params.ts Normal file
View File

@ -0,0 +1,3 @@
export const aggCount = 30;
export const runs = 100;
export const runsNoble = 10;