Merge pull request #57 from ChainSafe/dapplion/benchmarks

Add benchmark results

README.md

@@ -85,14 +85,35 @@ try {
 
 The API is identical for all implementations.
 
+## Benchmarks
+
+- `blst`: [src/blst](src/blst)
+- `herumi`: [src/herumi](src/herumi)
+- `noble`: [noble-bls12-381](https://github.com/paulmillr/noble-bls12-381)
+
+Results are in `ops/sec (x times slower)`, where `x times slower` = times slower than fastest implementation (`blst`).
+
+| Function - `ops/sec`      | `blst` |   `herumi`   |   `noble`\*   |
+| ------------------------- | :----: | :----------: | :-----------: |
+| `verify`                  | 443.75 | 46.658 (x9)  | 12.355 (x36)  |
+| `verifyAggregate` (30)    | 438.68 | 46.615 (x9)  | 9.8803 (x44)  |
+| `verifyMultiple` (30)     | 35.138 | 3.4332 (x10) | 0.9217 (x38)  |
+| `aggregate` (pubkeys, 30) | 15761  | 2603.5 (x6)  | 42.956 (x366) |
+| `aggregate` (sigs, 30)    | 6587.8 | 1018.7 (x6)  |       -       |
+
+\*`noble` methods include serialization and deserialization to bytes, which may impact all benchmarks specially `aggregate`.
+\*\* `blst` and `herumi` performed 100 runs each, `noble` 10 runs.
+
+Results from CI run https://github.com/ChainSafe/bls/runs/1488856560?check_suite_focus=true#step:12:13
+
 ## Spec versioning
 
 | Version | Bls spec hash-to-curve version |
-| ------- | :--------------: |
+| ------- | :----------------------------: |
-| 5.x.x   |     draft #9     |
+| 5.x.x   |            draft #9            |
-| 2.x.x   |     draft #7     |
+| 2.x.x   |            draft #7            |
-| 1.x.x   |     draft #6     |
+| 1.x.x   |            draft #6            |
-| 0.3.x   | initial version  |
+| 0.3.x   |        initial version         |
 
 > [spec](https://github.com/ethereum/eth2.0-specs/blob/v1.0.0/specs/phase0/beacon-chain.md#bls-signatures)
 

benchmark/index.ts

@@ -2,8 +2,7 @@ import {runBenchmark} from "./runner";
 import {runForAllImplementations} from "../test/switch";
 import {PublicKey, Signature} from "../src/interface";
 import {range, randomMessage} from "../test/util";
-
+import {aggCount, runs} from "./params";
-const aggCount = 30;
 
 (async function () {
   await runForAllImplementations(async (bls, implementation) => {
@@ -25,12 +24,13 @@ const aggCount = 30;
       testRunner: ({pk, msg, sig}) => {
         return sig.verify(pk, msg);
       },
+      runs,
     });
 
     // Fast aggregate
 
     await runBenchmark<{pks: PublicKey[]; msg: Uint8Array; sig: Signature}, boolean>({
-      id: `${implementation} verifyAggregate`,
+      id: `${implementation} verifyAggregate (${aggCount})`,
 
       prepareTest: () => {
         const msg = randomMessage();
@@ -52,6 +52,36 @@ const aggCount = 30;
       testRunner: ({pks, msg, sig}) => {
         return sig.verifyAggregate(pks, msg);
       },
+      runs,
+    });
+
+    // Verify multiple
+
+    await runBenchmark<{pks: PublicKey[]; msgs: Uint8Array[]; sig: Signature}, boolean>({
+      id: `${implementation} verifyMultiple (${aggCount})`,
+
+      prepareTest: () => {
+        const dataArr = range(aggCount).map(() => {
+          const sk = bls.SecretKey.fromKeygen();
+          const pk = sk.toPublicKey();
+          const msg = randomMessage();
+          const sig = sk.sign(msg);
+          return {pk, msg, sig};
+        });
+
+        const pks = dataArr.map((data) => data.pk);
+        const msgs = dataArr.map((data) => data.msg);
+        const sig = bls.Signature.aggregate(dataArr.map((data) => data.sig));
+
+        return {
+          input: {pks, msgs, sig},
+          resultCheck: (valid) => valid === true,
+        };
+      },
+      testRunner: ({pks, msgs, sig}) => {
+        return sig.verifyMultiple(pks, msgs);
+      },
+      runs,
     });
 
     // Aggregate pubkeys
@@ -67,6 +97,7 @@ const aggCount = 30;
       testRunner: (pks) => {
         bls.PublicKey.aggregate(pks);
       },
+      runs,
     });
 
     // Aggregate sigs
@@ -87,6 +118,7 @@ const aggCount = 30;
       testRunner: (sigs) => {
         bls.Signature.aggregate(sigs);
       },
+      runs,
     });
   });
 })();

benchmark/noble.ts

@@ -2,9 +2,7 @@ import {runBenchmark} from "./runner";
 import {range, randomMessage} from "../test/util";
 import {generateRandomSecretKey} from "@chainsafe/bls-keygen";
 import * as noble from "noble-bls12-381";
-
+import {aggCount, runsNoble} from "./params";
-const aggCount = 30;
-const nobleRuns = 10;
 
 (async function () {
   // verify
@@ -26,13 +24,13 @@ const nobleRuns = 10;
     testRunner: async ({pk, msg, sig}) => {
       return await noble.verify(sig, msg, pk);
     },
-    runs: nobleRuns,
+    runs: runsNoble,
   });
 
   // Fast aggregate
 
   await runBenchmark<{pks: Uint8Array[]; msg: Uint8Array; sig: Uint8Array}, boolean>({
-    id: `noble verifyAggregate`,
+    id: `noble verifyAggregate (${aggCount})`,
 
     prepareTest: async () => {
       const msg = randomMessage();
@@ -57,7 +55,38 @@ const nobleRuns = 10;
       const pk = noble.aggregatePublicKeys(pks);
       return await noble.verify(sig, msg, pk);
     },
-    runs: nobleRuns,
+    runs: runsNoble,
+  });
+
+  // Verify multiple
+
+  await runBenchmark<{pks: Uint8Array[]; msgs: Uint8Array[]; sig: Uint8Array}, boolean>({
+    id: `noble verifyMultiple (${aggCount})`,
+
+    prepareTest: async () => {
+      const dataArr = await Promise.all(
+        range(aggCount).map(async () => {
+          const sk = generateRandomSecretKey();
+          const pk = noble.getPublicKey(sk);
+          const msg = randomMessage();
+          const sig = await noble.sign(msg, sk);
+          return {pk, msg, sig};
+        })
+      );
+
+      const pks = dataArr.map((data) => data.pk);
+      const msgs = dataArr.map((data) => data.msg);
+      const sig = noble.aggregateSignatures(dataArr.map((data) => data.sig));
+
+      return {
+        input: {pks, msgs, sig},
+        resultCheck: (valid: boolean) => valid === true,
+      };
+    },
+    testRunner: async ({pks, msgs, sig}) => {
+      return await noble.verifyBatch(msgs, pks, sig);
+    },
+    runs: runsNoble,
   });
 
   // Aggregate pubkeys
@@ -73,6 +102,6 @@ const nobleRuns = 10;
     testRunner: async (pks) => {
       noble.aggregatePublicKeys(pks);
     },
-    runs: nobleRuns,
+    runs: runsNoble,
   });
 })();

benchmark/params.ts

@@ -0,0 +1,3 @@
+export const aggCount = 30;
+export const runs = 100;
+export const runsNoble = 10;