Merge pull request #51 from ChainSafe/dapplion/noble-benchmark

Benchmark noble-bls12-381

.github/workflows/test.yml

@@ -29,4 +29,4 @@ jobs:
         run: yarn test:web
 
       - name: Benchmark
-        run: yarn benchmark
+        run: yarn benchmark:all

package.json

@@ -42,7 +42,8 @@
     "test:spec": "mocha --colors -r ts-node/register 'test/spec/**/*.test.ts'",
     "test": "yarn run test:unit && yarn run test:spec",
     "coverage": "codecov -F bls",
-    "benchmark": "ts-node test/benchmark"
+    "benchmark": "ts-node test/benchmark",
+    "benchmark:all": "cd test/benchmark && yarn install && yarn benchmark"
   },
   "dependencies": {
     "@chainsafe/bls-keygen": "^0.3.0",

test/benchmark/index.ts

@@ -1,100 +1,92 @@
 import {runBenchmark} from "./runner";
 import {runForAllImplementations} from "../switch";
 import {IPublicKey, ISignature} from "../../src/interface";
-import {randomBytes} from "../../src/helpers";
+import {range, randomMessage} from "../util";
 
-runForAllImplementations((bls, implementation) => {
-  const aggCount = 30;
+const aggCount = 30;
 
-  // verify
+(async function () {
+  await runForAllImplementations(async (bls, implementation) => {
+    // verify
 
-  runBenchmark<{pk: IPublicKey; msg: Uint8Array; sig: ISignature}, boolean>({
-    id: `${implementation} verify`,
+    await runBenchmark<{pk: IPublicKey; msg: Uint8Array; sig: ISignature}, boolean>({
+      id: `${implementation} verify`,
 
-    prepareTest: () => {
-      const msg = randomMsg();
-      const sk = bls.SecretKey.fromKeygen();
-      const pk = sk.toPublicKey();
-      const sig = sk.sign(msg);
-      return {
-        input: {pk, msg, sig},
-        resultCheck: (valid) => valid === true,
-      };
-    },
-    testRunner: ({pk, msg, sig}) => {
-      return sig.verify(pk, msg);
-    },
-  });
-
-  // Fast aggregate
-
-  runBenchmark<{pks: IPublicKey[]; msg: Uint8Array; sig: ISignature}, boolean>({
-    id: `${implementation} verifyAggregate`,
-
-    prepareTest: () => {
-      const msg = randomMsg();
-      const dataArr = range(aggCount).map(() => {
+      prepareTest: () => {
+        const msg = randomMessage();
         const sk = bls.SecretKey.fromKeygen();
         const pk = sk.toPublicKey();
         const sig = sk.sign(msg);
-        return {pk, sig};
-      });
+        return {
+          input: {pk, msg, sig},
+          resultCheck: (valid) => valid === true,
+        };
+      },
+      testRunner: ({pk, msg, sig}) => {
+        return sig.verify(pk, msg);
+      },
+    });
 
-      const pks = dataArr.map((data) => data.pk);
-      const sig = bls.Signature.aggregate(dataArr.map((data) => data.sig));
+    // Fast aggregate
 
-      return {
-        input: {pks, msg, sig},
-        resultCheck: (valid) => valid === true,
-      };
-    },
-    testRunner: ({pks, msg, sig}) => {
-      return sig.verifyAggregate(pks, msg);
-    },
+    await runBenchmark<{pks: IPublicKey[]; msg: Uint8Array; sig: ISignature}, boolean>({
+      id: `${implementation} verifyAggregate`,
+
+      prepareTest: () => {
+        const msg = randomMessage();
+        const dataArr = range(aggCount).map(() => {
+          const sk = bls.SecretKey.fromKeygen();
+          const pk = sk.toPublicKey();
+          const sig = sk.sign(msg);
+          return {pk, sig};
+        });
+
+        const pks = dataArr.map((data) => data.pk);
+        const sig = bls.Signature.aggregate(dataArr.map((data) => data.sig));
+
+        return {
+          input: {pks, msg, sig},
+          resultCheck: (valid) => valid === true,
+        };
+      },
+      testRunner: ({pks, msg, sig}) => {
+        return sig.verifyAggregate(pks, msg);
+      },
+    });
+
+    // Aggregate pubkeys
+
+    await runBenchmark<IPublicKey[], void>({
+      id: `${implementation} aggregate pubkeys (${aggCount})`,
+
+      prepareTest: () => {
+        return {
+          input: range(aggCount).map(() => bls.SecretKey.fromKeygen().toPublicKey()),
+        };
+      },
+      testRunner: (pks) => {
+        bls.PublicKey.aggregate(pks);
+      },
+    });
+
+    // Aggregate sigs
+
+    await runBenchmark<ISignature[], void>({
+      id: `${implementation} aggregate signatures (${aggCount})`,
+
+      prepareTest: () => {
+        const msg = randomMessage();
+        const sigs = range(aggCount).map(() => {
+          const sk = bls.SecretKey.fromKeygen();
+          return sk.sign(msg);
+        });
+        return {
+          input: sigs,
+        };
+      },
+      testRunner: (sigs) => {
+        bls.Signature.aggregate(sigs);
+      },
+    });
   });
-
-  // Aggregate pubkeys
-
-  runBenchmark<IPublicKey[], void>({
-    id: `${implementation} aggregate pubkeys (${aggCount})`,
-
-    prepareTest: () => {
-      return {
-        input: range(aggCount).map(() => bls.SecretKey.fromKeygen().toPublicKey()),
-      };
-    },
-    testRunner: (pks) => {
-      bls.PublicKey.aggregate(pks);
-    },
-  });
-
-  // Aggregate sigs
-
-  runBenchmark<ISignature[], void>({
-    id: `${implementation} aggregate signatures (${aggCount})`,
-
-    prepareTest: () => {
-      const msg = randomMsg();
-      const sigs = range(aggCount).map(() => {
-        const sk = bls.SecretKey.fromKeygen();
-        return sk.sign(msg);
-      });
-      return {
-        input: sigs,
-      };
-    },
-    testRunner: (sigs) => {
-      bls.Signature.aggregate(sigs);
-    },
-  });
-});
-
-function range(n: number): number[] {
-  const nums: number[] = [];
-  for (let i = 0; i < n; i++) nums.push(i);
-  return nums;
-}
-
-function randomMsg(): Uint8Array {
-  return randomBytes(32);
-}
+})();

test/benchmark/noble.ts

@@ -0,0 +1,78 @@
+import {runBenchmark} from "./runner";
+import {range, randomMessage} from "../util";
+import {generateRandomSecretKey} from "@chainsafe/bls-keygen";
+import * as noble from "noble-bls12-381";
+
+const aggCount = 30;
+const nobleRuns = 10;
+
+(async function () {
+  // verify
+
+  await runBenchmark<{pk: Uint8Array; msg: Uint8Array; sig: Uint8Array}, boolean>({
+    id: `noble verify`,
+
+    prepareTest: async () => {
+      const priv = generateRandomSecretKey();
+      const msg = randomMessage();
+      const sig = await noble.sign(msg, priv);
+      const pk = noble.getPublicKey(priv);
+
+      return {
+        input: {pk, msg, sig},
+        resultCheck: (valid: boolean) => valid === true,
+      };
+    },
+    testRunner: async ({pk, msg, sig}) => {
+      return await noble.verify(sig, msg, pk);
+    },
+    runs: nobleRuns,
+  });
+
+  // Fast aggregate
+
+  await runBenchmark<{pks: Uint8Array[]; msg: Uint8Array; sig: Uint8Array}, boolean>({
+    id: `noble verifyAggregate`,
+
+    prepareTest: async () => {
+      const msg = randomMessage();
+      const dataArr = await Promise.all(
+        range(aggCount).map(async () => {
+          const sk = generateRandomSecretKey();
+          const pk = noble.getPublicKey(sk);
+          const sig = await noble.sign(msg, sk);
+          return {pk, sig};
+        })
+      );
+
+      const pks = dataArr.map((data) => data.pk);
+      const sig = noble.aggregateSignatures(dataArr.map((data) => data.sig));
+
+      return {
+        input: {pks, msg, sig},
+        resultCheck: (valid: boolean) => valid === true,
+      };
+    },
+    testRunner: async ({pks, msg, sig}) => {
+      const pk = noble.aggregatePublicKeys(pks);
+      return await noble.verify(sig, msg, pk);
+    },
+    runs: nobleRuns,
+  });
+
+  // Aggregate pubkeys
+
+  await runBenchmark<Uint8Array[], void>({
+    id: `noble aggregate pubkeys (${aggCount})`,
+
+    prepareTest: () => {
+      return {
+        input: range(aggCount).map(() => noble.getPublicKey(generateRandomSecretKey())),
+      };
+    },
+    testRunner: async (pks) => {
+      noble.aggregatePublicKeys(pks);
+    },
+    runs: nobleRuns,
+  });
+})();

test/benchmark/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "bls-libs-benchmark",
+  "version": "1.0.0",
+  "main": "index.js",
+  "license": "MIT",
+  "scripts": {
+    "benchmark": "ts-node index && ts-node noble"
+  },
+  "dependencies": {
+    "noble-bls12-381": "^0.6.1"
+  }
+}

test/benchmark/runner.ts

@@ -1,21 +1,23 @@
-export function runBenchmark<T, R>({
+type PromiseOptional<T> = T | Promise<T>;
+
+export async function runBenchmark<T, R>({
   prepareTest,
   testRunner,
   runs = 100,
   id,
 }: {
-  prepareTest: (i: number) => {input: T; resultCheck?: (result: R) => boolean};
-  testRunner: (input: T) => R;
+  prepareTest: (i: number) => PromiseOptional<{input: T; resultCheck?: (result: R) => boolean}>;
+  testRunner: (input: T) => PromiseOptional<R>;
   runs?: number;
   id: string;
-}): void {
+}): Promise<void> {
   const diffsNanoSec: bigint[] = [];
 
   for (let i = 0; i < runs; i++) {
-    const {input, resultCheck} = prepareTest(i);
+    const {input, resultCheck} = await prepareTest(i);
 
     const start = process.hrtime.bigint();
-    const result = testRunner(input);
+    const result = await testRunner(input);
     const end = process.hrtime.bigint();
 
     if (resultCheck && !resultCheck(result)) throw Error("Result fails check test");

test/benchmark/yarn.lock

@@ -0,0 +1,8 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+noble-bls12-381@^0.6.1:
+  version "0.6.1"
+  resolved "https://registry.yarnpkg.com/noble-bls12-381/-/noble-bls12-381-0.6.1.tgz#b44bb5443b4b5c409723f19a8288155f0b3ad126"
+  integrity sha512-Dt0lq24ez75HqOqNIsxxbzfY7YOuwArtE3H6Clp1XbwnY4Ga1OjFbTaXq5aDBE3+ab1wLK11s0b3yR3+RiWWqw==

test/util.ts

@@ -7,3 +7,9 @@ export function randomMessage(): Uint8Array {
 export function getN<T>(n: number, getter: () => T): T[] {
   return Array.from({length: n}, () => getter());
 }
+
+export function range(n: number): number[] {
+  const nums: number[] = [];
+  for (let i = 0; i < n; i++) nums.push(i);
+  return nums;
+}