Merge pull request #86 from ChainSafe/dapplion/speedup-benchmark

Speedup benchmarks

benchmark/index.ts

@@ -1,166 +1,84 @@
 import {runBenchmark} from "./runner";
 import {runForAllImplementations} from "../test/switch";
-import {PublicKey, Signature, SecretKey} from "../src/interface";
+import {PublicKey, Signature} from "../src/interface";
-import {range, randomMessage} from "../test/util";
+import {aggCount} from "./params";
-import {aggCount, runs} from "./params";
 
 (async function () {
   await runForAllImplementations(async (bls, implementation) => {
+    const msgSame = Buffer.alloc(32, 255);
+    const sameMessage: {pk: PublicKey; msg: Uint8Array; sig: Signature}[] = [];
+    const diffMessage: {pk: PublicKey; msg: Uint8Array; sig: Signature}[] = [];
+
+    for (let i = 0; i < aggCount; i++) {
+      const msg = Buffer.alloc(32, i + 1);
+      const sk = bls.SecretKey.fromBytes(Buffer.alloc(32, i + 1));
+      const pk = sk.toPublicKey();
+      sameMessage.push({pk, msg: msgSame, sig: sk.sign(msgSame)});
+      diffMessage.push({pk, msg, sig: sk.sign(msg)});
+    }
+    const {pk, msg, sig} = diffMessage[0];
+    const sameMessageSig = bls.Signature.aggregate(sameMessage.map((s) => s.sig));
+    const diffMessageSig = bls.Signature.aggregate(diffMessage.map((s) => s.sig));
+
     // verify
 
-    await runBenchmark<{pk: PublicKey; msg: Uint8Array; sig: Signature}, boolean>({
+    await runBenchmark({
       id: `${implementation} verify`,
-
+      prepareTest: () => ({pk, msg, sig}),
-      prepareTest: () => {
+      testRunner: ({pk, msg, sig}) => sig.verify(pk, msg),
-        const msg = randomMessage();
-        const sk = bls.SecretKey.fromKeygen();
-        const pk = sk.toPublicKey();
-        const sig = sk.sign(msg);
-        return {
-          input: {pk, msg, sig},
-          resultCheck: (valid) => valid === true,
-        };
-      },
-      testRunner: ({pk, msg, sig}) => {
-        return sig.verify(pk, msg);
-      },
-      runs,
     });
 
     // Fast aggregate
 
-    await runBenchmark<{pks: PublicKey[]; msg: Uint8Array; sig: Signature}, boolean>({
+    await runBenchmark({
       id: `${implementation} verifyAggregate (${aggCount})`,
-
+      prepareTest: () => ({pks: sameMessage.map((s) => s.pk), msg: msgSame, sig: sameMessageSig}),
-      prepareTest: () => {
+      testRunner: ({pks, msg, sig}) => sig.verifyAggregate(pks, msg),
-        const msg = randomMessage();
-        const dataArr = range(aggCount).map(() => {
-          const sk = bls.SecretKey.fromKeygen();
-          const pk = sk.toPublicKey();
-          const sig = sk.sign(msg);
-          return {pk, sig};
-        });
-
-        const pks = dataArr.map((data) => data.pk);
-        const sig = bls.Signature.aggregate(dataArr.map((data) => data.sig));
-
-        return {
-          input: {pks, msg, sig},
-          resultCheck: (valid) => valid === true,
-        };
-      },
-      testRunner: ({pks, msg, sig}) => {
-        return sig.verifyAggregate(pks, msg);
-      },
-      runs,
     });
 
     // Verify multiple
 
-    await runBenchmark<{pks: PublicKey[]; msgs: Uint8Array[]; sig: Signature}, boolean>({
+    await runBenchmark({
       id: `${implementation} verifyMultiple (${aggCount})`,
-
+      prepareTest: () => ({
-      prepareTest: () => {
+        pks: diffMessage.map((s) => s.pk),
-        const dataArr = range(aggCount).map(() => {
+        msgs: diffMessage.map((s) => s.msg),
-          const sk = bls.SecretKey.fromKeygen();
+        sig: diffMessageSig,
-          const pk = sk.toPublicKey();
+      }),
-          const msg = randomMessage();
+      testRunner: ({pks, msgs, sig}) => sig.verifyMultiple(pks, msgs),
-          const sig = sk.sign(msg);
-          return {pk, msg, sig};
-        });
-
-        const pks = dataArr.map((data) => data.pk);
-        const msgs = dataArr.map((data) => data.msg);
-        const sig = bls.Signature.aggregate(dataArr.map((data) => data.sig));
-
-        return {
-          input: {pks, msgs, sig},
-          resultCheck: (valid) => valid === true,
-        };
-      },
-      testRunner: ({pks, msgs, sig}) => {
-        return sig.verifyMultiple(pks, msgs);
-      },
-      runs,
     });
 
     // Verify multiple signatures
 
     await runBenchmark({
       id: `${implementation} verifyMultipleSignatures (${aggCount})`,
-
+      prepareTest: () => diffMessage,
-      prepareTest: () => {
+      testRunner: (sets) =>
-        const sets = range(aggCount).map(() => {
+        bls.Signature.verifyMultipleSignatures(sets.map((s) => ({publicKey: s.pk, message: s.msg, signature: s.sig}))),
-          const sk = bls.SecretKey.fromKeygen();
-          const pk = sk.toPublicKey();
-          const msg = randomMessage();
-          const sig = sk.sign(msg);
-          return {publicKey: pk, message: msg, signature: sig};
-        });
-
-        return {
-          input: sets,
-          resultCheck: (valid) => valid === true,
-        };
-      },
-      testRunner: (sets) => {
-        return bls.Signature.verifyMultipleSignatures(sets);
-      },
-      runs,
     });
 
     // Aggregate pubkeys
 
-    await runBenchmark<PublicKey[], void>({
+    await runBenchmark({
       id: `${implementation} aggregate pubkeys (${aggCount})`,
-
+      prepareTest: () => diffMessage.map((s) => s.pk),
-      prepareTest: () => {
+      testRunner: (pks) => bls.PublicKey.aggregate(pks),
-        return {
-          input: range(aggCount).map(() => bls.SecretKey.fromKeygen().toPublicKey()),
-        };
-      },
-      testRunner: (pks) => {
-        bls.PublicKey.aggregate(pks);
-      },
-      runs,
     });
 
     // Aggregate sigs
 
-    await runBenchmark<Signature[], void>({
+    await runBenchmark({
       id: `${implementation} aggregate signatures (${aggCount})`,
-
+      prepareTest: () => diffMessage.map((s) => s.sig),
-      prepareTest: () => {
+      testRunner: (sigs) => bls.Signature.aggregate(sigs),
-        const msg = randomMessage();
-        const sigs = range(aggCount).map(() => {
-          const sk = bls.SecretKey.fromKeygen();
-          return sk.sign(msg);
-        });
-        return {
-          input: sigs,
-        };
-      },
-      testRunner: (sigs) => {
-        bls.Signature.aggregate(sigs);
-      },
-      runs,
     });
 
     // Sign
 
-    await runBenchmark<{sk: SecretKey; msg: Uint8Array}, void>({
+    await runBenchmark({
       id: `${implementation} sign`,
-
+      prepareTest: () => ({sk: bls.SecretKey.fromKeygen(), msg: msgSame}),
-      prepareTest: () => ({
+      testRunner: ({sk, msg}) => sk.sign(msg),
-        input: {
-          sk: bls.SecretKey.fromKeygen(),
-          msg: randomMessage(),
-        },
-      }),
-      testRunner: ({sk, msg}) => {
-        sk.sign(msg);
-      },
-      runs,
     });
   });
 })();

benchmark/noble.ts

@@ -5,133 +5,105 @@ import * as noble from "noble-bls12-381";
 import {aggCount, runsNoble} from "./params";
 
 (async function () {
-  // verify
+  {
+    // verify
 
-  await runBenchmark<{pk: noble.PointG1; msg: noble.PointG2; sig: noble.PointG2}, boolean>({
+    const priv = generateRandomSecretKey();
-    id: `noble verify`,
+    const msg = randomMessage();
+    const pk = noble.PointG1.fromPrivateKey(priv);
+    const sig = noble.PointG2.fromSignature(await noble.sign(msg, priv));
 
-    prepareTest: async () => {
+    await runBenchmark<{pk: noble.PointG1; msg: noble.PointG2; sig: noble.PointG2}, boolean>({
-      const priv = generateRandomSecretKey();
+      id: `noble verify`,
-      const msg = randomMessage();
+      prepareTest: async () => ({pk, msg: await noble.PointG2.hashToCurve(msg), sig}),
-      const pk = noble.PointG1.fromPrivateKey(priv);
+      testRunner: async ({pk, msg, sig}) => await noble.verify(sig, msg, pk),
-      const sig = noble.PointG2.fromSignature(await noble.sign(msg, priv));
+      runs: runsNoble,
+    });
+  }
 
-      return {
+  {
-        input: {pk, msg: await noble.PointG2.hashToCurve(msg), sig},
+    // Fast aggregate
-        resultCheck: (valid: boolean) => valid === true,
-      };
-    },
-    testRunner: async ({pk, msg, sig}) => {
-      return await noble.verify(sig, msg, pk);
-    },
-    runs: runsNoble,
-  });
 
-  // Fast aggregate
+    const msg = randomMessage();
+    const dataArr = await Promise.all(
+      range(aggCount).map(async () => {
+        const sk = generateRandomSecretKey();
+        const pk = noble.PointG1.fromPrivateKey(sk);
+        const sig = noble.PointG2.fromSignature(await noble.sign(msg, sk));
+        return {pk, sig};
+      })
+    );
 
-  await runBenchmark<{pks: noble.PointG1[]; msg: noble.PointG2; sig: noble.PointG2}, boolean>({
+    const pks = dataArr.map((data) => data.pk);
-    id: `noble verifyAggregate (${aggCount})`,
+    const sig = (noble.aggregateSignatures(dataArr.map((data) => data.sig)) as any) as noble.PointG2;
 
-    prepareTest: async () => {
+    await runBenchmark({
-      const msg = randomMessage();
+      id: `noble verifyAggregate (${aggCount})`,
-      const dataArr = await Promise.all(
+      prepareTest: async () => ({pks, msg: await noble.PointG2.hashToCurve(msg), sig}),
-        range(aggCount).map(async () => {
+      testRunner: async ({pks, msg, sig}) =>
-          const sk = generateRandomSecretKey();
+        await noble.verify(sig, msg, (noble.aggregatePublicKeys(pks) as any) as noble.PointG1),
-          const pk = noble.PointG1.fromPrivateKey(sk);
+      runs: runsNoble,
-          const sig = noble.PointG2.fromSignature(await noble.sign(msg, sk));
+    });
-          return {pk, sig};
+  }
-        })
-      );
 
-      const pks = dataArr.map((data) => data.pk);
+  {
-      const sig = noble.aggregateSignatures(dataArr.map((data) => data.sig)) as any as noble.PointG2;
+    // Verify multiple
 
-      return {
+    const dataArr = await Promise.all(
-        input: {pks, msg: await noble.PointG2.hashToCurve(msg), sig},
+      range(aggCount).map(async () => {
-        resultCheck: (valid: boolean) => valid === true,
+        const sk = generateRandomSecretKey();
-      };
+        const pk = noble.PointG1.fromPrivateKey(sk);
-    },
+        const msg = randomMessage();
-    testRunner: async ({pks, msg, sig}) => {
+        const sig = noble.PointG2.fromSignature(await noble.sign(msg, sk));
-      const pk = noble.aggregatePublicKeys(pks) as any as noble.PointG1;
+        return {pk, msg: await noble.PointG2.hashToCurve(msg), sig};
-      return await noble.verify(sig, msg, pk);
+      })
-    },
+    );
-    runs: runsNoble,
-  });
 
-  // // Verify multiple
+    const pks = dataArr.map((data) => data.pk);
+    const msgs = dataArr.map((data) => data.msg);
+    const sig = (noble.aggregateSignatures(dataArr.map((data) => data.sig)) as any) as noble.PointG2;
 
-  await runBenchmark<{pks: noble.PointG1[]; msgs: noble.PointG2[]; sig: noble.PointG2}, boolean>({
+    await runBenchmark({
-    id: `noble verifyMultiple (${aggCount})`,
+      id: `noble verifyMultiple (${aggCount})`,
+      prepareTest: async () => ({pks, msgs, sig}),
+      testRunner: async ({pks, msgs, sig}) => await noble.verifyBatch(msgs, pks, sig),
+      runs: runsNoble,
+    });
+  }
 
-    prepareTest: async () => {
+  {
-      const dataArr = await Promise.all(
+    // Aggregate pubkeys
-        range(aggCount).map(async () => {
-          const sk = generateRandomSecretKey();
-          const pk = noble.PointG1.fromPrivateKey(sk);
-          const msg = randomMessage();
-          const sig = noble.PointG2.fromSignature(await noble.sign(msg, sk));
-          return {pk, msg: await noble.PointG2.hashToCurve(msg), sig};
-        })
-      );
 
-      const pks = dataArr.map((data) => data.pk);
+    const pubkeys = range(aggCount).map(() => noble.PointG1.fromPrivateKey(generateRandomSecretKey()));
-      const msgs = dataArr.map((data) => data.msg);
-      const sig = noble.aggregateSignatures(dataArr.map((data) => data.sig)) as any as noble.PointG2;
 
-      return {
+    await runBenchmark({
-        input: {pks, msgs, sig},
+      id: `noble aggregate pubkeys (${aggCount})`,
-        resultCheck: (valid: boolean) => valid === true,
+      prepareTest: () => pubkeys,
-      };
+      testRunner: async (pks) => noble.aggregatePublicKeys(pks),
-    },
+      runs: runsNoble,
-    testRunner: async ({pks, msgs, sig}) => {
+    });
-      return await noble.verifyBatch(msgs, pks, sig);
+  }
-    },
-    runs: runsNoble,
-  });
 
-  // Aggregate pubkeys
+  const hashes = await Promise.all(
+    range(aggCount)
+      .map(() => generateRandomSecretKey())
+      .map(noble.PointG2.hashToCurve)
+  );
 
-  await runBenchmark<noble.PointG1[], void>({
+  await runBenchmark({
-    id: `noble aggregate pubkeys (${aggCount})`,
-
-    prepareTest: () => {
-      return {
-        input: range(aggCount).map(() => noble.PointG1.fromPrivateKey(generateRandomSecretKey())),
-      };
-    },
-    testRunner: async (pks) => {
-      noble.aggregatePublicKeys(pks);
-    },
-    runs: runsNoble,
-  });
-
-  await runBenchmark<noble.PointG2[], void>({
     id: `noble aggregate signatures (${aggCount})`,
-
+    prepareTest: () => hashes,
-    prepareTest: async () => {
+    testRunner: async (sigs) => noble.aggregateSignatures(sigs),
-      const hashes = range(aggCount).map(() => generateRandomSecretKey()).map(noble.PointG2.hashToCurve);
-      return {
-        input: await Promise.all(hashes),
-      };
-    },
-    testRunner: async (sigs) => {
-      noble.aggregateSignatures(sigs);
-    },
     runs: runsNoble,
   });
 
-  await runBenchmark<{sk: Uint8Array; msg: noble.PointG2}, void>({
+  const sk = generateRandomSecretKey();
-    id: `noble sign`,
+  const msg = await noble.PointG2.hashToCurve(randomMessage());
 
-    prepareTest: async () => ({
+  await runBenchmark({
-      input: {
+    id: `noble sign`,
-        sk: generateRandomSecretKey(),
+    prepareTest: () => ({sk, msg}),
-        msg: await noble.PointG2.hashToCurve(randomMessage()),
+    testRunner: async ({sk, msg}) => await noble.sign(msg, sk),
-      },
-    }),
-    testRunner: async ({sk, msg}) => {
-      await noble.sign(msg, sk);
-    },
     runs: runsNoble,
   });
 })();

benchmark/runner.ts

@@ -6,7 +6,7 @@ export async function runBenchmark<T, R>({
   runs = 100,
   id,
 }: {
-  prepareTest: (i: number) => PromiseOptional<{input: T; resultCheck?: (result: R) => boolean}>;
+  prepareTest: (i: number) => PromiseOptional<T>;
   testRunner: (input: T) => PromiseOptional<R>;
   runs?: number;
   id: string;
@@ -14,13 +14,12 @@ export async function runBenchmark<T, R>({
   const diffsNanoSec: bigint[] = [];
 
   for (let i = 0; i < runs; i++) {
-    const {input, resultCheck} = await prepareTest(i);
+    const input = await prepareTest(i);
 
     const start = process.hrtime.bigint();
     const result = await testRunner(input);
     const end = process.hrtime.bigint();
 
-    if (resultCheck && !resultCheck(result)) throw Error("Result fails check test");
     diffsNanoSec.push(end - start);
   }