feat(benches): Criterion + Iai Benchmarking [RFC] (#131)

* ⚙️ benches

* 📝 docs

* 🏗️ file_db benches and checkpoint fixes

* 🔨 fix github action env vars

* ✅ benchmark env vars

* ⚙️ sync benchmarks

* ✨ cargo fmt touchups

.github/workflows/test.yml

@@ -6,12 +6,15 @@ on:
   pull_request:
     branches: [ "master" ]
 
+env:
+  MAINNET_RPC_URL: ${{ secrets.MAINNET_RPC_URL }}
+  GOERLI_RPC_URL: ${{ secrets.GOERLI_RPC_URL }}
+
 jobs:
   check:
-    name: check
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
@@ -23,10 +26,9 @@ jobs:
           command: check
 
   test:
-    name: test
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
@@ -39,10 +41,9 @@ jobs:
           args: --all
 
   fmt:
-    name: fmt
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
@@ -55,11 +56,25 @@ jobs:
           command: fmt
           args: --all -- --check
 
-  clippy:
-    name: clippy
+  benches:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly
+          override: true
+          components: rustfmt
+      - uses: Swatinem/rust-cache@v2
+      - uses: actions-rs/cargo@v1
+        with:
+          command: bench
+
+  clippy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal

.gitignore

@@ -1 +1,3 @@
 /target
+
+*.env

Cargo.toml

@@ -1,10 +1,10 @@
 [package]
 name = "helios"
-version = "0.1.1"
+version = "0.1.2"
 edition = "2021"
+autobenches = false
 
 [workspace]
-
 members = [
     "cli",
     "client",
@@ -28,6 +28,15 @@ home = "0.5.4"
 ethers = "1.0.2"
 env_logger = "0.9.0"
 log = "0.4.17"
+tracing-test = "0.2.3"
+criterion = { version = "0.4", features = [ "async_tokio", "plotters" ]}
+plotters = "0.3.3"
+tempfile = "3.3.0"
+hex = "0.4.3"
+
+######################################
+# Examples
+######################################
 
 [[example]]
 name = "checkpoints"
@@ -45,3 +54,22 @@ path = "examples/client.rs"
 name = "config"
 path = "examples/config.rs"
 
+######################################
+# Benchmarks
+######################################
+
+[[bench]]
+name = "file_db"
+harness = false
+
+[[bench]]
+name = "get_balance"
+harness = false
+
+[[bench]]
+name = "get_code"
+harness = false
+
+[[bench]]
+name = "sync"
+harness = false

README.md

@@ -8,7 +8,6 @@ Helios converts an untrusted centralized RPC endpoint into a safe unmanipulable
 
 The entire size of Helios's binary is 13Mb and should be easy to compile into WebAssembly. This makes it a perfect target to embed directly inside wallets and dapps.
 
-
 ## Installing
 
 First install `heliosup`, Helios's installer:
@@ -19,7 +18,6 @@ curl https://raw.githubusercontent.com/a16z/helios/master/heliosup/install | bas
 
 To install Helios, run `heliosup`.
 
-
 ## Usage
 
 To run Helios, run the below command, replacing `$ETH_RPC_URL` with an RPC provider URL such as Alchemy or Infura:
@@ -40,7 +38,6 @@ Helios also provides documentation of its supported RPC methods in the [rpc.md](
 
 Helios is still experimental software. While we hope you try it out, we do not suggest adding it as your main RPC in wallets yet. Sending high-value transactions from a wallet connected to Helios is discouraged.
 
-
 ### Additional Options
 
 `--consensus-rpc` or `-c` can be used to set a custom consensus layer rpc endpoint. This must be a consenus node that supports the light client beaconchain api. We recommend using Nimbus for this. If no consensus rpc is supplied, it defaults to `https://www.lightclientdata.org` which is run by us.
@@ -68,7 +65,6 @@ This can be run like so: `helios --load-external-fallback` (or `helios -l` with
 
 `--help` or `-h` prints the help message.
 
-
 ### Configuration Files
 
 All configuration options can be set on a per-network level in `~/.helios/helios.toml`. Here is an example config file:
@@ -85,7 +81,6 @@ execution_rpc = "https://eth-goerli.g.alchemy.com/v2/XXXXX"
 checkpoint = "0xb5c375696913865d7c0e166d87bc7c772b6210dc9edf149f4c7ddc6da0dd4495"
 ```
 
-
 ### Using Helios as a Library
 
 Helios can be imported into any Rust project. Helios requires the Rust nightly toolchain to compile.
@@ -148,16 +143,20 @@ async fn main() -> Result<()> {
 }
 ```
 
+## Benchmarks
+
+Benchmarks are defined in the [benches](./benches/) subdirectory. They are built using the [criterion](https://github.com/bheisler/criterion.rs) statistics-driven benchmarking library.
+
+To run all benchmarks, you can use `cargo bench`. To run a specific benchmark, you can use `cargo bench --bench <name>`, where `<name>` is one of the benchmarks defined in the [Cargo.toml](./Cargo.toml) file under a `[[bench]]` section.
+
 ## Contributing
 
 All contributions to Helios are welcome. Before opening a PR, please submit an issue detailing the bug or feature. When opening a PR, please ensure that your contribution builds on the nightly rust toolchain, has been linted with `cargo fmt`, and contains tests when applicable.
 
-
 ## Telegram
 
 If you are having trouble with Helios or are considering contributing, feel free to join our telegram [here](https://t.me/+IntDY_gZJSRkNTJj).
 
-
 ## Disclaimer
 
 _This code is being provided as is. No guarantee, representation or warranty is being made, express or implied, as to the safety or correctness of the code. It has not been audited and as such there can be no assurance it will work as intended, and users may experience delays, failures, errors, omissions or loss of transmitted information. Nothing in this repo should be construed as investment advice or legal advice for any particular facts or circumstances and is not meant to replace competent counsel. It is strongly advised for you to contact a reputable attorney in your jurisdiction for any questions or concerns with respect thereto. a16z is not liable for any use of the foregoing, and users should proceed with caution and use at their own risk. See a16z.com/disclosures for more info._

benches/file_db.rs

@@ -0,0 +1,43 @@
+use client::database::Database;
+use criterion::{criterion_group, criterion_main, Criterion};
+use helios::prelude::FileDB;
+use tempfile::tempdir;
+
+mod harness;
+
+criterion_main!(file_db);
+criterion_group! {
+    name = file_db;
+    config = Criterion::default();
+    targets = save_checkpoint, load_checkpoint
+}
+
+/// Benchmark saving/writing a checkpoint to the file db.
+pub fn save_checkpoint(c: &mut Criterion) {
+    c.bench_function("save_checkpoint", |b| {
+        let checkpoint = vec![1, 2, 3];
+        b.iter(|| {
+            let temp_dir = tempdir().unwrap().into_path();
+            let db = FileDB::new(temp_dir);
+            db.save_checkpoint(checkpoint.clone()).unwrap();
+        })
+    });
+}
+
+/// Benchmark loading a checkpoint from the file db.
+pub fn load_checkpoint(c: &mut Criterion) {
+    c.bench_function("load_checkpoint", |b| {
+        // First write to the db
+        let temp_dir = tempdir().unwrap().into_path();
+        let db = FileDB::new(temp_dir.clone());
+        let written_checkpoint = vec![1, 2, 3];
+        db.save_checkpoint(written_checkpoint.clone()).unwrap();
+
+        // Then read from the db
+        b.iter(|| {
+            let db = FileDB::new(temp_dir.clone());
+            let checkpoint = db.load_checkpoint().unwrap();
+            assert_eq!(checkpoint, written_checkpoint.clone());
+        })
+    });
+}

benches/get_balance.rs

@@ -0,0 +1,63 @@
+use criterion::{criterion_group, criterion_main, Criterion};
+use ethers::prelude::*;
+use helios::types::BlockTag;
+use std::str::FromStr;
+
+mod harness;
+
+criterion_main!(get_balance);
+criterion_group! {
+    name = get_balance;
+    config = Criterion::default().sample_size(10);
+    targets = bench_mainnet_get_balance, bench_goerli_get_balance
+}
+
+/// Benchmark mainnet get balance.
+/// Address: 0x00000000219ab540356cbb839cbe05303d7705fa (beacon chain deposit address)
+pub fn bench_mainnet_get_balance(c: &mut Criterion) {
+    c.bench_function("get_balance", |b| {
+        // Create a new multi-threaded tokio runtime.
+        let rt = tokio::runtime::Builder::new_multi_thread()
+            .enable_all()
+            .build()
+            .unwrap();
+
+        // Construct a mainnet client using our harness and tokio runtime.
+        let client = std::sync::Arc::new(harness::construct_mainnet_client(&rt).unwrap());
+
+        // Get the beacon chain deposit contract address.
+        let addr = Address::from_str("0x00000000219ab540356cbb839cbe05303d7705fa").unwrap();
+        let block = BlockTag::Latest;
+
+        // Execute the benchmark asynchronously.
+        b.to_async(rt).iter(|| async {
+            let inner = std::sync::Arc::clone(&client);
+            inner.get_balance(&addr, block).await.unwrap()
+        })
+    });
+}
+
+/// Benchmark goerli get balance.
+/// Address: 0xB4FBF271143F4FBf7B91A5ded31805e42b2208d6 (goerli weth)
+pub fn bench_goerli_get_balance(c: &mut Criterion) {
+    c.bench_function("get_balance", |b| {
+        // Create a new multi-threaded tokio runtime.
+        let rt = tokio::runtime::Builder::new_multi_thread()
+            .enable_all()
+            .build()
+            .unwrap();
+
+        // Construct a goerli client using our harness and tokio runtime.
+        let client = std::sync::Arc::new(harness::construct_goerli_client(&rt).unwrap());
+
+        // Get the beacon chain deposit contract address.
+        let addr = Address::from_str("0xB4FBF271143F4FBf7B91A5ded31805e42b2208d6").unwrap();
+        let block = BlockTag::Latest;
+
+        // Execute the benchmark asynchronously.
+        b.to_async(rt).iter(|| async {
+            let inner = std::sync::Arc::clone(&client);
+            inner.get_balance(&addr, block).await.unwrap()
+        })
+    });
+}

benches/get_code.rs

@@ -0,0 +1,63 @@
+use criterion::{criterion_group, criterion_main, Criterion};
+use ethers::prelude::*;
+use helios::types::BlockTag;
+use std::str::FromStr;
+
+mod harness;
+
+criterion_main!(get_code);
+criterion_group! {
+    name = get_code;
+    config = Criterion::default().sample_size(10);
+    targets = bench_mainnet_get_code, bench_goerli_get_code
+}
+
+/// Benchmark mainnet get code call.
+/// Address: 0x00000000219ab540356cbb839cbe05303d7705fa (beacon chain deposit address)
+pub fn bench_mainnet_get_code(c: &mut Criterion) {
+    c.bench_function("get_code", |b| {
+        // Create a new multi-threaded tokio runtime.
+        let rt = tokio::runtime::Builder::new_multi_thread()
+            .enable_all()
+            .build()
+            .unwrap();
+
+        // Construct a mainnet client using our harness and tokio runtime.
+        let client = std::sync::Arc::new(harness::construct_mainnet_client(&rt).unwrap());
+
+        // Get the beacon chain deposit contract address.
+        let addr = Address::from_str("0x00000000219ab540356cbb839cbe05303d7705fa").unwrap();
+        let block = BlockTag::Latest;
+
+        // Execute the benchmark asynchronously.
+        b.to_async(rt).iter(|| async {
+            let inner = std::sync::Arc::clone(&client);
+            inner.get_code(&addr, block).await.unwrap()
+        })
+    });
+}
+
+/// Benchmark goerli get code call.
+/// Address: 0xB4FBF271143F4FBf7B91A5ded31805e42b2208d6 (goerli weth)
+pub fn bench_goerli_get_code(c: &mut Criterion) {
+    c.bench_function("get_code", |b| {
+        // Create a new multi-threaded tokio runtime.
+        let rt = tokio::runtime::Builder::new_multi_thread()
+            .enable_all()
+            .build()
+            .unwrap();
+
+        // Construct a goerli client using our harness and tokio runtime.
+        let client = std::sync::Arc::new(harness::construct_goerli_client(&rt).unwrap());
+
+        // Get the beacon chain deposit contract address.
+        let addr = Address::from_str("0xB4FBF271143F4FBf7B91A5ded31805e42b2208d6").unwrap();
+        let block = BlockTag::Latest;
+
+        // Execute the benchmark asynchronously.
+        b.to_async(rt).iter(|| async {
+            let inner = std::sync::Arc::clone(&client);
+            inner.get_code(&addr, block).await.unwrap()
+        })
+    });
+}

benches/harness.rs

@@ -0,0 +1,113 @@
+#![allow(dead_code)]
+
+use std::{str::FromStr, sync::Arc};
+
+use ::client::Client;
+use ethers::{
+    abi::Address,
+    types::{H256, U256},
+};
+use helios::{client, config::networks, prelude::FileDB, types::BlockTag};
+
+/// Fetches the latest mainnet checkpoint from the fallback service.
+///
+/// Uses the [CheckpointFallback](config::CheckpointFallback).
+/// The `build` method will fetch a list of [CheckpointFallbackService](config::CheckpointFallbackService)s from a community-mainained list by ethPandaOps.
+/// This list is NOT guaranteed to be secure, but is provided in good faith.
+/// The raw list can be found here: https://github.com/ethpandaops/checkpoint-sync-health-checks/blob/master/_data/endpoints.yaml
+pub async fn fetch_mainnet_checkpoint() -> eyre::Result<H256> {
+    let cf = config::CheckpointFallback::new().build().await.unwrap();
+    cf.fetch_latest_checkpoint(&networks::Network::MAINNET)
+        .await
+}
+
+/// Constructs a mainnet [Client](client::Client) for benchmark usage.
+///
+/// Requires a [Runtime](tokio::runtime::Runtime) to be passed in by reference.
+/// The client is parameterized with a [FileDB](client::FileDB).
+/// It will also use the environment variable `MAINNET_RPC_URL` to connect to a mainnet node.
+/// The client will use `https://www.lightclientdata.org` as the consensus RPC.
+pub fn construct_mainnet_client(
+    rt: &tokio::runtime::Runtime,
+) -> eyre::Result<client::Client<client::FileDB>> {
+    rt.block_on(inner_construct_mainnet_client())
+}
+
+pub async fn inner_construct_mainnet_client() -> eyre::Result<client::Client<client::FileDB>> {
+    let benchmark_rpc_url = std::env::var("MAINNET_RPC_URL")?;
+    let mut client = client::ClientBuilder::new()
+        .network(networks::Network::MAINNET)
+        .consensus_rpc("https://www.lightclientdata.org")
+        .execution_rpc(&benchmark_rpc_url)
+        .load_external_fallback()
+        .build()?;
+    client.start().await?;
+    Ok(client)
+}
+
+pub async fn construct_mainnet_client_with_checkpoint(
+    checkpoint: &str,
+) -> eyre::Result<client::Client<client::FileDB>> {
+    let benchmark_rpc_url = std::env::var("MAINNET_RPC_URL")?;
+    let mut client = client::ClientBuilder::new()
+        .network(networks::Network::MAINNET)
+        .consensus_rpc("https://www.lightclientdata.org")
+        .execution_rpc(&benchmark_rpc_url)
+        .checkpoint(checkpoint)
+        .build()?;
+    client.start().await?;
+    Ok(client)
+}
+
+/// Create a tokio multi-threaded runtime.
+///
+/// # Panics
+///
+/// Panics if the runtime cannot be created.
+pub fn construct_runtime() -> tokio::runtime::Runtime {
+    tokio::runtime::Builder::new_multi_thread()
+        .enable_all()
+        .build()
+        .unwrap()
+}
+
+/// Constructs a goerli client for benchmark usage.
+///
+/// Requires a [Runtime](tokio::runtime::Runtime) to be passed in by reference.
+/// The client is parameterized with a [FileDB](client::FileDB).
+/// It will also use the environment variable `GOERLI_RPC_URL` to connect to a mainnet node.
+/// The client will use `http://testing.prater.beacon-api.nimbus.team` as the consensus RPC.
+pub fn construct_goerli_client(
+    rt: &tokio::runtime::Runtime,
+) -> eyre::Result<client::Client<client::FileDB>> {
+    rt.block_on(async {
+        let benchmark_rpc_url = std::env::var("GOERLI_RPC_URL")?;
+        let mut client = client::ClientBuilder::new()
+            .network(networks::Network::GOERLI)
+            .consensus_rpc("http://testing.prater.beacon-api.nimbus.team")
+            .execution_rpc(&benchmark_rpc_url)
+            .load_external_fallback()
+            .build()?;
+        client.start().await?;
+        Ok(client)
+    })
+}
+
+/// Gets the balance of the given address on mainnet.
+pub fn get_balance(
+    rt: &tokio::runtime::Runtime,
+    client: Arc<Client<FileDB>>,
+    address: &str,
+) -> eyre::Result<U256> {
+    rt.block_on(async {
+        let block = BlockTag::Latest;
+        let address = Address::from_str(address)?;
+        client.get_balance(&address, block).await
+    })
+}
+
+// h/t @ https://github.com/smrpn
+// rev: https://github.com/smrpn/casbin-rs/commit/7a0a75d8075440ee65acdac3ee9c0de6fcbd5c48
+pub fn await_future<F: std::future::Future<Output = T>, T>(future: F) -> T {
+    tokio::runtime::Runtime::new().unwrap().block_on(future)
+}

benches/sync.rs

@@ -0,0 +1,81 @@
+use criterion::{criterion_group, criterion_main, Criterion};
+use ethers::types::Address;
+use helios::types::BlockTag;
+
+mod harness;
+
+criterion_main!(sync);
+criterion_group! {
+    name = sync;
+    config = Criterion::default().sample_size(10);
+    targets =
+        bench_full_sync,
+        bench_full_sync_with_call,
+        bench_full_sync_checkpoint_fallback,
+        bench_full_sync_with_call_checkpoint_fallback,
+}
+
+/// Benchmark full client sync.
+pub fn bench_full_sync(c: &mut Criterion) {
+    // Externally, let's fetch the latest checkpoint from our fallback service so as not to benchmark the checkpoint fetch.
+    let checkpoint = harness::await_future(harness::fetch_mainnet_checkpoint()).unwrap();
+    let checkpoint = hex::encode(checkpoint);
+
+    // On client construction, it will sync to the latest checkpoint using our fetched checkpoint.
+    c.bench_function("full_sync", |b| {
+        b.to_async(harness::construct_runtime()).iter(|| async {
+            let _client = std::sync::Arc::new(
+                harness::construct_mainnet_client_with_checkpoint(&checkpoint)
+                    .await
+                    .unwrap(),
+            );
+        })
+    });
+}
+
+/// Benchmark full client sync.
+/// Address: 0x00000000219ab540356cbb839cbe05303d7705fa (beacon chain deposit address)
+pub fn bench_full_sync_with_call(c: &mut Criterion) {
+    // Externally, let's fetch the latest checkpoint from our fallback service so as not to benchmark the checkpoint fetch.
+    let checkpoint = harness::await_future(harness::fetch_mainnet_checkpoint()).unwrap();
+    let checkpoint = hex::encode(checkpoint);
+
+    // On client construction, it will sync to the latest checkpoint using our fetched checkpoint.
+    c.bench_function("full_sync_call", |b| {
+        b.to_async(harness::construct_runtime()).iter(|| async {
+            let client = std::sync::Arc::new(
+                harness::construct_mainnet_client_with_checkpoint(&checkpoint)
+                    .await
+                    .unwrap(),
+            );
+            let addr = "0x00000000219ab540356cbb839cbe05303d7705fa"
+                .parse::<Address>()
+                .unwrap();
+            let block = BlockTag::Latest;
+            client.get_balance(&addr, block).await.unwrap()
+        })
+    });
+}
+
+/// Benchmark full client sync with checkpoint fallback.
+pub fn bench_full_sync_checkpoint_fallback(c: &mut Criterion) {
+    c.bench_function("full_sync_fallback", |b| {
+        let rt = harness::construct_runtime();
+        b.iter(|| {
+            let _client = std::sync::Arc::new(harness::construct_mainnet_client(&rt).unwrap());
+        })
+    });
+}
+
+/// Benchmark full client sync with a call and checkpoint fallback.
+/// Address: 0x00000000219ab540356cbb839cbe05303d7705fa (beacon chain deposit address)
+pub fn bench_full_sync_with_call_checkpoint_fallback(c: &mut Criterion) {
+    c.bench_function("full_sync_call", |b| {
+        let addr = "0x00000000219ab540356cbb839cbe05303d7705fa";
+        let rt = harness::construct_runtime();
+        b.iter(|| {
+            let client = std::sync::Arc::new(harness::construct_mainnet_client(&rt).unwrap());
+            harness::get_balance(&rt, client, addr).unwrap();
+        })
+    });
+}

client/src/database.rs

@@ -1,9 +1,14 @@
-use std::{fs, io::Write, path::PathBuf};
+use std::{
+    fs,
+    io::{Read, Write},
+    path::PathBuf,
+};
 
 use eyre::Result;
 
 pub trait Database {
     fn save_checkpoint(&self, checkpoint: Vec<u8>) -> Result<()>;
+    fn load_checkpoint(&self) -> Result<Vec<u8>>;
 }
 
 pub struct FileDB {
@@ -30,4 +35,15 @@ impl Database for FileDB {
 
         Ok(())
     }
+
+    fn load_checkpoint(&self) -> Result<Vec<u8>> {
+        let mut f = fs::OpenOptions::new()
+            .read(true)
+            .open(self.data_dir.join("checkpoint"))?;
+
+        let mut buf = Vec::new();
+        f.read_to_end(&mut buf)?;
+
+        Ok(buf)
+    }
 }

config/src/checkpoints.rs

@@ -117,10 +117,18 @@ impl CheckpointFallback {
         &self,
         network: &crate::networks::Network,
     ) -> eyre::Result<H256> {
-        let services = &self.services[network];
+        let services = &self.get_healthy_fallback_services(network);
         Self::fetch_latest_checkpoint_from_services(&services[..]).await
     }
 
+    async fn query_service(endpoint: &str) -> Option<RawSlotResponse> {
+        let client = reqwest::Client::new();
+        let constructed_url = Self::construct_url(endpoint);
+        let res = client.get(&constructed_url).send().await.ok()?;
+        let raw: RawSlotResponse = res.json().await.ok()?;
+        Some(raw)
+    }
+
     /// Fetch the latest checkpoint from a list of checkpoint fallback services.
     pub async fn fetch_latest_checkpoint_from_services(
         services: &[CheckpointFallbackService],
@@ -131,14 +139,15 @@ impl CheckpointFallback {
             .map(|service| {
                 let service = service.clone();
                 tokio::spawn(async move {
-                    let client = reqwest::Client::new();
-                    let constructed_url = Self::construct_url(&service.endpoint);
-                    let res = client.get(&constructed_url).send().await?;
-                    let raw: RawSlotResponse = res.json().await?;
+                    match Self::query_service(&service.endpoint).await {
+                        Some(raw) => {
                             if raw.data.slots.is_empty() {
                                 return Err(eyre::eyre!("no slots"));
                             }
                             Ok(raw.data.slots[0].clone())
+                        }
+                        None => Err(eyre::eyre!("failed to query service")),
+                    }
                 })
             })
             .collect();
@@ -230,6 +239,22 @@ impl CheckpointFallback {
             .collect()
     }
 
+    /// Returns a list of healthchecked checkpoint fallback services.
+    ///
+    /// ### Warning
+    ///
+    /// These services are not trustworthy and may act with malice by returning invalid checkpoints.
+    pub fn get_healthy_fallback_services(
+        &self,
+        network: &networks::Network,
+    ) -> Vec<CheckpointFallbackService> {
+        self.services[network]
+            .iter()
+            .filter(|service| service.state)
+            .cloned()
+            .collect::<Vec<CheckpointFallbackService>>()
+    }
+
     /// Returns the raw checkpoint fallback service objects for a given network.
     pub fn get_fallback_services(
         &self,