Merge branch 'master' into @refcell/p2p

.github/workflows/benchmarks.yml

@@ -0,0 +1,25 @@
+name: benchmarks
+
+on:
+  push:
+    branches: [ "master" ]
+
+env:
+  MAINNET_RPC_URL: ${{ secrets.MAINNET_RPC_URL }}
+  GOERLI_RPC_URL: ${{ secrets.GOERLI_RPC_URL }}
+
+jobs:
+  benches:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly
+          override: true
+          components: rustfmt
+      - uses: Swatinem/rust-cache@v2
+      - uses: actions-rs/cargo@v1
+        with:
+          command: bench

.github/workflows/test.yml

@@ -8,10 +8,9 @@ on:
 
 jobs:
   check:
-    name: check
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
@@ -23,10 +22,9 @@ jobs:
           command: check
 
   test:
-    name: test
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
@@ -39,10 +37,9 @@ jobs:
           args: --all
 
   fmt:
-    name: fmt
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
@@ -56,10 +53,9 @@ jobs:
           args: --all -- --check
 
   clippy:
-    name: clippy
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal

.gitignore

@@ -1 +1,3 @@
 /target
+
+*.env

Cargo.toml

@@ -1,10 +1,10 @@
 [package]
 name = "helios"
-version = "0.1.1"
+version = "0.1.2"
 edition = "2021"
+autobenches = false
 
 [workspace]
-
 members = [
     "cli",
     "client",
@@ -28,6 +28,15 @@ home = "0.5.4"
 ethers = "1.0.2"
 env_logger = "0.9.0"
 log = "0.4.17"
+tracing-test = "0.2.3"
+criterion = { version = "0.4", features = [ "async_tokio", "plotters" ]}
+plotters = "0.3.3"
+tempfile = "3.3.0"
+hex = "0.4.3"
+
+######################################
+# Examples
+######################################
 
 [[example]]
 name = "checkpoints"
@@ -45,3 +54,22 @@ path = "examples/client.rs"
 name = "config"
 path = "examples/config.rs"
 
+######################################
+# Benchmarks
+######################################
+
+[[bench]]
+name = "file_db"
+harness = false
+
+[[bench]]
+name = "get_balance"
+harness = false
+
+[[bench]]
+name = "get_code"
+harness = false
+
+[[bench]]
+name = "sync"
+harness = false

README.md

@@ -8,7 +8,6 @@ Helios converts an untrusted centralized RPC endpoint into a safe unmanipulable
 
 The entire size of Helios's binary is 13Mb and should be easy to compile into WebAssembly. This makes it a perfect target to embed directly inside wallets and dapps.
 
-
 ## Installing
 
 First install `heliosup`, Helios's installer:
@@ -19,7 +18,6 @@ curl https://raw.githubusercontent.com/a16z/helios/master/heliosup/install | bas
 
 To install Helios, run `heliosup`.
 
-
 ## Usage
 
 To run Helios, run the below command, replacing `$ETH_RPC_URL` with an RPC provider URL such as Alchemy or Infura:
@@ -32,14 +30,14 @@ helios --execution-rpc $ETH_RPC_URL
 
 Helios will now run a local RPC server at `http://127.0.0.1:8545`.
 
-Helios also provides examples in the [`examples/`](./examples/) directory. To run an example, you can execute `cargo run --example <example_name>` from inside the helios repository.
+Helios provides examples in the [`examples/`](./examples/) directory. To run an example, you can execute `cargo run --example <example_name>` from inside the helios repository.
 
+Helios also provides documentation of its supported RPC methods in the [rpc.md](./rpc.md) file.
 
 ### Warning
 
 Helios is still experimental software. While we hope you try it out, we do not suggest adding it as your main RPC in wallets yet. Sending high-value transactions from a wallet connected to Helios is discouraged.
 
-
 ### Additional Options
 
 `--consensus-rpc` or `-c` can be used to set a custom consensus layer rpc endpoint. This must be a consenus node that supports the light client beaconchain api. We recommend using Nimbus for this. If no consensus rpc is supplied, it defaults to `https://www.lightclientdata.org` which is run by us.
@@ -67,7 +65,6 @@ This can be run like so: `helios --load-external-fallback` (or `helios -l` with
 
 `--help` or `-h` prints the help message.
 
-
 ### Configuration Files
 
 All configuration options can be set on a per-network level in `~/.helios/helios.toml`. Here is an example config file:
@@ -84,6 +81,8 @@ execution_rpc = "https://eth-goerli.g.alchemy.com/v2/XXXXX"
 checkpoint = "0xb5c375696913865d7c0e166d87bc7c772b6210dc9edf149f4c7ddc6da0dd4495"
 ```
 
+A comprehensive breakdown of config options is available in the [config.md](./config.md) file.
+
 
 ### Using Helios as a Library
 
@@ -147,16 +146,20 @@ async fn main() -> Result<()> {
 }
 ```
 
+## Benchmarks
+
+Benchmarks are defined in the [benches](./benches/) subdirectory. They are built using the [criterion](https://github.com/bheisler/criterion.rs) statistics-driven benchmarking library.
+
+To run all benchmarks, you can use `cargo bench`. To run a specific benchmark, you can use `cargo bench --bench <name>`, where `<name>` is one of the benchmarks defined in the [Cargo.toml](./Cargo.toml) file under a `[[bench]]` section.
+
 ## Contributing
 
 All contributions to Helios are welcome. Before opening a PR, please submit an issue detailing the bug or feature. When opening a PR, please ensure that your contribution builds on the nightly rust toolchain, has been linted with `cargo fmt`, and contains tests when applicable.
 
-
 ## Telegram
 
 If you are having trouble with Helios or are considering contributing, feel free to join our telegram [here](https://t.me/+IntDY_gZJSRkNTJj).
 
-
 ## Disclaimer
 
 _This code is being provided as is. No guarantee, representation or warranty is being made, express or implied, as to the safety or correctness of the code. It has not been audited and as such there can be no assurance it will work as intended, and users may experience delays, failures, errors, omissions or loss of transmitted information. Nothing in this repo should be construed as investment advice or legal advice for any particular facts or circumstances and is not meant to replace competent counsel. It is strongly advised for you to contact a reputable attorney in your jurisdiction for any questions or concerns with respect thereto. a16z is not liable for any use of the foregoing, and users should proceed with caution and use at their own risk. See a16z.com/disclosures for more info._

benches/file_db.rs

@@ -0,0 +1,43 @@
+use client::database::Database;
+use criterion::{criterion_group, criterion_main, Criterion};
+use helios::prelude::FileDB;
+use tempfile::tempdir;
+
+mod harness;
+
+criterion_main!(file_db);
+criterion_group! {
+    name = file_db;
+    config = Criterion::default();
+    targets = save_checkpoint, load_checkpoint
+}
+
+/// Benchmark saving/writing a checkpoint to the file db.
+pub fn save_checkpoint(c: &mut Criterion) {
+    c.bench_function("save_checkpoint", |b| {
+        let checkpoint = vec![1, 2, 3];
+        b.iter(|| {
+            let temp_dir = tempdir().unwrap().into_path();
+            let db = FileDB::new(temp_dir);
+            db.save_checkpoint(checkpoint.clone()).unwrap();
+        })
+    });
+}
+
+/// Benchmark loading a checkpoint from the file db.
+pub fn load_checkpoint(c: &mut Criterion) {
+    c.bench_function("load_checkpoint", |b| {
+        // First write to the db
+        let temp_dir = tempdir().unwrap().into_path();
+        let db = FileDB::new(temp_dir.clone());
+        let written_checkpoint = vec![1, 2, 3];
+        db.save_checkpoint(written_checkpoint.clone()).unwrap();
+
+        // Then read from the db
+        b.iter(|| {
+            let db = FileDB::new(temp_dir.clone());
+            let checkpoint = db.load_checkpoint().unwrap();
+            assert_eq!(checkpoint, written_checkpoint.clone());
+        })
+    });
+}

benches/get_balance.rs

@@ -0,0 +1,63 @@
+use criterion::{criterion_group, criterion_main, Criterion};
+use ethers::prelude::*;
+use helios::types::BlockTag;
+use std::str::FromStr;
+
+mod harness;
+
+criterion_main!(get_balance);
+criterion_group! {
+    name = get_balance;
+    config = Criterion::default().sample_size(10);
+    targets = bench_mainnet_get_balance, bench_goerli_get_balance
+}
+
+/// Benchmark mainnet get balance.
+/// Address: 0x00000000219ab540356cbb839cbe05303d7705fa (beacon chain deposit address)
+pub fn bench_mainnet_get_balance(c: &mut Criterion) {
+    c.bench_function("get_balance", |b| {
+        // Create a new multi-threaded tokio runtime.
+        let rt = tokio::runtime::Builder::new_multi_thread()
+            .enable_all()
+            .build()
+            .unwrap();
+
+        // Construct a mainnet client using our harness and tokio runtime.
+        let client = std::sync::Arc::new(harness::construct_mainnet_client(&rt).unwrap());
+
+        // Get the beacon chain deposit contract address.
+        let addr = Address::from_str("0x00000000219ab540356cbb839cbe05303d7705fa").unwrap();
+        let block = BlockTag::Latest;
+
+        // Execute the benchmark asynchronously.
+        b.to_async(rt).iter(|| async {
+            let inner = std::sync::Arc::clone(&client);
+            inner.get_balance(&addr, block).await.unwrap()
+        })
+    });
+}
+
+/// Benchmark goerli get balance.
+/// Address: 0xB4FBF271143F4FBf7B91A5ded31805e42b2208d6 (goerli weth)
+pub fn bench_goerli_get_balance(c: &mut Criterion) {
+    c.bench_function("get_balance", |b| {
+        // Create a new multi-threaded tokio runtime.
+        let rt = tokio::runtime::Builder::new_multi_thread()
+            .enable_all()
+            .build()
+            .unwrap();
+
+        // Construct a goerli client using our harness and tokio runtime.
+        let client = std::sync::Arc::new(harness::construct_goerli_client(&rt).unwrap());
+
+        // Get the beacon chain deposit contract address.
+        let addr = Address::from_str("0xB4FBF271143F4FBf7B91A5ded31805e42b2208d6").unwrap();
+        let block = BlockTag::Latest;
+
+        // Execute the benchmark asynchronously.
+        b.to_async(rt).iter(|| async {
+            let inner = std::sync::Arc::clone(&client);
+            inner.get_balance(&addr, block).await.unwrap()
+        })
+    });
+}

benches/get_code.rs

@@ -0,0 +1,63 @@
+use criterion::{criterion_group, criterion_main, Criterion};
+use ethers::prelude::*;
+use helios::types::BlockTag;
+use std::str::FromStr;
+
+mod harness;
+
+criterion_main!(get_code);
+criterion_group! {
+    name = get_code;
+    config = Criterion::default().sample_size(10);
+    targets = bench_mainnet_get_code, bench_goerli_get_code
+}
+
+/// Benchmark mainnet get code call.
+/// Address: 0x00000000219ab540356cbb839cbe05303d7705fa (beacon chain deposit address)
+pub fn bench_mainnet_get_code(c: &mut Criterion) {
+    c.bench_function("get_code", |b| {
+        // Create a new multi-threaded tokio runtime.
+        let rt = tokio::runtime::Builder::new_multi_thread()
+            .enable_all()
+            .build()
+            .unwrap();
+
+        // Construct a mainnet client using our harness and tokio runtime.
+        let client = std::sync::Arc::new(harness::construct_mainnet_client(&rt).unwrap());
+
+        // Get the beacon chain deposit contract address.
+        let addr = Address::from_str("0x00000000219ab540356cbb839cbe05303d7705fa").unwrap();
+        let block = BlockTag::Latest;
+
+        // Execute the benchmark asynchronously.
+        b.to_async(rt).iter(|| async {
+            let inner = std::sync::Arc::clone(&client);
+            inner.get_code(&addr, block).await.unwrap()
+        })
+    });
+}
+
+/// Benchmark goerli get code call.
+/// Address: 0xB4FBF271143F4FBf7B91A5ded31805e42b2208d6 (goerli weth)
+pub fn bench_goerli_get_code(c: &mut Criterion) {
+    c.bench_function("get_code", |b| {
+        // Create a new multi-threaded tokio runtime.
+        let rt = tokio::runtime::Builder::new_multi_thread()
+            .enable_all()
+            .build()
+            .unwrap();
+
+        // Construct a goerli client using our harness and tokio runtime.
+        let client = std::sync::Arc::new(harness::construct_goerli_client(&rt).unwrap());
+
+        // Get the beacon chain deposit contract address.
+        let addr = Address::from_str("0xB4FBF271143F4FBf7B91A5ded31805e42b2208d6").unwrap();
+        let block = BlockTag::Latest;
+
+        // Execute the benchmark asynchronously.
+        b.to_async(rt).iter(|| async {
+            let inner = std::sync::Arc::clone(&client);
+            inner.get_code(&addr, block).await.unwrap()
+        })
+    });
+}

benches/harness.rs

@@ -0,0 +1,113 @@
+#![allow(dead_code)]
+
+use std::{str::FromStr, sync::Arc};
+
+use ::client::Client;
+use ethers::{
+    abi::Address,
+    types::{H256, U256},
+};
+use helios::{client, config::networks, prelude::FileDB, types::BlockTag};
+
+/// Fetches the latest mainnet checkpoint from the fallback service.
+///
+/// Uses the [CheckpointFallback](config::CheckpointFallback).
+/// The `build` method will fetch a list of [CheckpointFallbackService](config::CheckpointFallbackService)s from a community-mainained list by ethPandaOps.
+/// This list is NOT guaranteed to be secure, but is provided in good faith.
+/// The raw list can be found here: https://github.com/ethpandaops/checkpoint-sync-health-checks/blob/master/_data/endpoints.yaml
+pub async fn fetch_mainnet_checkpoint() -> eyre::Result<H256> {
+    let cf = config::CheckpointFallback::new().build().await.unwrap();
+    cf.fetch_latest_checkpoint(&networks::Network::MAINNET)
+        .await
+}
+
+/// Constructs a mainnet [Client](client::Client) for benchmark usage.
+///
+/// Requires a [Runtime](tokio::runtime::Runtime) to be passed in by reference.
+/// The client is parameterized with a [FileDB](client::FileDB).
+/// It will also use the environment variable `MAINNET_RPC_URL` to connect to a mainnet node.
+/// The client will use `https://www.lightclientdata.org` as the consensus RPC.
+pub fn construct_mainnet_client(
+    rt: &tokio::runtime::Runtime,
+) -> eyre::Result<client::Client<client::FileDB>> {
+    rt.block_on(inner_construct_mainnet_client())
+}
+
+pub async fn inner_construct_mainnet_client() -> eyre::Result<client::Client<client::FileDB>> {
+    let benchmark_rpc_url = std::env::var("MAINNET_RPC_URL")?;
+    let mut client = client::ClientBuilder::new()
+        .network(networks::Network::MAINNET)
+        .consensus_rpc("https://www.lightclientdata.org")
+        .execution_rpc(&benchmark_rpc_url)
+        .load_external_fallback()
+        .build()?;
+    client.start().await?;
+    Ok(client)
+}
+
+pub async fn construct_mainnet_client_with_checkpoint(
+    checkpoint: &str,
+) -> eyre::Result<client::Client<client::FileDB>> {
+    let benchmark_rpc_url = std::env::var("MAINNET_RPC_URL")?;
+    let mut client = client::ClientBuilder::new()
+        .network(networks::Network::MAINNET)
+        .consensus_rpc("https://www.lightclientdata.org")
+        .execution_rpc(&benchmark_rpc_url)
+        .checkpoint(checkpoint)
+        .build()?;
+    client.start().await?;
+    Ok(client)
+}
+
+/// Create a tokio multi-threaded runtime.
+///
+/// # Panics
+///
+/// Panics if the runtime cannot be created.
+pub fn construct_runtime() -> tokio::runtime::Runtime {
+    tokio::runtime::Builder::new_multi_thread()
+        .enable_all()
+        .build()
+        .unwrap()
+}
+
+/// Constructs a goerli client for benchmark usage.
+///
+/// Requires a [Runtime](tokio::runtime::Runtime) to be passed in by reference.
+/// The client is parameterized with a [FileDB](client::FileDB).
+/// It will also use the environment variable `GOERLI_RPC_URL` to connect to a mainnet node.
+/// The client will use `http://testing.prater.beacon-api.nimbus.team` as the consensus RPC.
+pub fn construct_goerli_client(
+    rt: &tokio::runtime::Runtime,
+) -> eyre::Result<client::Client<client::FileDB>> {
+    rt.block_on(async {
+        let benchmark_rpc_url = std::env::var("GOERLI_RPC_URL")?;
+        let mut client = client::ClientBuilder::new()
+            .network(networks::Network::GOERLI)
+            .consensus_rpc("http://testing.prater.beacon-api.nimbus.team")
+            .execution_rpc(&benchmark_rpc_url)
+            .load_external_fallback()
+            .build()?;
+        client.start().await?;
+        Ok(client)
+    })
+}
+
+/// Gets the balance of the given address on mainnet.
+pub fn get_balance(
+    rt: &tokio::runtime::Runtime,
+    client: Arc<Client<FileDB>>,
+    address: &str,
+) -> eyre::Result<U256> {
+    rt.block_on(async {
+        let block = BlockTag::Latest;
+        let address = Address::from_str(address)?;
+        client.get_balance(&address, block).await
+    })
+}
+
+// h/t @ https://github.com/smrpn
+// rev: https://github.com/smrpn/casbin-rs/commit/7a0a75d8075440ee65acdac3ee9c0de6fcbd5c48
+pub fn await_future<F: std::future::Future<Output = T>, T>(future: F) -> T {
+    tokio::runtime::Runtime::new().unwrap().block_on(future)
+}

benches/sync.rs

@@ -0,0 +1,81 @@
+use criterion::{criterion_group, criterion_main, Criterion};
+use ethers::types::Address;
+use helios::types::BlockTag;
+
+mod harness;
+
+criterion_main!(sync);
+criterion_group! {
+    name = sync;
+    config = Criterion::default().sample_size(10);
+    targets =
+        bench_full_sync,
+        bench_full_sync_with_call,
+        bench_full_sync_checkpoint_fallback,
+        bench_full_sync_with_call_checkpoint_fallback,
+}
+
+/// Benchmark full client sync.
+pub fn bench_full_sync(c: &mut Criterion) {
+    // Externally, let's fetch the latest checkpoint from our fallback service so as not to benchmark the checkpoint fetch.
+    let checkpoint = harness::await_future(harness::fetch_mainnet_checkpoint()).unwrap();
+    let checkpoint = hex::encode(checkpoint);
+
+    // On client construction, it will sync to the latest checkpoint using our fetched checkpoint.
+    c.bench_function("full_sync", |b| {
+        b.to_async(harness::construct_runtime()).iter(|| async {
+            let _client = std::sync::Arc::new(
+                harness::construct_mainnet_client_with_checkpoint(&checkpoint)
+                    .await
+                    .unwrap(),
+            );
+        })
+    });
+}
+
+/// Benchmark full client sync.
+/// Address: 0x00000000219ab540356cbb839cbe05303d7705fa (beacon chain deposit address)
+pub fn bench_full_sync_with_call(c: &mut Criterion) {
+    // Externally, let's fetch the latest checkpoint from our fallback service so as not to benchmark the checkpoint fetch.
+    let checkpoint = harness::await_future(harness::fetch_mainnet_checkpoint()).unwrap();
+    let checkpoint = hex::encode(checkpoint);
+
+    // On client construction, it will sync to the latest checkpoint using our fetched checkpoint.
+    c.bench_function("full_sync_call", |b| {
+        b.to_async(harness::construct_runtime()).iter(|| async {
+            let client = std::sync::Arc::new(
+                harness::construct_mainnet_client_with_checkpoint(&checkpoint)
+                    .await
+                    .unwrap(),
+            );
+            let addr = "0x00000000219ab540356cbb839cbe05303d7705fa"
+                .parse::<Address>()
+                .unwrap();
+            let block = BlockTag::Latest;
+            client.get_balance(&addr, block).await.unwrap()
+        })
+    });
+}
+
+/// Benchmark full client sync with checkpoint fallback.
+pub fn bench_full_sync_checkpoint_fallback(c: &mut Criterion) {
+    c.bench_function("full_sync_fallback", |b| {
+        let rt = harness::construct_runtime();
+        b.iter(|| {
+            let _client = std::sync::Arc::new(harness::construct_mainnet_client(&rt).unwrap());
+        })
+    });
+}
+
+/// Benchmark full client sync with a call and checkpoint fallback.
+/// Address: 0x00000000219ab540356cbb839cbe05303d7705fa (beacon chain deposit address)
+pub fn bench_full_sync_with_call_checkpoint_fallback(c: &mut Criterion) {
+    c.bench_function("full_sync_call", |b| {
+        let addr = "0x00000000219ab540356cbb839cbe05303d7705fa";
+        let rt = harness::construct_runtime();
+        b.iter(|| {
+            let client = std::sync::Arc::new(harness::construct_mainnet_client(&rt).unwrap());
+            harness::get_balance(&rt, client, addr).unwrap();
+        })
+    });
+}

client/src/client.rs

@@ -340,12 +340,35 @@ impl<DB: Database> Client<DB> {
         self.node.read().await.get_nonce(address, block).await
     }
 
+    pub async fn get_block_transaction_count_by_hash(&self, hash: &Vec<u8>) -> Result<u64> {
+        self.node
+            .read()
+            .await
+            .get_block_transaction_count_by_hash(hash)
+    }
+
+    pub async fn get_block_transaction_count_by_number(&self, block: BlockTag) -> Result<u64> {
+        self.node
+            .read()
+            .await
+            .get_block_transaction_count_by_number(block)
+    }
+
     pub async fn get_code(&self, address: &Address, block: BlockTag) -> Result<Vec<u8>> {
         self.node.read().await.get_code(address, block).await
     }
 
-    pub async fn get_storage_at(&self, address: &Address, slot: H256) -> Result<U256> {
+    pub async fn get_storage_at(
-        self.node.read().await.get_storage_at(address, slot).await
+        &self,
+        address: &Address,
+        slot: H256,
+        block: BlockTag,
+    ) -> Result<U256> {
+        self.node
+            .read()
+            .await
+            .get_storage_at(address, slot, block)
+            .await
     }
 
     pub async fn send_raw_transaction(&self, bytes: &[u8]) -> Result<H256> {

client/src/database.rs

@@ -1,9 +1,14 @@
-use std::{fs, io::Write, path::PathBuf};
+use std::{
+    fs,
+    io::{Read, Write},
+    path::PathBuf,
+};
 
 use eyre::Result;
 
 pub trait Database {
     fn save_checkpoint(&self, checkpoint: Vec<u8>) -> Result<()>;
+    fn load_checkpoint(&self) -> Result<Vec<u8>>;
 }
 
 pub struct FileDB {
@@ -30,4 +35,15 @@ impl Database for FileDB {
 
         Ok(())
     }
+
+    fn load_checkpoint(&self) -> Result<Vec<u8>> {
+        let mut f = fs::OpenOptions::new()
+            .read(true)
+            .open(self.data_dir.join("checkpoint"))?;
+
+        let mut buf = Vec::new();
+        f.read_to_end(&mut buf)?;
+
+        Ok(buf)
+    }
 }

client/src/node.rs

@@ -155,6 +155,20 @@ impl Node {
         Ok(account.nonce)
     }
 
+    pub fn get_block_transaction_count_by_hash(&self, hash: &Vec<u8>) -> Result<u64> {
+        let payload = self.get_payload_by_hash(hash)?;
+        let transaction_count = payload.1.transactions.len();
+
+        Ok(transaction_count as u64)
+    }
+
+    pub fn get_block_transaction_count_by_number(&self, block: BlockTag) -> Result<u64> {
+        let payload = self.get_payload(block)?;
+        let transaction_count = payload.transactions.len();
+
+        Ok(transaction_count as u64)
+    }
+
     pub async fn get_code(&self, address: &Address, block: BlockTag) -> Result<Vec<u8>> {
         self.check_blocktag_age(&block)?;
 
@@ -163,10 +177,15 @@ impl Node {
         Ok(account.code)
     }
 
-    pub async fn get_storage_at(&self, address: &Address, slot: H256) -> Result<U256> {
+    pub async fn get_storage_at(
+        &self,
+        address: &Address,
+        slot: H256,
+        block: BlockTag,
+    ) -> Result<U256> {
         self.check_head_age()?;
 
-        let payload = self.get_payload(BlockTag::Latest)?;
+        let payload = self.get_payload(block)?;
         let account = self
             .execution
             .get_account(address, Some(&[slot]), payload)
@@ -243,19 +262,11 @@ impl Node {
         hash: &Vec<u8>,
         full_tx: bool,
     ) -> Result<Option<ExecutionBlock>> {
-        let payloads = self
+        let payload = self.get_payload_by_hash(hash);
-            .payloads
-            .iter()
-            .filter(|entry| &entry.1.block_hash.to_vec() == hash)
-            .collect::<Vec<(&u64, &ExecutionPayload)>>();
 
-        if let Some(payload_entry) = payloads.get(0) {
+        match payload {
-            self.execution
+            Ok(payload) => self.execution.get_block(payload.1, full_tx).await.map(Some),
-                .get_block(payload_entry.1, full_tx)
+            Err(_) => Ok(None),
-                .await
-                .map(Some)
-        } else {
-            Ok(None)
         }
     }
 
@@ -291,6 +302,19 @@ impl Node {
         }
     }
 
+    fn get_payload_by_hash(&self, hash: &Vec<u8>) -> Result<(&u64, &ExecutionPayload)> {
+        let payloads = self
+            .payloads
+            .iter()
+            .filter(|entry| &entry.1.block_hash.to_vec() == hash)
+            .collect::<Vec<(&u64, &ExecutionPayload)>>();
+
+        payloads
+            .get(0)
+            .cloned()
+            .ok_or(eyre!("Block not found by hash"))
+    }
+
     fn check_head_age(&self) -> Result<(), NodeError> {
         let synced_slot = self.consensus.get_header().slot;
         let expected_slot = self.consensus.expected_current_slot();

client/src/rpc.rs

@@ -57,6 +57,11 @@ trait EthRpc {
     async fn get_balance(&self, address: &str, block: BlockTag) -> Result<String, Error>;
     #[method(name = "getTransactionCount")]
     async fn get_transaction_count(&self, address: &str, block: BlockTag) -> Result<String, Error>;
+    #[method(name = "getBlockTransactionCountByHash")]
+    async fn get_block_transaction_count_by_hash(&self, hash: &str) -> Result<String, Error>;
+    #[method(name = "getBlockTransactionCountByNumber")]
+    async fn get_block_transaction_count_by_number(&self, block: BlockTag)
+        -> Result<String, Error>;
     #[method(name = "getCode")]
     async fn get_code(&self, address: &str, block: BlockTag) -> Result<String, Error>;
     #[method(name = "call")]
@@ -94,6 +99,13 @@ trait EthRpc {
     async fn get_transaction_by_hash(&self, hash: &str) -> Result<Option<Transaction>, Error>;
     #[method(name = "getLogs")]
     async fn get_logs(&self, filter: Filter) -> Result<Vec<Log>, Error>;
+    #[method(name = "getStorageAt")]
+    async fn get_storage_at(
+        &self,
+        address: &str,
+        slot: H256,
+        block: BlockTag,
+    ) -> Result<String, Error>;
 }
 
 #[rpc(client, server, namespace = "net")]
@@ -126,6 +138,23 @@ impl EthRpcServer for RpcInner {
         Ok(format!("0x{nonce:x}"))
     }
 
+    async fn get_block_transaction_count_by_hash(&self, hash: &str) -> Result<String, Error> {
+        let hash = convert_err(hex_str_to_bytes(hash))?;
+        let node = self.node.read().await;
+        let transaction_count = convert_err(node.get_block_transaction_count_by_hash(&hash))?;
+
+        Ok(u64_to_hex_string(transaction_count))
+    }
+
+    async fn get_block_transaction_count_by_number(
+        &self,
+        block: BlockTag,
+    ) -> Result<String, Error> {
+        let node = self.node.read().await;
+        let transaction_count = convert_err(node.get_block_transaction_count_by_number(block))?;
+        Ok(u64_to_hex_string(transaction_count))
+    }
+
     async fn get_code(&self, address: &str, block: BlockTag) -> Result<String, Error> {
         let address = convert_err(Address::from_str(address))?;
         let node = self.node.read().await;
@@ -227,6 +256,19 @@ impl EthRpcServer for RpcInner {
         let node = self.node.read().await;
         convert_err(node.get_logs(&filter).await)
     }
+
+    async fn get_storage_at(
+        &self,
+        address: &str,
+        slot: H256,
+        block: BlockTag,
+    ) -> Result<String, Error> {
+        let address = convert_err(Address::from_str(address))?;
+        let node = self.node.read().await;
+        let storage = convert_err(node.get_storage_at(&address, slot, block).await)?;
+
+        Ok(format_hex(&storage))
+    }
 }
 
 #[async_trait]

config.md

@@ -0,0 +1,74 @@
+## Helios Configuration
+
+All configuration options can be set on a per-network level in `~/.helios/helios.toml`.
+
+#### Comprehensive Example
+
+```toml
+[mainnet]
+# The consensus rpc to use. This should be a trusted rpc endpoint. Defaults to "https://www.lightclientdata.org".
+consensus_rpc = "https://www.lightclientdata.org"
+# [REQUIRED] The execution rpc to use. This should be a trusted rpc endpoint.
+execution_rpc = "https://eth-mainnet.g.alchemy.com/v2/XXXXX"
+# The port to run the JSON-RPC server on. By default, Helios will use port 8545.
+rpc_port = 8545
+# The latest checkpoint. This should be a trusted checkpoint that is no greater than ~2 weeks old.
+# If you are unsure what checkpoint to use, you can skip this option and set either `load_external_fallback` or `fallback` values (described below) to fetch a checkpoint. Though this is not recommended and less secure.
+checkpoint = "0x85e6151a246e8fdba36db27a0c7678a575346272fe978c9281e13a8b26cdfa68"
+# The directory to store the checkpoint database in. If not provided, Helios will use "~/.helios/data/mainnet", where `mainnet` is the network.
+# It is recommended to set this directory to a persistent location mapped to a fast storage device.
+data_dir = "/home/user/.helios/mainnet"
+# The maximum age of a checkpoint in seconds. If the checkpoint is older than this, Helios will attempt to fetch a new checkpoint.
+max_checkpoint_age = 86400
+# A checkpoint fallback is used if no checkpoint is provided or the given checkpoint is too old.
+# This is expected to be a trusted checkpoint sync api (like provided in https://github.com/ethpandaops/checkpoint-sync-health-checks/blob/master/_data/endpoints.yaml).
+fallback = "https://sync-mainnet.beaconcha.in"
+# If no checkpoint is provided, or the checkpoint is too old, Helios will attempt to dynamically fetch a checkpoint from a maintained list of checkpoint sync apis.
+# NOTE: This is an insecure feature and not recommended for production use. Checkpoint manipulation is possible.
+load_external_fallback = true
+
+[goerli]
+# The consensus rpc to use. This should be a trusted rpc endpoint. Defaults to Nimbus testnet.
+consensus_rpc = "http://testing.prater.beacon-api.nimbus.team"
+# [REQUIRED] The execution rpc to use. This should be a trusted rpc endpoint.
+execution_rpc = "https://eth-goerli.g.alchemy.com/v2/XXXXX"
+# The port to run the JSON-RPC server on. By default, Helios will use port 8545.
+rpc_port = 8545
+# The latest checkpoint. This should be a trusted checkpoint that is no greater than ~2 weeks old.
+# If you are unsure what checkpoint to use, you can skip this option and set either `load_external_fallback` or `fallback` values (described below) to fetch a checkpoint. Though this is not recommended and less secure.
+checkpoint = "0xb5c375696913865d7c0e166d87bc7c772b6210dc9edf149f4c7ddc6da0dd4495"
+# The directory to store the checkpoint database in. If not provided, Helios will use "~/.helios/data/goerli", where `goerli` is the network.
+# It is recommended to set this directory to a persistent location mapped to a fast storage device.
+data_dir = "/home/user/.helios/goerli"
+# The maximum age of a checkpoint in seconds. If the checkpoint is older than this, Helios will attempt to fetch a new checkpoint.
+max_checkpoint_age = 86400
+# A checkpoint fallback is used if no checkpoint is provided or the given checkpoint is too old.
+# This is expected to be a trusted checkpoint sync api (like provided in https://github.com/ethpandaops/checkpoint-sync-health-checks/blob/master/_data/endpoints.yaml).
+fallback = "https://sync-goerli.beaconcha.in"
+# If no checkpoint is provided, or the checkpoint is too old, Helios will attempt to dynamically fetch a checkpoint from a maintained list of checkpoint sync apis.
+# NOTE: This is an insecure feature and not recommended for production use. Checkpoint manipulation is possible.
+load_external_fallback = true
+```
+
+
+#### Options
+
+All configuration options below are available on a per-network level, where network is specified by a header (eg `[mainnet]` or `[goerli]`). Many of these options can be configured through cli flags as well. See [README.md](./README.md#additional-options) or run `helios --help` for more information.
+
+
+- `consensus_rpc` - The URL of the consensus RPC endpoint used to fetch the latest beacon chain head and sync status. This must be a consenus node that supports the light client beaconchain api. We recommend using Nimbus for this. If no consensus rpc is supplied, it defaults to `https://www.lightclientdata.org` which is run by us.
+
+- `execution_rpc` - The URL of the execution RPC endpoint used to fetch the latest execution chain head and sync status. This must be an execution node that supports the light client execution api. We recommend using Geth for this.
+
+- `rpc_port` - The port to run the JSON-RPC server on. By default, Helios will use port 8545.
+
+- `checkpoint` - The latest checkpoint. This should be a trusted checkpoint that is no greater than ~2 weeks old. If you are unsure what checkpoint to use, you can skip this option and set either `load_external_fallback` or `fallback` values (described below) to fetch a checkpoint. Though this is not recommended and less secure.
+
+- `data_dir` - The directory to store the checkpoint database in. If not provided, Helios will use "~/.helios/data/<NETWORK>", where `<NETWORK>` is the network. It is recommended to set this directory to a persistent location mapped to a fast storage device.
+
+- `max_checkpoint_age` - The maximum age of a checkpoint in seconds. If the checkpoint is older than this, Helios will attempt to fetch a new checkpoint.
+
+- `fallback` - A checkpoint fallback is used if no checkpoint is provided or the given checkpoint is too old. This is expected to be a trusted checkpoint sync api (eg https://sync-mainnet.beaconcha.in). An extensive list of checkpoint sync apis can be found here: https://github.com/ethpandaops/checkpoint-sync-health-checks/blob/master/_data/endpoints.yaml.
+
+- `load_external_fallback` - If no checkpoint is provided, or the checkpoint is too old, Helios will attempt to dynamically fetch a checkpoint from a maintained list of checkpoint sync apis. NOTE: This is an insecure feature and not recommended for production use. Checkpoint manipulation is possible.
+

config/src/checkpoints.rs

@@ -117,10 +117,18 @@ impl CheckpointFallback {
         &self,
         network: &crate::networks::Network,
     ) -> eyre::Result<H256> {
-        let services = &self.services[network];
+        let services = &self.get_healthy_fallback_services(network);
         Self::fetch_latest_checkpoint_from_services(&services[..]).await
     }
 
+    async fn query_service(endpoint: &str) -> Option<RawSlotResponse> {
+        let client = reqwest::Client::new();
+        let constructed_url = Self::construct_url(endpoint);
+        let res = client.get(&constructed_url).send().await.ok()?;
+        let raw: RawSlotResponse = res.json().await.ok()?;
+        Some(raw)
+    }
+
     /// Fetch the latest checkpoint from a list of checkpoint fallback services.
     pub async fn fetch_latest_checkpoint_from_services(
         services: &[CheckpointFallbackService],
@@ -131,14 +139,15 @@ impl CheckpointFallback {
             .map(|service| {
                 let service = service.clone();
                 tokio::spawn(async move {
-                    let client = reqwest::Client::new();
+                    match Self::query_service(&service.endpoint).await {
-                    let constructed_url = Self::construct_url(&service.endpoint);
+                        Some(raw) => {
-                    let res = client.get(&constructed_url).send().await?;
+                            if raw.data.slots.is_empty() {
-                    let raw: RawSlotResponse = res.json().await?;
+                                return Err(eyre::eyre!("no slots"));
-                    if raw.data.slots.is_empty() {
+                            }
-                        return Err(eyre::eyre!("no slots"));
+                            Ok(raw.data.slots[0].clone())
+                        }
+                        None => Err(eyre::eyre!("failed to query service")),
                     }
-                    Ok(raw.data.slots[0].clone())
                 })
             })
             .collect();
@@ -230,6 +239,22 @@ impl CheckpointFallback {
             .collect()
     }
 
+    /// Returns a list of healthchecked checkpoint fallback services.
+    ///
+    /// ### Warning
+    ///
+    /// These services are not trustworthy and may act with malice by returning invalid checkpoints.
+    pub fn get_healthy_fallback_services(
+        &self,
+        network: &networks::Network,
+    ) -> Vec<CheckpointFallbackService> {
+        self.services[network]
+            .iter()
+            .filter(|service| service.state)
+            .cloned()
+            .collect::<Vec<CheckpointFallbackService>>()
+    }
+
     /// Returns the raw checkpoint fallback service objects for a given network.
     pub fn get_fallback_services(
         &self,

examples/readme.rs

@@ -9,7 +9,7 @@ use helios::{client::ClientBuilder, config::networks::Network, types::BlockTag};
 async fn main() -> Result<()> {
     env_logger::Builder::from_env(Env::default().default_filter_or("info")).init();
 
-    let untrusted_rpc_url = "https://mainnet.infura.io/v3/<YOUR_API_KEY>";
+    let untrusted_rpc_url = "https://eth-mainnet.g.alchemy.com/v2/<YOUR_API_KEY>";
     log::info!("Using untrusted RPC URL [REDACTED]");
 
     let consensus_rpc = "https://www.lightclientdata.org";

rpc.md

@@ -0,0 +1,25 @@
+# Helios Remote Procedure Calls
+
+Helios provides a variety of RPC methods for interacting with the Ethereum network. These methods are exposed via the `Client` struct.  The RPC methods follow the [Ethereum JSON RPC Spec](https://ethereum.github.io/execution-apis/api-documentation). See [examples](./examples/readme.rs) of running remote procedure calls with Helios.
+
+## RPC Methods
+
+| RPC Method | Client Function | Description | Example |
+| ---------- | --------------- | ----------- | ------- |
+| `eth_getBalance` | `get_balance` | Returns the balance of the account given an address. | `client.get_balance(&self, address: &str, block: BlockTag)` |
+| `eth_getTransactionCount` | `get_nonce` | Returns the number of transactions sent from the given address. | `client.get_nonce(&self, address: &str, block: BlockTag)` |
+| `eth_getCode` | `get_code` | Returns the code at a given address. | `client.get_code(&self, address: &str, block: BlockTag)` |
+| `eth_call` | `call` | Executes a new message call immediately without creating a transaction on the blockchain. | `client.call(&self, opts: CallOpts, block: BlockTag)` |
+| `eth_estimateGas` | `estimate_gas` | Generates and returns an estimate of how much gas is necessary to allow the transaction to complete. | `client.estimate_gas(&self, opts: CallOpts)` |
+| `eth_getChainId` | `chain_id` | Returns the chain ID of the current network. | `client.chain_id(&self)` |
+| `eth_gasPrice` | `gas_price` | Returns the current price per gas in wei. | `client.gas_price(&self)` |
+| `eth_maxPriorityFeePerGas` | `max_priority_fee_per_gas` | Returns the current max priority fee per gas in wei. | `client.max_priority_fee_per_gas(&self)` |
+| `eth_blockNumber` | `block_number` | Returns the number of the most recent block. | `client.block_number(&self)` |
+| `eth_getBlockByNumber` | `get_block_by_number` | Returns the information of a block by number. | `get_block_by_number(&self, block: BlockTag, full_tx: bool)` |
+| `eth_getBlockByHash` | `get_block_by_hash` | Returns the information of a block by hash. | `get_block_by_hash(&self, hash: &str, full_tx: bool)` |
+| `eth_sendRawTransaction` | `send_raw_transaction` | Submits a raw transaction to the network. | `client.send_raw_transaction(&self, bytes: &str)` |
+| `eth_getTransactionReceipt` | `get_transaction_receipt` | Returns the receipt of a transaction by transaction hash. | `client.get_transaction_receipt(&self, hash: &str)` |
+| `eth_getLogs` | `get_logs` | Returns an array of logs matching the filter. | `client.get_logs(&self, filter: Filter)` |
+| `eth_getStorageAt` | `get_storage_at` | Returns the value from a storage position at a given address. | `client.get_storage_at(&self, address: &str, slot: H256, block: BlockTag)` |
+| `eth_getBlockTransactionCountByHash` | `get_block_transaction_count_by_hash` | Returns the number of transactions in a block from a block matching the transaction hash. | `client.get_block_transaction_count_by_hash(&self, hash: &str)` |
+| `eth_getBlockTransactionCountByNumber` | `get_block_transaction_count_by_number` | Returns the number of transactions in a block from a block matching the block number. | `client.get_block_transaction_count_by_number(&self, block: BlockTag)` |

src/lib.rs

@@ -60,7 +60,7 @@ pub mod config {
 
 pub mod types {
     pub use common::types::BlockTag;
-    pub use execution::types::CallOpts;
+    pub use execution::types::{CallOpts, ExecutionBlock};
 }
 
 pub mod errors {