feat: fee history (#211)

* client get_fee_history

* node get_fee_history

* errors: InvalidBaseGaseFee

* execution get_fee_history

* http rpc get_fee_history

* moc rpc get_fee_history and json file

* module add get_fee_history

* update exec

* test feehistory

* update execution with logging + better logic

* fee history config loader

* rust fmt client

* rustfmt node

* rustfmt error

* rustfmt execution

* rustfmt http and moc rpc

* rustfmt mod.rs

* fee history formating

* correct typos

* use env var

* InvalidGasUsedRatio error

* check gas used ratio

* remove logging

* Update execution/src/errors.rs

Co-authored-by: refcell.eth <abigger87@gmail.com>

* Update execution/src/execution.rs

Co-authored-by: refcell.eth <abigger87@gmail.com>

* adding block and payload errors

* using error

* handle error in test

* fix: evm panic on slot not found (#208)

* fixes, but test fails

* fix: cleanup and example (#210)

* clean up fee history

* bump time dep in chrono, thx dependabot

* add benches to pr

* sleep

* fmt ✨

* place benching behind a man flag

---------

Co-authored-by: SFYLL <santiagoflood@hotmail.fr>
Co-authored-by: SFYLL <39958632+SFYLL@users.noreply.github.com>
Co-authored-by: refcell.eth <abigger87@gmail.com>

.github/workflows/benchmarks.yml

@@ -1,8 +1,9 @@
 name: benchmarks
 
 on:
-  push:
+  workflow_bench:
-    branches: [ "master" ]
+  # push:
+  #   branches: [ "master" ]
 
 env:
   MAINNET_RPC_URL: ${{ secrets.MAINNET_RPC_URL }}

.github/workflows/test.yml

@@ -6,6 +6,10 @@ on:
   pull_request:
     branches: [ "master" ]
 
+env:
+  MAINNET_RPC_URL: ${{ secrets.MAINNET_RPC_URL }}
+  GOERLI_RPC_URL: ${{ secrets.GOERLI_RPC_URL }}
+
 jobs:
   check:
     runs-on: ubuntu-latest

benches/get_balance.rs

@@ -48,7 +48,14 @@ pub fn bench_goerli_get_balance(c: &mut Criterion) {
             .unwrap();
 
         // Construct a goerli client using our harness and tokio runtime.
-        let client = std::sync::Arc::new(harness::construct_goerli_client(&rt).unwrap());
+        let gc = match harness::construct_goerli_client(&rt) {
+            Ok(gc) => gc,
+            Err(e) => {
+                println!("failed to construct goerli client: {}", e);
+                std::process::exit(1);
+            }
+        };
+        let client = std::sync::Arc::new(gc);
 
         // Get the beacon chain deposit contract address.
         let addr = Address::from_str("0xB4FBF271143F4FBf7B91A5ded31805e42b2208d6").unwrap();

client/src/client.rs

@@ -3,7 +3,9 @@ use std::sync::Arc;
 use config::networks::Network;
 use consensus::errors::ConsensusError;
 use ethers::prelude::{Address, U256};
-use ethers::types::{Filter, Log, SyncingStatus, Transaction, TransactionReceipt, H256};
+use ethers::types::{
+    FeeHistory, Filter, Log, SyncingStatus, Transaction, TransactionReceipt, H256,
+};
 use eyre::{eyre, Result};
 
 use common::types::BlockTag;
@@ -255,8 +257,8 @@ impl<DB: Database> Client<DB> {
 
         if let Err(err) = sync_res {
             match err {
-                NodeError::ConsensusSyncError(err) => match err.downcast_ref().unwrap() {
+                NodeError::ConsensusSyncError(err) => match err.downcast_ref() {
-                    ConsensusError::CheckpointTooOld => {
+                    Some(ConsensusError::CheckpointTooOld) => {
                         warn!(
                             "failed to sync consensus node with checkpoint: 0x{}",
                             hex::encode(
@@ -501,6 +503,19 @@ impl<DB: Database> Client<DB> {
         self.node.read().await.get_block_number()
     }
 
+    pub async fn get_fee_history(
+        &self,
+        block_count: u64,
+        last_block: u64,
+        reward_percentiles: &[f64],
+    ) -> Result<Option<FeeHistory>> {
+        self.node
+            .read()
+            .await
+            .get_fee_history(block_count, last_block, reward_percentiles)
+            .await
+    }
+
     pub async fn get_block_by_number(
         &self,
         block: BlockTag,

client/src/node.rs

@@ -4,7 +4,7 @@ use std::time::Duration;
 
 use ethers::prelude::{Address, U256};
 use ethers::types::{
-    Filter, Log, SyncProgress, SyncingStatus, Transaction, TransactionReceipt, H256,
+    FeeHistory, Filter, Log, SyncProgress, SyncingStatus, Transaction, TransactionReceipt, H256,
 };
 use eyre::{eyre, Result};
 
@@ -300,6 +300,17 @@ impl Node {
         }
     }
 
+    pub async fn get_fee_history(
+        &self,
+        block_count: u64,
+        last_block: u64,
+        reward_percentiles: &[f64],
+    ) -> Result<Option<FeeHistory>> {
+        self.execution
+            .get_fee_history(block_count, last_block, reward_percentiles, &self.payloads)
+            .await
+    }
+
     pub async fn get_block_by_hash(
         &self,
         hash: &Vec<u8>,

consensus/Cargo.toml

@@ -16,7 +16,7 @@ bytes = "1.2.1"
 toml = "0.5.9"
 async-trait = "0.1.57"
 log = "0.4.17"
-chrono = "0.4.22"
+chrono = "0.4.23"
 thiserror = "1.0.37"
 reqwest = { version = "0.11.13", features = ["json"] }
 

execution/src/errors.rs

@@ -26,6 +26,16 @@ pub enum ExecutionError {
     TooManyLogsToProve(usize, usize),
     #[error("execution rpc is for the incorect network")]
     IncorrectRpcNetwork(),
+    #[error("Invalid base gas fee helios {0} vs rpc endpoint {1} at block {2}")]
+    InvalidBaseGaseFee(U256, U256, u64),
+    #[error("Invalid gas used ratio of helios {0} vs rpc endpoint {1} at block {2}")]
+    InvalidGasUsedRatio(f64, f64, u64),
+    #[error("Block {0} not found")]
+    BlockNotFoundError(u64),
+    #[error("Helios Execution Payload is empty")]
+    EmptyExecutionPayload(),
+    #[error("User query for block {0} but helios oldest block is {1}")]
+    InvalidBlockRange(u64, u64),
 }
 
 /// Errors that can occur during evm.rs calls

execution/src/execution.rs

@@ -3,7 +3,7 @@ use std::str::FromStr;
 
 use ethers::abi::AbiEncode;
 use ethers::prelude::{Address, U256};
-use ethers::types::{Filter, Log, Transaction, TransactionReceipt, H256};
+use ethers::types::{FeeHistory, Filter, Log, Transaction, TransactionReceipt, H256};
 use ethers::utils::keccak256;
 use ethers::utils::rlp::{encode, Encodable, RlpStream};
 use eyre::Result;
@@ -345,6 +345,130 @@ impl<R: ExecutionRpc> ExecutionClient<R> {
         }
         Ok(logs)
     }
+
+    pub async fn get_fee_history(
+        &self,
+        block_count: u64,
+        last_block: u64,
+        _reward_percentiles: &[f64],
+        payloads: &BTreeMap<u64, ExecutionPayload>,
+    ) -> Result<Option<FeeHistory>> {
+        // Extract the latest and oldest block numbers from the payloads
+        let helios_latest_block_number = *payloads
+            .last_key_value()
+            .ok_or(ExecutionError::EmptyExecutionPayload())?
+            .0;
+        let helios_oldest_block_number = *payloads
+            .first_key_value()
+            .ok_or(ExecutionError::EmptyExecutionPayload())?
+            .0;
+
+        // Case where all requested blocks are earlier than Helios' latest block number
+        // So helios can't prove anything in this range
+        if last_block < helios_oldest_block_number {
+            return Err(
+                ExecutionError::InvalidBlockRange(last_block, helios_latest_block_number).into(),
+            );
+        }
+
+        // If the requested block is more recent than helios' latest block
+        // we can only return up to helios' latest block
+        let mut request_latest_block = last_block;
+        if request_latest_block > helios_latest_block_number {
+            request_latest_block = helios_latest_block_number;
+        }
+
+        // Requested oldest block is further out than what helios' synced
+        let mut request_oldest_block = request_latest_block - block_count;
+        if request_oldest_block < helios_oldest_block_number {
+            request_oldest_block = helios_oldest_block_number;
+        }
+
+        // Construct a fee history
+        let mut fee_history = FeeHistory {
+            oldest_block: U256::from(request_oldest_block),
+            base_fee_per_gas: vec![],
+            gas_used_ratio: vec![],
+            reward: payloads.iter().map(|_| vec![]).collect::<Vec<Vec<U256>>>(),
+        };
+        for block_id in request_oldest_block..=request_latest_block {
+            let execution_payload = payloads
+                .get(&block_id)
+                .ok_or(ExecutionError::EmptyExecutionPayload())?;
+            let converted_base_fee_per_gas = ethers::types::U256::from_little_endian(
+                &execution_payload.base_fee_per_gas.to_bytes_le(),
+            );
+            fee_history
+                .base_fee_per_gas
+                .push(converted_base_fee_per_gas);
+            let gas_used_ratio_helios = ((execution_payload.gas_used as f64
+                / execution_payload.gas_limit as f64)
+                * 10.0_f64.powi(12))
+            .round()
+                / 10.0_f64.powi(12);
+            fee_history.gas_used_ratio.push(gas_used_ratio_helios);
+        }
+
+        // TODO: Maybe place behind a query option param?
+        // Optionally verify the computed fee history using the rpc
+        // verify_fee_history(
+        //     &self.rpc,
+        //     &fee_history,
+        //     fee_history.base_fee_per_gas.len(),
+        //     request_latest_block,
+        //     reward_percentiles,
+        // ).await?;
+
+        Ok(Some(fee_history))
+    }
+}
+
+/// Verifies a fee history against an rpc.
+pub async fn verify_fee_history(
+    rpc: &impl ExecutionRpc,
+    calculated_fee_history: &FeeHistory,
+    block_count: u64,
+    request_latest_block: u64,
+    reward_percentiles: &[f64],
+) -> Result<()> {
+    let fee_history = rpc
+        .get_fee_history(block_count, request_latest_block, reward_percentiles)
+        .await?;
+
+    for (_pos, _base_fee_per_gas) in fee_history.base_fee_per_gas.iter().enumerate() {
+        // Break at last iteration
+        // Otherwise, this would add an additional block
+        if _pos == block_count as usize {
+            continue;
+        }
+
+        // Check base fee per gas
+        let block_to_check = (fee_history.oldest_block + _pos as u64).as_u64();
+        let fee_to_check = calculated_fee_history.base_fee_per_gas[_pos];
+        let gas_ratio_to_check = calculated_fee_history.gas_used_ratio[_pos];
+        if *_base_fee_per_gas != fee_to_check {
+            return Err(ExecutionError::InvalidBaseGaseFee(
+                fee_to_check,
+                *_base_fee_per_gas,
+                block_to_check,
+            )
+            .into());
+        }
+
+        // Check gas used ratio
+        let rpc_gas_used_rounded =
+            (fee_history.gas_used_ratio[_pos] * 10.0_f64.powi(12)).round() / 10.0_f64.powi(12);
+        if gas_ratio_to_check != rpc_gas_used_rounded {
+            return Err(ExecutionError::InvalidGasUsedRatio(
+                gas_ratio_to_check,
+                rpc_gas_used_rounded,
+                block_to_check,
+            )
+            .into());
+        }
+    }
+
+    Ok(())
 }
 
 fn encode_receipt(receipt: &TransactionReceipt) -> Vec<u8> {

execution/src/rpc/http_rpc.rs

@@ -6,8 +6,8 @@ use ethers::providers::{HttpRateLimitRetryPolicy, Middleware, Provider, RetryCli
 use ethers::types::transaction::eip2718::TypedTransaction;
 use ethers::types::transaction::eip2930::AccessList;
 use ethers::types::{
-    BlockId, Bytes, EIP1186ProofResponse, Eip1559TransactionRequest, Filter, Log, Transaction,
+    BlockId, BlockNumber, Bytes, EIP1186ProofResponse, Eip1559TransactionRequest, FeeHistory,
-    TransactionReceipt, H256, U256,
+    Filter, Log, Transaction, TransactionReceipt, H256, U256,
 };
 use eyre::Result;
 
@@ -140,4 +140,18 @@ impl ExecutionRpc for HttpRpc {
             .map_err(|e| RpcError::new("chain_id", e))?
             .as_u64())
     }
+
+    async fn get_fee_history(
+        &self,
+        block_count: u64,
+        last_block: u64,
+        reward_percentiles: &[f64],
+    ) -> Result<FeeHistory> {
+        let block = BlockNumber::from(last_block);
+        Ok(self
+            .provider
+            .fee_history(block_count, block, reward_percentiles)
+            .await
+            .map_err(|e| RpcError::new("fee_history", e))?)
+    }
 }

execution/src/rpc/mock_rpc.rs

@@ -3,8 +3,8 @@ use std::{fs::read_to_string, path::PathBuf};
 use async_trait::async_trait;
 use common::utils::hex_str_to_bytes;
 use ethers::types::{
-    transaction::eip2930::AccessList, Address, EIP1186ProofResponse, Filter, Log, Transaction,
+    transaction::eip2930::AccessList, Address, EIP1186ProofResponse, FeeHistory, Filter, Log,
-    TransactionReceipt, H256,
+    Transaction, TransactionReceipt, H256,
 };
 use eyre::{eyre, Result};
 
@@ -66,4 +66,14 @@ impl ExecutionRpc for MockRpc {
     async fn chain_id(&self) -> Result<u64> {
         Err(eyre!("not implemented"))
     }
+
+    async fn get_fee_history(
+        &self,
+        _block_count: u64,
+        _last_block: u64,
+        _reward_percentiles: &[f64],
+    ) -> Result<FeeHistory> {
+        let fee_history = read_to_string(self.path.join("fee_history.json"))?;
+        Ok(serde_json::from_str(&fee_history)?)
+    }
 }

execution/src/rpc/mod.rs

@@ -1,7 +1,7 @@
 use async_trait::async_trait;
 use ethers::types::{
-    transaction::eip2930::AccessList, Address, EIP1186ProofResponse, Filter, Log, Transaction,
+    transaction::eip2930::AccessList, Address, EIP1186ProofResponse, FeeHistory, Filter, Log,
-    TransactionReceipt, H256,
+    Transaction, TransactionReceipt, H256,
 };
 use eyre::Result;
 
@@ -31,4 +31,10 @@ pub trait ExecutionRpc: Send + Clone + Sync + 'static {
     async fn get_transaction(&self, tx_hash: &H256) -> Result<Option<Transaction>>;
     async fn get_logs(&self, filter: &Filter) -> Result<Vec<Log>>;
     async fn chain_id(&self) -> Result<u64>;
+    async fn get_fee_history(
+        &self,
+        block_count: u64,
+        last_block: u64,
+        reward_percentiles: &[f64],
+    ) -> Result<FeeHistory>;
 }

execution/testdata/fee_history.json

@@ -0,0 +1,35 @@
+{
+    "id": "1",
+    "jsonrpc": "2.0",
+    "result": {
+      "oldestBlock": 10762137,
+      "reward": [
+        [
+          "0x4a817c7ee",
+          "0x4a817c7ee"
+        ], [
+          "0x773593f0",
+          "0x773593f5"
+        ], [
+          "0x0",
+          "0x0"
+        ], [
+          "0x773593f5",
+          "0x773bae75"
+        ]
+      ],
+      "baseFeePerGas": [
+        "0x12",
+        "0x10",
+        "0x10",
+        "0xe",
+        "0xd"
+      ],
+      "gasUsedRatio": [
+        0.026089875,
+        0.406803,
+        0,
+        0.0866665
+      ]
+    }
+  }

tests/feehistory.rs

@@ -0,0 +1,122 @@
+use env_logger::Env;
+use eyre::Result;
+use helios::{config::networks::Network, prelude::*};
+use std::time::Duration;
+use std::{env, path::PathBuf};
+
+#[tokio::test]
+async fn feehistory() -> Result<()> {
+    env_logger::Builder::from_env(Env::default().default_filter_or("info")).init();
+
+    // Client Configuration
+    let api_key = env::var("MAINNET_RPC_URL").expect("MAINNET_RPC_URL env variable missing");
+    let checkpoint = "0x4d9b87a319c52e54068b7727a93dd3d52b83f7336ed93707bcdf7b37aefce700";
+    let consensus_rpc = "https://www.lightclientdata.org";
+    let data_dir = "/tmp/helios";
+    log::info!("Using consensus RPC URL: {}", consensus_rpc);
+
+    // Instantiate Client
+    let mut client: Client<FileDB> = ClientBuilder::new()
+        .network(Network::MAINNET)
+        .consensus_rpc(consensus_rpc)
+        .execution_rpc(&api_key)
+        .checkpoint(checkpoint)
+        .load_external_fallback()
+        .data_dir(PathBuf::from(data_dir))
+        .build()?;
+
+    log::info!(
+        "Built client on \"{}\" with external checkpoint fallbacks",
+        Network::MAINNET
+    );
+
+    client.start().await?;
+
+    // Wait for syncing
+    std::thread::sleep(Duration::from_secs(5));
+
+    // Get inputs for fee_history calls
+    let head_block_num = client.get_block_number().await?;
+    log::info!("head_block_num: {}", &head_block_num);
+    let block = BlockTag::Latest;
+    let block_number = BlockTag::Number(head_block_num);
+    log::info!("block {:?} and block_number {:?}", block, block_number);
+    let reward_percentiles: Vec<f64> = vec![];
+
+    // Get fee history for 1 block back from latest
+    let fee_history = client
+        .get_fee_history(1, head_block_num, &reward_percentiles)
+        .await?
+        .unwrap();
+    assert_eq!(fee_history.base_fee_per_gas.len(), 2);
+    assert_eq!(fee_history.oldest_block.as_u64(), head_block_num - 1);
+
+    // Fetch 10000 delta, helios will return as many as it can
+    let fee_history = match client
+        .get_fee_history(10_000, head_block_num, &reward_percentiles)
+        .await?
+    {
+        Some(fee_history) => fee_history,
+        None => panic!(
+            "empty gas fee returned with inputs: Block count: {:?}, Head Block #: {:?}, Reward Percentiles: {:?}",
+            10_000, head_block_num, &reward_percentiles
+        ),
+    };
+    assert!(
+        !fee_history.base_fee_per_gas.is_empty(),
+        "fee_history.base_fee_per_gas.len() {:?}",
+        fee_history.base_fee_per_gas.len()
+    );
+
+    // Fetch 10000 blocks in the past
+    // Helios will error since it won't have those historical blocks
+    let fee_history = client
+        .get_fee_history(1, head_block_num - 10_000, &reward_percentiles)
+        .await;
+    assert!(fee_history.is_err(), "fee_history() {fee_history:?}");
+
+    // Fetch 20 block away
+    // Should return array of size 21: our 20 block of interest + the next one
+    // The oldest block should be 19 block away, including it
+    let fee_history = client
+        .get_fee_history(20, head_block_num, &reward_percentiles)
+        .await?
+        .unwrap();
+    assert_eq!(
+        fee_history.base_fee_per_gas.len(),
+        21,
+        "fee_history.base_fee_per_gas.len() {:?} vs 21",
+        fee_history.base_fee_per_gas.len()
+    );
+    assert_eq!(
+        fee_history.oldest_block.as_u64(),
+        head_block_num - 20,
+        "fee_history.oldest_block.as_u64() {:?} vs head_block_num {:?} - 19",
+        fee_history.oldest_block.as_u64(),
+        head_block_num
+    );
+
+    // Fetch whatever blocks ahead, but that will fetch one block behind.
+    // This should return an answer of size two as Helios will cap this request to the newest block it knows
+    // we refresh parameters to make sure head_block_num is in line with newest block of our payload
+    let head_block_num = client.get_block_number().await?;
+    let fee_history = client
+        .get_fee_history(1, head_block_num + 1000, &reward_percentiles)
+        .await?
+        .unwrap();
+    assert_eq!(
+        fee_history.base_fee_per_gas.len(),
+        2,
+        "fee_history.base_fee_per_gas.len() {:?} vs 2",
+        fee_history.base_fee_per_gas.len()
+    );
+    assert_eq!(
+        fee_history.oldest_block.as_u64(),
+        head_block_num - 1,
+        "fee_history.oldest_block.as_u64() {:?} vs head_block_num {:?}",
+        fee_history.oldest_block.as_u64(),
+        head_block_num
+    );
+
+    Ok(())
+}