From 5a32f306861cad1e2642521e2b895e1d24e3e820 Mon Sep 17 00:00:00 2001 From: Noah Citron Date: Sat, 11 Mar 2023 01:59:29 -0500 Subject: [PATCH] feat: fee history (#211) * client get_fee_history * node get_fee_history * errors: InvalidBaseGaseFee * execution get_fee_history * http rpc get_fee_history * moc rpc get_fee_history and json file * module add get_fee_history * update exec * test feehistory * update execution with logging + better logic * fee history config loader * rust fmt client * rustfmt node * rustfmt error * rustfmt execution * rustfmt http and moc rpc * rustfmt mod.rs * fee history formating * correct typos * use env var * InvalidGasUsedRatio error * check gas used ratio * remove logging * Update execution/src/errors.rs Co-authored-by: refcell.eth * Update execution/src/execution.rs Co-authored-by: refcell.eth * adding block and payload errors * using error * handle error in test * fix: evm panic on slot not found (#208) * fixes, but test fails * fix: cleanup and example (#210) * clean up fee history * bump time dep in chrono, thx dependabot * add benches to pr * sleep * fmt :sparkles: * place benching behind a man flag --------- Co-authored-by: SFYLL Co-authored-by: SFYLL <39958632+SFYLL@users.noreply.github.com> Co-authored-by: refcell.eth --- .github/workflows/benchmarks.yml | 5 +- .github/workflows/test.yml | 4 + benches/get_balance.rs | 9 +- client/src/client.rs | 21 ++++- client/src/node.rs | 13 ++- consensus/Cargo.toml | 2 +- execution/src/errors.rs | 10 +++ execution/src/execution.rs | 126 +++++++++++++++++++++++++++- execution/src/rpc/http_rpc.rs | 18 +++- execution/src/rpc/mock_rpc.rs | 14 +++- execution/src/rpc/mod.rs | 10 ++- execution/testdata/fee_history.json | 35 ++++++++ tests/feehistory.rs | 122 +++++++++++++++++++++++++++ 13 files changed, 374 insertions(+), 15 deletions(-) create mode 100644 execution/testdata/fee_history.json create mode 100644 tests/feehistory.rs diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 4507a62..b7f967a 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -1,8 +1,9 @@ name: benchmarks on: - push: - branches: [ "master" ] + workflow_bench: + # push: + # branches: [ "master" ] env: MAINNET_RPC_URL: ${{ secrets.MAINNET_RPC_URL }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9f0a89c..61b2c6c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,6 +6,10 @@ on: pull_request: branches: [ "master" ] +env: + MAINNET_RPC_URL: ${{ secrets.MAINNET_RPC_URL }} + GOERLI_RPC_URL: ${{ secrets.GOERLI_RPC_URL }} + jobs: check: runs-on: ubuntu-latest diff --git a/benches/get_balance.rs b/benches/get_balance.rs index 19cd314..cc02512 100644 --- a/benches/get_balance.rs +++ b/benches/get_balance.rs @@ -48,7 +48,14 @@ pub fn bench_goerli_get_balance(c: &mut Criterion) { .unwrap(); // Construct a goerli client using our harness and tokio runtime. - let client = std::sync::Arc::new(harness::construct_goerli_client(&rt).unwrap()); + let gc = match harness::construct_goerli_client(&rt) { + Ok(gc) => gc, + Err(e) => { + println!("failed to construct goerli client: {}", e); + std::process::exit(1); + } + }; + let client = std::sync::Arc::new(gc); // Get the beacon chain deposit contract address. let addr = Address::from_str("0xB4FBF271143F4FBf7B91A5ded31805e42b2208d6").unwrap(); diff --git a/client/src/client.rs b/client/src/client.rs index 77abdce..a2e7f6a 100644 --- a/client/src/client.rs +++ b/client/src/client.rs @@ -3,7 +3,9 @@ use std::sync::Arc; use config::networks::Network; use consensus::errors::ConsensusError; use ethers::prelude::{Address, U256}; -use ethers::types::{Filter, Log, SyncingStatus, Transaction, TransactionReceipt, H256}; +use ethers::types::{ + FeeHistory, Filter, Log, SyncingStatus, Transaction, TransactionReceipt, H256, +}; use eyre::{eyre, Result}; use common::types::BlockTag; @@ -255,8 +257,8 @@ impl Client { if let Err(err) = sync_res { match err { - NodeError::ConsensusSyncError(err) => match err.downcast_ref().unwrap() { - ConsensusError::CheckpointTooOld => { + NodeError::ConsensusSyncError(err) => match err.downcast_ref() { + Some(ConsensusError::CheckpointTooOld) => { warn!( "failed to sync consensus node with checkpoint: 0x{}", hex::encode( @@ -501,6 +503,19 @@ impl Client { self.node.read().await.get_block_number() } + pub async fn get_fee_history( + &self, + block_count: u64, + last_block: u64, + reward_percentiles: &[f64], + ) -> Result> { + self.node + .read() + .await + .get_fee_history(block_count, last_block, reward_percentiles) + .await + } + pub async fn get_block_by_number( &self, block: BlockTag, diff --git a/client/src/node.rs b/client/src/node.rs index 6f06cee..d2091cb 100644 --- a/client/src/node.rs +++ b/client/src/node.rs @@ -4,7 +4,7 @@ use std::time::Duration; use ethers::prelude::{Address, U256}; use ethers::types::{ - Filter, Log, SyncProgress, SyncingStatus, Transaction, TransactionReceipt, H256, + FeeHistory, Filter, Log, SyncProgress, SyncingStatus, Transaction, TransactionReceipt, H256, }; use eyre::{eyre, Result}; @@ -300,6 +300,17 @@ impl Node { } } + pub async fn get_fee_history( + &self, + block_count: u64, + last_block: u64, + reward_percentiles: &[f64], + ) -> Result> { + self.execution + .get_fee_history(block_count, last_block, reward_percentiles, &self.payloads) + .await + } + pub async fn get_block_by_hash( &self, hash: &Vec, diff --git a/consensus/Cargo.toml b/consensus/Cargo.toml index c261ff4..6c5576f 100644 --- a/consensus/Cargo.toml +++ b/consensus/Cargo.toml @@ -16,7 +16,7 @@ bytes = "1.2.1" toml = "0.5.9" async-trait = "0.1.57" log = "0.4.17" -chrono = "0.4.22" +chrono = "0.4.23" thiserror = "1.0.37" reqwest = { version = "0.11.13", features = ["json"] } diff --git a/execution/src/errors.rs b/execution/src/errors.rs index 12b52b1..58da3a6 100644 --- a/execution/src/errors.rs +++ b/execution/src/errors.rs @@ -26,6 +26,16 @@ pub enum ExecutionError { TooManyLogsToProve(usize, usize), #[error("execution rpc is for the incorect network")] IncorrectRpcNetwork(), + #[error("Invalid base gas fee helios {0} vs rpc endpoint {1} at block {2}")] + InvalidBaseGaseFee(U256, U256, u64), + #[error("Invalid gas used ratio of helios {0} vs rpc endpoint {1} at block {2}")] + InvalidGasUsedRatio(f64, f64, u64), + #[error("Block {0} not found")] + BlockNotFoundError(u64), + #[error("Helios Execution Payload is empty")] + EmptyExecutionPayload(), + #[error("User query for block {0} but helios oldest block is {1}")] + InvalidBlockRange(u64, u64), } /// Errors that can occur during evm.rs calls diff --git a/execution/src/execution.rs b/execution/src/execution.rs index c103437..a7e8b03 100644 --- a/execution/src/execution.rs +++ b/execution/src/execution.rs @@ -3,7 +3,7 @@ use std::str::FromStr; use ethers::abi::AbiEncode; use ethers::prelude::{Address, U256}; -use ethers::types::{Filter, Log, Transaction, TransactionReceipt, H256}; +use ethers::types::{FeeHistory, Filter, Log, Transaction, TransactionReceipt, H256}; use ethers::utils::keccak256; use ethers::utils::rlp::{encode, Encodable, RlpStream}; use eyre::Result; @@ -345,6 +345,130 @@ impl ExecutionClient { } Ok(logs) } + + pub async fn get_fee_history( + &self, + block_count: u64, + last_block: u64, + _reward_percentiles: &[f64], + payloads: &BTreeMap, + ) -> Result> { + // Extract the latest and oldest block numbers from the payloads + let helios_latest_block_number = *payloads + .last_key_value() + .ok_or(ExecutionError::EmptyExecutionPayload())? + .0; + let helios_oldest_block_number = *payloads + .first_key_value() + .ok_or(ExecutionError::EmptyExecutionPayload())? + .0; + + // Case where all requested blocks are earlier than Helios' latest block number + // So helios can't prove anything in this range + if last_block < helios_oldest_block_number { + return Err( + ExecutionError::InvalidBlockRange(last_block, helios_latest_block_number).into(), + ); + } + + // If the requested block is more recent than helios' latest block + // we can only return up to helios' latest block + let mut request_latest_block = last_block; + if request_latest_block > helios_latest_block_number { + request_latest_block = helios_latest_block_number; + } + + // Requested oldest block is further out than what helios' synced + let mut request_oldest_block = request_latest_block - block_count; + if request_oldest_block < helios_oldest_block_number { + request_oldest_block = helios_oldest_block_number; + } + + // Construct a fee history + let mut fee_history = FeeHistory { + oldest_block: U256::from(request_oldest_block), + base_fee_per_gas: vec![], + gas_used_ratio: vec![], + reward: payloads.iter().map(|_| vec![]).collect::>>(), + }; + for block_id in request_oldest_block..=request_latest_block { + let execution_payload = payloads + .get(&block_id) + .ok_or(ExecutionError::EmptyExecutionPayload())?; + let converted_base_fee_per_gas = ethers::types::U256::from_little_endian( + &execution_payload.base_fee_per_gas.to_bytes_le(), + ); + fee_history + .base_fee_per_gas + .push(converted_base_fee_per_gas); + let gas_used_ratio_helios = ((execution_payload.gas_used as f64 + / execution_payload.gas_limit as f64) + * 10.0_f64.powi(12)) + .round() + / 10.0_f64.powi(12); + fee_history.gas_used_ratio.push(gas_used_ratio_helios); + } + + // TODO: Maybe place behind a query option param? + // Optionally verify the computed fee history using the rpc + // verify_fee_history( + // &self.rpc, + // &fee_history, + // fee_history.base_fee_per_gas.len(), + // request_latest_block, + // reward_percentiles, + // ).await?; + + Ok(Some(fee_history)) + } +} + +/// Verifies a fee history against an rpc. +pub async fn verify_fee_history( + rpc: &impl ExecutionRpc, + calculated_fee_history: &FeeHistory, + block_count: u64, + request_latest_block: u64, + reward_percentiles: &[f64], +) -> Result<()> { + let fee_history = rpc + .get_fee_history(block_count, request_latest_block, reward_percentiles) + .await?; + + for (_pos, _base_fee_per_gas) in fee_history.base_fee_per_gas.iter().enumerate() { + // Break at last iteration + // Otherwise, this would add an additional block + if _pos == block_count as usize { + continue; + } + + // Check base fee per gas + let block_to_check = (fee_history.oldest_block + _pos as u64).as_u64(); + let fee_to_check = calculated_fee_history.base_fee_per_gas[_pos]; + let gas_ratio_to_check = calculated_fee_history.gas_used_ratio[_pos]; + if *_base_fee_per_gas != fee_to_check { + return Err(ExecutionError::InvalidBaseGaseFee( + fee_to_check, + *_base_fee_per_gas, + block_to_check, + ) + .into()); + } + + // Check gas used ratio + let rpc_gas_used_rounded = + (fee_history.gas_used_ratio[_pos] * 10.0_f64.powi(12)).round() / 10.0_f64.powi(12); + if gas_ratio_to_check != rpc_gas_used_rounded { + return Err(ExecutionError::InvalidGasUsedRatio( + gas_ratio_to_check, + rpc_gas_used_rounded, + block_to_check, + ) + .into()); + } + } + + Ok(()) } fn encode_receipt(receipt: &TransactionReceipt) -> Vec { diff --git a/execution/src/rpc/http_rpc.rs b/execution/src/rpc/http_rpc.rs index 2007f3a..9c0f9b2 100644 --- a/execution/src/rpc/http_rpc.rs +++ b/execution/src/rpc/http_rpc.rs @@ -6,8 +6,8 @@ use ethers::providers::{HttpRateLimitRetryPolicy, Middleware, Provider, RetryCli use ethers::types::transaction::eip2718::TypedTransaction; use ethers::types::transaction::eip2930::AccessList; use ethers::types::{ - BlockId, Bytes, EIP1186ProofResponse, Eip1559TransactionRequest, Filter, Log, Transaction, - TransactionReceipt, H256, U256, + BlockId, BlockNumber, Bytes, EIP1186ProofResponse, Eip1559TransactionRequest, FeeHistory, + Filter, Log, Transaction, TransactionReceipt, H256, U256, }; use eyre::Result; @@ -140,4 +140,18 @@ impl ExecutionRpc for HttpRpc { .map_err(|e| RpcError::new("chain_id", e))? .as_u64()) } + + async fn get_fee_history( + &self, + block_count: u64, + last_block: u64, + reward_percentiles: &[f64], + ) -> Result { + let block = BlockNumber::from(last_block); + Ok(self + .provider + .fee_history(block_count, block, reward_percentiles) + .await + .map_err(|e| RpcError::new("fee_history", e))?) + } } diff --git a/execution/src/rpc/mock_rpc.rs b/execution/src/rpc/mock_rpc.rs index e7ede3c..42bec5e 100644 --- a/execution/src/rpc/mock_rpc.rs +++ b/execution/src/rpc/mock_rpc.rs @@ -3,8 +3,8 @@ use std::{fs::read_to_string, path::PathBuf}; use async_trait::async_trait; use common::utils::hex_str_to_bytes; use ethers::types::{ - transaction::eip2930::AccessList, Address, EIP1186ProofResponse, Filter, Log, Transaction, - TransactionReceipt, H256, + transaction::eip2930::AccessList, Address, EIP1186ProofResponse, FeeHistory, Filter, Log, + Transaction, TransactionReceipt, H256, }; use eyre::{eyre, Result}; @@ -66,4 +66,14 @@ impl ExecutionRpc for MockRpc { async fn chain_id(&self) -> Result { Err(eyre!("not implemented")) } + + async fn get_fee_history( + &self, + _block_count: u64, + _last_block: u64, + _reward_percentiles: &[f64], + ) -> Result { + let fee_history = read_to_string(self.path.join("fee_history.json"))?; + Ok(serde_json::from_str(&fee_history)?) + } } diff --git a/execution/src/rpc/mod.rs b/execution/src/rpc/mod.rs index f2157a8..d4dd8a5 100644 --- a/execution/src/rpc/mod.rs +++ b/execution/src/rpc/mod.rs @@ -1,7 +1,7 @@ use async_trait::async_trait; use ethers::types::{ - transaction::eip2930::AccessList, Address, EIP1186ProofResponse, Filter, Log, Transaction, - TransactionReceipt, H256, + transaction::eip2930::AccessList, Address, EIP1186ProofResponse, FeeHistory, Filter, Log, + Transaction, TransactionReceipt, H256, }; use eyre::Result; @@ -31,4 +31,10 @@ pub trait ExecutionRpc: Send + Clone + Sync + 'static { async fn get_transaction(&self, tx_hash: &H256) -> Result>; async fn get_logs(&self, filter: &Filter) -> Result>; async fn chain_id(&self) -> Result; + async fn get_fee_history( + &self, + block_count: u64, + last_block: u64, + reward_percentiles: &[f64], + ) -> Result; } diff --git a/execution/testdata/fee_history.json b/execution/testdata/fee_history.json new file mode 100644 index 0000000..edf4d79 --- /dev/null +++ b/execution/testdata/fee_history.json @@ -0,0 +1,35 @@ +{ + "id": "1", + "jsonrpc": "2.0", + "result": { + "oldestBlock": 10762137, + "reward": [ + [ + "0x4a817c7ee", + "0x4a817c7ee" + ], [ + "0x773593f0", + "0x773593f5" + ], [ + "0x0", + "0x0" + ], [ + "0x773593f5", + "0x773bae75" + ] + ], + "baseFeePerGas": [ + "0x12", + "0x10", + "0x10", + "0xe", + "0xd" + ], + "gasUsedRatio": [ + 0.026089875, + 0.406803, + 0, + 0.0866665 + ] + } + } \ No newline at end of file diff --git a/tests/feehistory.rs b/tests/feehistory.rs new file mode 100644 index 0000000..db61103 --- /dev/null +++ b/tests/feehistory.rs @@ -0,0 +1,122 @@ +use env_logger::Env; +use eyre::Result; +use helios::{config::networks::Network, prelude::*}; +use std::time::Duration; +use std::{env, path::PathBuf}; + +#[tokio::test] +async fn feehistory() -> Result<()> { + env_logger::Builder::from_env(Env::default().default_filter_or("info")).init(); + + // Client Configuration + let api_key = env::var("MAINNET_RPC_URL").expect("MAINNET_RPC_URL env variable missing"); + let checkpoint = "0x4d9b87a319c52e54068b7727a93dd3d52b83f7336ed93707bcdf7b37aefce700"; + let consensus_rpc = "https://www.lightclientdata.org"; + let data_dir = "/tmp/helios"; + log::info!("Using consensus RPC URL: {}", consensus_rpc); + + // Instantiate Client + let mut client: Client = ClientBuilder::new() + .network(Network::MAINNET) + .consensus_rpc(consensus_rpc) + .execution_rpc(&api_key) + .checkpoint(checkpoint) + .load_external_fallback() + .data_dir(PathBuf::from(data_dir)) + .build()?; + + log::info!( + "Built client on \"{}\" with external checkpoint fallbacks", + Network::MAINNET + ); + + client.start().await?; + + // Wait for syncing + std::thread::sleep(Duration::from_secs(5)); + + // Get inputs for fee_history calls + let head_block_num = client.get_block_number().await?; + log::info!("head_block_num: {}", &head_block_num); + let block = BlockTag::Latest; + let block_number = BlockTag::Number(head_block_num); + log::info!("block {:?} and block_number {:?}", block, block_number); + let reward_percentiles: Vec = vec![]; + + // Get fee history for 1 block back from latest + let fee_history = client + .get_fee_history(1, head_block_num, &reward_percentiles) + .await? + .unwrap(); + assert_eq!(fee_history.base_fee_per_gas.len(), 2); + assert_eq!(fee_history.oldest_block.as_u64(), head_block_num - 1); + + // Fetch 10000 delta, helios will return as many as it can + let fee_history = match client + .get_fee_history(10_000, head_block_num, &reward_percentiles) + .await? + { + Some(fee_history) => fee_history, + None => panic!( + "empty gas fee returned with inputs: Block count: {:?}, Head Block #: {:?}, Reward Percentiles: {:?}", + 10_000, head_block_num, &reward_percentiles + ), + }; + assert!( + !fee_history.base_fee_per_gas.is_empty(), + "fee_history.base_fee_per_gas.len() {:?}", + fee_history.base_fee_per_gas.len() + ); + + // Fetch 10000 blocks in the past + // Helios will error since it won't have those historical blocks + let fee_history = client + .get_fee_history(1, head_block_num - 10_000, &reward_percentiles) + .await; + assert!(fee_history.is_err(), "fee_history() {fee_history:?}"); + + // Fetch 20 block away + // Should return array of size 21: our 20 block of interest + the next one + // The oldest block should be 19 block away, including it + let fee_history = client + .get_fee_history(20, head_block_num, &reward_percentiles) + .await? + .unwrap(); + assert_eq!( + fee_history.base_fee_per_gas.len(), + 21, + "fee_history.base_fee_per_gas.len() {:?} vs 21", + fee_history.base_fee_per_gas.len() + ); + assert_eq!( + fee_history.oldest_block.as_u64(), + head_block_num - 20, + "fee_history.oldest_block.as_u64() {:?} vs head_block_num {:?} - 19", + fee_history.oldest_block.as_u64(), + head_block_num + ); + + // Fetch whatever blocks ahead, but that will fetch one block behind. + // This should return an answer of size two as Helios will cap this request to the newest block it knows + // we refresh parameters to make sure head_block_num is in line with newest block of our payload + let head_block_num = client.get_block_number().await?; + let fee_history = client + .get_fee_history(1, head_block_num + 1000, &reward_percentiles) + .await? + .unwrap(); + assert_eq!( + fee_history.base_fee_per_gas.len(), + 2, + "fee_history.base_fee_per_gas.len() {:?} vs 2", + fee_history.base_fee_per_gas.len() + ); + assert_eq!( + fee_history.oldest_block.as_u64(), + head_block_num - 1, + "fee_history.oldest_block.as_u64() {:?} vs head_block_num {:?}", + fee_history.oldest_block.as_u64(), + head_block_num + ); + + Ok(()) +}