feat: fee history (#211)

* client get_fee_history

* node get_fee_history

* errors: InvalidBaseGaseFee

* execution get_fee_history

* http rpc get_fee_history

* moc rpc get_fee_history and json file

* module add get_fee_history

* update exec

* test feehistory

* update execution with logging + better logic

* fee history config loader

* rust fmt client

* rustfmt node

* rustfmt error

* rustfmt execution

* rustfmt http and moc rpc

* rustfmt mod.rs

* fee history formating

* correct typos

* use env var

* InvalidGasUsedRatio error

* check gas used ratio

* remove logging

* Update execution/src/errors.rs

Co-authored-by: refcell.eth <abigger87@gmail.com>

* Update execution/src/execution.rs

Co-authored-by: refcell.eth <abigger87@gmail.com>

* adding block and payload errors

* using error

* handle error in test

* fix: evm panic on slot not found (#208)

* fixes, but test fails

* fix: cleanup and example (#210)

* clean up fee history

* bump time dep in chrono, thx dependabot

* add benches to pr

* sleep

* fmt 

* place benching behind a man flag

---------

Co-authored-by: SFYLL <santiagoflood@hotmail.fr>
Co-authored-by: SFYLL <39958632+SFYLL@users.noreply.github.com>
Co-authored-by: refcell.eth <abigger87@gmail.com>
This commit is contained in:
Noah Citron 2023-03-11 01:59:29 -05:00 committed by GitHub
parent 3c471c2bef
commit 5a32f30686
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 374 additions and 15 deletions

View File

@ -1,8 +1,9 @@
name: benchmarks name: benchmarks
on: on:
push: workflow_bench:
branches: [ "master" ] # push:
# branches: [ "master" ]
env: env:
MAINNET_RPC_URL: ${{ secrets.MAINNET_RPC_URL }} MAINNET_RPC_URL: ${{ secrets.MAINNET_RPC_URL }}

View File

@ -6,6 +6,10 @@ on:
pull_request: pull_request:
branches: [ "master" ] branches: [ "master" ]
env:
MAINNET_RPC_URL: ${{ secrets.MAINNET_RPC_URL }}
GOERLI_RPC_URL: ${{ secrets.GOERLI_RPC_URL }}
jobs: jobs:
check: check:
runs-on: ubuntu-latest runs-on: ubuntu-latest

View File

@ -48,7 +48,14 @@ pub fn bench_goerli_get_balance(c: &mut Criterion) {
.unwrap(); .unwrap();
// Construct a goerli client using our harness and tokio runtime. // Construct a goerli client using our harness and tokio runtime.
let client = std::sync::Arc::new(harness::construct_goerli_client(&rt).unwrap()); let gc = match harness::construct_goerli_client(&rt) {
Ok(gc) => gc,
Err(e) => {
println!("failed to construct goerli client: {}", e);
std::process::exit(1);
}
};
let client = std::sync::Arc::new(gc);
// Get the beacon chain deposit contract address. // Get the beacon chain deposit contract address.
let addr = Address::from_str("0xB4FBF271143F4FBf7B91A5ded31805e42b2208d6").unwrap(); let addr = Address::from_str("0xB4FBF271143F4FBf7B91A5ded31805e42b2208d6").unwrap();

View File

@ -3,7 +3,9 @@ use std::sync::Arc;
use config::networks::Network; use config::networks::Network;
use consensus::errors::ConsensusError; use consensus::errors::ConsensusError;
use ethers::prelude::{Address, U256}; use ethers::prelude::{Address, U256};
use ethers::types::{Filter, Log, SyncingStatus, Transaction, TransactionReceipt, H256}; use ethers::types::{
FeeHistory, Filter, Log, SyncingStatus, Transaction, TransactionReceipt, H256,
};
use eyre::{eyre, Result}; use eyre::{eyre, Result};
use common::types::BlockTag; use common::types::BlockTag;
@ -255,8 +257,8 @@ impl<DB: Database> Client<DB> {
if let Err(err) = sync_res { if let Err(err) = sync_res {
match err { match err {
NodeError::ConsensusSyncError(err) => match err.downcast_ref().unwrap() { NodeError::ConsensusSyncError(err) => match err.downcast_ref() {
ConsensusError::CheckpointTooOld => { Some(ConsensusError::CheckpointTooOld) => {
warn!( warn!(
"failed to sync consensus node with checkpoint: 0x{}", "failed to sync consensus node with checkpoint: 0x{}",
hex::encode( hex::encode(
@ -501,6 +503,19 @@ impl<DB: Database> Client<DB> {
self.node.read().await.get_block_number() self.node.read().await.get_block_number()
} }
pub async fn get_fee_history(
&self,
block_count: u64,
last_block: u64,
reward_percentiles: &[f64],
) -> Result<Option<FeeHistory>> {
self.node
.read()
.await
.get_fee_history(block_count, last_block, reward_percentiles)
.await
}
pub async fn get_block_by_number( pub async fn get_block_by_number(
&self, &self,
block: BlockTag, block: BlockTag,

View File

@ -4,7 +4,7 @@ use std::time::Duration;
use ethers::prelude::{Address, U256}; use ethers::prelude::{Address, U256};
use ethers::types::{ use ethers::types::{
Filter, Log, SyncProgress, SyncingStatus, Transaction, TransactionReceipt, H256, FeeHistory, Filter, Log, SyncProgress, SyncingStatus, Transaction, TransactionReceipt, H256,
}; };
use eyre::{eyre, Result}; use eyre::{eyre, Result};
@ -300,6 +300,17 @@ impl Node {
} }
} }
pub async fn get_fee_history(
&self,
block_count: u64,
last_block: u64,
reward_percentiles: &[f64],
) -> Result<Option<FeeHistory>> {
self.execution
.get_fee_history(block_count, last_block, reward_percentiles, &self.payloads)
.await
}
pub async fn get_block_by_hash( pub async fn get_block_by_hash(
&self, &self,
hash: &Vec<u8>, hash: &Vec<u8>,

View File

@ -16,7 +16,7 @@ bytes = "1.2.1"
toml = "0.5.9" toml = "0.5.9"
async-trait = "0.1.57" async-trait = "0.1.57"
log = "0.4.17" log = "0.4.17"
chrono = "0.4.22" chrono = "0.4.23"
thiserror = "1.0.37" thiserror = "1.0.37"
reqwest = { version = "0.11.13", features = ["json"] } reqwest = { version = "0.11.13", features = ["json"] }

View File

@ -26,6 +26,16 @@ pub enum ExecutionError {
TooManyLogsToProve(usize, usize), TooManyLogsToProve(usize, usize),
#[error("execution rpc is for the incorect network")] #[error("execution rpc is for the incorect network")]
IncorrectRpcNetwork(), IncorrectRpcNetwork(),
#[error("Invalid base gas fee helios {0} vs rpc endpoint {1} at block {2}")]
InvalidBaseGaseFee(U256, U256, u64),
#[error("Invalid gas used ratio of helios {0} vs rpc endpoint {1} at block {2}")]
InvalidGasUsedRatio(f64, f64, u64),
#[error("Block {0} not found")]
BlockNotFoundError(u64),
#[error("Helios Execution Payload is empty")]
EmptyExecutionPayload(),
#[error("User query for block {0} but helios oldest block is {1}")]
InvalidBlockRange(u64, u64),
} }
/// Errors that can occur during evm.rs calls /// Errors that can occur during evm.rs calls

View File

@ -3,7 +3,7 @@ use std::str::FromStr;
use ethers::abi::AbiEncode; use ethers::abi::AbiEncode;
use ethers::prelude::{Address, U256}; use ethers::prelude::{Address, U256};
use ethers::types::{Filter, Log, Transaction, TransactionReceipt, H256}; use ethers::types::{FeeHistory, Filter, Log, Transaction, TransactionReceipt, H256};
use ethers::utils::keccak256; use ethers::utils::keccak256;
use ethers::utils::rlp::{encode, Encodable, RlpStream}; use ethers::utils::rlp::{encode, Encodable, RlpStream};
use eyre::Result; use eyre::Result;
@ -345,6 +345,130 @@ impl<R: ExecutionRpc> ExecutionClient<R> {
} }
Ok(logs) Ok(logs)
} }
pub async fn get_fee_history(
&self,
block_count: u64,
last_block: u64,
_reward_percentiles: &[f64],
payloads: &BTreeMap<u64, ExecutionPayload>,
) -> Result<Option<FeeHistory>> {
// Extract the latest and oldest block numbers from the payloads
let helios_latest_block_number = *payloads
.last_key_value()
.ok_or(ExecutionError::EmptyExecutionPayload())?
.0;
let helios_oldest_block_number = *payloads
.first_key_value()
.ok_or(ExecutionError::EmptyExecutionPayload())?
.0;
// Case where all requested blocks are earlier than Helios' latest block number
// So helios can't prove anything in this range
if last_block < helios_oldest_block_number {
return Err(
ExecutionError::InvalidBlockRange(last_block, helios_latest_block_number).into(),
);
}
// If the requested block is more recent than helios' latest block
// we can only return up to helios' latest block
let mut request_latest_block = last_block;
if request_latest_block > helios_latest_block_number {
request_latest_block = helios_latest_block_number;
}
// Requested oldest block is further out than what helios' synced
let mut request_oldest_block = request_latest_block - block_count;
if request_oldest_block < helios_oldest_block_number {
request_oldest_block = helios_oldest_block_number;
}
// Construct a fee history
let mut fee_history = FeeHistory {
oldest_block: U256::from(request_oldest_block),
base_fee_per_gas: vec![],
gas_used_ratio: vec![],
reward: payloads.iter().map(|_| vec![]).collect::<Vec<Vec<U256>>>(),
};
for block_id in request_oldest_block..=request_latest_block {
let execution_payload = payloads
.get(&block_id)
.ok_or(ExecutionError::EmptyExecutionPayload())?;
let converted_base_fee_per_gas = ethers::types::U256::from_little_endian(
&execution_payload.base_fee_per_gas.to_bytes_le(),
);
fee_history
.base_fee_per_gas
.push(converted_base_fee_per_gas);
let gas_used_ratio_helios = ((execution_payload.gas_used as f64
/ execution_payload.gas_limit as f64)
* 10.0_f64.powi(12))
.round()
/ 10.0_f64.powi(12);
fee_history.gas_used_ratio.push(gas_used_ratio_helios);
}
// TODO: Maybe place behind a query option param?
// Optionally verify the computed fee history using the rpc
// verify_fee_history(
// &self.rpc,
// &fee_history,
// fee_history.base_fee_per_gas.len(),
// request_latest_block,
// reward_percentiles,
// ).await?;
Ok(Some(fee_history))
}
}
/// Verifies a fee history against an rpc.
pub async fn verify_fee_history(
rpc: &impl ExecutionRpc,
calculated_fee_history: &FeeHistory,
block_count: u64,
request_latest_block: u64,
reward_percentiles: &[f64],
) -> Result<()> {
let fee_history = rpc
.get_fee_history(block_count, request_latest_block, reward_percentiles)
.await?;
for (_pos, _base_fee_per_gas) in fee_history.base_fee_per_gas.iter().enumerate() {
// Break at last iteration
// Otherwise, this would add an additional block
if _pos == block_count as usize {
continue;
}
// Check base fee per gas
let block_to_check = (fee_history.oldest_block + _pos as u64).as_u64();
let fee_to_check = calculated_fee_history.base_fee_per_gas[_pos];
let gas_ratio_to_check = calculated_fee_history.gas_used_ratio[_pos];
if *_base_fee_per_gas != fee_to_check {
return Err(ExecutionError::InvalidBaseGaseFee(
fee_to_check,
*_base_fee_per_gas,
block_to_check,
)
.into());
}
// Check gas used ratio
let rpc_gas_used_rounded =
(fee_history.gas_used_ratio[_pos] * 10.0_f64.powi(12)).round() / 10.0_f64.powi(12);
if gas_ratio_to_check != rpc_gas_used_rounded {
return Err(ExecutionError::InvalidGasUsedRatio(
gas_ratio_to_check,
rpc_gas_used_rounded,
block_to_check,
)
.into());
}
}
Ok(())
} }
fn encode_receipt(receipt: &TransactionReceipt) -> Vec<u8> { fn encode_receipt(receipt: &TransactionReceipt) -> Vec<u8> {

View File

@ -6,8 +6,8 @@ use ethers::providers::{HttpRateLimitRetryPolicy, Middleware, Provider, RetryCli
use ethers::types::transaction::eip2718::TypedTransaction; use ethers::types::transaction::eip2718::TypedTransaction;
use ethers::types::transaction::eip2930::AccessList; use ethers::types::transaction::eip2930::AccessList;
use ethers::types::{ use ethers::types::{
BlockId, Bytes, EIP1186ProofResponse, Eip1559TransactionRequest, Filter, Log, Transaction, BlockId, BlockNumber, Bytes, EIP1186ProofResponse, Eip1559TransactionRequest, FeeHistory,
TransactionReceipt, H256, U256, Filter, Log, Transaction, TransactionReceipt, H256, U256,
}; };
use eyre::Result; use eyre::Result;
@ -140,4 +140,18 @@ impl ExecutionRpc for HttpRpc {
.map_err(|e| RpcError::new("chain_id", e))? .map_err(|e| RpcError::new("chain_id", e))?
.as_u64()) .as_u64())
} }
async fn get_fee_history(
&self,
block_count: u64,
last_block: u64,
reward_percentiles: &[f64],
) -> Result<FeeHistory> {
let block = BlockNumber::from(last_block);
Ok(self
.provider
.fee_history(block_count, block, reward_percentiles)
.await
.map_err(|e| RpcError::new("fee_history", e))?)
}
} }

View File

@ -3,8 +3,8 @@ use std::{fs::read_to_string, path::PathBuf};
use async_trait::async_trait; use async_trait::async_trait;
use common::utils::hex_str_to_bytes; use common::utils::hex_str_to_bytes;
use ethers::types::{ use ethers::types::{
transaction::eip2930::AccessList, Address, EIP1186ProofResponse, Filter, Log, Transaction, transaction::eip2930::AccessList, Address, EIP1186ProofResponse, FeeHistory, Filter, Log,
TransactionReceipt, H256, Transaction, TransactionReceipt, H256,
}; };
use eyre::{eyre, Result}; use eyre::{eyre, Result};
@ -66,4 +66,14 @@ impl ExecutionRpc for MockRpc {
async fn chain_id(&self) -> Result<u64> { async fn chain_id(&self) -> Result<u64> {
Err(eyre!("not implemented")) Err(eyre!("not implemented"))
} }
async fn get_fee_history(
&self,
_block_count: u64,
_last_block: u64,
_reward_percentiles: &[f64],
) -> Result<FeeHistory> {
let fee_history = read_to_string(self.path.join("fee_history.json"))?;
Ok(serde_json::from_str(&fee_history)?)
}
} }

View File

@ -1,7 +1,7 @@
use async_trait::async_trait; use async_trait::async_trait;
use ethers::types::{ use ethers::types::{
transaction::eip2930::AccessList, Address, EIP1186ProofResponse, Filter, Log, Transaction, transaction::eip2930::AccessList, Address, EIP1186ProofResponse, FeeHistory, Filter, Log,
TransactionReceipt, H256, Transaction, TransactionReceipt, H256,
}; };
use eyre::Result; use eyre::Result;
@ -31,4 +31,10 @@ pub trait ExecutionRpc: Send + Clone + Sync + 'static {
async fn get_transaction(&self, tx_hash: &H256) -> Result<Option<Transaction>>; async fn get_transaction(&self, tx_hash: &H256) -> Result<Option<Transaction>>;
async fn get_logs(&self, filter: &Filter) -> Result<Vec<Log>>; async fn get_logs(&self, filter: &Filter) -> Result<Vec<Log>>;
async fn chain_id(&self) -> Result<u64>; async fn chain_id(&self) -> Result<u64>;
async fn get_fee_history(
&self,
block_count: u64,
last_block: u64,
reward_percentiles: &[f64],
) -> Result<FeeHistory>;
} }

35
execution/testdata/fee_history.json vendored Normal file
View File

@ -0,0 +1,35 @@
{
"id": "1",
"jsonrpc": "2.0",
"result": {
"oldestBlock": 10762137,
"reward": [
[
"0x4a817c7ee",
"0x4a817c7ee"
], [
"0x773593f0",
"0x773593f5"
], [
"0x0",
"0x0"
], [
"0x773593f5",
"0x773bae75"
]
],
"baseFeePerGas": [
"0x12",
"0x10",
"0x10",
"0xe",
"0xd"
],
"gasUsedRatio": [
0.026089875,
0.406803,
0,
0.0866665
]
}
}

122
tests/feehistory.rs Normal file
View File

@ -0,0 +1,122 @@
use env_logger::Env;
use eyre::Result;
use helios::{config::networks::Network, prelude::*};
use std::time::Duration;
use std::{env, path::PathBuf};
#[tokio::test]
async fn feehistory() -> Result<()> {
env_logger::Builder::from_env(Env::default().default_filter_or("info")).init();
// Client Configuration
let api_key = env::var("MAINNET_RPC_URL").expect("MAINNET_RPC_URL env variable missing");
let checkpoint = "0x4d9b87a319c52e54068b7727a93dd3d52b83f7336ed93707bcdf7b37aefce700";
let consensus_rpc = "https://www.lightclientdata.org";
let data_dir = "/tmp/helios";
log::info!("Using consensus RPC URL: {}", consensus_rpc);
// Instantiate Client
let mut client: Client<FileDB> = ClientBuilder::new()
.network(Network::MAINNET)
.consensus_rpc(consensus_rpc)
.execution_rpc(&api_key)
.checkpoint(checkpoint)
.load_external_fallback()
.data_dir(PathBuf::from(data_dir))
.build()?;
log::info!(
"Built client on \"{}\" with external checkpoint fallbacks",
Network::MAINNET
);
client.start().await?;
// Wait for syncing
std::thread::sleep(Duration::from_secs(5));
// Get inputs for fee_history calls
let head_block_num = client.get_block_number().await?;
log::info!("head_block_num: {}", &head_block_num);
let block = BlockTag::Latest;
let block_number = BlockTag::Number(head_block_num);
log::info!("block {:?} and block_number {:?}", block, block_number);
let reward_percentiles: Vec<f64> = vec![];
// Get fee history for 1 block back from latest
let fee_history = client
.get_fee_history(1, head_block_num, &reward_percentiles)
.await?
.unwrap();
assert_eq!(fee_history.base_fee_per_gas.len(), 2);
assert_eq!(fee_history.oldest_block.as_u64(), head_block_num - 1);
// Fetch 10000 delta, helios will return as many as it can
let fee_history = match client
.get_fee_history(10_000, head_block_num, &reward_percentiles)
.await?
{
Some(fee_history) => fee_history,
None => panic!(
"empty gas fee returned with inputs: Block count: {:?}, Head Block #: {:?}, Reward Percentiles: {:?}",
10_000, head_block_num, &reward_percentiles
),
};
assert!(
!fee_history.base_fee_per_gas.is_empty(),
"fee_history.base_fee_per_gas.len() {:?}",
fee_history.base_fee_per_gas.len()
);
// Fetch 10000 blocks in the past
// Helios will error since it won't have those historical blocks
let fee_history = client
.get_fee_history(1, head_block_num - 10_000, &reward_percentiles)
.await;
assert!(fee_history.is_err(), "fee_history() {fee_history:?}");
// Fetch 20 block away
// Should return array of size 21: our 20 block of interest + the next one
// The oldest block should be 19 block away, including it
let fee_history = client
.get_fee_history(20, head_block_num, &reward_percentiles)
.await?
.unwrap();
assert_eq!(
fee_history.base_fee_per_gas.len(),
21,
"fee_history.base_fee_per_gas.len() {:?} vs 21",
fee_history.base_fee_per_gas.len()
);
assert_eq!(
fee_history.oldest_block.as_u64(),
head_block_num - 20,
"fee_history.oldest_block.as_u64() {:?} vs head_block_num {:?} - 19",
fee_history.oldest_block.as_u64(),
head_block_num
);
// Fetch whatever blocks ahead, but that will fetch one block behind.
// This should return an answer of size two as Helios will cap this request to the newest block it knows
// we refresh parameters to make sure head_block_num is in line with newest block of our payload
let head_block_num = client.get_block_number().await?;
let fee_history = client
.get_fee_history(1, head_block_num + 1000, &reward_percentiles)
.await?
.unwrap();
assert_eq!(
fee_history.base_fee_per_gas.len(),
2,
"fee_history.base_fee_per_gas.len() {:?} vs 2",
fee_history.base_fee_per_gas.len()
);
assert_eq!(
fee_history.oldest_block.as_u64(),
head_block_num - 1,
"fee_history.oldest_block.as_u64() {:?} vs head_block_num {:?}",
fee_history.oldest_block.as_u64(),
head_block_num
);
Ok(())
}