Robust gas oracles (#1222)

* Pass reqwest Client to constructors

* Add Median oracle aggregator

* DRY

* Weighted median

* Add cache layer

* Simplify lifetimes

* Add with_client constructors

* Update GasNow urls

* Add u256_from_f64_saturating

* Add polygon oracle

* Fixes

* Fix lints

* Remove dbg statements
This commit is contained in:
Remco Bloemen 2022-05-06 08:16:43 -07:00 committed by GitHub
parent ce3ebaefa0
commit 18b4ef4e47
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 503 additions and 38 deletions

12
Cargo.lock generated
View File

@ -1292,6 +1292,7 @@ dependencies = [
"ethers-providers",
"ethers-signers",
"ethers-solc",
"futures-locks",
"futures-util",
"hex",
"instant",
@ -1588,6 +1589,17 @@ version = "0.3.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc4045962a5a5e935ee2fdedaa4e08284547402885ab326734432bed5d12966b"
[[package]]
name = "futures-locks"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3eb42d4fb72227be5778429f9ef5240a38a358925a49f05b5cf702ce7c7e558a"
dependencies = [
"futures-channel",
"futures-task",
"tokio",
]
[[package]]
name = "futures-macro"
version = "0.3.21"

View File

@ -21,6 +21,9 @@ pub use address_or_bytes::AddressOrBytes;
mod path_or_string;
pub use path_or_string::PathOrString;
mod u256;
pub use u256::*;
mod i256;
pub use i256::{Sign, I256};

View File

@ -0,0 +1,121 @@
use ethabi::ethereum_types::U256;
/// Convert a floating point value to its nearest f64 integer.
///
/// It is saturating, so values $\ge 2^{256}$ will be rounded
/// to [`U256::max_value()`] and values $< 0$ to zero. This includes
/// positive and negative infinity.
///
/// TODO: Move to ethabi::ethereum_types::U256.
/// TODO: Add [`super::I256`] version.
///
/// # Panics
///
/// Panics if `f` is NaN.
pub fn u256_from_f64_saturating(mut f: f64) -> U256 {
if f.is_nan() {
panic!("NaN is not a valid value for U256");
}
if f < 0.5 {
return U256::zero()
}
if f >= 1.157_920_892_373_162e77_f64 {
return U256::max_value()
}
// All non-normal cases should have been handled above
assert!(f.is_normal());
// Turn nearest rounding into truncated rounding
f += 0.5;
// Parse IEEE-754 double into U256
// Sign should be zero, exponent should be >= 0.
let bits = f.to_bits();
let sign = bits >> 63;
assert!(sign == 0);
let biased_exponent = (bits >> 52) & 0x7ff;
assert!(biased_exponent >= 1023);
let exponent = biased_exponent - 1023;
let fraction = bits & 0xfffffffffffff;
let mantissa = 0x10000000000000 | fraction;
if exponent > 255 {
U256::max_value()
} else if exponent < 52 {
// Truncate mantissa
U256([mantissa, 0, 0, 0]) >> (52 - exponent)
} else {
U256([mantissa, 0, 0, 0]) << (exponent - 52)
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::f64;
#[test]
fn test_small_integers() {
for i in 0..=255 {
let f = i as f64;
let u = u256_from_f64_saturating(f);
assert_eq!(u, U256::from(i));
}
}
#[test]
fn test_small_integers_round_down() {
for i in 0..=255 {
let f = (i as f64) + 0.499;
let u = u256_from_f64_saturating(f);
assert_eq!(u, U256::from(i));
}
}
#[test]
fn test_small_integers_round_up() {
for i in 0..=255 {
let f = (i as f64) - 0.5;
let u = u256_from_f64_saturating(f);
assert_eq!(u, U256::from(i));
}
}
#[test]
fn test_infinities() {
assert_eq!(u256_from_f64_saturating(f64::INFINITY), U256::max_value());
assert_eq!(u256_from_f64_saturating(f64::NEG_INFINITY), U256::zero());
}
#[test]
fn test_saturating() {
assert_eq!(u256_from_f64_saturating(-1.0), U256::zero());
assert_eq!(u256_from_f64_saturating(1e90_f64), U256::max_value());
}
#[test]
fn test_large() {
// Check with e.g. `python3 -c 'print(int(1.0e36))'`
assert_eq!(
u256_from_f64_saturating(1.0e36_f64),
U256::from_dec_str("1000000000000000042420637374017961984").unwrap()
);
assert_eq!(
u256_from_f64_saturating(f64::consts::PI * 2.0e60_f64),
U256::from_dec_str("6283185307179586084560863929317662625677330590403879287914496")
.unwrap()
);
assert_eq!(
u256_from_f64_saturating(5.78960446186581e76_f64),
U256::from_dec_str(
"57896044618658097711785492504343953926634992332820282019728792003956564819968"
)
.unwrap()
);
assert_eq!(
u256_from_f64_saturating(1.157920892373161e77_f64),
U256::from_dec_str(
"115792089237316105435040506505232477503392813560534822796089932171514352762880"
)
.unwrap()
);
}
}

View File

@ -24,6 +24,7 @@ async-trait = { version = "0.1.50", default-features = false }
serde = { version = "1.0.124", default-features = false, features = ["derive"] }
thiserror = { version = "1.0.31", default-features = false }
futures-util = { version = "^0.3" }
futures-locks = { version = "0.7" }
tracing = { version = "0.1.34", default-features = false }
tracing-futures = { version = "0.2.5", default-features = false }

View File

@ -1,12 +1,9 @@
use crate::gas_oracle::{GasCategory, GasOracle, GasOracleError, GWEI_TO_WEI};
use async_trait::async_trait;
use ethers_core::types::U256;
use reqwest::{
header::{HeaderMap, HeaderValue, AUTHORIZATION},
Client, ClientBuilder,
};
use reqwest::{header::AUTHORIZATION, Client};
use serde::Deserialize;
use std::{collections::HashMap, convert::TryInto, iter::FromIterator};
use std::{collections::HashMap, convert::TryInto};
use url::Url;
const BLOCKNATIVE_GAS_PRICE_ENDPOINT: &str = "https://api.blocknative.com/gasprices/blockprices";
@ -26,6 +23,7 @@ fn gas_category_to_confidence(gas_category: &GasCategory) -> u64 {
pub struct BlockNative {
client: Client,
url: Url,
api_key: String,
gas_category: GasCategory,
}
@ -84,13 +82,16 @@ pub struct BaseFeeEstimate {
}
impl BlockNative {
/// Creates a new [BlockNative](https://www.blocknative.com/gas-estimator) gas oracle
pub fn new(api_key: &str) -> Self {
let header_value = HeaderValue::from_str(api_key).unwrap();
let headers = HeaderMap::from_iter([(AUTHORIZATION, header_value)]);
let client = ClientBuilder::new().default_headers(headers).build().unwrap();
/// Creates a new [BlockNative](https://www.blocknative.com/gas-estimator) gas oracle.
pub fn new(api_key: String) -> Self {
Self::with_client(Client::new(), api_key)
}
/// Same as [`Self::new`] but with a custom [`Client`].
pub fn with_client(client: Client, api_key: String) -> Self {
Self {
client,
api_key,
url: BLOCKNATIVE_GAS_PRICE_ENDPOINT.try_into().unwrap(),
gas_category: GasCategory::Standard,
}
@ -105,7 +106,15 @@ impl BlockNative {
/// Perform request to Blocknative, decode response
pub async fn request(&self) -> Result<BlockNativeGasResponse, GasOracleError> {
Ok(self.client.get(self.url.as_ref()).send().await?.error_for_status()?.json().await?)
self.client
.get(self.url.as_ref())
.header(AUTHORIZATION, &self.api_key)
.send()
.await?
.error_for_status()?
.json()
.await
.map_err(GasOracleError::HttpClientError)
}
}

View File

@ -0,0 +1,70 @@
use crate::gas_oracle::{GasOracle, GasOracleError};
use async_trait::async_trait;
use ethers_core::types::U256;
use futures_locks::RwLock;
use std::{
fmt::Debug,
future::Future,
time::{Duration, Instant},
};
#[derive(Debug)]
pub struct Cache<T: GasOracle> {
inner: T,
validity: Duration,
fee: Cached<U256>,
eip1559: Cached<(U256, U256)>,
}
#[derive(Default, Debug)]
struct Cached<T: Clone>(RwLock<Option<(Instant, T)>>);
impl<T: Clone> Cached<T> {
async fn get<F, E, Fut>(&self, validity: Duration, fetch: F) -> Result<T, E>
where
F: FnOnce() -> Fut,
Fut: Future<Output = Result<T, E>>,
{
// Try with a read lock
{
let lock = self.0.read().await;
if let Some((last_fetch, value)) = lock.as_ref() {
if Instant::now().duration_since(*last_fetch) < validity {
return Ok(value.clone())
}
}
}
// Acquire a write lock
{
let mut lock = self.0.write().await;
// Check again, a concurrent thread may have raced us to the write.
if let Some((last_fetch, value)) = lock.as_ref() {
if Instant::now().duration_since(*last_fetch) < validity {
return Ok(value.clone())
}
}
// Set a fresh value
let value = fetch().await?;
*lock = Some((Instant::now(), value.clone()));
Ok(value)
}
}
}
impl<T: GasOracle> Cache<T> {
pub fn new(validity: Duration, inner: T) -> Self {
Self { inner, validity, fee: Cached::default(), eip1559: Cached::default() }
}
}
#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
impl<T: GasOracle> GasOracle for Cache<T> {
async fn fetch(&self) -> Result<U256, GasOracleError> {
self.fee.get(self.validity, || self.inner.fetch()).await
}
async fn estimate_eip1559_fees(&self) -> Result<(U256, U256), GasOracleError> {
self.eip1559.get(self.validity, || self.inner.estimate_eip1559_fees()).await
}
}

View File

@ -61,15 +61,17 @@ pub struct EthGasStationResponse {
impl EthGasStation {
/// Creates a new [EthGasStation](https://docs.ethgasstation.info/) gas oracle
pub fn new(api_key: Option<&'static str>) -> Self {
let url = match api_key {
Some(key) => format!("{}?api-key={}", ETH_GAS_STATION_URL_PREFIX, key),
None => ETH_GAS_STATION_URL_PREFIX.to_string(),
};
pub fn new(api_key: Option<&str>) -> Self {
Self::with_client(Client::new(), api_key)
}
let url = Url::parse(&url).expect("invalid url");
EthGasStation { client: Client::new(), url, gas_category: GasCategory::Standard }
/// Same as [`Self::new`] but with a custom [`Client`].
pub fn with_client(client: Client, api_key: Option<&str>) -> Self {
let mut url = Url::parse(ETH_GAS_STATION_URL_PREFIX).expect("invalid url");
if let Some(key) = api_key {
url.query_pairs_mut().append_pair("api-key", key);
}
EthGasStation { client, url, gas_category: GasCategory::Standard }
}
/// Sets the gas price category to be used when fetching the gas price.
@ -84,6 +86,12 @@ impl EthGasStation {
}
}
impl Default for EthGasStation {
fn default() -> Self {
Self::new(None)
}
}
#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
impl GasOracle for EthGasStation {

View File

@ -18,12 +18,6 @@ pub struct Etherchain {
gas_category: GasCategory,
}
impl Default for Etherchain {
fn default() -> Self {
Self::new()
}
}
#[derive(Clone, Debug, Deserialize, PartialEq, PartialOrd)]
#[serde(rename_all = "camelCase")]
pub struct EtherchainResponse {
@ -38,9 +32,14 @@ pub struct EtherchainResponse {
impl Etherchain {
/// Creates a new [Etherchain](https://etherchain.org/tools/gasPriceOracle) gas price oracle.
pub fn new() -> Self {
Self::with_client(Client::new())
}
/// Same as [`Self::new`] but with a custom [`Client`].
pub fn with_client(client: Client) -> Self {
let url = Url::parse(ETHERCHAIN_URL).expect("invalid url");
Etherchain { client: Client::new(), url, gas_category: GasCategory::Standard }
Etherchain { client, url, gas_category: GasCategory::Standard }
}
/// Sets the gas price category to be used when fetching the gas price.
@ -55,6 +54,12 @@ impl Etherchain {
}
}
impl Default for Etherchain {
fn default() -> Self {
Self::new()
}
}
#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
impl GasOracle for Etherchain {

View File

@ -7,9 +7,9 @@ use url::Url;
use crate::gas_oracle::{GasCategory, GasOracle, GasOracleError};
const GAS_NOW_URL: &str = "https://www.gasnow.org/api/v3/gas/price";
const GAS_NOW_URL: &str = "https://etherchain.org/api/gasnow";
/// A client over HTTP for the [GasNow](https://www.gasnow.org/api/v1/gas/price) gas tracker API
/// A client over HTTP for the [Etherchain GasNow](https://etherchain.org/tools/gasnow) gas tracker API
/// that implements the `GasOracle` trait
#[derive(Clone, Debug)]
pub struct GasNow {
@ -18,12 +18,6 @@ pub struct GasNow {
gas_category: GasCategory,
}
impl Default for GasNow {
fn default() -> Self {
Self::new()
}
}
#[derive(Deserialize)]
struct GasNowResponseWrapper {
data: GasNowResponse,
@ -38,11 +32,16 @@ pub struct GasNowResponse {
}
impl GasNow {
/// Creates a new [GasNow](https://gasnow.org) gas price oracle.
/// Creates a new [Etherchain GasNow](https://etherchain.org/tools/gasnow) gas price oracle.
pub fn new() -> Self {
Self::with_client(Client::new())
}
/// Same as [`Self::new`] but with a custom [`Client`].
pub fn with_client(client: Client) -> Self {
let url = Url::parse(GAS_NOW_URL).expect("invalid url");
Self { client: Client::new(), url, gas_category: GasCategory::Standard }
Self { url, gas_category: GasCategory::Standard }
}
/// Sets the gas price category to be used when fetching the gas price.
@ -63,6 +62,12 @@ impl GasNow {
}
}
impl Default for GasNow {
fn default() -> Self {
Self::new(Client::new())
}
}
#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
impl GasOracle for GasNow {

View File

@ -0,0 +1,125 @@
use crate::gas_oracle::{GasOracle, GasOracleError};
use async_trait::async_trait;
use ethers_core::types::U256;
use futures_util::future::join_all;
use std::{fmt::Debug, future::Future};
use tracing::warn;
#[derive(Default, Debug)]
pub struct Median {
oracles: Vec<(f32, Box<dyn GasOracle>)>,
}
/// Computes the median gas price from a selection of oracles.
///
/// Don't forget to set a timeout on the source oracles. By default
/// the reqwest based oracles will never time out.
impl Median {
pub fn new() -> Self {
Self::default()
}
pub fn add<T: 'static + GasOracle>(&mut self, oracle: T) {
self.add_weighted(1.0, oracle)
}
pub fn add_weighted<T: 'static + GasOracle>(&mut self, weight: f32, oracle: T) {
assert!(weight > 0.0);
self.oracles.push((weight, Box::new(oracle)));
}
pub async fn query_all<'a, Fn, Fut, O>(
&'a self,
mut f: Fn,
) -> Result<Vec<(f32, O)>, GasOracleError>
where
Fn: FnMut(&'a dyn GasOracle) -> Fut,
Fut: Future<Output = Result<O, GasOracleError>>,
{
// Process the oracles in parallel
let futures = self.oracles.iter().map(|(_, oracle)| f(oracle.as_ref()));
let results = join_all(futures).await;
// Filter out any errors
let values =
self.oracles.iter().zip(results).filter_map(
|((weight, oracle), result)| match result {
Ok(value) => Some((*weight, value)),
Err(err) => {
warn!("Failed to fetch gas price from {:?}: {}", oracle, err);
None
}
},
);
let values = values.collect::<Vec<_>>();
if values.is_empty() {
return Err(GasOracleError::NoValues)
}
Ok(values)
}
}
#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
impl GasOracle for Median {
async fn fetch(&self) -> Result<U256, GasOracleError> {
let mut values = self.query_all(|oracle| oracle.fetch()).await?;
// `query_all` guarantees `values` is not empty
Ok(*weighted_fractile_by_key(0.5, &mut values, |fee| fee).unwrap())
}
async fn estimate_eip1559_fees(&self) -> Result<(U256, U256), GasOracleError> {
let mut values = self.query_all(|oracle| oracle.estimate_eip1559_fees()).await?;
// `query_all` guarantees `values` is not empty
Ok((
weighted_fractile_by_key(0.5, &mut values, |(max_fee, _)| max_fee).unwrap().0,
weighted_fractile_by_key(0.5, &mut values, |(_, priority_fee)| priority_fee).unwrap().1,
))
}
}
/// Weighted fractile by key
///
/// Sort the values in place by key and return the weighted fractile value such
/// that `fractile` fraction of the values by weight are less than or equal to
/// the value.
///
/// Returns None if the values are empty.
///
/// Note: it doesn't handle NaNs or other special float values.
///
/// See <https://en.wikipedia.org/wiki/Percentile#The_weighted_percentile_method>
///
/// # Panics
///
/// Panics if [`fractile`] is not in the range $[0, 1]$.
fn weighted_fractile_by_key<'a, T, F, K>(
fractile: f32,
values: &'a mut [(f32, T)],
mut key: F,
) -> Option<&'a T>
where
F: for<'b> FnMut(&'b T) -> &'b K,
K: Ord,
{
assert!((0.0..=1.0).contains(&fractile));
if values.is_empty() {
return None
}
let weight_rank = fractile * values.iter().map(|(weight, _)| *weight).sum::<f32>();
values.sort_unstable_by(|a, b| key(&a.1).cmp(key(&b.1)));
let mut cumulative_weight = 0.0_f32;
for (weight, value) in values.iter() {
cumulative_weight += *weight;
if cumulative_weight >= weight_rank {
return Some(value)
}
}
// By the last element, cumulative_weight == weight_rank and we should have
// returned already. Assume there is a slight rounding error causing
// cumulative_weight to be slightly less than expected. In this case the last
// element is appropriate. (This is not exactly right, since the last
// elements may have zero weight.)
// `values` is not empty.
Some(&values.last().unwrap().1)
}

View File

@ -13,6 +13,15 @@ pub use etherscan::Etherscan;
mod middleware;
pub use middleware::{GasOracleMiddleware, MiddlewareError};
mod median;
pub use median::Median;
mod cache;
pub use cache::Cache;
mod polygon;
pub use polygon::Polygon;
use ethers_core::types::U256;
use async_trait::async_trait;
@ -58,6 +67,12 @@ pub enum GasOracleError {
#[error("EIP-1559 gas estimation not supported")]
Eip1559EstimationNotSupported,
#[error("None of the oracles returned a value")]
NoValues,
#[error("Chain is not supported by the oracle")]
UnsupportedChain,
}
/// `GasOracle` is a trait that an underlying gas oracle needs to implement.

View File

@ -0,0 +1,91 @@
use crate::gas_oracle::{GasCategory, GasOracle, GasOracleError};
use async_trait::async_trait;
use ethers_core::types::{u256_from_f64_saturating, Chain, U256};
use reqwest::Client;
use serde::Deserialize;
use url::Url;
const GAS_PRICE_ENDPOINT: &str = "https://gasstation-mainnet.matic.network/v2";
const MUMBAI_GAS_PRICE_ENDPOINT: &str = "https://gasstation-mumbai.matic.today/v2";
/// The [Polygon](https://docs.polygon.technology/docs/develop/tools/polygon-gas-station/) gas station API
/// Queries over HTTP and implements the `GasOracle` trait
#[derive(Clone, Debug)]
pub struct Polygon {
client: Client,
url: Url,
gas_category: GasCategory,
}
/// The response from the Polygon gas station API.
/// Gas prices are in Gwei.
#[derive(Debug, Deserialize, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct Response {
estimated_base_fee: f64,
safe_low: GasEstimate,
standard: GasEstimate,
fast: GasEstimate,
}
#[derive(Clone, Copy, Debug, Deserialize, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct GasEstimate {
max_priority_fee: f64,
max_fee: f64,
}
impl Polygon {
pub fn new(chain: Chain) -> Result<Self, GasOracleError> {
Self::with_client(Client::new(), chain)
}
pub fn with_client(client: Client, chain: Chain) -> Result<Self, GasOracleError> {
// TODO: Sniff chain from chain id.
let url = match chain {
Chain::Polygon => Url::parse(GAS_PRICE_ENDPOINT).unwrap(),
Chain::PolygonMumbai => Url::parse(MUMBAI_GAS_PRICE_ENDPOINT).unwrap(),
_ => return Err(GasOracleError::UnsupportedChain),
};
Ok(Self { client, url, gas_category: GasCategory::Standard })
}
/// Sets the gas price category to be used when fetching the gas price.
#[must_use]
pub fn category(mut self, gas_category: GasCategory) -> Self {
self.gas_category = gas_category;
self
}
/// Perform request to Blocknative, decode response
pub async fn request(&self) -> Result<(f64, GasEstimate), GasOracleError> {
let response: Response =
self.client.get(self.url.as_ref()).send().await?.error_for_status()?.json().await?;
let estimate = match self.gas_category {
GasCategory::SafeLow => response.safe_low,
GasCategory::Standard => response.standard,
GasCategory::Fast => response.fast,
GasCategory::Fastest => response.fast,
};
Ok((response.estimated_base_fee, estimate))
}
}
#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
impl GasOracle for Polygon {
async fn fetch(&self) -> Result<U256, GasOracleError> {
let (base_fee, estimate) = self.request().await?;
let fee = base_fee + estimate.max_priority_fee;
Ok(from_gwei(fee))
}
async fn estimate_eip1559_fees(&self) -> Result<(U256, U256), GasOracleError> {
let (_, estimate) = self.request().await?;
Ok((from_gwei(estimate.max_fee), from_gwei(estimate.max_priority_fee)))
}
}
fn from_gwei(gwei: f64) -> U256 {
u256_from_f64_saturating(gwei * 1.0e18_f64)
}

View File

@ -57,7 +57,7 @@ async fn using_gas_oracle() {
#[tokio::test]
async fn eth_gas_station() {
// initialize and fetch gas estimates from EthGasStation
let eth_gas_station_oracle = EthGasStation::new(None);
let eth_gas_station_oracle = EthGasStation::default();
let data = eth_gas_station_oracle.fetch().await;
assert!(data.is_ok());
}
@ -83,7 +83,7 @@ async fn etherscan() {
#[tokio::test]
async fn etherchain() {
// initialize and fetch gas estimates from Etherchain
let etherchain_oracle = Etherchain::new().category(GasCategory::Fast);
let etherchain_oracle = Etherchain::default().category(GasCategory::Fast);
let data = etherchain_oracle.fetch().await;
assert!(data.is_ok());
}