feat(etherscan, middleware): implement gas endpoints and use in oracle middleware (#621)

* feat(etherscan, middleware): implement gas-related endpoints and use them in gas oracle

* cleanup deps

* better comment

* revertme: add debug print

* Revert "revertme: add debug print"

This reverts commit 63993642c4.

* make etherscan tests serial

* run each etherscan test at least 200ms

* 200ms -> 250ms

* Revert "200ms -> 250ms"

This reverts commit 29c7f6ed9a.

* add separate etherscan key for celo tests

* 200ms -> 250ms

* add Chain::Sepolia to match

.github/workflows/ci.yml

@@ -10,7 +10,8 @@ name: Tests
 # so that we do not get rate limited by Etherscan (and it's free to generate as
 # many as you want)
 env:
-  ETHERSCAN_API_KEY: I5BXNZYP5GEDWFINGVEZKYIVU2695NPQZB
+  ETHERSCAN_API_KEY_ETHEREUM: I5BXNZYP5GEDWFINGVEZKYIVU2695NPQZB
+  ETHERSCAN_API_KEY_CELO: B13XSMUT6Q3Q4WZ5DNQR8RXDBA2KNTMT4M
   RINKEBY_PRIVATE_KEY: "a046a5b763923d437855a6fe64962569c9a378efba5c84920212c4b6ae270df5"
 
 jobs:
@@ -56,6 +57,7 @@ jobs:
       - name: cargo test
         run: |
             export PATH=$HOME/bin:$PATH
+            export ETHERSCAN_API_KEY=$ETHERSCAN_API_KEY_ETHEREUM
             cargo test
 
   feature-tests:
@@ -103,6 +105,7 @@ jobs:
       - name: cargo test (Celo)
         run: |
             export PATH=$HOME/bin:$PATH
+            export ETHERSCAN_API_KEY=$ETHERSCAN_API_KEY_CELO
             cargo test --all-features
 
   lint:

Cargo.lock

@@ -1073,7 +1073,9 @@ dependencies = [
  "ethers-core",
  "reqwest",
  "serde",
+ "serde-aux",
  "serde_json",
+ "serial_test",
  "thiserror",
  "tokio",
 ]
@@ -1085,6 +1087,7 @@ dependencies = [
  "async-trait",
  "ethers-contract",
  "ethers-core",
+ "ethers-etherscan",
  "ethers-providers",
  "ethers-signers",
  "ethers-solc",
@@ -1095,8 +1098,8 @@ dependencies = [
  "rand 0.8.4",
  "reqwest",
  "serde",
- "serde-aux",
  "serde_json",
+ "serial_test",
  "thiserror",
  "tokio",
  "tracing",
@@ -2759,6 +2762,28 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "serial_test"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e0bccbcf40c8938196944a3da0e133e031a33f4d6b72db3bda3cc556e361905d"
+dependencies = [
+ "lazy_static",
+ "parking_lot",
+ "serial_test_derive",
+]
+
+[[package]]
+name = "serial_test_derive"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b2acd6defeddb41eb60bb468f8825d0cfd0c2a76bc03bfd235b6a1dc4f6a1ad5"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "sha-1"
 version = "0.9.8"

ethers-core/src/types/chain.rs

@@ -14,6 +14,7 @@ pub enum Chain {
     PolygonMumbai,
     Avalanche,
     AvalancheFuji,
+    Sepolia,
 }
 
 impl fmt::Display for Chain {
@@ -35,6 +36,7 @@ impl From<Chain> for u32 {
             Chain::PolygonMumbai => 80001,
             Chain::Avalanche => 43114,
             Chain::AvalancheFuji => 43113,
+            Chain::Sepolia => 11155111,
         }
     }
 }

ethers-etherscan/Cargo.toml

@@ -18,10 +18,12 @@ ethers-core = { version = "^0.6.0", path = "../ethers-core", default-features =
 reqwest = { version = "0.11.6", features = ["json"] }
 serde = { version = "1.0.124", default-features = false, features = ["derive"] }
 serde_json = { version = "1.0.64", default-features = false }
+serde-aux = { version = "3.0.1", default-features = false }
 thiserror = "1.0.29"
 
 [dev-dependencies]
-tokio = { version = "1.5", features = ["macros", "rt-multi-thread"] }
+tokio = { version = "1.5", features = ["macros", "rt-multi-thread", "time"] }
+serial_test = "0.5.1"
 
 [package.metadata.docs.rs]
 all-features = true

ethers-etherscan/src/contract.rs

@@ -1,8 +1,11 @@
-use crate::{Client, Response, Result};
-use ethers_core::abi::{Abi, Address};
-use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 
+use serde::{Deserialize, Serialize};
+
+use ethers_core::abi::{Abi, Address};
+
+use crate::{Client, Response, Result};
+
 /// Arguments for verifying contracts
 #[derive(Debug, Clone, Serialize)]
 pub struct VerifyContract {
@@ -251,34 +254,49 @@ impl Client {
 
 #[cfg(test)]
 mod tests {
-    use crate::{contract::VerifyContract, Client};
+    use std::time::Duration;
+
+    use serial_test::serial;
+
     use ethers_core::types::Chain;
 
+    use crate::{contract::VerifyContract, tests::run_at_least_duration, Client};
+
     #[tokio::test]
+    #[serial]
     #[ignore]
     async fn can_fetch_contract_abi() {
+        run_at_least_duration(Duration::from_millis(250), async {
             let client = Client::new_from_env(Chain::Mainnet).unwrap();
 
             let _abi = client
                 .contract_abi("0xBB9bc244D798123fDe783fCc1C72d3Bb8C189413".parse().unwrap())
                 .await
                 .unwrap();
+        })
+        .await;
     }
 
     #[tokio::test]
+    #[serial]
     #[ignore]
     async fn can_fetch_contract_source_code() {
+        run_at_least_duration(Duration::from_millis(250), async {
             let client = Client::new_from_env(Chain::Mainnet).unwrap();
 
             let _meta = client
                 .contract_source_code("0xBB9bc244D798123fDe783fCc1C72d3Bb8C189413".parse().unwrap())
                 .await
                 .unwrap();
+        })
+        .await
     }
 
     #[tokio::test]
+    #[serial]
     #[ignore]
     async fn can_verify_contract() {
+        run_at_least_duration(Duration::from_millis(250), async {
             // TODO this needs further investigation
 
             // https://etherscan.io/address/0x9e744c9115b74834c0f33f4097f40c02a9ac5c33#code
@@ -296,5 +314,6 @@ mod tests {
                     .runs(200);
 
             let _resp = client.submit_contract_verification(&contract).await;
+        }).await
     }
 }

ethers-etherscan/src/errors.rs

@@ -9,6 +9,8 @@ pub enum EtherscanError {
     ExecutionFailed(String),
     #[error("tx receipt failed")]
     TransactionReceiptFailed,
+    #[error("gas estimation failed")]
+    GasEstimationFailed,
     #[error("bad status code {0}")]
     BadStatusCode(String),
     #[error(transparent)]

ethers-etherscan/src/gas.rs

@@ -0,0 +1,130 @@
+use std::{collections::HashMap, str::FromStr};
+
+use serde::{de, Deserialize};
+use serde_aux::prelude::*;
+
+use ethers_core::types::U256;
+
+use crate::{Client, EtherscanError, Response, Result};
+
+#[derive(Deserialize)]
+#[serde(rename_all = "PascalCase")]
+pub struct GasOracle {
+    #[serde(deserialize_with = "deserialize_number_from_string")]
+    pub safe_gas_price: u64,
+    #[serde(deserialize_with = "deserialize_number_from_string")]
+    pub propose_gas_price: u64,
+    #[serde(deserialize_with = "deserialize_number_from_string")]
+    pub fast_gas_price: u64,
+    #[serde(deserialize_with = "deserialize_number_from_string")]
+    pub last_block: u64,
+    #[serde(deserialize_with = "deserialize_number_from_string")]
+    #[serde(rename = "suggestBaseFee")]
+    pub suggested_base_fee: f64,
+    #[serde(deserialize_with = "deserialize_f64_vec")]
+    #[serde(rename = "gasUsedRatio")]
+    pub gas_used_ratio: Vec<f64>,
+}
+
+fn deserialize_f64_vec<'de, D>(deserializer: D) -> core::result::Result<Vec<f64>, D::Error>
+where
+    D: de::Deserializer<'de>,
+{
+    let str_sequence = String::deserialize(deserializer)?;
+    str_sequence
+        .split(',')
+        .map(|item| f64::from_str(item).map_err(|err| de::Error::custom(err.to_string())))
+        .collect()
+}
+
+impl Client {
+    /// Returns the estimated time, in seconds, for a transaction to be confirmed on the blockchain
+    /// for the specified gas price
+    pub async fn gas_estimate(&self, gas_price: U256) -> Result<u32> {
+        let query = self.create_query(
+            "gastracker",
+            "gasestimate",
+            HashMap::from([("gasprice", gas_price.to_string())]),
+        );
+        let response: Response<String> = self.get_json(&query).await?;
+
+        if response.status == "1" {
+            Ok(u32::from_str(&response.result).map_err(|_| EtherscanError::GasEstimationFailed)?)
+        } else {
+            Err(EtherscanError::GasEstimationFailed)
+        }
+    }
+
+    /// Returns the current Safe, Proposed and Fast gas prices
+    /// Post EIP-1559 changes:
+    /// - Safe/Proposed/Fast gas price recommendations are now modeled as Priority Fees.
+    /// - New field `suggestBaseFee`, the baseFee of the next pending block
+    /// - New field `gasUsedRatio`, to estimate how busy the network is
+    pub async fn gas_oracle(&self) -> Result<GasOracle> {
+        let query = self.create_query("gastracker", "gasoracle", serde_json::Value::Null);
+        let response: Response<GasOracle> = self.get_json(&query).await?;
+
+        Ok(response.result)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::time::Duration;
+
+    use serial_test::serial;
+
+    use ethers_core::types::Chain;
+
+    use crate::tests::run_at_least_duration;
+
+    use super::*;
+
+    #[tokio::test]
+    #[serial]
+    async fn gas_estimate_success() {
+        run_at_least_duration(Duration::from_millis(250), async {
+            let client = Client::new_from_env(Chain::Mainnet).unwrap();
+
+            let result = client.gas_estimate(2000000000u32.into()).await;
+
+            assert!(result.is_ok());
+        })
+        .await
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn gas_estimate_error() {
+        run_at_least_duration(Duration::from_millis(250), async {
+            let client = Client::new_from_env(Chain::Mainnet).unwrap();
+
+            let err = client.gas_estimate(7123189371829732819379218u128.into()).await.unwrap_err();
+
+            assert!(matches!(err, EtherscanError::GasEstimationFailed));
+        })
+        .await
+    }
+
+    #[tokio::test]
+    #[serial]
+    async fn gas_oracle_success() {
+        run_at_least_duration(Duration::from_millis(250), async {
+            let client = Client::new_from_env(Chain::Mainnet).unwrap();
+
+            let result = client.gas_oracle().await;
+
+            assert!(result.is_ok());
+
+            let oracle = result.unwrap();
+
+            assert!(oracle.safe_gas_price > 0);
+            assert!(oracle.propose_gas_price > 0);
+            assert!(oracle.fast_gas_price > 0);
+            assert!(oracle.last_block > 0);
+            assert!(oracle.suggested_base_fee > 0.0);
+            assert!(oracle.gas_used_ratio.len() > 0);
+        })
+        .await
+    }
+}

ethers-etherscan/src/lib.rs

@@ -1,14 +1,17 @@
 //! Bindings for [etherscan.io web api](https://docs.etherscan.io/)
 
-pub mod contract;
+use std::borrow::Cow;
-pub mod errors;
+
-mod transaction;
+use reqwest::{header, Url};
+use serde::{de::DeserializeOwned, Deserialize, Serialize};
 
 use errors::EtherscanError;
 use ethers_core::{abi::Address, types::Chain};
-use reqwest::{header, Url};
+
-use serde::{de::DeserializeOwned, Deserialize, Serialize};
+pub mod contract;
-use std::borrow::Cow;
+pub mod errors;
+pub mod gas;
+pub mod transaction;
 
 pub type Result<T> = std::result::Result<T, EtherscanError>;
 
@@ -75,7 +78,7 @@ impl Client {
             Chain::Mainnet | Chain::Ropsten | Chain::Kovan | Chain::Rinkeby | Chain::Goerli => {
                 std::env::var("ETHERSCAN_API_KEY")?
             }
-            Chain::XDai => String::default(),
+            Chain::XDai | Chain::Sepolia => String::default(),
         };
         Self::new(chain, api_key)
     }
@@ -172,9 +175,15 @@ struct Query<'a, T: Serialize> {
 
 #[cfg(test)]
 mod tests {
-    use crate::{Client, EtherscanError};
+    use std::{
+        future::Future,
+        time::{Duration, SystemTime},
+    };
+
     use ethers_core::types::Chain;
 
+    use crate::{Client, EtherscanError};
+
     #[test]
     fn chain_not_supported() {
         let err = Client::new_from_env(Chain::XDai).unwrap_err();
@@ -182,4 +191,12 @@ mod tests {
         assert!(matches!(err, EtherscanError::ChainNotSupported(_)));
         assert_eq!(err.to_string(), "chain XDai not supported");
     }
+
+    pub async fn run_at_least_duration(duration: Duration, block: impl Future) {
+        let start = SystemTime::now();
+        block.await;
+        if let Some(sleep) = duration.checked_sub(start.elapsed().unwrap()) {
+            tokio::time::sleep(sleep).await;
+        }
+    }
 }

ethers-etherscan/src/transaction.rs

@@ -19,9 +19,6 @@ struct TransactionReceiptStatus {
 impl Client {
     /// Returns the status of a contract execution
     pub async fn check_contract_execution_status(&self, tx_hash: impl AsRef<str>) -> Result<()> {
-        let mut map = HashMap::new();
-        map.insert("txhash", tx_hash.as_ref());
-
         let query = self.create_query(
             "transaction",
             "getstatus",
@@ -55,11 +52,18 @@ impl Client {
 
 #[cfg(test)]
 mod tests {
+    use std::time::Duration;
+
+    use serial_test::serial;
+
+    use crate::{tests::run_at_least_duration, Chain};
+
     use super::*;
-    use crate::Chain;
 
     #[tokio::test]
+    #[serial]
     async fn check_contract_execution_status_success() {
+        run_at_least_duration(Duration::from_millis(250), async {
             let client = Client::new_from_env(Chain::Mainnet).unwrap();
 
             let status = client
@@ -69,10 +73,14 @@ mod tests {
                 .await;
 
             assert!(status.is_ok());
+        })
+        .await
     }
 
     #[tokio::test]
+    #[serial]
     async fn check_contract_execution_status_error() {
+        run_at_least_duration(Duration::from_millis(250), async {
             let client = Client::new_from_env(Chain::Mainnet).unwrap();
 
             let err = client
@@ -84,10 +92,14 @@ mod tests {
 
             assert!(matches!(err, EtherscanError::ExecutionFailed(_)));
             assert_eq!(err.to_string(), "contract execution call failed: Bad jump destination");
+        })
+        .await
     }
 
     #[tokio::test]
+    #[serial]
     async fn check_transaction_receipt_status_success() {
+        run_at_least_duration(Duration::from_millis(250), async {
             let client = Client::new_from_env(Chain::Mainnet).unwrap();
 
             let success = client
@@ -97,10 +109,14 @@ mod tests {
                 .await;
 
             assert!(success.is_ok());
+        })
+        .await
     }
 
     #[tokio::test]
+    #[serial]
     async fn check_transaction_receipt_status_failed() {
+        run_at_least_duration(Duration::from_millis(250), async {
             let client = Client::new_from_env(Chain::Mainnet).unwrap();
 
             let err = client
@@ -111,5 +127,7 @@ mod tests {
                 .unwrap_err();
 
             assert!(matches!(err, EtherscanError::TransactionReceiptFailed));
+        })
+        .await
     }
 }

ethers-middleware/Cargo.toml

@@ -16,6 +16,7 @@ rustdoc-args = ["--cfg", "docsrs"]
 [dependencies]
 ethers-contract = { version = "^0.6.0", path = "../ethers-contract", default-features = false, features = ["abigen"] }
 ethers-core = { version = "^0.6.0", path = "../ethers-core", default-features = false }
+ethers-etherscan = { version = "^0.2.0", path = "../ethers-etherscan", default-features = false }
 ethers-providers = { version = "^0.6.0", path = "../ethers-providers", default-features = false }
 ethers-signers = { version = "^0.6.0", path = "../ethers-signers", default-features = false }
 
@@ -27,7 +28,6 @@ tracing = { version = "0.1.29", default-features = false }
 tracing-futures = { version = "0.2.5", default-features = false }
 
 # for gas oracles
-serde-aux = { version = "3.0.1", default-features = false }
 reqwest = { version = "0.11.6", default-features = false, features = ["json", "rustls-tls"] }
 url = { version = "2.2.2", default-features = false }
 
@@ -43,6 +43,7 @@ rand = { version = "0.8.4", default-features = false }
 ethers-providers = { version = "^0.6.0", path = "../ethers-providers", default-features = false, features = ["ws", "rustls"] }
 once_cell = "1.8.0"
 ethers-solc = { version = "^0.1.0", path = "../ethers-solc", default-features = false }
+serial_test = "0.5.1"
 
 [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
 tokio = { version = "1.5", default-features = false, features = ["rt", "macros", "time"] }

ethers-middleware/src/gas_oracle/etherscan.rs

@@ -1,73 +1,22 @@
-use ethers_core::types::U256;
-
 use async_trait::async_trait;
-use reqwest::Client;
+
-use serde::Deserialize;
+use ethers_core::types::U256;
-use serde_aux::prelude::*;
+use ethers_etherscan::Client;
-use url::Url;
 
 use crate::gas_oracle::{GasCategory, GasOracle, GasOracleError, GWEI_TO_WEI};
 
-const ETHERSCAN_URL_PREFIX: &str =
-    "https://api.etherscan.io/api?module=gastracker&action=gasoracle";
-
 /// A client over HTTP for the [Etherscan](https://api.etherscan.io/api?module=gastracker&action=gasoracle) gas tracker API
 /// that implements the `GasOracle` trait
 #[derive(Clone, Debug)]
 pub struct Etherscan {
     client: Client,
-    url: Url,
     gas_category: GasCategory,
 }
 
-#[derive(Deserialize)]
-struct EtherscanResponseWrapper {
-    result: EtherscanResponse,
-}
-
-#[derive(Clone, Debug, Deserialize, PartialEq, PartialOrd)]
-#[serde(rename_all = "PascalCase")]
-pub struct EtherscanResponse {
-    #[serde(deserialize_with = "deserialize_number_from_string")]
-    pub safe_gas_price: u64,
-    #[serde(deserialize_with = "deserialize_number_from_string")]
-    pub propose_gas_price: u64,
-    #[serde(deserialize_with = "deserialize_number_from_string")]
-    pub fast_gas_price: u64,
-    #[serde(deserialize_with = "deserialize_number_from_string")]
-    pub last_block: u64,
-    #[serde(deserialize_with = "deserialize_number_from_string")]
-    #[serde(rename = "suggestBaseFee")]
-    pub suggested_base_fee: f64,
-    #[serde(deserialize_with = "deserialize_f64_vec")]
-    #[serde(rename = "gasUsedRatio")]
-    pub gas_used_ratio: Vec<f64>,
-}
-
-use serde::de;
-use std::str::FromStr;
-fn deserialize_f64_vec<'de, D>(deserializer: D) -> Result<Vec<f64>, D::Error>
-where
-    D: de::Deserializer<'de>,
-{
-    let str_sequence = String::deserialize(deserializer)?;
-    str_sequence
-        .split(',')
-        .map(|item| f64::from_str(item).map_err(|err| de::Error::custom(err.to_string())))
-        .collect()
-}
-
 impl Etherscan {
     /// Creates a new [Etherscan](https://etherscan.io/gastracker) gas price oracle.
-    pub fn new(api_key: Option<&str>) -> Self {
+    pub fn new(client: Client) -> Self {
-        let url = match api_key {
+        Etherscan { client, gas_category: GasCategory::Standard }
-            Some(key) => format!("{}&apikey={}", ETHERSCAN_URL_PREFIX, key),
-            None => ETHERSCAN_URL_PREFIX.to_string(),
-        };
-
-        let url = Url::parse(&url).expect("invalid url");
-
-        Etherscan { client: Client::new(), url, gas_category: GasCategory::Standard }
     }
 
     /// Sets the gas price category to be used when fetching the gas price.
@@ -75,17 +24,6 @@ impl Etherscan {
         self.gas_category = gas_category;
         self
     }
-
-    pub async fn query(&self) -> Result<EtherscanResponse, GasOracleError> {
-        let res = self
-            .client
-            .get(self.url.as_ref())
-            .send()
-            .await?
-            .json::<EtherscanResponseWrapper>()
-            .await?;
-        Ok(res.result)
-    }
 }
 
 #[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
@@ -96,12 +34,12 @@ impl GasOracle for Etherscan {
             return Err(GasOracleError::GasCategoryNotSupported)
         }
 
-        let res = self.query().await?;
+        let result = self.client.gas_oracle().await?;
 
         match self.gas_category {
-            GasCategory::SafeLow => Ok(U256::from(res.safe_gas_price * GWEI_TO_WEI)),
+            GasCategory::SafeLow => Ok(U256::from(result.safe_gas_price * GWEI_TO_WEI)),
-            GasCategory::Standard => Ok(U256::from(res.propose_gas_price * GWEI_TO_WEI)),
+            GasCategory::Standard => Ok(U256::from(result.propose_gas_price * GWEI_TO_WEI)),
-            GasCategory::Fast => Ok(U256::from(res.fast_gas_price * GWEI_TO_WEI)),
+            GasCategory::Fast => Ok(U256::from(result.fast_gas_price * GWEI_TO_WEI)),
             _ => Err(GasOracleError::GasCategoryNotSupported),
         }
     }

ethers-middleware/src/gas_oracle/mod.rs

@@ -35,6 +35,11 @@ pub enum GasOracleError {
     #[error(transparent)]
     HttpClientError(#[from] ReqwestError),
 
+    /// An internal error in the Etherscan client request made from the underlying
+    /// gas oracle
+    #[error(transparent)]
+    EtherscanError(#[from] ethers_etherscan::errors::EtherscanError),
+
     /// An internal error thrown when the required gas category is not
     /// supported by the gas oracle API
     #[error("gas category not supported")]

ethers-middleware/tests/gas_oracle.rs

@@ -10,6 +10,7 @@ use ethers_middleware::gas_oracle::{
     GasOracleMiddleware,
 };
 use ethers_providers::{Http, Middleware, Provider};
+use serial_test::serial;
 
 #[derive(Debug)]
 struct FakeGasOracle {
@@ -60,20 +61,20 @@ async fn eth_gas_station() {
 }
 
 #[tokio::test]
+#[serial]
 async fn etherscan() {
-    let api_key = std::env::var("ETHERSCAN_API_KEY").unwrap();
+    let etherscan_client = ethers_etherscan::Client::new_from_env(Chain::Mainnet).unwrap();
-    let api_key = Some(api_key.as_str());
 
     // initialize and fetch gas estimates from Etherscan
     // since etherscan does not support `fastest` category, we expect an error
-    let etherscan_oracle = Etherscan::new(api_key).category(GasCategory::Fastest);
+    let etherscan_oracle = Etherscan::new(etherscan_client.clone()).category(GasCategory::Fastest);
     let data = etherscan_oracle.fetch().await;
     assert!(data.is_err());
 
     // but fetching the `standard` gas price should work fine
-    let etherscan_oracle_2 = Etherscan::new(api_key).category(GasCategory::SafeLow);
+    let etherscan_oracle = Etherscan::new(etherscan_client).category(GasCategory::SafeLow);
 
-    let data = etherscan_oracle_2.fetch().await;
+    let data = etherscan_oracle.fetch().await;
     assert!(data.is_ok());
 }