From c436d19a9c6ccbb8b00b5367aa8bb057d2ad9f0d Mon Sep 17 00:00:00 2001 From: Bjerg Date: Wed, 6 Apr 2022 04:01:44 +0200 Subject: [PATCH] feat(etherscan): add caching (#1108) * feat(etherscan): add caching * feat: write to cache if not exists * fix: flush `BufWriter` * fix: fix serialization * fix: read cache without truncating the file * chore: remove comments * feat: rate limit errors * test: fix tests * test: fix tests * fix: don't fail if cache doesn't exist * fix: catch all rate limits * feat: add ttl * feat: also cache when contracts are not verified * chore: clippy Co-authored-by: Georgios Konstantopoulos --- ethers-etherscan/src/contract.rs | 51 +++++++++++- ethers-etherscan/src/errors.rs | 16 ++-- ethers-etherscan/src/lib.rs | 121 ++++++++++++++++++++++++++-- ethers-etherscan/src/transaction.rs | 2 +- 4 files changed, 175 insertions(+), 15 deletions(-) diff --git a/ethers-etherscan/src/contract.rs b/ethers-etherscan/src/contract.rs index 29a63ea0..10195ecc 100644 --- a/ethers-etherscan/src/contract.rs +++ b/ethers-etherscan/src/contract.rs @@ -138,8 +138,8 @@ impl Default for CodeFormat { } #[derive(Debug, Serialize, Deserialize)] +#[serde(transparent)] pub struct ContractMetadata { - #[serde(flatten)] pub items: Vec, } @@ -284,12 +284,36 @@ impl Client { /// # } /// ``` pub async fn contract_abi(&self, address: Address) -> Result { + // apply caching + if let Some(ref cache) = self.cache { + // If this is None, then we have a cache miss + if let Some(src) = cache.get_abi(address) { + // If this is None, then the contract is not verified + return match src { + Some(src) => Ok(src), + None => Err(EtherscanError::ContractCodeNotVerified(address)), + } + } + } + let query = self.create_query("contract", "getabi", HashMap::from([("address", address)])); let resp: Response = self.get_json(&query).await?; + if resp.result.starts_with("Max rate limit reached") { + return Err(EtherscanError::RateLimitExceeded) + } if resp.result.starts_with("Contract source code not verified") { + if let Some(ref cache) = self.cache { + let _ = cache.set_abi(address, None); + } return Err(EtherscanError::ContractCodeNotVerified(address)) } - Ok(serde_json::from_str(&resp.result)?) + let abi = serde_json::from_str(&resp.result)?; + + if let Some(ref cache) = self.cache { + let _ = cache.set_abi(address, Some(&abi)); + } + + Ok(abi) } /// Get Contract Source Code for Verified Contract Source Codes @@ -307,13 +331,34 @@ impl Client { /// # } /// ``` pub async fn contract_source_code(&self, address: Address) -> Result { + // apply caching + if let Some(ref cache) = self.cache { + // If this is None, then we have a cache miss + if let Some(src) = cache.get_source(address) { + // If this is None, then the contract is not verified + return match src { + Some(src) => Ok(src), + None => Err(EtherscanError::ContractCodeNotVerified(address)), + } + } + } + let query = self.create_query("contract", "getsourcecode", HashMap::from([("address", address)])); let response: Response> = self.get_json(&query).await?; if response.result.iter().any(|item| item.abi == "Contract source code not verified") { + if let Some(ref cache) = self.cache { + let _ = cache.set_source(address, None); + } return Err(EtherscanError::ContractCodeNotVerified(address)) } - Ok(ContractMetadata { items: response.result }) + let res = ContractMetadata { items: response.result }; + + if let Some(ref cache) = self.cache { + let _ = cache.set_source(address, Some(&res)); + } + + Ok(res) } } diff --git a/ethers-etherscan/src/errors.rs b/ethers-etherscan/src/errors.rs index cad7a808..d70f632c 100644 --- a/ethers-etherscan/src/errors.rs +++ b/ethers-etherscan/src/errors.rs @@ -3,17 +3,17 @@ use std::env::VarError; #[derive(Debug, thiserror::Error)] pub enum EtherscanError { - #[error("chain {0} not supported")] + #[error("Chain {0} not supported")] ChainNotSupported(Chain), - #[error("contract execution call failed: {0}")] + #[error("Contract execution call failed: {0}")] ExecutionFailed(String), - #[error("balance failed")] + #[error("Balance failed")] BalanceFailed, - #[error("tx receipt failed")] + #[error("Transaction receipt failed")] TransactionReceiptFailed, - #[error("gas estimation failed")] + #[error("Gas estimation failed")] GasEstimationFailed, - #[error("bad status code {0}")] + #[error("Bad status code: {0}")] BadStatusCode(String), #[error(transparent)] EnvVarNotFound(#[from] VarError), @@ -23,8 +23,12 @@ pub enum EtherscanError { Serde(#[from] serde_json::Error), #[error("Contract source code not verified: {0}")] ContractCodeNotVerified(Address), + #[error("Rate limit exceeded")] + RateLimitExceeded, #[error(transparent)] IO(#[from] std::io::Error), #[error("Local networks (e.g. ganache, geth --dev) cannot be indexed by etherscan")] LocalNetworksNotSupported, + #[error("Unknown error: {0}")] + Unknown(String), } diff --git a/ethers-etherscan/src/lib.rs b/ethers-etherscan/src/lib.rs index 2255ac00..fea67802 100644 --- a/ethers-etherscan/src/lib.rs +++ b/ethers-etherscan/src/lib.rs @@ -1,12 +1,21 @@ //! Bindings for [etherscan.io web api](https://docs.etherscan.io/) -use std::borrow::Cow; +use std::{ + borrow::Cow, + io::Write, + path::PathBuf, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; +use contract::ContractMetadata; use reqwest::{header, Url}; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use errors::EtherscanError; -use ethers_core::{abi::Address, types::Chain}; +use ethers_core::{ + abi::{Abi, Address}, + types::Chain, +}; pub mod account; pub mod contract; @@ -28,9 +37,92 @@ pub struct Client { etherscan_api_url: Url, /// Etherscan base endpoint like etherscan_url: Url, + /// Path to where ABI files should be cached + cache: Option, +} + +/// A wrapper around an Etherscan cache object with an expiry +#[derive(Clone, Debug, Deserialize, Serialize)] +struct CacheEnvelope { + expiry: u64, + data: T, +} + +/// Simple cache for etherscan requests +#[derive(Clone, Debug)] +struct Cache { + root: PathBuf, + ttl: Duration, +} + +impl Cache { + fn new(root: PathBuf, ttl: Duration) -> Self { + Self { root, ttl } + } + + fn get_abi(&self, address: Address) -> Option> { + self.get("abi", address) + } + + fn set_abi(&self, address: Address, abi: Option<&Abi>) { + self.set("abi", address, abi) + } + + fn get_source(&self, address: Address) -> Option> { + self.get("sources", address) + } + + fn set_source(&self, address: Address, source: Option<&ContractMetadata>) { + self.set("sources", address, source) + } + + fn set(&self, prefix: &str, address: Address, item: T) { + let path = self.root.join(prefix).join(format!("{:?}.json", address)); + let writer = std::fs::File::create(path).ok().map(std::io::BufWriter::new); + if let Some(mut writer) = writer { + let _ = serde_json::to_writer( + &mut writer, + &CacheEnvelope { + expiry: SystemTime::now() + .checked_add(self.ttl) + .expect("cache ttl overflowed") + .duration_since(UNIX_EPOCH) + .expect("system time is before unix epoch") + .as_secs(), + data: item, + }, + ); + let _ = writer.flush(); + } + } + + fn get(&self, prefix: &str, address: Address) -> Option { + let path = self.root.join(prefix).join(format!("{:?}.json", address)); + let reader = std::io::BufReader::new(std::fs::File::open(path).ok()?); + if let Ok(inner) = serde_json::from_reader::<_, CacheEnvelope>(reader) { + // If this does not return None then we have passed the expiry + if SystemTime::now().checked_sub(Duration::from_secs(inner.expiry)).is_some() { + return None + } + + return Some(inner.data) + } + None + } } impl Client { + pub fn new_cached( + chain: Chain, + api_key: impl Into, + cache_root: Option, + cache_ttl: Duration, + ) -> Result { + let mut this = Self::new(chain, api_key)?; + this.cache = cache_root.map(|root| Cache::new(root, cache_ttl)); + Ok(this) + } + /// Create a new client with the correct endpoints based on the chain and provided API key pub fn new(chain: Chain, api_key: impl Into) -> Result { let (etherscan_api_url, etherscan_url) = match chain { @@ -101,6 +193,7 @@ impl Client { api_key: api_key.into(), etherscan_api_url: etherscan_api_url.expect("is valid http"), etherscan_url: etherscan_url.expect("is valid http"), + cache: None, }) } @@ -180,7 +273,7 @@ impl Client { /// Execute an API GET request with parameters async fn get_json(&self, query: &Q) -> Result> { - Ok(self + let res: ResponseData = self .client .get(self.etherscan_api_url.clone()) .header(header::ACCEPT, "application/json") @@ -188,7 +281,18 @@ impl Client { .send() .await? .json() - .await?) + .await?; + + match res { + ResponseData::Error { result, .. } => { + if result.starts_with("Max rate limit reached") { + Err(EtherscanError::RateLimitExceeded) + } else { + Err(EtherscanError::Unknown(result)) + } + } + ResponseData::Success(res) => Ok(res), + } } fn create_query( @@ -214,6 +318,13 @@ pub struct Response { pub result: T, } +#[derive(Deserialize, Debug, Clone)] +#[serde(untagged)] +pub enum ResponseData { + Success(Response), + Error { status: String, message: String, result: String }, +} + /// The type that gets serialized as query #[derive(Debug, Serialize)] struct Query<'a, T: Serialize> { @@ -240,7 +351,7 @@ mod tests { let err = Client::new_from_env(Chain::XDai).unwrap_err(); assert!(matches!(err, EtherscanError::ChainNotSupported(_))); - assert_eq!(err.to_string(), "chain xdai not supported"); + assert_eq!(err.to_string(), "Chain xdai not supported"); } #[test] diff --git a/ethers-etherscan/src/transaction.rs b/ethers-etherscan/src/transaction.rs index 694aaebd..e916d55a 100644 --- a/ethers-etherscan/src/transaction.rs +++ b/ethers-etherscan/src/transaction.rs @@ -91,7 +91,7 @@ mod tests { .unwrap_err(); assert!(matches!(err, EtherscanError::ExecutionFailed(_))); - assert_eq!(err.to_string(), "contract execution call failed: Bad jump destination"); + assert_eq!(err.to_string(), "Contract execution call failed: Bad jump destination"); }) .await }