feat(etherscan): parse SourceCode field (#1747)

* wip

* feat(etherscan): parse SourceCode field

* feat: add project builder method

* chore: dependencies

* docs

* refactor: impls

* refactor: create verify submodule

* refactor: use untagged enum

* test: add more assertions

* docs

* feat: add more utility methods

* feat: deserialize ABI and improve SourceCode deserialization

* fix: lookup_compiler_version

* refactor: source tree methods

* docs

* chore: add solc feature

* fix: test

* chore: use only optional dependency

* chore: re-add dev-dependency

ethers-etherscan/Cargo.toml

@@ -19,6 +19,7 @@ keywords = ["ethereum", "web3", "etherscan", "ethers"]
 
 [dependencies]
 ethers-core = { version = "^0.17.0", path = "../ethers-core", default-features = false }
+ethers-solc = { version = "^0.17.0", path = "../ethers-solc", default-features = false, optional = true }
 reqwest = { version = "0.11.12", default-features = false, features = ["json"] }
 serde = { version = "1.0.124", default-features = false, features = ["derive"] }
 serde_json = { version = "1.0.64", default-features = false }

ethers-etherscan/src/lib.rs

@@ -24,6 +24,7 @@ pub mod gas;
 pub mod source_tree;
 pub mod transaction;
 pub mod utils;
+pub mod verify;
 
 pub(crate) type Result<T> = std::result::Result<T, EtherscanError>;
 
@@ -155,35 +156,15 @@ impl Client {
         format!("{}token/{:?}", self.etherscan_url, token_hash)
     }
 
-    /// Execute an API POST request with a form
+    /// Execute an GET request with parameters.
-    async fn post_form<T: DeserializeOwned, Form: Serialize>(
+    async fn get_json<T: DeserializeOwned, Q: Serialize>(&self, query: &Q) -> Result<Response<T>> {
-        &self,
+        let res = self.get(query).await?;
-        form: &Form,
+        self.sanitize_response(res)
-    ) -> Result<Response<T>> {
-        trace!(target: "etherscan", "POST FORM {}", self.etherscan_api_url);
-        let response = self
-            .client
-            .post(self.etherscan_api_url.clone())
-            .header(header::CONTENT_TYPE, "application/x-www-form-urlencoded")
-            .form(form)
-            .send()
-            .await?
-            .text()
-            .await?;
-
-        serde_json::from_str(&response).map_err(|err| {
-            error!(target: "etherscan", ?response, "Failed to deserialize response: {}", err);
-            if is_blocked_by_cloudflare_response(&response) {
-                EtherscanError::BlockedByCloudflare
-            } else {
-                EtherscanError::Serde(err)
-            }
-        })
     }
 
-    /// Execute an API GET request with parameters
+    /// Execute a GET request with parameters, without sanity checking the response.
-    async fn get_json<T: DeserializeOwned, Q: Serialize>(&self, query: &Q) -> Result<Response<T>> {
+    async fn get<Q: Serialize>(&self, query: &Q) -> Result<String> {
-        trace!(target: "etherscan", "GET JSON {}", self.etherscan_api_url);
+        trace!(target: "etherscan", "GET {}", self.etherscan_api_url);
         let response = self
             .client
             .get(self.etherscan_api_url.clone())
@@ -193,17 +174,42 @@ impl Client {
             .await?
             .text()
             .await?;
+        Ok(response)
+    }
 
-        let response: ResponseData<T> = serde_json::from_str(&response).map_err(|err| {
+    /// Execute a POST request with a form.
-            error!(target: "etherscan", ?response, "Failed to deserialize response: {}", err);
+    async fn post_form<T: DeserializeOwned, F: Serialize>(&self, form: &F) -> Result<Response<T>> {
-            if is_blocked_by_cloudflare_response(&response) {
+        let res = self.post(form).await?;
+        self.sanitize_response(res)
+    }
+
+    /// Execute a POST request with a form, without sanity checking the response.
+    async fn post<F: Serialize>(&self, form: &F) -> Result<String> {
+        trace!(target: "etherscan", "POST {}", self.etherscan_api_url);
+        let response = self
+            .client
+            .post(self.etherscan_api_url.clone())
+            .form(form)
+            .send()
+            .await?
+            .text()
+            .await?;
+        Ok(response)
+    }
+
+    /// Perform sanity checks on a response and deserialize it into a [Response].
+    fn sanitize_response<T: DeserializeOwned>(&self, res: impl AsRef<str>) -> Result<Response<T>> {
+        let res = res.as_ref();
+        let res: ResponseData<T> = serde_json::from_str(res).map_err(|err| {
+            error!(target: "etherscan", ?res, "Failed to deserialize response: {}", err);
+            if is_blocked_by_cloudflare_response(res) {
                 EtherscanError::BlockedByCloudflare
             } else {
                 EtherscanError::Serde(err)
             }
         })?;
 
-        match response {
+        match res {
             ResponseData::Error { result, .. } => {
                 if result.starts_with("Max rate limit reached") {
                     Err(EtherscanError::RateLimitExceeded)

ethers-etherscan/src/utils.rs

@@ -1,29 +1,70 @@
+use crate::{contract::SourceCodeMetadata, EtherscanError, Result};
+use ethers_core::{abi::Abi, types::Address};
 use semver::Version;
-
+use serde::{Deserialize, Deserializer};
-use crate::{EtherscanError, Result};
 
 static SOLC_BIN_LIST_URL: &str =
     "https://raw.githubusercontent.com/ethereum/solc-bin/gh-pages/bin/list.txt";
 
-/// Given the compiler version  lookup the build metadata
+/// Given a Solc [Version], lookup the build metadata and return the full SemVer.
-/// and return full semver
+/// e.g. `0.8.13` -> `0.8.13+commit.abaa5c0e`
-/// i.e. `0.8.13` -> `0.8.13+commit.abaa5c0e`
 pub async fn lookup_compiler_version(version: &Version) -> Result<Version> {
     let response = reqwest::get(SOLC_BIN_LIST_URL).await?.text().await?;
-    let version = format!("{}", version);
+    // Ignore extra metadata (`pre` or `build`)
+    let version = format!("{}.{}.{}", version.major, version.minor, version.patch);
     let v = response
         .lines()
         .find(|l| !l.contains("nightly") && l.contains(&version))
-        .map(|l| l.trim_start_matches("soljson-v").trim_end_matches(".js").to_owned())
+        .map(|l| l.trim_start_matches("soljson-v").trim_end_matches(".js"))
         .ok_or(EtherscanError::MissingSolcVersion(version))?;
 
     Ok(v.parse().expect("failed to parse semver"))
 }
 
+/// Return None if empty, otherwise parse as [Address].
+pub fn deserialize_address_opt<'de, D: Deserializer<'de>>(
+    deserializer: D,
+) -> std::result::Result<Option<Address>, D::Error> {
+    let s = String::deserialize(deserializer)?;
+    if s.is_empty() {
+        Ok(None)
+    } else {
+        let addr: Address = s.parse().map_err(serde::de::Error::custom)?;
+        Ok(Some(addr))
+    }
+}
+
+/// Deserializes as JSON:
+///
+/// `{ "SourceCode": "{{ .. }}", ..}`
+///
+/// or
+///
+/// `{ "SourceCode": "..", .. }`
+pub fn deserialize_stringified_source_code<'de, D: Deserializer<'de>>(
+    deserializer: D,
+) -> std::result::Result<SourceCodeMetadata, D::Error> {
+    let s = String::deserialize(deserializer)?;
+    if s.starts_with("{{") && s.ends_with("}}") {
+        let s = &s[1..s.len() - 1];
+        serde_json::from_str(s).map_err(serde::de::Error::custom)
+    } else {
+        Ok(SourceCodeMetadata::SourceCode(s))
+    }
+}
+
+/// Deserializes as JSON: "\[...\]"
+pub fn deserialize_stringified_abi<'de, D: Deserializer<'de>>(
+    deserializer: D,
+) -> std::result::Result<Abi, D::Error> {
+    let s = String::deserialize(deserializer)?;
+    serde_json::from_str(&s).map_err(serde::de::Error::custom)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::tests::run_at_least_duration;
+    use crate::{contract::SourceCodeLanguage, tests::run_at_least_duration};
     use semver::{BuildMetadata, Prerelease};
     use serial_test::serial;
     use std::time::Duration;
@@ -53,4 +94,62 @@ mod tests {
         })
         .await
     }
+
+    #[test]
+    fn can_deserialize_address_opt() {
+        #[derive(Deserialize)]
+        struct Test {
+            #[serde(deserialize_with = "deserialize_address_opt")]
+            address: Option<Address>,
+        }
+
+        // https://api.etherscan.io/api?module=contract&action=getsourcecode&address=0xBB9bc244D798123fDe783fCc1C72d3Bb8C189413
+        let json = r#"{"address":""}"#;
+        let de: Test = serde_json::from_str(json).unwrap();
+        assert_eq!(de.address, None);
+
+        // https://api.etherscan.io/api?module=contract&action=getsourcecode&address=0xDef1C0ded9bec7F1a1670819833240f027b25EfF
+        let json = r#"{"address":"0x4af649ffde640ceb34b1afaba3e0bb8e9698cb01"}"#;
+        let de: Test = serde_json::from_str(json).unwrap();
+        let expected = "0x4af649ffde640ceb34b1afaba3e0bb8e9698cb01".parse().unwrap();
+        assert_eq!(de.address, Some(expected));
+    }
+
+    #[test]
+    fn can_deserialize_stringified_abi() {
+        #[derive(Deserialize)]
+        struct Test {
+            #[serde(deserialize_with = "deserialize_stringified_abi")]
+            abi: Abi,
+        }
+
+        let json = r#"{"abi": "[]"}"#;
+        let de: Test = serde_json::from_str(json).unwrap();
+        assert_eq!(de.abi, Abi::default());
+    }
+
+    #[test]
+    fn can_deserialize_stringified_source_code() {
+        #[derive(Deserialize)]
+        struct Test {
+            #[serde(deserialize_with = "deserialize_stringified_source_code")]
+            source_code: SourceCodeMetadata,
+        }
+
+        let src = "source code text";
+
+        let json = r#"{
+            "source_code": "{{ \"language\": \"Solidity\", \"sources\": {\"Contract\": { \"content\": \"source code text\" } } }}"
+        }"#;
+        let de: Test = serde_json::from_str(json).unwrap();
+        assert!(matches!(de.source_code.language().unwrap(), SourceCodeLanguage::Solidity));
+        assert_eq!(de.source_code.sources().len(), 1);
+        assert_eq!(de.source_code.sources().get("Contract").unwrap().content, src);
+        #[cfg(feature = "ethers-solc")]
+        assert!(matches!(de.source_code.settings().unwrap(), None));
+
+        let json = r#"{"source_code": "source code text"}"#;
+        let de: Test = serde_json::from_str(json).unwrap();
+        assert_eq!(de.source_code.source_code(), src);
+    }
 }

ethers-etherscan/src/verify.rs

@@ -0,0 +1,207 @@
+use crate::{Client, Response, Result};
+use ethers_core::types::Address;
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+
+/// Arguments for verifying contracts
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct VerifyContract {
+    #[serde(rename = "contractaddress")]
+    pub address: Address,
+    #[serde(rename = "sourceCode")]
+    pub source: String,
+    #[serde(rename = "codeformat")]
+    pub code_format: CodeFormat,
+    /// if codeformat=solidity-standard-json-input, then expected as
+    /// `erc20.sol:erc20`
+    #[serde(rename = "contractname")]
+    pub contract_name: String,
+    #[serde(rename = "compilerversion")]
+    pub compiler_version: String,
+    /// applicable when codeformat=solidity-single-file
+    #[serde(rename = "optimizationUsed", skip_serializing_if = "Option::is_none")]
+    pub optimization_used: Option<String>,
+    /// applicable when codeformat=solidity-single-file
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub runs: Option<String>,
+    /// NOTE: there is a typo in the etherscan API `constructorArguements`
+    #[serde(rename = "constructorArguements", skip_serializing_if = "Option::is_none")]
+    pub constructor_arguments: Option<String>,
+    /// applicable when codeformat=solidity-single-file
+    #[serde(rename = "evmversion", skip_serializing_if = "Option::is_none")]
+    pub evm_version: Option<String>,
+    #[serde(flatten)]
+    pub other: HashMap<String, String>,
+}
+
+impl VerifyContract {
+    pub fn new(
+        address: Address,
+        contract_name: String,
+        source: String,
+        compiler_version: String,
+    ) -> Self {
+        Self {
+            address,
+            source,
+            code_format: Default::default(),
+            contract_name,
+            compiler_version,
+            optimization_used: None,
+            runs: None,
+            constructor_arguments: None,
+            evm_version: None,
+            other: Default::default(),
+        }
+    }
+
+    #[must_use]
+    pub fn runs(mut self, runs: u32) -> Self {
+        self.runs = Some(format!("{}", runs));
+        self
+    }
+
+    #[must_use]
+    pub fn optimization(self, optimization: bool) -> Self {
+        if optimization {
+            self.optimized()
+        } else {
+            self.not_optimized()
+        }
+    }
+
+    #[must_use]
+    pub fn optimized(mut self) -> Self {
+        self.optimization_used = Some("1".to_string());
+        self
+    }
+
+    #[must_use]
+    pub fn not_optimized(mut self) -> Self {
+        self.optimization_used = Some("0".to_string());
+        self
+    }
+
+    #[must_use]
+    pub fn code_format(mut self, code_format: CodeFormat) -> Self {
+        self.code_format = code_format;
+        self
+    }
+
+    #[must_use]
+    pub fn evm_version(mut self, evm_version: impl Into<String>) -> Self {
+        self.evm_version = Some(evm_version.into());
+        self
+    }
+
+    #[must_use]
+    pub fn constructor_arguments(
+        mut self,
+        constructor_arguments: Option<impl Into<String>>,
+    ) -> Self {
+        self.constructor_arguments = constructor_arguments.map(|s| {
+            s.into()
+                .trim()
+                // TODO is this correct?
+                .trim_start_matches("0x")
+                .to_string()
+        });
+        self
+    }
+}
+
+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+pub enum CodeFormat {
+    #[serde(rename = "solidity-single-file")]
+    SingleFile,
+
+    #[default]
+    #[serde(rename = "solidity-standard-json-input")]
+    StandardJsonInput,
+}
+
+impl AsRef<str> for CodeFormat {
+    fn as_ref(&self) -> &str {
+        match self {
+            CodeFormat::SingleFile => "solidity-single-file",
+            CodeFormat::StandardJsonInput => "solidity-standard-json-input",
+        }
+    }
+}
+
+impl Client {
+    /// Submit Source Code for Verification
+    pub async fn submit_contract_verification(
+        &self,
+        contract: &VerifyContract,
+    ) -> Result<Response<String>> {
+        let body = self.create_query("contract", "verifysourcecode", contract);
+        self.post_form(&body).await
+    }
+
+    /// Check Source Code Verification Status with receipt received from
+    /// `[Self::submit_contract_verification]`
+    pub async fn check_contract_verification_status(
+        &self,
+        guid: impl AsRef<str>,
+    ) -> Result<Response<String>> {
+        let body = self.create_query(
+            "contract",
+            "checkverifystatus",
+            HashMap::from([("guid", guid.as_ref())]),
+        );
+        self.post_form(&body).await
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::{tests::run_at_least_duration, Client};
+    use ethers_core::types::Chain;
+    use ethers_solc::{Project, ProjectPathsConfig};
+    use serial_test::serial;
+    use std::{path::PathBuf, time::Duration};
+
+    #[allow(unused)]
+    fn init_tracing() {
+        tracing_subscriber::fmt()
+            .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
+            .init();
+    }
+
+    #[tokio::test]
+    #[serial]
+    #[ignore]
+    async fn can_flatten_and_verify_contract() {
+        init_tracing();
+        run_at_least_duration(Duration::from_millis(250), async {
+            let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("resources");
+            let paths = ProjectPathsConfig::builder()
+                .sources(&root)
+                .build()
+                .expect("failed to resolve project paths");
+            let project = Project::builder()
+                .paths(paths)
+                .build()
+                .expect("failed to build the project");
+
+            let address = "0x9e744c9115b74834c0f33f4097f40c02a9ac5c33".parse().unwrap();
+            let compiler_version = "v0.5.17+commit.d19bba13";
+            let constructor_args = "0x000000000000000000000000000000000000000000000000000000000000006000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000005f5e1000000000000000000000000000000000000000000000000000000000000000007596179537761700000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000035941590000000000000000000000000000000000000000000000000000000000";
+            let contract = project.flatten(&root.join("UniswapExchange.sol")).expect("failed to flatten contract");
+            let contract_name = "UniswapExchange".to_owned();
+
+            let client = Client::new_from_env(Chain::Mainnet).unwrap();
+
+            let contract =
+                VerifyContract::new(address, contract_name, contract, compiler_version.to_string())
+                    .constructor_arguments(Some(constructor_args))
+                    .optimization(true)
+                    .runs(200);
+            let resp = client.submit_contract_verification(&contract).await.expect("failed to send the request");
+            assert_ne!(resp.result, "Error!"); // `Error!` result means that request was malformatted
+        })
+        .await
+    }
+}