From 23fb877c16c70a81825506ba54ea1c2bf915e19a Mon Sep 17 00:00:00 2001 From: Georgios Konstantopoulos Date: Wed, 3 Nov 2021 10:05:09 +0200 Subject: [PATCH] feat(solc): Multiple Solc Version detection (#551) * test: ensure that compilation succeeds * feat: add helper for parsing version req from a source * feat: detect the latest compatible solc version for a VersionReq * default to always enabling svm/async * test: add project with multiple contract versions * fix: always serde-default solc gas estimates * fix: normalize evm version in settings before compiling * feat: auto-detect version and compile if svm+async are on * chore: warnings * test: add a lock to ensure that there are no file conflicts when downloading solc * test: add tests for finding solc installations * chore: add features to ethers-rs config * chore: s/first/latest on finding solc version fn docs Co-authored-by: Matthias Seitz Co-authored-by: Matthias Seitz --- Cargo.toml | 4 + ethers-solc/src/artifacts.rs | 46 +++- ethers-solc/src/compile.rs | 205 +++++++++++++++++- ethers-solc/src/error.rs | 7 + ethers-solc/src/lib.rs | 88 +++++++- .../test-contract-versions/caret-0.4.14.sol | 5 + .../greater-equal-0.5.0.sol | 5 + .../test-contract-versions/pinned-0.4.14.sol | 5 + .../test-contract-versions/range-0.5.0.sol | 5 + .../test-contract-versions/simple-0.4.14.sol | 5 + ethers-solc/tests/project.rs | 13 +- 11 files changed, 372 insertions(+), 16 deletions(-) create mode 100644 ethers-solc/test-data/test-contract-versions/caret-0.4.14.sol create mode 100644 ethers-solc/test-data/test-contract-versions/greater-equal-0.5.0.sol create mode 100644 ethers-solc/test-data/test-contract-versions/pinned-0.4.14.sol create mode 100644 ethers-solc/test-data/test-contract-versions/range-0.5.0.sol create mode 100644 ethers-solc/test-data/test-contract-versions/simple-0.4.14.sol diff --git a/Cargo.toml b/Cargo.toml index e2ac4802..32493514 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,6 +72,10 @@ ledger = ["ethers-signers/ledger"] yubi = ["ethers-signers/yubi"] ## contracts abigen = ["ethers-contract/abigen"] +## solc +solc-async = ["ethers-solc/async"] +solc-full = ["ethers-solc/full"] + [dependencies] diff --git a/ethers-solc/src/artifacts.rs b/ethers-solc/src/artifacts.rs index 44bf8f95..64f65d9b 100644 --- a/ethers-solc/src/artifacts.rs +++ b/ethers-solc/src/artifacts.rs @@ -47,6 +47,15 @@ impl CompilerInput { self.settings.optimizer.runs(runs); self } + + /// Normalizes the EVM version used in the settings to be up to the latest one + /// supported by the provided compiler version. + pub fn normalize_evm_version(mut self, version: &Version) -> Self { + if let Some(ref mut evm_version) = self.settings.evm_version { + self.settings.evm_version = evm_version.normalize_version(version); + } + self + } } impl Default for CompilerInput { @@ -210,8 +219,8 @@ pub enum EvmVersion { Petersburg, Istanbul, Berlin, - London, Byzantium, + London, } impl EvmVersion { @@ -612,7 +621,9 @@ pub struct DeployedBytecode { #[derive(Clone, Debug, Serialize, Deserialize, Eq, PartialEq)] pub struct GasEstimates { pub creation: Creation, + #[serde(default)] pub external: BTreeMap, + #[serde(default)] pub internal: BTreeMap, } @@ -871,4 +882,37 @@ mod tests { }); } } + + #[test] + fn test_evm_version_normalization() { + for (solc_version, evm_version, expected) in &[ + // Ensure 0.4.21 it always returns None + ("0.4.20", EvmVersion::Homestead, None), + // Constantinople clipping + ("0.4.21", EvmVersion::Homestead, Some(EvmVersion::Homestead)), + ("0.4.21", EvmVersion::Constantinople, Some(EvmVersion::Constantinople)), + ("0.4.21", EvmVersion::London, Some(EvmVersion::Constantinople)), + // Petersburg + ("0.5.5", EvmVersion::Homestead, Some(EvmVersion::Homestead)), + ("0.5.5", EvmVersion::Petersburg, Some(EvmVersion::Petersburg)), + ("0.5.5", EvmVersion::London, Some(EvmVersion::Petersburg)), + // Istanbul + ("0.5.14", EvmVersion::Homestead, Some(EvmVersion::Homestead)), + ("0.5.14", EvmVersion::Istanbul, Some(EvmVersion::Istanbul)), + ("0.5.14", EvmVersion::London, Some(EvmVersion::Istanbul)), + // Berlin + ("0.8.5", EvmVersion::Homestead, Some(EvmVersion::Homestead)), + ("0.8.5", EvmVersion::Berlin, Some(EvmVersion::Berlin)), + ("0.8.5", EvmVersion::London, Some(EvmVersion::Berlin)), + // London + ("0.8.7", EvmVersion::Homestead, Some(EvmVersion::Homestead)), + ("0.8.7", EvmVersion::London, Some(EvmVersion::London)), + ("0.8.7", EvmVersion::London, Some(EvmVersion::London)), + ] { + assert_eq!( + &evm_version.normalize_version(&Version::from_str(solc_version).unwrap()), + expected + ) + } + } } diff --git a/ethers-solc/src/compile.rs b/ethers-solc/src/compile.rs index 76de8870..9d4d7a1e 100644 --- a/ethers-solc/src/compile.rs +++ b/ethers-solc/src/compile.rs @@ -1,8 +1,9 @@ use crate::{ + artifacts::Source, error::{Result, SolcError}, CompilerInput, CompilerOutput, }; -use semver::Version; +use semver::{Version, VersionReq}; use serde::{de::DeserializeOwned, Serialize}; use std::{ io::BufRead, @@ -34,10 +35,34 @@ pub const BERLIN_SOLC: Version = Version::new(0, 8, 5); /// https://blog.soliditylang.org/2021/08/11/solidity-0.8.7-release-announcement/ pub const LONDON_SOLC: Version = Version::new(0, 8, 7); +#[cfg(any(test, all(feature = "svm", feature = "async")))] +use once_cell::sync::Lazy; + +#[cfg(any(test, feature = "tests"))] +use std::sync::Mutex; +#[cfg(any(test, feature = "tests"))] +static LOCK: Lazy> = Lazy::new(|| Mutex::new(())); + +#[cfg(all(feature = "svm", feature = "async"))] +/// A list of upstream Solc releases, used to check which version +/// we should download. +pub static RELEASES: Lazy> = Lazy::new(|| { + // Try to download the releases, if it fails default to empty + match tokio::runtime::Runtime::new() + .expect("could not create tokio rt to get remote releases") + // TODO: Can we make this future timeout at a small time amount so that + // we do not degrade startup performance if the consumer has a weak network? + .block_on(svm::all_versions()) + { + Ok(inner) => inner, + Err(_) => Vec::new(), + } +}); + /// Abstraction over `solc` command line utility /// /// Supports sync and async functions. -#[derive(Debug, Clone, Eq, PartialEq)] +#[derive(Debug, Clone, Eq, PartialEq, PartialOrd, Ord)] pub struct Solc(pub PathBuf); impl Default for Solc { @@ -87,6 +112,74 @@ impl Solc { Ok(solc) } + /// Assuming the `versions` array is sorted, it returns the latest element which satisfies + /// the provided [`VersionReq`] + pub fn find_matching_installation( + versions: &[Version], + required_version: &VersionReq, + ) -> Option { + // iterate in reverse to find the last match + versions.iter().rev().find(|version| required_version.matches(version)).cloned() + } + + /// Given a Solidity source, it detects the latest compiler version which can be used + /// to build it, and returns it. + /// + /// If the required compiler version is not installed, it also proceeds to install it. + #[cfg(all(feature = "svm", feature = "async"))] + pub fn detect_version(source: &Source) -> Result { + // detects the required solc version + let sol_version = Self::version_req(source)?; + + #[cfg(any(test, feature = "tests"))] + // take the lock in tests, we use this to enforce that + // a test does not run while a compiler version is being installed + let _lock = LOCK.lock(); + + // load the local / remote versions + let versions = svm::installed_versions().unwrap_or_default(); + let local_versions = Self::find_matching_installation(&versions, &sol_version); + let remote_versions = Self::find_matching_installation(&RELEASES, &sol_version); + + // if there's a better upstream version than the one we have, install it + Ok(match (local_versions, remote_versions) { + (Some(local), None) => local, + (Some(local), Some(remote)) => { + if remote > local { + Self::blocking_install(&remote)?; + remote + } else { + local + } + } + (None, Some(version)) => { + Self::blocking_install(&version)?; + version + } + // do nothing otherwise + _ => return Err(SolcError::VersionNotFound), + }) + } + + /// Parses the given source looking for the `pragma` definition and + /// returns the corresponding SemVer version requirement. + pub fn version_req(source: &Source) -> Result { + let version = crate::utils::find_version_pragma(&source.content) + .ok_or(SolcError::PragmaNotFound)? + .replace(" ", ","); + + // Somehow, Solidity semver without an operator is considered to be "exact", + // but lack of operator automatically marks the operator as Caret, so we need + // to manually patch it? :shrug: + let exact = !matches!(&version[0..1], "*" | "^" | "=" | ">" | "<" | "~"); + let mut version = VersionReq::parse(&version)?; + if exact { + version.comparators[0].op = semver::Op::Exact; + } + + Ok(version) + } + /// Installs the provided version of Solc in the machine under the svm dir /// # Example /// ```no_run @@ -172,7 +265,6 @@ impl Solc { &self, path: impl AsRef, ) -> Result { - use crate::artifacts::Source; self.async_compile(&CompilerInput::with_sources(Source::async_read_all_from(path).await?)) .await } @@ -275,12 +367,6 @@ mod tests { let _version = Version::from_str("0.6.6+commit.6c089d02.Linux.gcc").unwrap(); } - #[test] - #[ignore] - fn can_find_solc() { - let _solc = Solc::find_svm_installed_version("0.8.9").unwrap(); - } - #[cfg(feature = "async")] #[tokio::test] async fn async_solc_version_works() { @@ -305,4 +391,105 @@ mod tests { let other = solc().async_compile(&serde_json::json!(input)).await.unwrap(); assert_eq!(out, other); } + + #[test] + fn test_version_req() { + let versions = ["=0.1.2", "^0.5.6", ">=0.7.1", ">0.8.0"]; + let sources = versions.iter().map(|version| source(version)); + + sources.zip(versions).for_each(|(source, version)| { + let version_req = Solc::version_req(&source).unwrap(); + assert_eq!(version_req, VersionReq::from_str(version).unwrap()); + }); + + // Solidity defines version ranges with a space, whereas the semver package + // requires them to be separated with a comma + let version_range = ">=0.8.0 <0.9.0"; + let source = source(version_range); + let version_req = Solc::version_req(&source).unwrap(); + assert_eq!(version_req, VersionReq::from_str(">=0.8.0,<0.9.0").unwrap()); + } + + #[test] + // This test might be a bit hard to maintain + #[cfg(all(feature = "svm", feature = "async"))] + fn test_detect_version() { + for (pragma, expected) in [ + // pinned + ("=0.4.14", "0.4.14"), + // pinned too + ("0.4.14", "0.4.14"), + // The latest patch is 0.4.26 + ("^0.4.14", "0.4.26"), + // latest version above 0.5.0 -> we have to + // update this test whenever there's a new sol + // version. that's ok! good reminder to check the + // patch notes. + (">=0.5.0", "0.8.9"), + // range + (">=0.4.0 <0.5.0", "0.4.26"), + ] + .iter() + { + // println!("Checking {}", pragma); + let source = source(pragma); + let res = Solc::detect_version(&source).unwrap(); + assert_eq!(res, Version::from_str(expected).unwrap()); + } + } + + #[test] + #[cfg(feature = "full")] + fn test_find_installed_version_path() { + // this test does not take the lock by default, so we need to manually + // add it here. + let _lock = LOCK.lock(); + let ver = "0.8.6"; + let version = Version::from_str(ver).unwrap(); + if !svm::installed_versions().unwrap().contains(&version) { + Solc::blocking_install(&version).unwrap(); + } + let res = Solc::find_svm_installed_version(&version.to_string()).unwrap().unwrap(); + let expected = svm::SVM_HOME.join(ver).join(format!("solc-{}", ver)); + assert_eq!(res.0, expected); + } + + #[test] + fn does_not_find_not_installed_version() { + let ver = "1.1.1"; + let version = Version::from_str(ver).unwrap(); + let res = Solc::find_svm_installed_version(&version.to_string()).unwrap(); + assert!(res.is_none()); + } + + #[test] + fn test_find_latest_matching_installation() { + let versions = ["0.4.24", "0.5.1", "0.5.2"] + .iter() + .map(|version| Version::from_str(version).unwrap()) + .collect::>(); + + let required = VersionReq::from_str(">=0.4.24").unwrap(); + + let got = Solc::find_matching_installation(&versions, &required).unwrap(); + assert_eq!(got, versions[2]); + } + + #[test] + fn test_no_matching_installation() { + let versions = ["0.4.24", "0.5.1", "0.5.2"] + .iter() + .map(|version| Version::from_str(version).unwrap()) + .collect::>(); + + let required = VersionReq::from_str(">=0.6.0").unwrap(); + let got = Solc::find_matching_installation(&versions, &required); + assert!(got.is_none()); + } + + ///// helpers + + fn source(version: &str) -> Source { + Source { content: format!("pragma solidity {};\n", version) } + } } diff --git a/ethers-solc/src/error.rs b/ethers-solc/src/error.rs index b7538e45..37ccd32c 100644 --- a/ethers-solc/src/error.rs +++ b/ethers-solc/src/error.rs @@ -8,6 +8,10 @@ pub enum SolcError { /// Internal solc error #[error("Solc Error: {0}")] SolcError(String), + #[error("missing pragma from solidity file")] + PragmaNotFound, + #[error("could not find solc version locally or upstream")] + VersionNotFound, #[error(transparent)] SemverError(#[from] semver::Error), /// Deserialization error @@ -16,6 +20,9 @@ pub enum SolcError { /// Deserialization error #[error(transparent)] Io(#[from] std::io::Error), + #[cfg(feature = "svm")] + #[error(transparent)] + SvmError(#[from] svm::SolcVmError), } impl SolcError { diff --git a/ethers-solc/src/lib.rs b/ethers-solc/src/lib.rs index b9ad04e6..eb5d03f8 100644 --- a/ethers-solc/src/lib.rs +++ b/ethers-solc/src/lib.rs @@ -93,8 +93,58 @@ impl Project { /// /// NOTE: this does not check if the contracts were successfully compiled, see /// `CompilerOutput::has_error` instead. + + /// NB: If the `svm` feature is enabled, this function will automatically detect + /// solc versions across files. pub fn compile(&self) -> Result { - let mut sources = self.sources()?; + let sources = self.sources()?; + + #[cfg(not(all(feature = "svm", feature = "async")))] + { + self.compile_with_version(&self.solc, sources) + } + #[cfg(all(feature = "svm", feature = "async"))] + self.svm_compile(sources) + } + + #[cfg(all(feature = "svm", feature = "async"))] + fn svm_compile(&self, sources: Sources) -> Result { + // split them by version + let mut sources_by_version = BTreeMap::new(); + for (path, source) in sources.into_iter() { + // will detect and install the solc version + let version = Solc::detect_version(&source)?; + // gets the solc binary for that version, it is expected tha this will succeed + // AND find the solc since it was installed right above + let solc = Solc::find_svm_installed_version(version.to_string())? + .expect("solc should have been installed"); + let entry = sources_by_version.entry(solc).or_insert_with(BTreeMap::new); + entry.insert(path, source); + } + + // run the compilation step for each version + let mut res = CompilerOutput::default(); + for (solc, sources) in sources_by_version { + let output = self.compile_with_version(&solc, sources)?; + if let ProjectCompileOutput::Compiled((compiled, _)) = output { + res.errors.extend(compiled.errors); + res.sources.extend(compiled.sources); + res.contracts.extend(compiled.contracts); + } + } + + Ok(if res.contracts.is_empty() { + ProjectCompileOutput::Unchanged + } else { + ProjectCompileOutput::Compiled((res, &self.ignored_error_codes)) + }) + } + + pub fn compile_with_version( + &self, + solc: &Solc, + mut sources: Sources, + ) -> Result { // add all libraries to the source set while keeping track of their actual disk path let mut source_name_path = HashMap::new(); let mut path_source_name = HashMap::new(); @@ -120,8 +170,8 @@ impl Project { // replace absolute path with source name to make solc happy let sources = apply_mappings(sources, path_source_name); - let input = CompilerInput::with_sources(sources); - let output = self.solc.compile(&input)?; + let input = CompilerInput::with_sources(sources).normalize_evm_version(&solc.version()?); + let output = solc.compile(&input)?; if output.has_error() { return Ok(ProjectCompileOutput::Compiled((output, &self.ignored_error_codes))) } @@ -248,3 +298,35 @@ impl<'a> fmt::Display for ProjectCompileOutput<'a> { } } } + +#[cfg(test)] +mod tests { + + #[test] + #[cfg(all(feature = "svm", feature = "async"))] + fn test_build_all_versions() { + use super::*; + + let paths = ProjectPathsConfig::builder() + .root("./test-data/test-contract-versions") + .sources("./test-data/test-contract-versions") + .build() + .unwrap(); + let project = Project::builder() + .paths(paths) + .ephemeral() + .artifacts(ArtifactOutput::Nothing) + .build() + .unwrap(); + let compiled = project.compile().unwrap(); + let contracts = match compiled { + ProjectCompileOutput::Compiled((out, _)) => { + assert!(!out.has_error()); + out.contracts + } + _ => panic!("must compile"), + }; + // Contracts A to F + assert_eq!(contracts.keys().count(), 5); + } +} diff --git a/ethers-solc/test-data/test-contract-versions/caret-0.4.14.sol b/ethers-solc/test-data/test-contract-versions/caret-0.4.14.sol new file mode 100644 index 00000000..e57e1293 --- /dev/null +++ b/ethers-solc/test-data/test-contract-versions/caret-0.4.14.sol @@ -0,0 +1,5 @@ +pragma solidity ^0.4.14; + +contract B { + function foo() public {} +} diff --git a/ethers-solc/test-data/test-contract-versions/greater-equal-0.5.0.sol b/ethers-solc/test-data/test-contract-versions/greater-equal-0.5.0.sol new file mode 100644 index 00000000..382894d2 --- /dev/null +++ b/ethers-solc/test-data/test-contract-versions/greater-equal-0.5.0.sol @@ -0,0 +1,5 @@ +pragma solidity >=0.5.0; + +contract C { + function foo() public {} +} diff --git a/ethers-solc/test-data/test-contract-versions/pinned-0.4.14.sol b/ethers-solc/test-data/test-contract-versions/pinned-0.4.14.sol new file mode 100644 index 00000000..28e6774e --- /dev/null +++ b/ethers-solc/test-data/test-contract-versions/pinned-0.4.14.sol @@ -0,0 +1,5 @@ +pragma solidity =0.4.14; + +contract D { + function foo() public {} +} diff --git a/ethers-solc/test-data/test-contract-versions/range-0.5.0.sol b/ethers-solc/test-data/test-contract-versions/range-0.5.0.sol new file mode 100644 index 00000000..9ac40a4a --- /dev/null +++ b/ethers-solc/test-data/test-contract-versions/range-0.5.0.sol @@ -0,0 +1,5 @@ +pragma solidity >=0.4.0 <0.5.0; + +contract E { + function foo() public {} +} diff --git a/ethers-solc/test-data/test-contract-versions/simple-0.4.14.sol b/ethers-solc/test-data/test-contract-versions/simple-0.4.14.sol new file mode 100644 index 00000000..8a8ec28b --- /dev/null +++ b/ethers-solc/test-data/test-contract-versions/simple-0.4.14.sol @@ -0,0 +1,5 @@ +pragma solidity =0.4.14; + +contract F { + function foo() public {} +} diff --git a/ethers-solc/tests/project.rs b/ethers-solc/tests/project.rs index 01b83de7..c6cd0367 100644 --- a/ethers-solc/tests/project.rs +++ b/ethers-solc/tests/project.rs @@ -25,7 +25,11 @@ fn can_compile_hardhat_sample() { // let paths = ProjectPathsConfig::hardhat(root).unwrap(); let project = Project::builder().paths(paths).build().unwrap(); - assert_ne!(project.compile().unwrap(), ProjectCompileOutput::Unchanged); + let compiled = project.compile().unwrap(); + match compiled { + ProjectCompileOutput::Compiled((out, _)) => assert!(!out.has_error()), + _ => panic!("must compile"), + } // nothing to compile assert_eq!(project.compile().unwrap(), ProjectCompileOutput::Unchanged); } @@ -46,10 +50,13 @@ fn can_compile_dapp_sample() { .root(root) .build() .unwrap(); - // let paths = ProjectPathsConfig::dapptools(root).unwrap(); let project = Project::builder().paths(paths).build().unwrap(); - assert_ne!(project.compile().unwrap(), ProjectCompileOutput::Unchanged); + let compiled = project.compile().unwrap(); + match compiled { + ProjectCompileOutput::Compiled((out, _)) => assert!(!out.has_error()), + _ => panic!("must compile"), + } // nothing to compile assert_eq!(project.compile().unwrap(), ProjectCompileOutput::Unchanged); }