feat(solc): Multiple Solc Version detection (#551)

* test: ensure that compilation succeeds

* feat: add helper for parsing version req from a source

* feat: detect the latest compatible solc version for a VersionReq

* default to always enabling svm/async

* test: add project with multiple contract versions

* fix: always serde-default solc gas estimates

* fix: normalize evm version in settings before compiling

* feat: auto-detect version and compile if svm+async are on

* chore: warnings

* test: add a lock to ensure that there are no file conflicts when downloading solc

* test: add tests for finding solc installations

* chore: add features to ethers-rs config

* chore: s/first/latest on finding solc version fn docs

Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>

Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>

Cargo.toml

@@ -72,6 +72,10 @@ ledger = ["ethers-signers/ledger"]
 yubi = ["ethers-signers/yubi"]
 ## contracts
 abigen = ["ethers-contract/abigen"]
+## solc
+solc-async = ["ethers-solc/async"]
+solc-full = ["ethers-solc/full"]
+
 
 
 [dependencies]

ethers-solc/src/artifacts.rs

@@ -47,6 +47,15 @@ impl CompilerInput {
         self.settings.optimizer.runs(runs);
         self
     }
+
+    /// Normalizes the EVM version used in the settings to be up to the latest one
+    /// supported by the provided compiler version.
+    pub fn normalize_evm_version(mut self, version: &Version) -> Self {
+        if let Some(ref mut evm_version) = self.settings.evm_version {
+            self.settings.evm_version = evm_version.normalize_version(version);
+        }
+        self
+    }
 }
 
 impl Default for CompilerInput {
@@ -210,8 +219,8 @@ pub enum EvmVersion {
     Petersburg,
     Istanbul,
     Berlin,
-    London,
     Byzantium,
+    London,
 }
 
 impl EvmVersion {
@@ -612,7 +621,9 @@ pub struct DeployedBytecode {
 #[derive(Clone, Debug, Serialize, Deserialize, Eq, PartialEq)]
 pub struct GasEstimates {
     pub creation: Creation,
+    #[serde(default)]
     pub external: BTreeMap<String, String>,
+    #[serde(default)]
     pub internal: BTreeMap<String, String>,
 }
 
@@ -871,4 +882,37 @@ mod tests {
             });
         }
     }
+
+    #[test]
+    fn test_evm_version_normalization() {
+        for (solc_version, evm_version, expected) in &[
+            // Ensure 0.4.21 it always returns None
+            ("0.4.20", EvmVersion::Homestead, None),
+            // Constantinople clipping
+            ("0.4.21", EvmVersion::Homestead, Some(EvmVersion::Homestead)),
+            ("0.4.21", EvmVersion::Constantinople, Some(EvmVersion::Constantinople)),
+            ("0.4.21", EvmVersion::London, Some(EvmVersion::Constantinople)),
+            // Petersburg
+            ("0.5.5", EvmVersion::Homestead, Some(EvmVersion::Homestead)),
+            ("0.5.5", EvmVersion::Petersburg, Some(EvmVersion::Petersburg)),
+            ("0.5.5", EvmVersion::London, Some(EvmVersion::Petersburg)),
+            // Istanbul
+            ("0.5.14", EvmVersion::Homestead, Some(EvmVersion::Homestead)),
+            ("0.5.14", EvmVersion::Istanbul, Some(EvmVersion::Istanbul)),
+            ("0.5.14", EvmVersion::London, Some(EvmVersion::Istanbul)),
+            // Berlin
+            ("0.8.5", EvmVersion::Homestead, Some(EvmVersion::Homestead)),
+            ("0.8.5", EvmVersion::Berlin, Some(EvmVersion::Berlin)),
+            ("0.8.5", EvmVersion::London, Some(EvmVersion::Berlin)),
+            // London
+            ("0.8.7", EvmVersion::Homestead, Some(EvmVersion::Homestead)),
+            ("0.8.7", EvmVersion::London, Some(EvmVersion::London)),
+            ("0.8.7", EvmVersion::London, Some(EvmVersion::London)),
+        ] {
+            assert_eq!(
+                &evm_version.normalize_version(&Version::from_str(solc_version).unwrap()),
+                expected
+            )
+        }
+    }
 }

ethers-solc/src/compile.rs

@@ -1,8 +1,9 @@
 use crate::{
+    artifacts::Source,
     error::{Result, SolcError},
     CompilerInput, CompilerOutput,
 };
-use semver::Version;
+use semver::{Version, VersionReq};
 use serde::{de::DeserializeOwned, Serialize};
 use std::{
     io::BufRead,
@@ -34,10 +35,34 @@ pub const BERLIN_SOLC: Version = Version::new(0, 8, 5);
 /// https://blog.soliditylang.org/2021/08/11/solidity-0.8.7-release-announcement/
 pub const LONDON_SOLC: Version = Version::new(0, 8, 7);
 
+#[cfg(any(test, all(feature = "svm", feature = "async")))]
+use once_cell::sync::Lazy;
+
+#[cfg(any(test, feature = "tests"))]
+use std::sync::Mutex;
+#[cfg(any(test, feature = "tests"))]
+static LOCK: Lazy<Mutex<()>> = Lazy::new(|| Mutex::new(()));
+
+#[cfg(all(feature = "svm", feature = "async"))]
+/// A list of upstream Solc releases, used to check which version
+/// we should download.
+pub static RELEASES: Lazy<Vec<Version>> = Lazy::new(|| {
+    // Try to download the releases, if it fails default to empty
+    match tokio::runtime::Runtime::new()
+        .expect("could not create tokio rt to get remote releases")
+        // TODO: Can we make this future timeout at a small time amount so that
+        // we do not degrade startup performance if the consumer has a weak network?
+        .block_on(svm::all_versions())
+    {
+        Ok(inner) => inner,
+        Err(_) => Vec::new(),
+    }
+});
+
 /// Abstraction over `solc` command line utility
 ///
 /// Supports sync and async functions.
-#[derive(Debug, Clone, Eq, PartialEq)]
+#[derive(Debug, Clone, Eq, PartialEq, PartialOrd, Ord)]
 pub struct Solc(pub PathBuf);
 
 impl Default for Solc {
@@ -87,6 +112,74 @@ impl Solc {
         Ok(solc)
     }
 
+    /// Assuming the `versions` array is sorted, it returns the latest element which satisfies
+    /// the provided [`VersionReq`]
+    pub fn find_matching_installation(
+        versions: &[Version],
+        required_version: &VersionReq,
+    ) -> Option<Version> {
+        // iterate in reverse to find the last match
+        versions.iter().rev().find(|version| required_version.matches(version)).cloned()
+    }
+
+    /// Given a Solidity source, it detects the latest compiler version which can be used
+    /// to build it, and returns it.
+    ///
+    /// If the required compiler version is not installed, it also proceeds to install it.
+    #[cfg(all(feature = "svm", feature = "async"))]
+    pub fn detect_version(source: &Source) -> Result<Version> {
+        // detects the required solc version
+        let sol_version = Self::version_req(source)?;
+
+        #[cfg(any(test, feature = "tests"))]
+        // take the lock in tests, we use this to enforce that
+        // a test does not run while a compiler version is being installed
+        let _lock = LOCK.lock();
+
+        // load the local / remote versions
+        let versions = svm::installed_versions().unwrap_or_default();
+        let local_versions = Self::find_matching_installation(&versions, &sol_version);
+        let remote_versions = Self::find_matching_installation(&RELEASES, &sol_version);
+
+        // if there's a better upstream version than the one we have, install it
+        Ok(match (local_versions, remote_versions) {
+            (Some(local), None) => local,
+            (Some(local), Some(remote)) => {
+                if remote > local {
+                    Self::blocking_install(&remote)?;
+                    remote
+                } else {
+                    local
+                }
+            }
+            (None, Some(version)) => {
+                Self::blocking_install(&version)?;
+                version
+            }
+            // do nothing otherwise
+            _ => return Err(SolcError::VersionNotFound),
+        })
+    }
+
+    /// Parses the given source looking for the `pragma` definition and
+    /// returns the corresponding SemVer version requirement.
+    pub fn version_req(source: &Source) -> Result<VersionReq> {
+        let version = crate::utils::find_version_pragma(&source.content)
+            .ok_or(SolcError::PragmaNotFound)?
+            .replace(" ", ",");
+
+        // Somehow, Solidity semver without an operator is considered to be "exact",
+        // but lack of operator automatically marks the operator as Caret, so we need
+        // to manually patch it? :shrug:
+        let exact = !matches!(&version[0..1], "*" | "^" | "=" | ">" | "<" | "~");
+        let mut version = VersionReq::parse(&version)?;
+        if exact {
+            version.comparators[0].op = semver::Op::Exact;
+        }
+
+        Ok(version)
+    }
+
     /// Installs the provided version of Solc in the machine under the svm dir
     /// # Example
     /// ```no_run
@@ -172,7 +265,6 @@ impl Solc {
         &self,
         path: impl AsRef<Path>,
     ) -> Result<CompilerOutput> {
-        use crate::artifacts::Source;
         self.async_compile(&CompilerInput::with_sources(Source::async_read_all_from(path).await?))
             .await
     }
@@ -275,12 +367,6 @@ mod tests {
         let _version = Version::from_str("0.6.6+commit.6c089d02.Linux.gcc").unwrap();
     }
 
-    #[test]
-    #[ignore]
-    fn can_find_solc() {
-        let _solc = Solc::find_svm_installed_version("0.8.9").unwrap();
-    }
-
     #[cfg(feature = "async")]
     #[tokio::test]
     async fn async_solc_version_works() {
@@ -305,4 +391,105 @@ mod tests {
         let other = solc().async_compile(&serde_json::json!(input)).await.unwrap();
         assert_eq!(out, other);
     }
+
+    #[test]
+    fn test_version_req() {
+        let versions = ["=0.1.2", "^0.5.6", ">=0.7.1", ">0.8.0"];
+        let sources = versions.iter().map(|version| source(version));
+
+        sources.zip(versions).for_each(|(source, version)| {
+            let version_req = Solc::version_req(&source).unwrap();
+            assert_eq!(version_req, VersionReq::from_str(version).unwrap());
+        });
+
+        // Solidity defines version ranges with a space, whereas the semver package
+        // requires them to be separated with a comma
+        let version_range = ">=0.8.0 <0.9.0";
+        let source = source(version_range);
+        let version_req = Solc::version_req(&source).unwrap();
+        assert_eq!(version_req, VersionReq::from_str(">=0.8.0,<0.9.0").unwrap());
+    }
+
+    #[test]
+    // This test might be a bit hard to maintain
+    #[cfg(all(feature = "svm", feature = "async"))]
+    fn test_detect_version() {
+        for (pragma, expected) in [
+            // pinned
+            ("=0.4.14", "0.4.14"),
+            // pinned too
+            ("0.4.14", "0.4.14"),
+            // The latest patch is 0.4.26
+            ("^0.4.14", "0.4.26"),
+            // latest version above 0.5.0 -> we have to
+            // update this test whenever there's a new sol
+            // version. that's ok! good reminder to check the
+            // patch notes.
+            (">=0.5.0", "0.8.9"),
+            // range
+            (">=0.4.0 <0.5.0", "0.4.26"),
+        ]
+        .iter()
+        {
+            // println!("Checking {}", pragma);
+            let source = source(pragma);
+            let res = Solc::detect_version(&source).unwrap();
+            assert_eq!(res, Version::from_str(expected).unwrap());
+        }
+    }
+
+    #[test]
+    #[cfg(feature = "full")]
+    fn test_find_installed_version_path() {
+        // this test does not take the lock by default, so we need to manually
+        // add it here.
+        let _lock = LOCK.lock();
+        let ver = "0.8.6";
+        let version = Version::from_str(ver).unwrap();
+        if !svm::installed_versions().unwrap().contains(&version) {
+            Solc::blocking_install(&version).unwrap();
+        }
+        let res = Solc::find_svm_installed_version(&version.to_string()).unwrap().unwrap();
+        let expected = svm::SVM_HOME.join(ver).join(format!("solc-{}", ver));
+        assert_eq!(res.0, expected);
+    }
+
+    #[test]
+    fn does_not_find_not_installed_version() {
+        let ver = "1.1.1";
+        let version = Version::from_str(ver).unwrap();
+        let res = Solc::find_svm_installed_version(&version.to_string()).unwrap();
+        assert!(res.is_none());
+    }
+
+    #[test]
+    fn test_find_latest_matching_installation() {
+        let versions = ["0.4.24", "0.5.1", "0.5.2"]
+            .iter()
+            .map(|version| Version::from_str(version).unwrap())
+            .collect::<Vec<_>>();
+
+        let required = VersionReq::from_str(">=0.4.24").unwrap();
+
+        let got = Solc::find_matching_installation(&versions, &required).unwrap();
+        assert_eq!(got, versions[2]);
+    }
+
+    #[test]
+    fn test_no_matching_installation() {
+        let versions = ["0.4.24", "0.5.1", "0.5.2"]
+            .iter()
+            .map(|version| Version::from_str(version).unwrap())
+            .collect::<Vec<_>>();
+
+        let required = VersionReq::from_str(">=0.6.0").unwrap();
+        let got = Solc::find_matching_installation(&versions, &required);
+        assert!(got.is_none());
+    }
+
+    ///// helpers
+
+    fn source(version: &str) -> Source {
+        Source { content: format!("pragma solidity {};\n", version) }
+    }
 }

ethers-solc/src/error.rs

@@ -8,6 +8,10 @@ pub enum SolcError {
     /// Internal solc error
     #[error("Solc Error: {0}")]
     SolcError(String),
+    #[error("missing pragma from solidity file")]
+    PragmaNotFound,
+    #[error("could not find solc version locally or upstream")]
+    VersionNotFound,
     #[error(transparent)]
     SemverError(#[from] semver::Error),
     /// Deserialization error
@@ -16,6 +20,9 @@ pub enum SolcError {
     /// Deserialization error
     #[error(transparent)]
     Io(#[from] std::io::Error),
+    #[cfg(feature = "svm")]
+    #[error(transparent)]
+    SvmError(#[from] svm::SolcVmError),
 }
 
 impl SolcError {

ethers-solc/src/lib.rs

@@ -93,8 +93,58 @@ impl Project {
     ///
     /// NOTE: this does not check if the contracts were successfully compiled, see
     /// `CompilerOutput::has_error` instead.
+
+    /// NB: If the `svm` feature is enabled, this function will automatically detect
+    /// solc versions across files.
     pub fn compile(&self) -> Result<ProjectCompileOutput> {
-        let mut sources = self.sources()?;
+        let sources = self.sources()?;
+
+        #[cfg(not(all(feature = "svm", feature = "async")))]
+        {
+            self.compile_with_version(&self.solc, sources)
+        }
+        #[cfg(all(feature = "svm", feature = "async"))]
+        self.svm_compile(sources)
+    }
+
+    #[cfg(all(feature = "svm", feature = "async"))]
+    fn svm_compile(&self, sources: Sources) -> Result<ProjectCompileOutput> {
+        // split them by version
+        let mut sources_by_version = BTreeMap::new();
+        for (path, source) in sources.into_iter() {
+            // will detect and install the solc version
+            let version = Solc::detect_version(&source)?;
+            // gets the solc binary for that version, it is expected tha this will succeed
+            // AND find the solc since it was installed right above
+            let solc = Solc::find_svm_installed_version(version.to_string())?
+                .expect("solc should have been installed");
+            let entry = sources_by_version.entry(solc).or_insert_with(BTreeMap::new);
+            entry.insert(path, source);
+        }
+
+        // run the compilation step for each version
+        let mut res = CompilerOutput::default();
+        for (solc, sources) in sources_by_version {
+            let output = self.compile_with_version(&solc, sources)?;
+            if let ProjectCompileOutput::Compiled((compiled, _)) = output {
+                res.errors.extend(compiled.errors);
+                res.sources.extend(compiled.sources);
+                res.contracts.extend(compiled.contracts);
+            }
+        }
+
+        Ok(if res.contracts.is_empty() {
+            ProjectCompileOutput::Unchanged
+        } else {
+            ProjectCompileOutput::Compiled((res, &self.ignored_error_codes))
+        })
+    }
+
+    pub fn compile_with_version(
+        &self,
+        solc: &Solc,
+        mut sources: Sources,
+    ) -> Result<ProjectCompileOutput> {
         // add all libraries to the source set while keeping track of their actual disk path
         let mut source_name_path = HashMap::new();
         let mut path_source_name = HashMap::new();
@@ -120,8 +170,8 @@ impl Project {
         // replace absolute path with source name to make solc happy
         let sources = apply_mappings(sources, path_source_name);
 
-        let input = CompilerInput::with_sources(sources);
+        let input = CompilerInput::with_sources(sources).normalize_evm_version(&solc.version()?);
-        let output = self.solc.compile(&input)?;
+        let output = solc.compile(&input)?;
         if output.has_error() {
             return Ok(ProjectCompileOutput::Compiled((output, &self.ignored_error_codes)))
         }
@@ -248,3 +298,35 @@ impl<'a> fmt::Display for ProjectCompileOutput<'a> {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+
+    #[test]
+    #[cfg(all(feature = "svm", feature = "async"))]
+    fn test_build_all_versions() {
+        use super::*;
+
+        let paths = ProjectPathsConfig::builder()
+            .root("./test-data/test-contract-versions")
+            .sources("./test-data/test-contract-versions")
+            .build()
+            .unwrap();
+        let project = Project::builder()
+            .paths(paths)
+            .ephemeral()
+            .artifacts(ArtifactOutput::Nothing)
+            .build()
+            .unwrap();
+        let compiled = project.compile().unwrap();
+        let contracts = match compiled {
+            ProjectCompileOutput::Compiled((out, _)) => {
+                assert!(!out.has_error());
+                out.contracts
+            }
+            _ => panic!("must compile"),
+        };
+        // Contracts A to F
+        assert_eq!(contracts.keys().count(), 5);
+    }
+}

ethers-solc/test-data/test-contract-versions/caret-0.4.14.sol

@@ -0,0 +1,5 @@
+pragma solidity ^0.4.14;
+
+contract B {
+    function foo() public {}
+}

ethers-solc/test-data/test-contract-versions/greater-equal-0.5.0.sol

@@ -0,0 +1,5 @@
+pragma solidity >=0.5.0;
+
+contract C {
+    function foo() public {}
+}

ethers-solc/test-data/test-contract-versions/pinned-0.4.14.sol

@@ -0,0 +1,5 @@
+pragma solidity =0.4.14;
+
+contract D {
+    function foo() public {}
+}

ethers-solc/test-data/test-contract-versions/range-0.5.0.sol

@@ -0,0 +1,5 @@
+pragma solidity >=0.4.0 <0.5.0;
+
+contract E {
+    function foo() public {}
+}

ethers-solc/test-data/test-contract-versions/simple-0.4.14.sol

@@ -0,0 +1,5 @@
+pragma solidity =0.4.14;
+
+contract F {
+    function foo() public {}
+}

ethers-solc/tests/project.rs

@@ -25,7 +25,11 @@ fn can_compile_hardhat_sample() {
     // let paths = ProjectPathsConfig::hardhat(root).unwrap();
 
     let project = Project::builder().paths(paths).build().unwrap();
-    assert_ne!(project.compile().unwrap(), ProjectCompileOutput::Unchanged);
+    let compiled = project.compile().unwrap();
+    match compiled {
+        ProjectCompileOutput::Compiled((out, _)) => assert!(!out.has_error()),
+        _ => panic!("must compile"),
+    }
     // nothing to compile
     assert_eq!(project.compile().unwrap(), ProjectCompileOutput::Unchanged);
 }
@@ -46,10 +50,13 @@ fn can_compile_dapp_sample() {
         .root(root)
         .build()
         .unwrap();
-    // let paths = ProjectPathsConfig::dapptools(root).unwrap();
 
     let project = Project::builder().paths(paths).build().unwrap();
-    assert_ne!(project.compile().unwrap(), ProjectCompileOutput::Unchanged);
+    let compiled = project.compile().unwrap();
+    match compiled {
+        ProjectCompileOutput::Compiled((out, _)) => assert!(!out.has_error()),
+        _ => panic!("must compile"),
+    }
     // nothing to compile
     assert_eq!(project.compile().unwrap(), ProjectCompileOutput::Unchanged);
 }