feat(solc): Multiple Solc Version detection (#551)

* test: ensure that compilation succeeds

* feat: add helper for parsing version req from a source

* feat: detect the latest compatible solc version for a VersionReq

* default to always enabling svm/async

* test: add project with multiple contract versions

* fix: always serde-default solc gas estimates

* fix: normalize evm version in settings before compiling

* feat: auto-detect version and compile if svm+async are on

* chore: warnings

* test: add a lock to ensure that there are no file conflicts when downloading solc

* test: add tests for finding solc installations

* chore: add features to ethers-rs config

* chore: s/first/latest on finding solc version fn docs

Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>

Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
This commit is contained in:
Georgios Konstantopoulos 2021-11-03 10:05:09 +02:00 committed by GitHub
parent 325b752144
commit 23fb877c16
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 372 additions and 16 deletions

View File

@ -72,6 +72,10 @@ ledger = ["ethers-signers/ledger"]
yubi = ["ethers-signers/yubi"] yubi = ["ethers-signers/yubi"]
## contracts ## contracts
abigen = ["ethers-contract/abigen"] abigen = ["ethers-contract/abigen"]
## solc
solc-async = ["ethers-solc/async"]
solc-full = ["ethers-solc/full"]
[dependencies] [dependencies]

View File

@ -47,6 +47,15 @@ impl CompilerInput {
self.settings.optimizer.runs(runs); self.settings.optimizer.runs(runs);
self self
} }
/// Normalizes the EVM version used in the settings to be up to the latest one
/// supported by the provided compiler version.
pub fn normalize_evm_version(mut self, version: &Version) -> Self {
if let Some(ref mut evm_version) = self.settings.evm_version {
self.settings.evm_version = evm_version.normalize_version(version);
}
self
}
} }
impl Default for CompilerInput { impl Default for CompilerInput {
@ -210,8 +219,8 @@ pub enum EvmVersion {
Petersburg, Petersburg,
Istanbul, Istanbul,
Berlin, Berlin,
London,
Byzantium, Byzantium,
London,
} }
impl EvmVersion { impl EvmVersion {
@ -612,7 +621,9 @@ pub struct DeployedBytecode {
#[derive(Clone, Debug, Serialize, Deserialize, Eq, PartialEq)] #[derive(Clone, Debug, Serialize, Deserialize, Eq, PartialEq)]
pub struct GasEstimates { pub struct GasEstimates {
pub creation: Creation, pub creation: Creation,
#[serde(default)]
pub external: BTreeMap<String, String>, pub external: BTreeMap<String, String>,
#[serde(default)]
pub internal: BTreeMap<String, String>, pub internal: BTreeMap<String, String>,
} }
@ -871,4 +882,37 @@ mod tests {
}); });
} }
} }
#[test]
fn test_evm_version_normalization() {
for (solc_version, evm_version, expected) in &[
// Ensure 0.4.21 it always returns None
("0.4.20", EvmVersion::Homestead, None),
// Constantinople clipping
("0.4.21", EvmVersion::Homestead, Some(EvmVersion::Homestead)),
("0.4.21", EvmVersion::Constantinople, Some(EvmVersion::Constantinople)),
("0.4.21", EvmVersion::London, Some(EvmVersion::Constantinople)),
// Petersburg
("0.5.5", EvmVersion::Homestead, Some(EvmVersion::Homestead)),
("0.5.5", EvmVersion::Petersburg, Some(EvmVersion::Petersburg)),
("0.5.5", EvmVersion::London, Some(EvmVersion::Petersburg)),
// Istanbul
("0.5.14", EvmVersion::Homestead, Some(EvmVersion::Homestead)),
("0.5.14", EvmVersion::Istanbul, Some(EvmVersion::Istanbul)),
("0.5.14", EvmVersion::London, Some(EvmVersion::Istanbul)),
// Berlin
("0.8.5", EvmVersion::Homestead, Some(EvmVersion::Homestead)),
("0.8.5", EvmVersion::Berlin, Some(EvmVersion::Berlin)),
("0.8.5", EvmVersion::London, Some(EvmVersion::Berlin)),
// London
("0.8.7", EvmVersion::Homestead, Some(EvmVersion::Homestead)),
("0.8.7", EvmVersion::London, Some(EvmVersion::London)),
("0.8.7", EvmVersion::London, Some(EvmVersion::London)),
] {
assert_eq!(
&evm_version.normalize_version(&Version::from_str(solc_version).unwrap()),
expected
)
}
}
} }

View File

@ -1,8 +1,9 @@
use crate::{ use crate::{
artifacts::Source,
error::{Result, SolcError}, error::{Result, SolcError},
CompilerInput, CompilerOutput, CompilerInput, CompilerOutput,
}; };
use semver::Version; use semver::{Version, VersionReq};
use serde::{de::DeserializeOwned, Serialize}; use serde::{de::DeserializeOwned, Serialize};
use std::{ use std::{
io::BufRead, io::BufRead,
@ -34,10 +35,34 @@ pub const BERLIN_SOLC: Version = Version::new(0, 8, 5);
/// https://blog.soliditylang.org/2021/08/11/solidity-0.8.7-release-announcement/ /// https://blog.soliditylang.org/2021/08/11/solidity-0.8.7-release-announcement/
pub const LONDON_SOLC: Version = Version::new(0, 8, 7); pub const LONDON_SOLC: Version = Version::new(0, 8, 7);
#[cfg(any(test, all(feature = "svm", feature = "async")))]
use once_cell::sync::Lazy;
#[cfg(any(test, feature = "tests"))]
use std::sync::Mutex;
#[cfg(any(test, feature = "tests"))]
static LOCK: Lazy<Mutex<()>> = Lazy::new(|| Mutex::new(()));
#[cfg(all(feature = "svm", feature = "async"))]
/// A list of upstream Solc releases, used to check which version
/// we should download.
pub static RELEASES: Lazy<Vec<Version>> = Lazy::new(|| {
// Try to download the releases, if it fails default to empty
match tokio::runtime::Runtime::new()
.expect("could not create tokio rt to get remote releases")
// TODO: Can we make this future timeout at a small time amount so that
// we do not degrade startup performance if the consumer has a weak network?
.block_on(svm::all_versions())
{
Ok(inner) => inner,
Err(_) => Vec::new(),
}
});
/// Abstraction over `solc` command line utility /// Abstraction over `solc` command line utility
/// ///
/// Supports sync and async functions. /// Supports sync and async functions.
#[derive(Debug, Clone, Eq, PartialEq)] #[derive(Debug, Clone, Eq, PartialEq, PartialOrd, Ord)]
pub struct Solc(pub PathBuf); pub struct Solc(pub PathBuf);
impl Default for Solc { impl Default for Solc {
@ -87,6 +112,74 @@ impl Solc {
Ok(solc) Ok(solc)
} }
/// Assuming the `versions` array is sorted, it returns the latest element which satisfies
/// the provided [`VersionReq`]
pub fn find_matching_installation(
versions: &[Version],
required_version: &VersionReq,
) -> Option<Version> {
// iterate in reverse to find the last match
versions.iter().rev().find(|version| required_version.matches(version)).cloned()
}
/// Given a Solidity source, it detects the latest compiler version which can be used
/// to build it, and returns it.
///
/// If the required compiler version is not installed, it also proceeds to install it.
#[cfg(all(feature = "svm", feature = "async"))]
pub fn detect_version(source: &Source) -> Result<Version> {
// detects the required solc version
let sol_version = Self::version_req(source)?;
#[cfg(any(test, feature = "tests"))]
// take the lock in tests, we use this to enforce that
// a test does not run while a compiler version is being installed
let _lock = LOCK.lock();
// load the local / remote versions
let versions = svm::installed_versions().unwrap_or_default();
let local_versions = Self::find_matching_installation(&versions, &sol_version);
let remote_versions = Self::find_matching_installation(&RELEASES, &sol_version);
// if there's a better upstream version than the one we have, install it
Ok(match (local_versions, remote_versions) {
(Some(local), None) => local,
(Some(local), Some(remote)) => {
if remote > local {
Self::blocking_install(&remote)?;
remote
} else {
local
}
}
(None, Some(version)) => {
Self::blocking_install(&version)?;
version
}
// do nothing otherwise
_ => return Err(SolcError::VersionNotFound),
})
}
/// Parses the given source looking for the `pragma` definition and
/// returns the corresponding SemVer version requirement.
pub fn version_req(source: &Source) -> Result<VersionReq> {
let version = crate::utils::find_version_pragma(&source.content)
.ok_or(SolcError::PragmaNotFound)?
.replace(" ", ",");
// Somehow, Solidity semver without an operator is considered to be "exact",
// but lack of operator automatically marks the operator as Caret, so we need
// to manually patch it? :shrug:
let exact = !matches!(&version[0..1], "*" | "^" | "=" | ">" | "<" | "~");
let mut version = VersionReq::parse(&version)?;
if exact {
version.comparators[0].op = semver::Op::Exact;
}
Ok(version)
}
/// Installs the provided version of Solc in the machine under the svm dir /// Installs the provided version of Solc in the machine under the svm dir
/// # Example /// # Example
/// ```no_run /// ```no_run
@ -172,7 +265,6 @@ impl Solc {
&self, &self,
path: impl AsRef<Path>, path: impl AsRef<Path>,
) -> Result<CompilerOutput> { ) -> Result<CompilerOutput> {
use crate::artifacts::Source;
self.async_compile(&CompilerInput::with_sources(Source::async_read_all_from(path).await?)) self.async_compile(&CompilerInput::with_sources(Source::async_read_all_from(path).await?))
.await .await
} }
@ -275,12 +367,6 @@ mod tests {
let _version = Version::from_str("0.6.6+commit.6c089d02.Linux.gcc").unwrap(); let _version = Version::from_str("0.6.6+commit.6c089d02.Linux.gcc").unwrap();
} }
#[test]
#[ignore]
fn can_find_solc() {
let _solc = Solc::find_svm_installed_version("0.8.9").unwrap();
}
#[cfg(feature = "async")] #[cfg(feature = "async")]
#[tokio::test] #[tokio::test]
async fn async_solc_version_works() { async fn async_solc_version_works() {
@ -305,4 +391,105 @@ mod tests {
let other = solc().async_compile(&serde_json::json!(input)).await.unwrap(); let other = solc().async_compile(&serde_json::json!(input)).await.unwrap();
assert_eq!(out, other); assert_eq!(out, other);
} }
#[test]
fn test_version_req() {
let versions = ["=0.1.2", "^0.5.6", ">=0.7.1", ">0.8.0"];
let sources = versions.iter().map(|version| source(version));
sources.zip(versions).for_each(|(source, version)| {
let version_req = Solc::version_req(&source).unwrap();
assert_eq!(version_req, VersionReq::from_str(version).unwrap());
});
// Solidity defines version ranges with a space, whereas the semver package
// requires them to be separated with a comma
let version_range = ">=0.8.0 <0.9.0";
let source = source(version_range);
let version_req = Solc::version_req(&source).unwrap();
assert_eq!(version_req, VersionReq::from_str(">=0.8.0,<0.9.0").unwrap());
}
#[test]
// This test might be a bit hard to maintain
#[cfg(all(feature = "svm", feature = "async"))]
fn test_detect_version() {
for (pragma, expected) in [
// pinned
("=0.4.14", "0.4.14"),
// pinned too
("0.4.14", "0.4.14"),
// The latest patch is 0.4.26
("^0.4.14", "0.4.26"),
// latest version above 0.5.0 -> we have to
// update this test whenever there's a new sol
// version. that's ok! good reminder to check the
// patch notes.
(">=0.5.0", "0.8.9"),
// range
(">=0.4.0 <0.5.0", "0.4.26"),
]
.iter()
{
// println!("Checking {}", pragma);
let source = source(pragma);
let res = Solc::detect_version(&source).unwrap();
assert_eq!(res, Version::from_str(expected).unwrap());
}
}
#[test]
#[cfg(feature = "full")]
fn test_find_installed_version_path() {
// this test does not take the lock by default, so we need to manually
// add it here.
let _lock = LOCK.lock();
let ver = "0.8.6";
let version = Version::from_str(ver).unwrap();
if !svm::installed_versions().unwrap().contains(&version) {
Solc::blocking_install(&version).unwrap();
}
let res = Solc::find_svm_installed_version(&version.to_string()).unwrap().unwrap();
let expected = svm::SVM_HOME.join(ver).join(format!("solc-{}", ver));
assert_eq!(res.0, expected);
}
#[test]
fn does_not_find_not_installed_version() {
let ver = "1.1.1";
let version = Version::from_str(ver).unwrap();
let res = Solc::find_svm_installed_version(&version.to_string()).unwrap();
assert!(res.is_none());
}
#[test]
fn test_find_latest_matching_installation() {
let versions = ["0.4.24", "0.5.1", "0.5.2"]
.iter()
.map(|version| Version::from_str(version).unwrap())
.collect::<Vec<_>>();
let required = VersionReq::from_str(">=0.4.24").unwrap();
let got = Solc::find_matching_installation(&versions, &required).unwrap();
assert_eq!(got, versions[2]);
}
#[test]
fn test_no_matching_installation() {
let versions = ["0.4.24", "0.5.1", "0.5.2"]
.iter()
.map(|version| Version::from_str(version).unwrap())
.collect::<Vec<_>>();
let required = VersionReq::from_str(">=0.6.0").unwrap();
let got = Solc::find_matching_installation(&versions, &required);
assert!(got.is_none());
}
///// helpers
fn source(version: &str) -> Source {
Source { content: format!("pragma solidity {};\n", version) }
}
} }

View File

@ -8,6 +8,10 @@ pub enum SolcError {
/// Internal solc error /// Internal solc error
#[error("Solc Error: {0}")] #[error("Solc Error: {0}")]
SolcError(String), SolcError(String),
#[error("missing pragma from solidity file")]
PragmaNotFound,
#[error("could not find solc version locally or upstream")]
VersionNotFound,
#[error(transparent)] #[error(transparent)]
SemverError(#[from] semver::Error), SemverError(#[from] semver::Error),
/// Deserialization error /// Deserialization error
@ -16,6 +20,9 @@ pub enum SolcError {
/// Deserialization error /// Deserialization error
#[error(transparent)] #[error(transparent)]
Io(#[from] std::io::Error), Io(#[from] std::io::Error),
#[cfg(feature = "svm")]
#[error(transparent)]
SvmError(#[from] svm::SolcVmError),
} }
impl SolcError { impl SolcError {

View File

@ -93,8 +93,58 @@ impl Project {
/// ///
/// NOTE: this does not check if the contracts were successfully compiled, see /// NOTE: this does not check if the contracts were successfully compiled, see
/// `CompilerOutput::has_error` instead. /// `CompilerOutput::has_error` instead.
/// NB: If the `svm` feature is enabled, this function will automatically detect
/// solc versions across files.
pub fn compile(&self) -> Result<ProjectCompileOutput> { pub fn compile(&self) -> Result<ProjectCompileOutput> {
let mut sources = self.sources()?; let sources = self.sources()?;
#[cfg(not(all(feature = "svm", feature = "async")))]
{
self.compile_with_version(&self.solc, sources)
}
#[cfg(all(feature = "svm", feature = "async"))]
self.svm_compile(sources)
}
#[cfg(all(feature = "svm", feature = "async"))]
fn svm_compile(&self, sources: Sources) -> Result<ProjectCompileOutput> {
// split them by version
let mut sources_by_version = BTreeMap::new();
for (path, source) in sources.into_iter() {
// will detect and install the solc version
let version = Solc::detect_version(&source)?;
// gets the solc binary for that version, it is expected tha this will succeed
// AND find the solc since it was installed right above
let solc = Solc::find_svm_installed_version(version.to_string())?
.expect("solc should have been installed");
let entry = sources_by_version.entry(solc).or_insert_with(BTreeMap::new);
entry.insert(path, source);
}
// run the compilation step for each version
let mut res = CompilerOutput::default();
for (solc, sources) in sources_by_version {
let output = self.compile_with_version(&solc, sources)?;
if let ProjectCompileOutput::Compiled((compiled, _)) = output {
res.errors.extend(compiled.errors);
res.sources.extend(compiled.sources);
res.contracts.extend(compiled.contracts);
}
}
Ok(if res.contracts.is_empty() {
ProjectCompileOutput::Unchanged
} else {
ProjectCompileOutput::Compiled((res, &self.ignored_error_codes))
})
}
pub fn compile_with_version(
&self,
solc: &Solc,
mut sources: Sources,
) -> Result<ProjectCompileOutput> {
// add all libraries to the source set while keeping track of their actual disk path // add all libraries to the source set while keeping track of their actual disk path
let mut source_name_path = HashMap::new(); let mut source_name_path = HashMap::new();
let mut path_source_name = HashMap::new(); let mut path_source_name = HashMap::new();
@ -120,8 +170,8 @@ impl Project {
// replace absolute path with source name to make solc happy // replace absolute path with source name to make solc happy
let sources = apply_mappings(sources, path_source_name); let sources = apply_mappings(sources, path_source_name);
let input = CompilerInput::with_sources(sources); let input = CompilerInput::with_sources(sources).normalize_evm_version(&solc.version()?);
let output = self.solc.compile(&input)?; let output = solc.compile(&input)?;
if output.has_error() { if output.has_error() {
return Ok(ProjectCompileOutput::Compiled((output, &self.ignored_error_codes))) return Ok(ProjectCompileOutput::Compiled((output, &self.ignored_error_codes)))
} }
@ -248,3 +298,35 @@ impl<'a> fmt::Display for ProjectCompileOutput<'a> {
} }
} }
} }
#[cfg(test)]
mod tests {
#[test]
#[cfg(all(feature = "svm", feature = "async"))]
fn test_build_all_versions() {
use super::*;
let paths = ProjectPathsConfig::builder()
.root("./test-data/test-contract-versions")
.sources("./test-data/test-contract-versions")
.build()
.unwrap();
let project = Project::builder()
.paths(paths)
.ephemeral()
.artifacts(ArtifactOutput::Nothing)
.build()
.unwrap();
let compiled = project.compile().unwrap();
let contracts = match compiled {
ProjectCompileOutput::Compiled((out, _)) => {
assert!(!out.has_error());
out.contracts
}
_ => panic!("must compile"),
};
// Contracts A to F
assert_eq!(contracts.keys().count(), 5);
}
}

View File

@ -0,0 +1,5 @@
pragma solidity ^0.4.14;
contract B {
function foo() public {}
}

View File

@ -0,0 +1,5 @@
pragma solidity >=0.5.0;
contract C {
function foo() public {}
}

View File

@ -0,0 +1,5 @@
pragma solidity =0.4.14;
contract D {
function foo() public {}
}

View File

@ -0,0 +1,5 @@
pragma solidity >=0.4.0 <0.5.0;
contract E {
function foo() public {}
}

View File

@ -0,0 +1,5 @@
pragma solidity =0.4.14;
contract F {
function foo() public {}
}

View File

@ -25,7 +25,11 @@ fn can_compile_hardhat_sample() {
// let paths = ProjectPathsConfig::hardhat(root).unwrap(); // let paths = ProjectPathsConfig::hardhat(root).unwrap();
let project = Project::builder().paths(paths).build().unwrap(); let project = Project::builder().paths(paths).build().unwrap();
assert_ne!(project.compile().unwrap(), ProjectCompileOutput::Unchanged); let compiled = project.compile().unwrap();
match compiled {
ProjectCompileOutput::Compiled((out, _)) => assert!(!out.has_error()),
_ => panic!("must compile"),
}
// nothing to compile // nothing to compile
assert_eq!(project.compile().unwrap(), ProjectCompileOutput::Unchanged); assert_eq!(project.compile().unwrap(), ProjectCompileOutput::Unchanged);
} }
@ -46,10 +50,13 @@ fn can_compile_dapp_sample() {
.root(root) .root(root)
.build() .build()
.unwrap(); .unwrap();
// let paths = ProjectPathsConfig::dapptools(root).unwrap();
let project = Project::builder().paths(paths).build().unwrap(); let project = Project::builder().paths(paths).build().unwrap();
assert_ne!(project.compile().unwrap(), ProjectCompileOutput::Unchanged); let compiled = project.compile().unwrap();
match compiled {
ProjectCompileOutput::Compiled((out, _)) => assert!(!out.has_error()),
_ => panic!("must compile"),
}
// nothing to compile // nothing to compile
assert_eq!(project.compile().unwrap(), ProjectCompileOutput::Unchanged); assert_eq!(project.compile().unwrap(), ProjectCompileOutput::Unchanged);
} }