From 8f9c47dbdbc17a117e78c8a4317beede87002613 Mon Sep 17 00:00:00 2001 From: Georgios Konstantopoulos Date: Sat, 13 Nov 2021 19:31:55 +0000 Subject: [PATCH] feat: auto-detect solc remappings (#574) * feat: add remappings detection * feat: add remappings to compiler settings * fix: check for error if no contract was compiled * feat: configure remappings and provide them to compiler settings * test: check remappings * chore: clippy lints * fix: tests (#578) * Update ethers-solc/src/remappings.rs Co-authored-by: Matthias Seitz Co-authored-by: Matthias Seitz --- Cargo.lock | 7 + ethers-solc/Cargo.toml | 1 + ethers-solc/src/artifacts.rs | 10 +- ethers-solc/src/compile.rs | 5 +- ethers-solc/src/config.rs | 38 ++- ethers-solc/src/error.rs | 4 + ethers-solc/src/lib.rs | 37 ++- ethers-solc/src/remappings.rs | 287 ++++++++++++++++++ .../lib/bar/src/Bar.sol | 3 + .../test-contract-remappings/src/Foo.sol | 5 + 10 files changed, 384 insertions(+), 13 deletions(-) create mode 100644 ethers-solc/src/remappings.rs create mode 100644 ethers-solc/test-data/test-contract-remappings/lib/bar/src/Bar.sol create mode 100644 ethers-solc/test-data/test-contract-remappings/src/Foo.sol diff --git a/Cargo.lock b/Cargo.lock index bcbb655e..6cee084d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1134,6 +1134,7 @@ dependencies = [ "colored", "ethers-core", "futures-util", + "glob", "hex", "home", "md-5", @@ -1369,6 +1370,12 @@ version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0a01e0497841a3b2db4f8afa483cce65f7e96a3498bd6c541734792aeac8fe7" +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + [[package]] name = "group" version = "0.10.0" diff --git a/ethers-solc/Cargo.toml b/ethers-solc/Cargo.toml index 369b7ee3..29598f49 100644 --- a/ethers-solc/Cargo.toml +++ b/ethers-solc/Cargo.toml @@ -28,6 +28,7 @@ thiserror = "1.0.30" hex = "0.4.3" colored = "2.0.0" svm = { package = "svm-rs", git = "https://github.com/roynalnaruto/svm-rs", optional = true } +glob = "0.3.0" [target.'cfg(not(target_arch = "wasm32"))'.dependencies] home = "0.5.3" diff --git a/ethers-solc/src/artifacts.rs b/ethers-solc/src/artifacts.rs index cc922679..c881b3ea 100644 --- a/ethers-solc/src/artifacts.rs +++ b/ethers-solc/src/artifacts.rs @@ -11,7 +11,7 @@ use std::{ str::FromStr, }; -use crate::{compile::*, utils}; +use crate::{compile::*, remappings::Remapping, utils}; use serde::{de::Visitor, Deserialize, Deserializer, Serialize, Serializer}; /// An ordered list of files and their source @@ -56,6 +56,11 @@ impl CompilerInput { } self } + + pub fn with_remappings(mut self, remappings: Vec) -> Self { + self.settings.remappings = remappings; + self + } } impl Default for CompilerInput { @@ -67,6 +72,8 @@ impl Default for CompilerInput { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct Settings { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub remappings: Vec, pub optimizer: Optimizer, #[serde(default, skip_serializing_if = "Option::is_none")] pub metadata: Option, @@ -177,6 +184,7 @@ impl Default for Settings { output_selection: Self::default_output_selection(), evm_version: Some(EvmVersion::Istanbul), libraries: Default::default(), + remappings: Default::default(), } .with_ast() } diff --git a/ethers-solc/src/compile.rs b/ethers-solc/src/compile.rs index ca2eec37..4a2bc2f5 100644 --- a/ethers-solc/src/compile.rs +++ b/ethers-solc/src/compile.rs @@ -472,7 +472,10 @@ mod tests { let _lock = LOCK.lock(); let ver = "0.8.6"; let version = Version::from_str(ver).unwrap(); - if !svm::installed_versions().unwrap().contains(&version) { + if svm::installed_versions() + .map(|versions| !versions.contains(&version)) + .unwrap_or_default() + { Solc::blocking_install(&version).unwrap(); } let res = Solc::find_svm_installed_version(&version.to_string()).unwrap().unwrap(); diff --git a/ethers-solc/src/config.rs b/ethers-solc/src/config.rs index d6a22c03..0e4af81d 100644 --- a/ethers-solc/src/config.rs +++ b/ethers-solc/src/config.rs @@ -2,6 +2,7 @@ use crate::{ artifacts::{CompactContractRef, Settings}, cache::SOLIDITY_FILES_CACHE_FILENAME, error::Result, + remappings::Remapping, CompilerOutput, Solc, }; use serde::{Deserialize, Serialize}; @@ -25,6 +26,8 @@ pub struct ProjectPathsConfig { pub tests: PathBuf, /// Where to look for libraries pub libraries: Vec, + /// The compiler remappings + pub remappings: Vec, } impl ProjectPathsConfig { @@ -33,22 +36,22 @@ impl ProjectPathsConfig { } /// Creates a new hardhat style config instance which points to the canonicalized root path - pub fn hardhat(root: impl AsRef) -> io::Result { + pub fn hardhat(root: impl AsRef) -> Result { PathStyle::HardHat.paths(root) } /// Creates a new dapptools style config instance which points to the canonicalized root path - pub fn dapptools(root: impl AsRef) -> io::Result { + pub fn dapptools(root: impl AsRef) -> Result { PathStyle::Dapptools.paths(root) } /// Creates a new config with the current directory as the root - pub fn current_hardhat() -> io::Result { + pub fn current_hardhat() -> Result { Self::hardhat(std::env::current_dir()?) } /// Creates a new config with the current directory as the root - pub fn current_dapptools() -> io::Result { + pub fn current_dapptools() -> Result { Self::dapptools(std::env::current_dir()?) } } @@ -60,23 +63,25 @@ pub enum PathStyle { } impl PathStyle { - pub fn paths(&self, root: impl AsRef) -> io::Result { + pub fn paths(&self, root: impl AsRef) -> Result { let root = std::fs::canonicalize(root)?; - match self { + Ok(match self { PathStyle::Dapptools => ProjectPathsConfig::builder() .sources(root.join("src")) .artifacts(root.join("out")) .lib(root.join("lib")) + .remappings(Remapping::find_many(&root.join("lib"))?) .root(root) - .build(), + .build()?, PathStyle::HardHat => ProjectPathsConfig::builder() .sources(root.join("contracts")) .artifacts(root.join("artifacts")) .lib(root.join("node_modules")) + .remappings(Remapping::find_many(&root.join("node_modules"))?) .root(root) - .build(), - } + .build()?, + }) } } @@ -88,6 +93,7 @@ pub struct ProjectPathsConfigBuilder { sources: Option, tests: Option, libraries: Option>, + remappings: Option>, } impl ProjectPathsConfigBuilder { @@ -135,6 +141,19 @@ impl ProjectPathsConfigBuilder { self } + pub fn remapping(mut self, remapping: Remapping) -> Self { + self.remappings.get_or_insert_with(Vec::new).push(remapping); + self + } + + pub fn remappings(mut self, remappings: impl IntoIterator) -> Self { + let our_remappings = self.remappings.get_or_insert_with(Vec::new); + for remapping in remappings.into_iter() { + our_remappings.push(remapping); + } + self + } + pub fn build(self) -> io::Result { let root = self.root.map(Ok).unwrap_or_else(std::env::current_dir)?; let root = std::fs::canonicalize(root)?; @@ -147,6 +166,7 @@ impl ProjectPathsConfigBuilder { sources: self.sources.unwrap_or_else(|| root.join("contracts")), tests: self.tests.unwrap_or_else(|| root.join("tests")), libraries: self.libraries.unwrap_or_default(), + remappings: self.remappings.unwrap_or_default(), root, }) } diff --git a/ethers-solc/src/error.rs b/ethers-solc/src/error.rs index 37ccd32c..509839c8 100644 --- a/ethers-solc/src/error.rs +++ b/ethers-solc/src/error.rs @@ -23,6 +23,10 @@ pub enum SolcError { #[cfg(feature = "svm")] #[error(transparent)] SvmError(#[from] svm::SolcVmError), + #[error("no contracts found under {0}")] + NoContracts(String), + #[error(transparent)] + PatternError(#[from] glob::PatternError), } impl SolcError { diff --git a/ethers-solc/src/lib.rs b/ethers-solc/src/lib.rs index 0446d107..b60b68b0 100644 --- a/ethers-solc/src/lib.rs +++ b/ethers-solc/src/lib.rs @@ -13,6 +13,8 @@ pub use compile::*; mod config; pub use config::{AllowedLibPaths, ArtifactOutput, ProjectPathsConfig, SolcConfig}; +pub mod remappings; + use crate::{artifacts::Source, cache::SolFilesCache}; pub mod error; @@ -139,7 +141,7 @@ impl Project { res.contracts.extend(compiled.contracts); } } - Ok(if res.contracts.is_empty() { + Ok(if res.contracts.is_empty() && res.errors.is_empty() { ProjectCompileOutput::Unchanged } else { ProjectCompileOutput::Compiled((res, &self.ignored_error_codes)) @@ -176,7 +178,9 @@ impl Project { // replace absolute path with source name to make solc happy let sources = apply_mappings(sources, path_source_name); - let input = CompilerInput::with_sources(sources).normalize_evm_version(&solc.version()?); + let input = CompilerInput::with_sources(sources) + .normalize_evm_version(&solc.version()?) + .with_remappings(self.paths.remappings.clone()); let output = solc.compile(&input)?; if output.has_error() { return Ok(ProjectCompileOutput::Compiled((output, &self.ignored_error_codes))) @@ -403,4 +407,33 @@ mod tests { }; assert_eq!(contracts.keys().count(), 3); } + + #[test] + #[cfg(all(feature = "svm", feature = "async"))] + fn test_build_remappings() { + use super::*; + + let root = std::fs::canonicalize("./test-data/test-contract-remappings").unwrap(); + let paths = ProjectPathsConfig::builder() + .root(&root) + .sources(root.join("src")) + .lib(root.join("lib")) + .build() + .unwrap(); + let project = Project::builder() + .paths(paths) + .ephemeral() + .artifacts(ArtifactOutput::Nothing) + .build() + .unwrap(); + let compiled = project.compile().unwrap(); + let contracts = match compiled { + ProjectCompileOutput::Compiled((out, _)) => { + assert!(!out.has_error()); + out.contracts + } + _ => panic!("must compile"), + }; + assert_eq!(contracts.keys().count(), 2); + } } diff --git a/ethers-solc/src/remappings.rs b/ethers-solc/src/remappings.rs new file mode 100644 index 00000000..ed421d12 --- /dev/null +++ b/ethers-solc/src/remappings.rs @@ -0,0 +1,287 @@ +use crate::{error::SolcError, Result}; +use serde::{Deserialize, Serialize}; +use std::fmt; + +const DAPPTOOLS_CONTRACTS_DIR: &str = "src"; +const JS_CONTRACTS_DIR: &str = "contracts"; + +/// The solidity compiler can only reference files that exist locally on your computer. +/// So importing directly from GitHub (as an example) is not possible. +/// +/// Let's imagine you want to use OpenZeppelin's amazing library of smart contracts, +/// @openzeppelin/contracts-ethereum-package: +/// +/// ```ignore +/// pragma solidity 0.5.11; +/// +/// import "@openzeppelin/contracts-ethereum-package/contracts/math/SafeMath.sol"; +/// +/// contract MyContract { +/// using SafeMath for uint256; +/// ... +/// } +/// ``` +/// +/// When using solc, you have to specify the following: +/// +/// "prefix" = the path that's used in your smart contract, i.e. +/// "@openzeppelin/contracts-ethereum-package" "target" = the absolute path of OpenZeppelin's +/// contracts downloaded on your computer +/// +/// The format looks like this: +/// `solc prefix=target ./MyContract.sol` +/// +/// solc --bin +/// @openzeppelin/contracts-ethereum-package=/Your/Absolute/Path/To/@openzeppelin/ +/// contracts-ethereum-package ./MyContract.sol +/// +/// [Source](https://ethereum.stackexchange.com/questions/74448/what-are-remappings-and-how-do-they-work-in-solidity) +#[derive(Clone, Debug, PartialEq, PartialOrd, Eq, Ord)] +pub struct Remapping { + pub name: String, + pub path: String, +} + +impl Serialize for Remapping { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::ser::Serializer, + { + serializer.serialize_str(&self.to_string()) + } +} + +impl<'de> Deserialize<'de> for Remapping { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::de::Deserializer<'de>, + { + let remapping = String::deserialize(deserializer)?; + let mut split = remapping.split('='); + let name = split + .next() + .ok_or_else(|| serde::de::Error::custom("no remapping prefix found"))? + .to_string(); + let path = split + .next() + .ok_or_else(|| serde::de::Error::custom("no remapping path found"))? + .to_string(); + Ok(Remapping { name, path }) + } +} + +// Remappings are printed as `prefix=target` +impl fmt::Display for Remapping { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}={}", self.name, self.path) + } +} + +impl Remapping { + /// Detects a remapping prioritizing Dapptools-style remappings over `contracts/`-style ones. + fn find(root: &str) -> Result { + Self::find_with_type(root, DAPPTOOLS_CONTRACTS_DIR) + .or_else(|_| Self::find_with_type(root, JS_CONTRACTS_DIR)) + } + + /// Given a path and the style of contracts dir, it proceeds to find + /// a `Remapping` for it. + fn find_with_type(name: &str, source: &str) -> Result { + let pattern = if name.contains(source) { + format!("{}/**/*.sol", name) + } else { + format!("{}/{}/**/*.sol", name, source) + }; + let mut dapptools_contracts = glob::glob(&pattern)?; + let next = dapptools_contracts.next(); + if next.is_some() { + let path = format!("{}/{}/", name, source); + let mut name = name.split('/').last().unwrap().to_string(); + name.push('/'); + Ok(Remapping { name, path }) + } else { + Err(SolcError::NoContracts(source.to_string())) + } + } + + pub fn find_many_str(path: &str) -> Result> { + let remappings = Self::find_many(path)?; + Ok(remappings.iter().map(|mapping| format!("{}={}", mapping.name, mapping.path)).collect()) + } + + /// Gets all the remappings detected + pub fn find_many(path: impl AsRef) -> Result> { + let path = path.as_ref(); + if !path.exists() { + // nothing to find + return Ok(Vec::new()) + } + let mut paths = std::fs::read_dir(path)?.into_iter().collect::>(); + + let mut remappings = Vec::new(); + while let Some(path) = paths.pop() { + let path = path?.path(); + + // get all the directories inside a file if it's a valid dir + if let Ok(dir) = std::fs::read_dir(&path) { + for inner in dir { + let inner = inner?; + let path = inner.path().display().to_string(); + let path = path.rsplit('/').next().unwrap().to_string(); + if path != DAPPTOOLS_CONTRACTS_DIR && path != JS_CONTRACTS_DIR { + paths.push(Ok(inner)); + } + } + } + + let remapping = Self::find(&path.display().to_string()); + if let Ok(remapping) = remapping { + // skip remappings that exist already + if let Some(ref mut found) = + remappings.iter_mut().find(|x: &&mut Remapping| x.name == remapping.name) + { + // always replace with the shortest length path + fn depth(path: &str, delim: char) -> usize { + path.matches(delim).count() + } + // if the one which exists is larger, we should replace it + // if not, ignore it + if depth(&found.path, '/') > depth(&remapping.path, '/') { + **found = remapping; + } + } else { + remappings.push(remapping); + } + } + } + + Ok(remappings) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // https://doc.rust-lang.org/rust-by-example/std_misc/fs.html + fn touch(path: &std::path::Path) -> std::io::Result<()> { + match std::fs::OpenOptions::new().create(true).write(true).open(path) { + Ok(_) => Ok(()), + Err(e) => Err(e), + } + } + + fn mkdir_or_touch(tmp: &std::path::Path, paths: &[&str]) { + for path in paths { + if path.ends_with(".sol") { + let path = tmp.join(path); + touch(&path).unwrap(); + } else { + let path = tmp.join(path); + std::fs::create_dir_all(&path).unwrap(); + } + } + } + + // helper function for converting path bufs to remapping strings + fn to_str(p: std::path::PathBuf) -> String { + format!("{}/", p.display()) + } + + #[test] + fn find_remapping_dapptools() { + let tmp_dir = tempdir::TempDir::new("lib").unwrap(); + let tmp_dir_path = tmp_dir.path(); + let paths = ["repo1/src/", "repo1/src/contract.sol"]; + mkdir_or_touch(tmp_dir_path, &paths[..]); + + let path = tmp_dir_path.join("repo1").display().to_string(); + Remapping::find_with_type(&path, JS_CONTRACTS_DIR).unwrap_err(); + let remapping = Remapping::find_with_type(&path, DAPPTOOLS_CONTRACTS_DIR).unwrap(); + + // repo1/=lib/repo1/src + assert_eq!(remapping.name, "repo1/"); + assert_eq!(remapping.path, format!("{}/src/", path)); + } + + #[test] + fn recursive_remappings() { + //let tmp_dir_path = PathBuf::from("."); // tempdir::TempDir::new("lib").unwrap(); + let tmp_dir = tempdir::TempDir::new("lib").unwrap(); + let tmp_dir_path = tmp_dir.path(); + let paths = [ + "repo1/src/", + "repo1/src/contract.sol", + "repo1/lib/", + "repo1/lib/ds-math/src/", + "repo1/lib/ds-math/src/contract.sol", + "repo1/lib/ds-math/lib/ds-test/src/", + "repo1/lib/ds-math/lib/ds-test/src/test.sol", + ]; + mkdir_or_touch(tmp_dir_path, &paths[..]); + + let path = tmp_dir_path.display().to_string(); + let mut remappings = Remapping::find_many(&path).unwrap(); + remappings.sort_unstable(); + + let mut expected = vec![ + Remapping { + name: "repo1/".to_string(), + path: to_str(tmp_dir_path.join("repo1").join("src")), + }, + Remapping { + name: "ds-math/".to_string(), + path: to_str(tmp_dir_path.join("repo1").join("lib").join("ds-math").join("src")), + }, + Remapping { + name: "ds-test/".to_string(), + path: to_str( + tmp_dir_path + .join("repo1") + .join("lib") + .join("ds-math") + .join("lib") + .join("ds-test") + .join("src"), + ), + }, + ]; + expected.sort_unstable(); + assert_eq!(remappings, expected); + } + + #[test] + fn remappings() { + let tmp_dir = tempdir::TempDir::new("lib").unwrap(); + let repo1 = tmp_dir.path().join("src_repo"); + let repo2 = tmp_dir.path().join("contracts_repo"); + + let dir1 = repo1.join("src"); + std::fs::create_dir_all(&dir1).unwrap(); + + let dir2 = repo2.join("contracts"); + std::fs::create_dir_all(&dir2).unwrap(); + + let contract1 = dir1.join("contract.sol"); + touch(&contract1).unwrap(); + + let contract2 = dir2.join("contract.sol"); + touch(&contract2).unwrap(); + + let path = tmp_dir.path().display().to_string(); + let mut remappings = Remapping::find_many(&path).unwrap(); + remappings.sort_unstable(); + let mut expected = vec![ + Remapping { + name: "src_repo/".to_string(), + path: format!("{}/", dir1.into_os_string().into_string().unwrap()), + }, + Remapping { + name: "contracts_repo/".to_string(), + path: format!("{}/", dir2.into_os_string().into_string().unwrap()), + }, + ]; + expected.sort_unstable(); + assert_eq!(remappings, expected); + } +} diff --git a/ethers-solc/test-data/test-contract-remappings/lib/bar/src/Bar.sol b/ethers-solc/test-data/test-contract-remappings/lib/bar/src/Bar.sol new file mode 100644 index 00000000..06579092 --- /dev/null +++ b/ethers-solc/test-data/test-contract-remappings/lib/bar/src/Bar.sol @@ -0,0 +1,3 @@ +pragma solidity 0.8.6; + +contract Bar {} diff --git a/ethers-solc/test-data/test-contract-remappings/src/Foo.sol b/ethers-solc/test-data/test-contract-remappings/src/Foo.sol new file mode 100644 index 00000000..2edb8f5d --- /dev/null +++ b/ethers-solc/test-data/test-contract-remappings/src/Foo.sol @@ -0,0 +1,5 @@ +pragma solidity 0.8.6; + +import "bar/Bar.sol"; + +contract Foo is Bar {}