feat: auto-detect solc remappings (#574)

* feat: add remappings detection

* feat: add remappings to compiler settings

* fix: check for error if no contract was compiled

* feat: configure remappings and provide them to compiler settings

* test: check remappings

* chore: clippy lints

* fix: tests (#578)

* Update ethers-solc/src/remappings.rs

Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>

Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>

Cargo.lock

@@ -1134,6 +1134,7 @@ dependencies = [
  "colored",
  "ethers-core",
  "futures-util",
+ "glob",
  "hex",
  "home",
  "md-5",
@@ -1369,6 +1370,12 @@ version = "0.25.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f0a01e0497841a3b2db4f8afa483cce65f7e96a3498bd6c541734792aeac8fe7"
 
+[[package]]
+name = "glob"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
+
 [[package]]
 name = "group"
 version = "0.10.0"

ethers-solc/Cargo.toml

@@ -28,6 +28,7 @@ thiserror = "1.0.30"
 hex = "0.4.3"
 colored = "2.0.0"
 svm = { package = "svm-rs", git = "https://github.com/roynalnaruto/svm-rs", optional = true }
+glob = "0.3.0"
 
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
 home = "0.5.3"

ethers-solc/src/artifacts.rs

@@ -11,7 +11,7 @@ use std::{
     str::FromStr,
 };
 
-use crate::{compile::*, utils};
+use crate::{compile::*, remappings::Remapping, utils};
 use serde::{de::Visitor, Deserialize, Deserializer, Serialize, Serializer};
 
 /// An ordered list of files and their source
@@ -56,6 +56,11 @@ impl CompilerInput {
         }
         self
     }
+
+    pub fn with_remappings(mut self, remappings: Vec<Remapping>) -> Self {
+        self.settings.remappings = remappings;
+        self
+    }
 }
 
 impl Default for CompilerInput {
@@ -67,6 +72,8 @@ impl Default for CompilerInput {
 #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct Settings {
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub remappings: Vec<Remapping>,
     pub optimizer: Optimizer,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub metadata: Option<Metadata>,
@@ -177,6 +184,7 @@ impl Default for Settings {
             output_selection: Self::default_output_selection(),
             evm_version: Some(EvmVersion::Istanbul),
             libraries: Default::default(),
+            remappings: Default::default(),
         }
         .with_ast()
     }

ethers-solc/src/compile.rs

@@ -472,7 +472,10 @@ mod tests {
         let _lock = LOCK.lock();
         let ver = "0.8.6";
         let version = Version::from_str(ver).unwrap();
-        if !svm::installed_versions().unwrap().contains(&version) {
+        if svm::installed_versions()
+            .map(|versions| !versions.contains(&version))
+            .unwrap_or_default()
+        {
             Solc::blocking_install(&version).unwrap();
         }
         let res = Solc::find_svm_installed_version(&version.to_string()).unwrap().unwrap();

ethers-solc/src/config.rs

@@ -2,6 +2,7 @@ use crate::{
     artifacts::{CompactContractRef, Settings},
     cache::SOLIDITY_FILES_CACHE_FILENAME,
     error::Result,
+    remappings::Remapping,
     CompilerOutput, Solc,
 };
 use serde::{Deserialize, Serialize};
@@ -25,6 +26,8 @@ pub struct ProjectPathsConfig {
     pub tests: PathBuf,
     /// Where to look for libraries
     pub libraries: Vec<PathBuf>,
+    /// The compiler remappings
+    pub remappings: Vec<Remapping>,
 }
 
 impl ProjectPathsConfig {
@@ -33,22 +36,22 @@ impl ProjectPathsConfig {
     }
 
     /// Creates a new hardhat style config instance which points to the canonicalized root path
-    pub fn hardhat(root: impl AsRef<Path>) -> io::Result<Self> {
+    pub fn hardhat(root: impl AsRef<Path>) -> Result<Self> {
         PathStyle::HardHat.paths(root)
     }
 
     /// Creates a new dapptools style config instance which points to the canonicalized root path
-    pub fn dapptools(root: impl AsRef<Path>) -> io::Result<Self> {
+    pub fn dapptools(root: impl AsRef<Path>) -> Result<Self> {
         PathStyle::Dapptools.paths(root)
     }
 
     /// Creates a new config with the current directory as the root
-    pub fn current_hardhat() -> io::Result<Self> {
+    pub fn current_hardhat() -> Result<Self> {
         Self::hardhat(std::env::current_dir()?)
     }
 
     /// Creates a new config with the current directory as the root
-    pub fn current_dapptools() -> io::Result<Self> {
+    pub fn current_dapptools() -> Result<Self> {
         Self::dapptools(std::env::current_dir()?)
     }
 }
@@ -60,23 +63,25 @@ pub enum PathStyle {
 }
 
 impl PathStyle {
-    pub fn paths(&self, root: impl AsRef<Path>) -> io::Result<ProjectPathsConfig> {
+    pub fn paths(&self, root: impl AsRef<Path>) -> Result<ProjectPathsConfig> {
         let root = std::fs::canonicalize(root)?;
 
-        match self {
+        Ok(match self {
             PathStyle::Dapptools => ProjectPathsConfig::builder()
                 .sources(root.join("src"))
                 .artifacts(root.join("out"))
                 .lib(root.join("lib"))
+                .remappings(Remapping::find_many(&root.join("lib"))?)
                 .root(root)
-                .build(),
+                .build()?,
             PathStyle::HardHat => ProjectPathsConfig::builder()
                 .sources(root.join("contracts"))
                 .artifacts(root.join("artifacts"))
                 .lib(root.join("node_modules"))
+                .remappings(Remapping::find_many(&root.join("node_modules"))?)
                 .root(root)
-                .build(),
+                .build()?,
-        }
+        })
     }
 }
 
@@ -88,6 +93,7 @@ pub struct ProjectPathsConfigBuilder {
     sources: Option<PathBuf>,
     tests: Option<PathBuf>,
     libraries: Option<Vec<PathBuf>>,
+    remappings: Option<Vec<Remapping>>,
 }
 
 impl ProjectPathsConfigBuilder {
@@ -135,6 +141,19 @@ impl ProjectPathsConfigBuilder {
         self
     }
 
+    pub fn remapping(mut self, remapping: Remapping) -> Self {
+        self.remappings.get_or_insert_with(Vec::new).push(remapping);
+        self
+    }
+
+    pub fn remappings(mut self, remappings: impl IntoIterator<Item = Remapping>) -> Self {
+        let our_remappings = self.remappings.get_or_insert_with(Vec::new);
+        for remapping in remappings.into_iter() {
+            our_remappings.push(remapping);
+        }
+        self
+    }
+
     pub fn build(self) -> io::Result<ProjectPathsConfig> {
         let root = self.root.map(Ok).unwrap_or_else(std::env::current_dir)?;
         let root = std::fs::canonicalize(root)?;
@@ -147,6 +166,7 @@ impl ProjectPathsConfigBuilder {
             sources: self.sources.unwrap_or_else(|| root.join("contracts")),
             tests: self.tests.unwrap_or_else(|| root.join("tests")),
             libraries: self.libraries.unwrap_or_default(),
+            remappings: self.remappings.unwrap_or_default(),
             root,
         })
     }

ethers-solc/src/error.rs

@@ -23,6 +23,10 @@ pub enum SolcError {
     #[cfg(feature = "svm")]
     #[error(transparent)]
     SvmError(#[from] svm::SolcVmError),
+    #[error("no contracts found under {0}")]
+    NoContracts(String),
+    #[error(transparent)]
+    PatternError(#[from] glob::PatternError),
 }
 
 impl SolcError {

ethers-solc/src/lib.rs

@@ -13,6 +13,8 @@ pub use compile::*;
 mod config;
 pub use config::{AllowedLibPaths, ArtifactOutput, ProjectPathsConfig, SolcConfig};
 
+pub mod remappings;
+
 use crate::{artifacts::Source, cache::SolFilesCache};
 
 pub mod error;
@@ -139,7 +141,7 @@ impl Project {
                 res.contracts.extend(compiled.contracts);
             }
         }
-        Ok(if res.contracts.is_empty() {
+        Ok(if res.contracts.is_empty() && res.errors.is_empty() {
             ProjectCompileOutput::Unchanged
         } else {
             ProjectCompileOutput::Compiled((res, &self.ignored_error_codes))
@@ -176,7 +178,9 @@ impl Project {
         // replace absolute path with source name to make solc happy
         let sources = apply_mappings(sources, path_source_name);
 
-        let input = CompilerInput::with_sources(sources).normalize_evm_version(&solc.version()?);
+        let input = CompilerInput::with_sources(sources)
+            .normalize_evm_version(&solc.version()?)
+            .with_remappings(self.paths.remappings.clone());
         let output = solc.compile(&input)?;
         if output.has_error() {
             return Ok(ProjectCompileOutput::Compiled((output, &self.ignored_error_codes)))
@@ -403,4 +407,33 @@ mod tests {
         };
         assert_eq!(contracts.keys().count(), 3);
     }
+
+    #[test]
+    #[cfg(all(feature = "svm", feature = "async"))]
+    fn test_build_remappings() {
+        use super::*;
+
+        let root = std::fs::canonicalize("./test-data/test-contract-remappings").unwrap();
+        let paths = ProjectPathsConfig::builder()
+            .root(&root)
+            .sources(root.join("src"))
+            .lib(root.join("lib"))
+            .build()
+            .unwrap();
+        let project = Project::builder()
+            .paths(paths)
+            .ephemeral()
+            .artifacts(ArtifactOutput::Nothing)
+            .build()
+            .unwrap();
+        let compiled = project.compile().unwrap();
+        let contracts = match compiled {
+            ProjectCompileOutput::Compiled((out, _)) => {
+                assert!(!out.has_error());
+                out.contracts
+            }
+            _ => panic!("must compile"),
+        };
+        assert_eq!(contracts.keys().count(), 2);
+    }
 }

ethers-solc/src/remappings.rs

@@ -0,0 +1,287 @@
+use crate::{error::SolcError, Result};
+use serde::{Deserialize, Serialize};
+use std::fmt;
+
+const DAPPTOOLS_CONTRACTS_DIR: &str = "src";
+const JS_CONTRACTS_DIR: &str = "contracts";
+
+/// The solidity compiler can only reference files that exist locally on your computer.
+/// So importing directly from GitHub (as an example) is not possible.
+///
+/// Let's imagine you want to use OpenZeppelin's amazing library of smart contracts,
+/// @openzeppelin/contracts-ethereum-package:
+///
+/// ```ignore
+/// pragma solidity 0.5.11;
+///
+/// import "@openzeppelin/contracts-ethereum-package/contracts/math/SafeMath.sol";
+///
+/// contract MyContract {
+///     using SafeMath for uint256;
+///     ...
+/// }
+/// ```
+///
+/// When using solc, you have to specify the following:
+///
+/// "prefix" = the path that's used in your smart contract, i.e.
+/// "@openzeppelin/contracts-ethereum-package" "target" = the absolute path of OpenZeppelin's
+/// contracts downloaded on your computer
+///
+/// The format looks like this:
+/// `solc prefix=target ./MyContract.sol`
+///
+/// solc --bin
+/// @openzeppelin/contracts-ethereum-package=/Your/Absolute/Path/To/@openzeppelin/
+/// contracts-ethereum-package ./MyContract.sol
+///
+/// [Source](https://ethereum.stackexchange.com/questions/74448/what-are-remappings-and-how-do-they-work-in-solidity)
+#[derive(Clone, Debug, PartialEq, PartialOrd, Eq, Ord)]
+pub struct Remapping {
+    pub name: String,
+    pub path: String,
+}
+
+impl Serialize for Remapping {
+    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
+    where
+        S: serde::ser::Serializer,
+    {
+        serializer.serialize_str(&self.to_string())
+    }
+}
+
+impl<'de> Deserialize<'de> for Remapping {
+    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
+    where
+        D: serde::de::Deserializer<'de>,
+    {
+        let remapping = String::deserialize(deserializer)?;
+        let mut split = remapping.split('=');
+        let name = split
+            .next()
+            .ok_or_else(|| serde::de::Error::custom("no remapping prefix found"))?
+            .to_string();
+        let path = split
+            .next()
+            .ok_or_else(|| serde::de::Error::custom("no remapping path found"))?
+            .to_string();
+        Ok(Remapping { name, path })
+    }
+}
+
+// Remappings are printed as `prefix=target`
+impl fmt::Display for Remapping {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}={}", self.name, self.path)
+    }
+}
+
+impl Remapping {
+    /// Detects a remapping prioritizing Dapptools-style remappings over `contracts/`-style ones.
+    fn find(root: &str) -> Result<Self> {
+        Self::find_with_type(root, DAPPTOOLS_CONTRACTS_DIR)
+            .or_else(|_| Self::find_with_type(root, JS_CONTRACTS_DIR))
+    }
+
+    /// Given a path and the style of contracts dir, it proceeds to find
+    /// a `Remapping` for it.
+    fn find_with_type(name: &str, source: &str) -> Result<Self> {
+        let pattern = if name.contains(source) {
+            format!("{}/**/*.sol", name)
+        } else {
+            format!("{}/{}/**/*.sol", name, source)
+        };
+        let mut dapptools_contracts = glob::glob(&pattern)?;
+        let next = dapptools_contracts.next();
+        if next.is_some() {
+            let path = format!("{}/{}/", name, source);
+            let mut name = name.split('/').last().unwrap().to_string();
+            name.push('/');
+            Ok(Remapping { name, path })
+        } else {
+            Err(SolcError::NoContracts(source.to_string()))
+        }
+    }
+
+    pub fn find_many_str(path: &str) -> Result<Vec<String>> {
+        let remappings = Self::find_many(path)?;
+        Ok(remappings.iter().map(|mapping| format!("{}={}", mapping.name, mapping.path)).collect())
+    }
+
+    /// Gets all the remappings detected
+    pub fn find_many(path: impl AsRef<std::path::Path>) -> Result<Vec<Self>> {
+        let path = path.as_ref();
+        if !path.exists() {
+            // nothing to find
+            return Ok(Vec::new())
+        }
+        let mut paths = std::fs::read_dir(path)?.into_iter().collect::<Vec<_>>();
+
+        let mut remappings = Vec::new();
+        while let Some(path) = paths.pop() {
+            let path = path?.path();
+
+            // get all the directories inside a file if it's a valid dir
+            if let Ok(dir) = std::fs::read_dir(&path) {
+                for inner in dir {
+                    let inner = inner?;
+                    let path = inner.path().display().to_string();
+                    let path = path.rsplit('/').next().unwrap().to_string();
+                    if path != DAPPTOOLS_CONTRACTS_DIR && path != JS_CONTRACTS_DIR {
+                        paths.push(Ok(inner));
+                    }
+                }
+            }
+
+            let remapping = Self::find(&path.display().to_string());
+            if let Ok(remapping) = remapping {
+                // skip remappings that exist already
+                if let Some(ref mut found) =
+                    remappings.iter_mut().find(|x: &&mut Remapping| x.name == remapping.name)
+                {
+                    // always replace with the shortest length path
+                    fn depth(path: &str, delim: char) -> usize {
+                        path.matches(delim).count()
+                    }
+                    // if the one which exists is larger, we should replace it
+                    // if not, ignore it
+                    if depth(&found.path, '/') > depth(&remapping.path, '/') {
+                        **found = remapping;
+                    }
+                } else {
+                    remappings.push(remapping);
+                }
+            }
+        }
+
+        Ok(remappings)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    // https://doc.rust-lang.org/rust-by-example/std_misc/fs.html
+    fn touch(path: &std::path::Path) -> std::io::Result<()> {
+        match std::fs::OpenOptions::new().create(true).write(true).open(path) {
+            Ok(_) => Ok(()),
+            Err(e) => Err(e),
+        }
+    }
+
+    fn mkdir_or_touch(tmp: &std::path::Path, paths: &[&str]) {
+        for path in paths {
+            if path.ends_with(".sol") {
+                let path = tmp.join(path);
+                touch(&path).unwrap();
+            } else {
+                let path = tmp.join(path);
+                std::fs::create_dir_all(&path).unwrap();
+            }
+        }
+    }
+
+    // helper function for converting path bufs to remapping strings
+    fn to_str(p: std::path::PathBuf) -> String {
+        format!("{}/", p.display())
+    }
+
+    #[test]
+    fn find_remapping_dapptools() {
+        let tmp_dir = tempdir::TempDir::new("lib").unwrap();
+        let tmp_dir_path = tmp_dir.path();
+        let paths = ["repo1/src/", "repo1/src/contract.sol"];
+        mkdir_or_touch(tmp_dir_path, &paths[..]);
+
+        let path = tmp_dir_path.join("repo1").display().to_string();
+        Remapping::find_with_type(&path, JS_CONTRACTS_DIR).unwrap_err();
+        let remapping = Remapping::find_with_type(&path, DAPPTOOLS_CONTRACTS_DIR).unwrap();
+
+        // repo1/=lib/repo1/src
+        assert_eq!(remapping.name, "repo1/");
+        assert_eq!(remapping.path, format!("{}/src/", path));
+    }
+
+    #[test]
+    fn recursive_remappings() {
+        //let tmp_dir_path = PathBuf::from("."); // tempdir::TempDir::new("lib").unwrap();
+        let tmp_dir = tempdir::TempDir::new("lib").unwrap();
+        let tmp_dir_path = tmp_dir.path();
+        let paths = [
+            "repo1/src/",
+            "repo1/src/contract.sol",
+            "repo1/lib/",
+            "repo1/lib/ds-math/src/",
+            "repo1/lib/ds-math/src/contract.sol",
+            "repo1/lib/ds-math/lib/ds-test/src/",
+            "repo1/lib/ds-math/lib/ds-test/src/test.sol",
+        ];
+        mkdir_or_touch(tmp_dir_path, &paths[..]);
+
+        let path = tmp_dir_path.display().to_string();
+        let mut remappings = Remapping::find_many(&path).unwrap();
+        remappings.sort_unstable();
+
+        let mut expected = vec![
+            Remapping {
+                name: "repo1/".to_string(),
+                path: to_str(tmp_dir_path.join("repo1").join("src")),
+            },
+            Remapping {
+                name: "ds-math/".to_string(),
+                path: to_str(tmp_dir_path.join("repo1").join("lib").join("ds-math").join("src")),
+            },
+            Remapping {
+                name: "ds-test/".to_string(),
+                path: to_str(
+                    tmp_dir_path
+                        .join("repo1")
+                        .join("lib")
+                        .join("ds-math")
+                        .join("lib")
+                        .join("ds-test")
+                        .join("src"),
+                ),
+            },
+        ];
+        expected.sort_unstable();
+        assert_eq!(remappings, expected);
+    }
+
+    #[test]
+    fn remappings() {
+        let tmp_dir = tempdir::TempDir::new("lib").unwrap();
+        let repo1 = tmp_dir.path().join("src_repo");
+        let repo2 = tmp_dir.path().join("contracts_repo");
+
+        let dir1 = repo1.join("src");
+        std::fs::create_dir_all(&dir1).unwrap();
+
+        let dir2 = repo2.join("contracts");
+        std::fs::create_dir_all(&dir2).unwrap();
+
+        let contract1 = dir1.join("contract.sol");
+        touch(&contract1).unwrap();
+
+        let contract2 = dir2.join("contract.sol");
+        touch(&contract2).unwrap();
+
+        let path = tmp_dir.path().display().to_string();
+        let mut remappings = Remapping::find_many(&path).unwrap();
+        remappings.sort_unstable();
+        let mut expected = vec![
+            Remapping {
+                name: "src_repo/".to_string(),
+                path: format!("{}/", dir1.into_os_string().into_string().unwrap()),
+            },
+            Remapping {
+                name: "contracts_repo/".to_string(),
+                path: format!("{}/", dir2.into_os_string().into_string().unwrap()),
+            },
+        ];
+        expected.sort_unstable();
+        assert_eq!(remappings, expected);
+    }
+}

ethers-solc/test-data/test-contract-remappings/lib/bar/src/Bar.sol

@@ -0,0 +1,3 @@
+pragma solidity 0.8.6;
+
+contract Bar {}

ethers-solc/test-data/test-contract-remappings/src/Foo.sol

@@ -0,0 +1,5 @@
+pragma solidity 0.8.6;
+
+import "bar/Bar.sol";
+
+contract Foo is Bar {}