From e5dbeb6b28e1cc02486df42c1d20b0d4bcf2e0e5 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 11 Mar 2022 17:43:48 +0100 Subject: [PATCH] revert: use simple change filter again (#1008) * chore: rollback dirty detection * docs: caching docs --- ethers-solc/src/cache.rs | 69 ++++++++---------------------- ethers-solc/src/compile/project.rs | 33 ++++++++++++++ 2 files changed, 50 insertions(+), 52 deletions(-) diff --git a/ethers-solc/src/cache.rs b/ethers-solc/src/cache.rs index faf07465..9c0edb8e 100644 --- a/ethers-solc/src/cache.rs +++ b/ethers-solc/src/cache.rs @@ -29,7 +29,7 @@ const ETHERS_FORMAT_VERSION: &str = "ethers-rs-sol-cache-2"; /// The file name of the default cache file pub const SOLIDITY_FILES_CACHE_FILENAME: &str = "solidity-files-cache.json"; -/// A hardhat compatible cache representation +/// A multi version cache file #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] pub struct SolFilesCache { #[serde(rename = "_format")] @@ -593,62 +593,35 @@ impl<'a, T: ArtifactOutput> ArtifactsCacheInner<'a, T> { } } - /// Returns only dirty sources that: + /// Returns only those sources that /// - are new /// - were changed /// - their imports were changed /// - their artifact is missing - /// This also includes their respective imports fn filter(&mut self, sources: Sources, version: &Version) -> Sources { self.fill_hashes(&sources); - - let mut imports_of_dirty = HashSet::new(); - // separates all source files that fit the criteria (dirty) from those that don't (clean) - let (mut dirty_sources, clean_sources) = sources + sources .into_iter() - .map(|(file, source)| self.filter_source(file, source, version)) - .fold( - (Sources::default(), Vec::new()), - |(mut dirty_sources, mut clean_sources), source| { - if source.dirty { - // mark all files that are imported by a dirty file - imports_of_dirty.extend(self.edges.all_imported_nodes(source.idx)); - dirty_sources.insert(source.file, source.source); - } else { - clean_sources.push(source); - } - - (dirty_sources, clean_sources) - }, - ); - - for clean_source in clean_sources { - let FilteredSource { file, source, idx, .. } = clean_source; - if imports_of_dirty.contains(&idx) { - // file is imported by a dirty file - dirty_sources.insert(file, source); - } else { - self.insert_filtered_source(file, source, version.clone()); - } - } - - // track dirty sources internally - for (file, source) in dirty_sources.iter() { - self.insert_new_cache_entry(file, source, version.clone()); - } - - dirty_sources + .filter_map(|(file, source)| self.requires_solc(file, source, version)) + .collect() } - /// Returns the state of the given source file. - fn filter_source(&self, file: PathBuf, source: Source, version: &Version) -> FilteredSource { - let idx = self.edges.node_id(&file); + /// Returns `Some` if the file _needs_ to be compiled and `None` if the artifact can be reu-used + fn requires_solc( + &mut self, + file: PathBuf, + source: Source, + version: &Version, + ) -> Option<(PathBuf, Source)> { if !self.is_dirty(&file, version) && self.edges.imports(&file).iter().all(|file| !self.is_dirty(file, version)) { - FilteredSource { file, source, idx, dirty: false } + self.insert_filtered_source(file, source, version.clone()); + None } else { - FilteredSource { file, source, idx, dirty: true } + self.insert_new_cache_entry(&file, &source, version.clone()); + + Some((file, source)) } } @@ -701,14 +674,6 @@ impl<'a, T: ArtifactOutput> ArtifactsCacheInner<'a, T> { } } -/// Helper type to represent the state of a source file -struct FilteredSource { - file: PathBuf, - source: Source, - idx: usize, - dirty: bool, -} - /// Abstraction over configured caching which can be either non-existent or an already loaded cache #[allow(clippy::large_enum_variant)] #[derive(Debug)] diff --git a/ethers-solc/src/compile/project.rs b/ethers-solc/src/compile/project.rs index 19e36a3c..31174ccb 100644 --- a/ethers-solc/src/compile/project.rs +++ b/ethers-solc/src/compile/project.rs @@ -73,6 +73,33 @@ //! file in the VFS under `dapp-bin/library/math.sol`. If the file is not available there, the //! source unit name will be passed to the Host Filesystem Loader, which will then look in //! `/project/dapp-bin/library/iterable_mapping.sol` +//! +//! +//! ### Caching and Change detection +//! +//! If caching is enabled in the [Project](crate::Project) a cache file will be created upon a +//! successful solc build. The [cache file](crate::SolFilesCache) stores metadata for all the files +//! that were provided to solc. +//! For every file the cache file contains a dedicated [cache +//! entry](crate::CacheEntry), which represents the state of the file. A solidity file can contain +//! several contracts, for every contract a separate [artifact](crate::Artifact) is emitted. +//! Therefor the entry also tracks all artifacts emitted by a file. A solidity file can also be +//! compiled with several solc versions. +//! +//! For example in `A(<=0.8.10) imports C(>0.4.0)` and +//! `B(0.8.11) imports C(>0.4.0)`, both `A` and `B` import `C` but there's no solc version that's +//! compatible with `A` and `B`, in which case two sets are compiled: [`A`, `C`] and [`B`, `C`]. +//! This is reflected in the cache entry which tracks the file's artifacts by version. +//! +//! The cache makes it possible to detect changes during recompilation, so that only the changed, +//! dirty, files need to be passed to solc. A file will be considered as dirty if: +//! - the file is new, not included in the existing cache +//! - the file was modified since the last compiler run, detected by comparing content hashes +//! - any of the imported files is dirty +//! - the file's artifacts don't exist, were deleted. +//! +//! Recompiling a project with cache enabled detects all files that meet these criteria and provides +//! solc with only these dirty files instead of the entire source set. use crate::{ artifact_output::Artifacts, @@ -283,7 +310,13 @@ impl CompilerSources { sources .into_iter() .map(|(solc, (version, sources))| { + tracing::trace!("Filtering {} sources for {}", sources.len(), version); let sources = cache.filter(sources, &version); + tracing::trace!( + "Detected {} dirty sources {:?}", + sources.len(), + sources.keys() + ); (solc, (version, sources)) }) .collect()