feat(solc): support customized output selection pruning (#1039)

* feat(solc): support customized output selection pruning

* chore(clippy): make clippy happy

ethers-solc/src/artifacts/mod.rs

@@ -26,7 +26,7 @@ pub mod output_selection;
 pub mod serde_helpers;
 use crate::{
     artifacts::output_selection::{ContractOutputSelection, OutputSelection},
-    cache::FilteredSources,
+    filter::FilteredSources,
 };
 pub use serde_helpers::{deserialize_bytes, deserialize_opt_bytes};
 

ethers-solc/src/cache.rs

@@ -1,8 +1,9 @@
 //! Support for compiling contracts
 use crate::{
-    artifacts::{output_selection::OutputSelection, Settings, Sources},
+    artifacts::Sources,
     config::SolcConfig,
     error::{Result, SolcError},
+    filter::{FilteredSource, FilteredSourceInfo, FilteredSources},
     resolver::GraphEdges,
     utils, ArtifactFile, ArtifactOutput, Artifacts, ArtifactsMap, Project, ProjectPathsConfig,
     Source,
@@ -14,7 +15,6 @@ use std::{
         btree_map::{BTreeMap, Entry},
         hash_map, BTreeSet, HashMap, HashSet,
     },
-    fmt,
     fs::{self},
     path::{Path, PathBuf},
     time::{Duration, UNIX_EPOCH},
@@ -729,157 +729,6 @@ impl<'a, T: ArtifactOutput> ArtifactsCacheInner<'a, T> {
     }
 }
 
-/// Container type for a set of [FilteredSource]
-#[derive(Debug, Clone, Eq, PartialEq)]
-pub struct FilteredSources(pub BTreeMap<PathBuf, FilteredSource>);
-
-impl FilteredSources {
-    pub fn is_empty(&self) -> bool {
-        self.0.is_empty()
-    }
-
-    pub fn len(&self) -> usize {
-        self.0.len()
-    }
-
-    /// Returns `true` if all files are dirty
-    pub fn all_dirty(&self) -> bool {
-        self.0.values().all(|s| s.is_dirty())
-    }
-
-    /// Returns all entries that are dirty
-    pub fn dirty(&self) -> impl Iterator<Item = (&PathBuf, &FilteredSource)> + '_ {
-        self.0.iter().filter(|(_, s)| s.is_dirty())
-    }
-
-    /// Returns all entries that are clean
-    pub fn clean(&self) -> impl Iterator<Item = (&PathBuf, &FilteredSource)> + '_ {
-        self.0.iter().filter(|(_, s)| !s.is_dirty())
-    }
-
-    /// Returns all dirty files
-    pub fn dirty_files(&self) -> impl Iterator<Item = &PathBuf> + fmt::Debug + '_ {
-        self.0.iter().filter_map(|(k, s)| s.is_dirty().then(|| k))
-    }
-
-    /// While solc needs all the files to compile the actual _dirty_ files, we can tell solc to
-    /// output everything for those dirty files as currently configured in the settings, but output
-    /// nothing for the other files that are _not_ dirty.
-    ///
-    /// This will modify the [OutputSelection] of the [Settings] so that we explicitly select the
-    /// files' output based on their state.
-    pub fn into_sources(self, settings: &mut Settings) -> Sources {
-        if !self.all_dirty() {
-            // settings can be optimized
-
-            tracing::trace!(
-                "Optimizing output selection for {}/{} sources",
-                self.clean().count(),
-                self.len()
-            );
-
-            let selection = settings
-                .output_selection
-                .as_mut()
-                .remove("*")
-                .unwrap_or_else(OutputSelection::default_file_output_selection);
-
-            for (file, source) in self.0.iter() {
-                if source.is_dirty() {
-                    settings
-                        .output_selection
-                        .as_mut()
-                        .insert(format!("{}", file.display()), selection.clone());
-                } else {
-                    tracing::trace!("Optimizing output for {}", file.display());
-                    settings.output_selection.as_mut().insert(
-                        format!("{}", file.display()),
-                        OutputSelection::empty_file_output_select(),
-                    );
-                }
-            }
-        }
-        self.into()
-    }
-}
-
-impl From<FilteredSources> for Sources {
-    fn from(sources: FilteredSources) -> Self {
-        sources.0.into_iter().map(|(k, v)| (k, v.into_source())).collect()
-    }
-}
-
-impl From<Sources> for FilteredSources {
-    fn from(s: Sources) -> Self {
-        FilteredSources(s.into_iter().map(|(key, val)| (key, FilteredSource::Dirty(val))).collect())
-    }
-}
-
-impl From<BTreeMap<PathBuf, FilteredSource>> for FilteredSources {
-    fn from(s: BTreeMap<PathBuf, FilteredSource>) -> Self {
-        FilteredSources(s)
-    }
-}
-
-impl AsRef<BTreeMap<PathBuf, FilteredSource>> for FilteredSources {
-    fn as_ref(&self) -> &BTreeMap<PathBuf, FilteredSource> {
-        &self.0
-    }
-}
-
-impl AsMut<BTreeMap<PathBuf, FilteredSource>> for FilteredSources {
-    fn as_mut(&mut self) -> &mut BTreeMap<PathBuf, FilteredSource> {
-        &mut self.0
-    }
-}
-
-/// Represents the state of a filtered [Source]
-#[derive(Debug, Clone, Eq, PartialEq)]
-pub enum FilteredSource {
-    /// A source that fits the _dirty_ criteria
-    Dirty(Source),
-    /// A source that does _not_ fit the _dirty_ criteria but is included in the filtered set
-    /// because a _dirty_ file pulls it in, either directly on indirectly.
-    Clean(Source),
-}
-
-impl FilteredSource {
-    /// Returns the underlying source
-    pub fn source(&self) -> &Source {
-        match self {
-            FilteredSource::Dirty(s) => s,
-            FilteredSource::Clean(s) => s,
-        }
-    }
-
-    /// Consumes the type and returns the underlying source
-    pub fn into_source(self) -> Source {
-        match self {
-            FilteredSource::Dirty(s) => s,
-            FilteredSource::Clean(s) => s,
-        }
-    }
-
-    /// Whether this file is actually dirt
-    pub fn is_dirty(&self) -> bool {
-        matches!(self, FilteredSource::Dirty(_))
-    }
-}
-
-/// Helper type that determines the state of a source file
-struct FilteredSourceInfo {
-    /// path to the source file
-    file: PathBuf,
-    /// contents of the file
-    source: Source,
-    /// idx in the [GraphEdges]
-    idx: usize,
-    /// whether this file is actually dirty
-    ///
-    /// See also [ArtifactsCacheInner::is_dirty()]
-    dirty: bool,
-}
-
 /// Abstraction over configured caching which can be either non-existent or an already loaded cache
 #[allow(clippy::large_enum_variant)]
 #[derive(Debug)]

ethers-solc/src/compile/project.rs

@@ -114,6 +114,7 @@ use crate::{
 };
 use rayon::prelude::*;
 
+use crate::filter::SparseOutputFileFilter;
 use std::{collections::btree_map::BTreeMap, path::PathBuf};
 
 #[derive(Debug)]
@@ -123,6 +124,8 @@ pub struct ProjectCompiler<'a, T: ArtifactOutput> {
     project: &'a Project<T>,
     /// how to compile all the sources
     sources: CompilerSources,
+    /// How to select solc [CompilerOutput] for files
+    sparse_output: SparseOutputFileFilter,
 }
 
 impl<'a, T: ArtifactOutput> ProjectCompiler<'a, T> {
@@ -162,7 +165,7 @@ impl<'a, T: ArtifactOutput> ProjectCompiler<'a, T> {
             CompilerSources::Sequential(sources_by_version)
         };
 
-        Ok(Self { edges, project, sources })
+        Ok(Self { edges, project, sources, sparse_output: Default::default() })
     }
 
     /// Compiles the sources with a pinned `Solc` instance
@@ -176,7 +179,14 @@ impl<'a, T: ArtifactOutput> ProjectCompiler<'a, T> {
         let sources_by_version = BTreeMap::from([(solc, (version, sources))]);
         let sources = CompilerSources::Sequential(sources_by_version);
 
-        Ok(Self { edges, project, sources })
+        Ok(Self { edges, project, sources, sparse_output: Default::default() })
+    }
+
+    /// Applies the specified [SparseOutputFileFilter] to be applied when selecting solc output for
+    /// specific files to be compiled
+    pub fn with_sparse_output(mut self, sparse_output: impl Into<SparseOutputFileFilter>) -> Self {
+        self.sparse_output = sparse_output.into();
+        self
     }
 
     /// Compiles all the sources of the `Project` in the appropriate mode
@@ -203,13 +213,13 @@ impl<'a, T: ArtifactOutput> ProjectCompiler<'a, T> {
     ///   - sets proper source unit names
     ///   - check cache
     fn preprocess(self) -> Result<PreprocessedState<'a, T>> {
-        let Self { edges, project, sources } = self;
+        let Self { edges, project, sources, sparse_output } = self;
 
         let mut cache = ArtifactsCache::new(project, edges)?;
         // retain and compile only dirty sources and all their imports
         let sources = sources.filtered(&mut cache);
 
-        Ok(PreprocessedState { sources, cache })
+        Ok(PreprocessedState { sources, cache, sparse_output })
     }
 }
 
@@ -222,14 +232,18 @@ struct PreprocessedState<'a, T: ArtifactOutput> {
     sources: FilteredCompilerSources,
     /// cache that holds [CacheEntry] object if caching is enabled and the project is recompiled
     cache: ArtifactsCache<'a, T>,
+    sparse_output: SparseOutputFileFilter,
 }
 
 impl<'a, T: ArtifactOutput> PreprocessedState<'a, T> {
     /// advance to the next state by compiling all sources
     fn compile(self) -> Result<CompiledState<'a, T>> {
-        let PreprocessedState { sources, cache } = self;
+        let PreprocessedState { sources, cache, sparse_output } = self;
-        let output =
+        let output = sources.compile(
-            sources.compile(&cache.project().solc_config.settings, &cache.project().paths)?;
+            &cache.project().solc_config.settings,
+            &cache.project().paths,
+            sparse_output,
+        )?;
 
         Ok(CompiledState { output, cache })
     }
@@ -351,13 +365,14 @@ impl FilteredCompilerSources {
         self,
         settings: &Settings,
         paths: &ProjectPathsConfig,
+        sparse_output: SparseOutputFileFilter,
     ) -> Result<AggregatedCompilerOutput> {
         match self {
             FilteredCompilerSources::Sequential(input) => {
-                compile_sequential(input, settings, paths)
+                compile_sequential(input, settings, paths, sparse_output)
             }
             FilteredCompilerSources::Parallel(input, j) => {
-                compile_parallel(input, j, settings, paths)
+                compile_parallel(input, j, settings, paths, sparse_output)
             }
         }
     }
@@ -377,6 +392,7 @@ fn compile_sequential(
     input: VersionedFilteredSources,
     settings: &Settings,
     paths: &ProjectPathsConfig,
+    sparse_output: SparseOutputFileFilter,
 ) -> Result<AggregatedCompilerOutput> {
     let mut aggregated = AggregatedCompilerOutput::default();
     tracing::trace!("compiling {} jobs sequentially", input.len());
@@ -402,7 +418,7 @@ fn compile_sequential(
         // depending on the composition of the filtered sources, the output selection can be
         // optimized
         let mut opt_settings = settings.clone();
-        let sources = filtered_sources.into_sources(&mut opt_settings);
+        let sources = sparse_output.sparse_sources(filtered_sources, &mut opt_settings);
 
         for input in CompilerInput::with_sources(sources) {
             let actually_dirty = input
@@ -450,6 +466,7 @@ fn compile_parallel(
     num_jobs: usize,
     settings: &Settings,
     paths: &ProjectPathsConfig,
+    sparse_output: SparseOutputFileFilter,
 ) -> Result<AggregatedCompilerOutput> {
     debug_assert!(num_jobs > 1);
     tracing::trace!(
@@ -475,7 +492,7 @@ fn compile_parallel(
         // depending on the composition of the filtered sources, the output selection can be
         // optimized
         let mut opt_settings = settings.clone();
-        let sources = filtered_sources.into_sources(&mut opt_settings);
+        let sources = sparse_output.sparse_sources(filtered_sources, &mut opt_settings);
 
         for input in CompilerInput::with_sources(sources) {
             let actually_dirty = input

ethers-solc/src/filter.rs

@@ -0,0 +1,252 @@
+//! Types to apply filter to input types
+
+use crate::{
+    artifacts::{output_selection::OutputSelection, Settings},
+    Source, Sources,
+};
+use std::{
+    collections::BTreeMap,
+    fmt,
+    fmt::Formatter,
+    path::{Path, PathBuf},
+};
+
+/// A predicate property that determines whether a file satisfies a certain condition
+pub trait FileFilter {
+    /// The predicate function that should return if the given `file` should be included.
+    fn is_match(&self, file: &Path) -> bool;
+}
+
+impl<F> FileFilter for F
+where
+    F: Fn(&Path) -> bool,
+{
+    fn is_match(&self, file: &Path) -> bool {
+        (self)(file)
+    }
+}
+
+/// An [InputFileFilter] that matches all solidity files that end with `.t.sol`
+#[derive(Default)]
+pub struct TestFileFilter {
+    _priv: (),
+}
+
+impl fmt::Debug for TestFileFilter {
+    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+        f.debug_struct("TestFileFilter").finish()
+    }
+}
+
+impl fmt::Display for TestFileFilter {
+    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+        f.write_str("TestFileFilter")
+    }
+}
+
+impl FileFilter for TestFileFilter {
+    fn is_match(&self, file: &Path) -> bool {
+        file.file_name().and_then(|s| s.to_str()).map(|s| s.ends_with(".t.sol")).unwrap_or_default()
+    }
+}
+
+/// A type that can apply a filter to a set of preprocessed [FilteredSources] in order to set sparse
+/// output for specific files
+pub enum SparseOutputFileFilter {
+    /// Sets the configured [OutputSelection] for dirty files only.
+    ///
+    /// In other words, we request the output of solc only for files that have been detected as
+    /// _dirty_.
+    AllDirty,
+    /// Apply an additional filter to [FilteredSources] to
+    Custom(Box<dyn FileFilter>),
+}
+
+impl SparseOutputFileFilter {
+    /// While solc needs all the files to compile the actual _dirty_ files, we can tell solc to
+    /// output everything for those dirty files as currently configured in the settings, but output
+    /// nothing for the other files that are _not_ dirty.
+    ///
+    /// This will modify the [OutputSelection] of the [Settings] so that we explicitly select the
+    /// files' output based on their state.
+    pub fn sparse_sources(&self, sources: FilteredSources, settings: &mut Settings) -> Sources {
+        fn apply(
+            sources: &FilteredSources,
+            settings: &mut Settings,
+            f: impl Fn(&PathBuf, &FilteredSource) -> bool,
+        ) {
+            let selection = settings
+                .output_selection
+                .as_mut()
+                .remove("*")
+                .unwrap_or_else(OutputSelection::default_file_output_selection);
+
+            for (file, source) in sources.0.iter() {
+                if f(file, source) {
+                    settings
+                        .output_selection
+                        .as_mut()
+                        .insert(format!("{}", file.display()), selection.clone());
+                } else {
+                    tracing::trace!("using pruned output selection for {}", file.display());
+                    settings.output_selection.as_mut().insert(
+                        format!("{}", file.display()),
+                        OutputSelection::empty_file_output_select(),
+                    );
+                }
+            }
+        }
+
+        match self {
+            SparseOutputFileFilter::AllDirty => {
+                if !sources.all_dirty() {
+                    // settings can be optimized
+                    tracing::trace!(
+                        "optimizing output selection for {}/{} sources",
+                        sources.clean().count(),
+                        sources.len()
+                    );
+                    apply(&sources, settings, |_, source| source.is_dirty())
+                }
+            }
+            SparseOutputFileFilter::Custom(f) => {
+                tracing::trace!("optimizing output selection with custom filter",);
+                apply(&sources, settings, |p, source| source.is_dirty() && f.is_match(p));
+            }
+        };
+        sources.into()
+    }
+}
+
+impl From<Box<dyn FileFilter>> for SparseOutputFileFilter {
+    fn from(f: Box<dyn FileFilter>) -> Self {
+        SparseOutputFileFilter::Custom(f)
+    }
+}
+
+impl Default for SparseOutputFileFilter {
+    fn default() -> Self {
+        SparseOutputFileFilter::AllDirty
+    }
+}
+
+impl fmt::Debug for SparseOutputFileFilter {
+    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+        match self {
+            SparseOutputFileFilter::AllDirty => f.write_str("AllDirty"),
+            SparseOutputFileFilter::Custom(_) => f.write_str("Custom"),
+        }
+    }
+}
+
+/// Container type for a set of [FilteredSource]
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct FilteredSources(pub BTreeMap<PathBuf, FilteredSource>);
+
+impl FilteredSources {
+    pub fn is_empty(&self) -> bool {
+        self.0.is_empty()
+    }
+
+    pub fn len(&self) -> usize {
+        self.0.len()
+    }
+
+    /// Returns `true` if all files are dirty
+    pub fn all_dirty(&self) -> bool {
+        self.0.values().all(|s| s.is_dirty())
+    }
+
+    /// Returns all entries that are dirty
+    pub fn dirty(&self) -> impl Iterator<Item = (&PathBuf, &FilteredSource)> + '_ {
+        self.0.iter().filter(|(_, s)| s.is_dirty())
+    }
+
+    /// Returns all entries that are clean
+    pub fn clean(&self) -> impl Iterator<Item = (&PathBuf, &FilteredSource)> + '_ {
+        self.0.iter().filter(|(_, s)| !s.is_dirty())
+    }
+
+    /// Returns all dirty files
+    pub fn dirty_files(&self) -> impl Iterator<Item = &PathBuf> + fmt::Debug + '_ {
+        self.0.iter().filter_map(|(k, s)| s.is_dirty().then(|| k))
+    }
+}
+
+impl From<FilteredSources> for Sources {
+    fn from(sources: FilteredSources) -> Self {
+        sources.0.into_iter().map(|(k, v)| (k, v.into_source())).collect()
+    }
+}
+
+impl From<Sources> for FilteredSources {
+    fn from(s: Sources) -> Self {
+        FilteredSources(s.into_iter().map(|(key, val)| (key, FilteredSource::Dirty(val))).collect())
+    }
+}
+
+impl From<BTreeMap<PathBuf, FilteredSource>> for FilteredSources {
+    fn from(s: BTreeMap<PathBuf, FilteredSource>) -> Self {
+        FilteredSources(s)
+    }
+}
+
+impl AsRef<BTreeMap<PathBuf, FilteredSource>> for FilteredSources {
+    fn as_ref(&self) -> &BTreeMap<PathBuf, FilteredSource> {
+        &self.0
+    }
+}
+
+impl AsMut<BTreeMap<PathBuf, FilteredSource>> for FilteredSources {
+    fn as_mut(&mut self) -> &mut BTreeMap<PathBuf, FilteredSource> {
+        &mut self.0
+    }
+}
+
+/// Represents the state of a filtered [Source]
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub enum FilteredSource {
+    /// A source that fits the _dirty_ criteria
+    Dirty(Source),
+    /// A source that does _not_ fit the _dirty_ criteria but is included in the filtered set
+    /// because a _dirty_ file pulls it in, either directly on indirectly.
+    Clean(Source),
+}
+
+impl FilteredSource {
+    /// Returns the underlying source
+    pub fn source(&self) -> &Source {
+        match self {
+            FilteredSource::Dirty(s) => s,
+            FilteredSource::Clean(s) => s,
+        }
+    }
+
+    /// Consumes the type and returns the underlying source
+    pub fn into_source(self) -> Source {
+        match self {
+            FilteredSource::Dirty(s) => s,
+            FilteredSource::Clean(s) => s,
+        }
+    }
+
+    /// Whether this file is actually dirt
+    pub fn is_dirty(&self) -> bool {
+        matches!(self, FilteredSource::Dirty(_))
+    }
+}
+
+/// Helper type that determines the state of a source file
+#[derive(Debug)]
+pub struct FilteredSourceInfo {
+    /// path to the source file
+    pub file: PathBuf,
+    /// contents of the file
+    pub source: Source,
+    /// idx in the [GraphEdges]
+    pub idx: usize,
+    /// whether this file is actually dirty
+    ///
+    /// See also [ArtifactsCacheInner::is_dirty()]
+    pub dirty: bool,
+}

ethers-solc/src/lib.rs

@@ -26,8 +26,10 @@ pub mod remappings;
 use crate::artifacts::Source;
 
 pub mod error;
+mod filter;
 pub mod report;
 pub mod utils;
+pub use filter::{FileFilter, TestFileFilter};
 
 use crate::{
     artifacts::{Contract, Sources},
@@ -287,6 +289,48 @@ impl<T: ArtifactOutput> Project<T> {
         project::ProjectCompiler::with_sources(self, Source::read_all(files)?)?.compile()
     }
 
+    /// Convenience function to compile only (re)compile files that match the provided [FileFilter].
+    /// Same as [`Self::svm_compile()`] but with only with those files as input that match
+    /// [FileFilter::is_match()].
+    ///
+    /// # Example - Only compile Test files
+    ///
+    /// ```
+    /// use ethers_solc::{Project, TestFileFilter};
+    /// # fn demo(project: Project) {
+    /// let project = Project::builder().build().unwrap();
+    /// let output = project
+    ///     .compile_sparse(
+    ///         TestFileFilter::default()
+    ///     ).unwrap();
+    /// # }
+    /// ```
+    ///
+    /// # Example - Apply a custom filter
+    ///
+    /// ```
+    /// use std::path::Path;
+    /// use ethers_solc::Project;
+    /// # fn demo(project: Project) {
+    /// let project = Project::builder().build().unwrap();
+    /// let output = project
+    ///     .compile_sparse(
+    ///         |path: &Path| path.ends_with("Greeter.sol")
+    ///     ).unwrap();
+    /// # }
+    /// ```
+    #[cfg(all(feature = "svm", feature = "async"))]
+    pub fn compile_sparse<F: FileFilter + 'static>(
+        &self,
+        filter: F,
+    ) -> Result<ProjectCompileOutput<T>> {
+        let sources =
+            Source::read_all(self.paths.input_files().into_iter().filter(|p| filter.is_match(p)))?;
+
+        let filter: Box<dyn FileFilter> = Box::new(filter);
+        project::ProjectCompiler::with_sources(self, sources)?.with_sparse_output(filter).compile()
+    }
+
     /// Compiles the given source files with the exact `Solc` executable
     ///
     /// First all libraries for the sources are resolved by scanning all their imports.