From 2d75f9f1e71ad753052052011b891fc21448d264 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 16 Mar 2022 15:36:35 +0100 Subject: [PATCH] feat(solc): support customized output selection pruning (#1039) * feat(solc): support customized output selection pruning * chore(clippy): make clippy happy --- ethers-solc/src/artifacts/mod.rs | 2 +- ethers-solc/src/cache.rs | 155 +----------------- ethers-solc/src/compile/project.rs | 39 +++-- ethers-solc/src/filter.rs | 252 +++++++++++++++++++++++++++++ ethers-solc/src/lib.rs | 44 +++++ 5 files changed, 327 insertions(+), 165 deletions(-) create mode 100644 ethers-solc/src/filter.rs diff --git a/ethers-solc/src/artifacts/mod.rs b/ethers-solc/src/artifacts/mod.rs index 61077e8f..ba745336 100644 --- a/ethers-solc/src/artifacts/mod.rs +++ b/ethers-solc/src/artifacts/mod.rs @@ -26,7 +26,7 @@ pub mod output_selection; pub mod serde_helpers; use crate::{ artifacts::output_selection::{ContractOutputSelection, OutputSelection}, - cache::FilteredSources, + filter::FilteredSources, }; pub use serde_helpers::{deserialize_bytes, deserialize_opt_bytes}; diff --git a/ethers-solc/src/cache.rs b/ethers-solc/src/cache.rs index 30e46bfe..f043b0c2 100644 --- a/ethers-solc/src/cache.rs +++ b/ethers-solc/src/cache.rs @@ -1,8 +1,9 @@ //! Support for compiling contracts use crate::{ - artifacts::{output_selection::OutputSelection, Settings, Sources}, + artifacts::Sources, config::SolcConfig, error::{Result, SolcError}, + filter::{FilteredSource, FilteredSourceInfo, FilteredSources}, resolver::GraphEdges, utils, ArtifactFile, ArtifactOutput, Artifacts, ArtifactsMap, Project, ProjectPathsConfig, Source, @@ -14,7 +15,6 @@ use std::{ btree_map::{BTreeMap, Entry}, hash_map, BTreeSet, HashMap, HashSet, }, - fmt, fs::{self}, path::{Path, PathBuf}, time::{Duration, UNIX_EPOCH}, @@ -729,157 +729,6 @@ impl<'a, T: ArtifactOutput> ArtifactsCacheInner<'a, T> { } } -/// Container type for a set of [FilteredSource] -#[derive(Debug, Clone, Eq, PartialEq)] -pub struct FilteredSources(pub BTreeMap); - -impl FilteredSources { - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } - - pub fn len(&self) -> usize { - self.0.len() - } - - /// Returns `true` if all files are dirty - pub fn all_dirty(&self) -> bool { - self.0.values().all(|s| s.is_dirty()) - } - - /// Returns all entries that are dirty - pub fn dirty(&self) -> impl Iterator + '_ { - self.0.iter().filter(|(_, s)| s.is_dirty()) - } - - /// Returns all entries that are clean - pub fn clean(&self) -> impl Iterator + '_ { - self.0.iter().filter(|(_, s)| !s.is_dirty()) - } - - /// Returns all dirty files - pub fn dirty_files(&self) -> impl Iterator + fmt::Debug + '_ { - self.0.iter().filter_map(|(k, s)| s.is_dirty().then(|| k)) - } - - /// While solc needs all the files to compile the actual _dirty_ files, we can tell solc to - /// output everything for those dirty files as currently configured in the settings, but output - /// nothing for the other files that are _not_ dirty. - /// - /// This will modify the [OutputSelection] of the [Settings] so that we explicitly select the - /// files' output based on their state. - pub fn into_sources(self, settings: &mut Settings) -> Sources { - if !self.all_dirty() { - // settings can be optimized - - tracing::trace!( - "Optimizing output selection for {}/{} sources", - self.clean().count(), - self.len() - ); - - let selection = settings - .output_selection - .as_mut() - .remove("*") - .unwrap_or_else(OutputSelection::default_file_output_selection); - - for (file, source) in self.0.iter() { - if source.is_dirty() { - settings - .output_selection - .as_mut() - .insert(format!("{}", file.display()), selection.clone()); - } else { - tracing::trace!("Optimizing output for {}", file.display()); - settings.output_selection.as_mut().insert( - format!("{}", file.display()), - OutputSelection::empty_file_output_select(), - ); - } - } - } - self.into() - } -} - -impl From for Sources { - fn from(sources: FilteredSources) -> Self { - sources.0.into_iter().map(|(k, v)| (k, v.into_source())).collect() - } -} - -impl From for FilteredSources { - fn from(s: Sources) -> Self { - FilteredSources(s.into_iter().map(|(key, val)| (key, FilteredSource::Dirty(val))).collect()) - } -} - -impl From> for FilteredSources { - fn from(s: BTreeMap) -> Self { - FilteredSources(s) - } -} - -impl AsRef> for FilteredSources { - fn as_ref(&self) -> &BTreeMap { - &self.0 - } -} - -impl AsMut> for FilteredSources { - fn as_mut(&mut self) -> &mut BTreeMap { - &mut self.0 - } -} - -/// Represents the state of a filtered [Source] -#[derive(Debug, Clone, Eq, PartialEq)] -pub enum FilteredSource { - /// A source that fits the _dirty_ criteria - Dirty(Source), - /// A source that does _not_ fit the _dirty_ criteria but is included in the filtered set - /// because a _dirty_ file pulls it in, either directly on indirectly. - Clean(Source), -} - -impl FilteredSource { - /// Returns the underlying source - pub fn source(&self) -> &Source { - match self { - FilteredSource::Dirty(s) => s, - FilteredSource::Clean(s) => s, - } - } - - /// Consumes the type and returns the underlying source - pub fn into_source(self) -> Source { - match self { - FilteredSource::Dirty(s) => s, - FilteredSource::Clean(s) => s, - } - } - - /// Whether this file is actually dirt - pub fn is_dirty(&self) -> bool { - matches!(self, FilteredSource::Dirty(_)) - } -} - -/// Helper type that determines the state of a source file -struct FilteredSourceInfo { - /// path to the source file - file: PathBuf, - /// contents of the file - source: Source, - /// idx in the [GraphEdges] - idx: usize, - /// whether this file is actually dirty - /// - /// See also [ArtifactsCacheInner::is_dirty()] - dirty: bool, -} - /// Abstraction over configured caching which can be either non-existent or an already loaded cache #[allow(clippy::large_enum_variant)] #[derive(Debug)] diff --git a/ethers-solc/src/compile/project.rs b/ethers-solc/src/compile/project.rs index b4c46a0f..5c68132c 100644 --- a/ethers-solc/src/compile/project.rs +++ b/ethers-solc/src/compile/project.rs @@ -114,6 +114,7 @@ use crate::{ }; use rayon::prelude::*; +use crate::filter::SparseOutputFileFilter; use std::{collections::btree_map::BTreeMap, path::PathBuf}; #[derive(Debug)] @@ -123,6 +124,8 @@ pub struct ProjectCompiler<'a, T: ArtifactOutput> { project: &'a Project, /// how to compile all the sources sources: CompilerSources, + /// How to select solc [CompilerOutput] for files + sparse_output: SparseOutputFileFilter, } impl<'a, T: ArtifactOutput> ProjectCompiler<'a, T> { @@ -162,7 +165,7 @@ impl<'a, T: ArtifactOutput> ProjectCompiler<'a, T> { CompilerSources::Sequential(sources_by_version) }; - Ok(Self { edges, project, sources }) + Ok(Self { edges, project, sources, sparse_output: Default::default() }) } /// Compiles the sources with a pinned `Solc` instance @@ -176,7 +179,14 @@ impl<'a, T: ArtifactOutput> ProjectCompiler<'a, T> { let sources_by_version = BTreeMap::from([(solc, (version, sources))]); let sources = CompilerSources::Sequential(sources_by_version); - Ok(Self { edges, project, sources }) + Ok(Self { edges, project, sources, sparse_output: Default::default() }) + } + + /// Applies the specified [SparseOutputFileFilter] to be applied when selecting solc output for + /// specific files to be compiled + pub fn with_sparse_output(mut self, sparse_output: impl Into) -> Self { + self.sparse_output = sparse_output.into(); + self } /// Compiles all the sources of the `Project` in the appropriate mode @@ -203,13 +213,13 @@ impl<'a, T: ArtifactOutput> ProjectCompiler<'a, T> { /// - sets proper source unit names /// - check cache fn preprocess(self) -> Result> { - let Self { edges, project, sources } = self; + let Self { edges, project, sources, sparse_output } = self; let mut cache = ArtifactsCache::new(project, edges)?; // retain and compile only dirty sources and all their imports let sources = sources.filtered(&mut cache); - Ok(PreprocessedState { sources, cache }) + Ok(PreprocessedState { sources, cache, sparse_output }) } } @@ -222,14 +232,18 @@ struct PreprocessedState<'a, T: ArtifactOutput> { sources: FilteredCompilerSources, /// cache that holds [CacheEntry] object if caching is enabled and the project is recompiled cache: ArtifactsCache<'a, T>, + sparse_output: SparseOutputFileFilter, } impl<'a, T: ArtifactOutput> PreprocessedState<'a, T> { /// advance to the next state by compiling all sources fn compile(self) -> Result> { - let PreprocessedState { sources, cache } = self; - let output = - sources.compile(&cache.project().solc_config.settings, &cache.project().paths)?; + let PreprocessedState { sources, cache, sparse_output } = self; + let output = sources.compile( + &cache.project().solc_config.settings, + &cache.project().paths, + sparse_output, + )?; Ok(CompiledState { output, cache }) } @@ -351,13 +365,14 @@ impl FilteredCompilerSources { self, settings: &Settings, paths: &ProjectPathsConfig, + sparse_output: SparseOutputFileFilter, ) -> Result { match self { FilteredCompilerSources::Sequential(input) => { - compile_sequential(input, settings, paths) + compile_sequential(input, settings, paths, sparse_output) } FilteredCompilerSources::Parallel(input, j) => { - compile_parallel(input, j, settings, paths) + compile_parallel(input, j, settings, paths, sparse_output) } } } @@ -377,6 +392,7 @@ fn compile_sequential( input: VersionedFilteredSources, settings: &Settings, paths: &ProjectPathsConfig, + sparse_output: SparseOutputFileFilter, ) -> Result { let mut aggregated = AggregatedCompilerOutput::default(); tracing::trace!("compiling {} jobs sequentially", input.len()); @@ -402,7 +418,7 @@ fn compile_sequential( // depending on the composition of the filtered sources, the output selection can be // optimized let mut opt_settings = settings.clone(); - let sources = filtered_sources.into_sources(&mut opt_settings); + let sources = sparse_output.sparse_sources(filtered_sources, &mut opt_settings); for input in CompilerInput::with_sources(sources) { let actually_dirty = input @@ -450,6 +466,7 @@ fn compile_parallel( num_jobs: usize, settings: &Settings, paths: &ProjectPathsConfig, + sparse_output: SparseOutputFileFilter, ) -> Result { debug_assert!(num_jobs > 1); tracing::trace!( @@ -475,7 +492,7 @@ fn compile_parallel( // depending on the composition of the filtered sources, the output selection can be // optimized let mut opt_settings = settings.clone(); - let sources = filtered_sources.into_sources(&mut opt_settings); + let sources = sparse_output.sparse_sources(filtered_sources, &mut opt_settings); for input in CompilerInput::with_sources(sources) { let actually_dirty = input diff --git a/ethers-solc/src/filter.rs b/ethers-solc/src/filter.rs new file mode 100644 index 00000000..3e68d038 --- /dev/null +++ b/ethers-solc/src/filter.rs @@ -0,0 +1,252 @@ +//! Types to apply filter to input types + +use crate::{ + artifacts::{output_selection::OutputSelection, Settings}, + Source, Sources, +}; +use std::{ + collections::BTreeMap, + fmt, + fmt::Formatter, + path::{Path, PathBuf}, +}; + +/// A predicate property that determines whether a file satisfies a certain condition +pub trait FileFilter { + /// The predicate function that should return if the given `file` should be included. + fn is_match(&self, file: &Path) -> bool; +} + +impl FileFilter for F +where + F: Fn(&Path) -> bool, +{ + fn is_match(&self, file: &Path) -> bool { + (self)(file) + } +} + +/// An [InputFileFilter] that matches all solidity files that end with `.t.sol` +#[derive(Default)] +pub struct TestFileFilter { + _priv: (), +} + +impl fmt::Debug for TestFileFilter { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct("TestFileFilter").finish() + } +} + +impl fmt::Display for TestFileFilter { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.write_str("TestFileFilter") + } +} + +impl FileFilter for TestFileFilter { + fn is_match(&self, file: &Path) -> bool { + file.file_name().and_then(|s| s.to_str()).map(|s| s.ends_with(".t.sol")).unwrap_or_default() + } +} + +/// A type that can apply a filter to a set of preprocessed [FilteredSources] in order to set sparse +/// output for specific files +pub enum SparseOutputFileFilter { + /// Sets the configured [OutputSelection] for dirty files only. + /// + /// In other words, we request the output of solc only for files that have been detected as + /// _dirty_. + AllDirty, + /// Apply an additional filter to [FilteredSources] to + Custom(Box), +} + +impl SparseOutputFileFilter { + /// While solc needs all the files to compile the actual _dirty_ files, we can tell solc to + /// output everything for those dirty files as currently configured in the settings, but output + /// nothing for the other files that are _not_ dirty. + /// + /// This will modify the [OutputSelection] of the [Settings] so that we explicitly select the + /// files' output based on their state. + pub fn sparse_sources(&self, sources: FilteredSources, settings: &mut Settings) -> Sources { + fn apply( + sources: &FilteredSources, + settings: &mut Settings, + f: impl Fn(&PathBuf, &FilteredSource) -> bool, + ) { + let selection = settings + .output_selection + .as_mut() + .remove("*") + .unwrap_or_else(OutputSelection::default_file_output_selection); + + for (file, source) in sources.0.iter() { + if f(file, source) { + settings + .output_selection + .as_mut() + .insert(format!("{}", file.display()), selection.clone()); + } else { + tracing::trace!("using pruned output selection for {}", file.display()); + settings.output_selection.as_mut().insert( + format!("{}", file.display()), + OutputSelection::empty_file_output_select(), + ); + } + } + } + + match self { + SparseOutputFileFilter::AllDirty => { + if !sources.all_dirty() { + // settings can be optimized + tracing::trace!( + "optimizing output selection for {}/{} sources", + sources.clean().count(), + sources.len() + ); + apply(&sources, settings, |_, source| source.is_dirty()) + } + } + SparseOutputFileFilter::Custom(f) => { + tracing::trace!("optimizing output selection with custom filter",); + apply(&sources, settings, |p, source| source.is_dirty() && f.is_match(p)); + } + }; + sources.into() + } +} + +impl From> for SparseOutputFileFilter { + fn from(f: Box) -> Self { + SparseOutputFileFilter::Custom(f) + } +} + +impl Default for SparseOutputFileFilter { + fn default() -> Self { + SparseOutputFileFilter::AllDirty + } +} + +impl fmt::Debug for SparseOutputFileFilter { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self { + SparseOutputFileFilter::AllDirty => f.write_str("AllDirty"), + SparseOutputFileFilter::Custom(_) => f.write_str("Custom"), + } + } +} + +/// Container type for a set of [FilteredSource] +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct FilteredSources(pub BTreeMap); + +impl FilteredSources { + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub fn len(&self) -> usize { + self.0.len() + } + + /// Returns `true` if all files are dirty + pub fn all_dirty(&self) -> bool { + self.0.values().all(|s| s.is_dirty()) + } + + /// Returns all entries that are dirty + pub fn dirty(&self) -> impl Iterator + '_ { + self.0.iter().filter(|(_, s)| s.is_dirty()) + } + + /// Returns all entries that are clean + pub fn clean(&self) -> impl Iterator + '_ { + self.0.iter().filter(|(_, s)| !s.is_dirty()) + } + + /// Returns all dirty files + pub fn dirty_files(&self) -> impl Iterator + fmt::Debug + '_ { + self.0.iter().filter_map(|(k, s)| s.is_dirty().then(|| k)) + } +} + +impl From for Sources { + fn from(sources: FilteredSources) -> Self { + sources.0.into_iter().map(|(k, v)| (k, v.into_source())).collect() + } +} + +impl From for FilteredSources { + fn from(s: Sources) -> Self { + FilteredSources(s.into_iter().map(|(key, val)| (key, FilteredSource::Dirty(val))).collect()) + } +} + +impl From> for FilteredSources { + fn from(s: BTreeMap) -> Self { + FilteredSources(s) + } +} + +impl AsRef> for FilteredSources { + fn as_ref(&self) -> &BTreeMap { + &self.0 + } +} + +impl AsMut> for FilteredSources { + fn as_mut(&mut self) -> &mut BTreeMap { + &mut self.0 + } +} + +/// Represents the state of a filtered [Source] +#[derive(Debug, Clone, Eq, PartialEq)] +pub enum FilteredSource { + /// A source that fits the _dirty_ criteria + Dirty(Source), + /// A source that does _not_ fit the _dirty_ criteria but is included in the filtered set + /// because a _dirty_ file pulls it in, either directly on indirectly. + Clean(Source), +} + +impl FilteredSource { + /// Returns the underlying source + pub fn source(&self) -> &Source { + match self { + FilteredSource::Dirty(s) => s, + FilteredSource::Clean(s) => s, + } + } + + /// Consumes the type and returns the underlying source + pub fn into_source(self) -> Source { + match self { + FilteredSource::Dirty(s) => s, + FilteredSource::Clean(s) => s, + } + } + + /// Whether this file is actually dirt + pub fn is_dirty(&self) -> bool { + matches!(self, FilteredSource::Dirty(_)) + } +} + +/// Helper type that determines the state of a source file +#[derive(Debug)] +pub struct FilteredSourceInfo { + /// path to the source file + pub file: PathBuf, + /// contents of the file + pub source: Source, + /// idx in the [GraphEdges] + pub idx: usize, + /// whether this file is actually dirty + /// + /// See also [ArtifactsCacheInner::is_dirty()] + pub dirty: bool, +} diff --git a/ethers-solc/src/lib.rs b/ethers-solc/src/lib.rs index 6224f205..b82f00ba 100644 --- a/ethers-solc/src/lib.rs +++ b/ethers-solc/src/lib.rs @@ -26,8 +26,10 @@ pub mod remappings; use crate::artifacts::Source; pub mod error; +mod filter; pub mod report; pub mod utils; +pub use filter::{FileFilter, TestFileFilter}; use crate::{ artifacts::{Contract, Sources}, @@ -287,6 +289,48 @@ impl Project { project::ProjectCompiler::with_sources(self, Source::read_all(files)?)?.compile() } + /// Convenience function to compile only (re)compile files that match the provided [FileFilter]. + /// Same as [`Self::svm_compile()`] but with only with those files as input that match + /// [FileFilter::is_match()]. + /// + /// # Example - Only compile Test files + /// + /// ``` + /// use ethers_solc::{Project, TestFileFilter}; + /// # fn demo(project: Project) { + /// let project = Project::builder().build().unwrap(); + /// let output = project + /// .compile_sparse( + /// TestFileFilter::default() + /// ).unwrap(); + /// # } + /// ``` + /// + /// # Example - Apply a custom filter + /// + /// ``` + /// use std::path::Path; + /// use ethers_solc::Project; + /// # fn demo(project: Project) { + /// let project = Project::builder().build().unwrap(); + /// let output = project + /// .compile_sparse( + /// |path: &Path| path.ends_with("Greeter.sol") + /// ).unwrap(); + /// # } + /// ``` + #[cfg(all(feature = "svm", feature = "async"))] + pub fn compile_sparse( + &self, + filter: F, + ) -> Result> { + let sources = + Source::read_all(self.paths.input_files().into_iter().filter(|p| filter.is_match(p)))?; + + let filter: Box = Box::new(filter); + project::ProjectCompiler::with_sources(self, sources)?.with_sparse_output(filter).compile() + } + /// Compiles the given source files with the exact `Solc` executable /// /// First all libraries for the sources are resolved by scanning all their imports.