LumeWeb
/
ethers-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(solc): support customized output selection pruning (#1039) 

* feat(solc): support customized output selection pruning

* chore(clippy): make clippy happy
This commit is contained in:
Matthias Seitz 2022-03-16 15:36:35 +01:00 committed by GitHub
parent 5d14198fbe
commit 2d75f9f1e7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 327 additions and 165 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ethers-solc/src/artifacts/mod.rs
View File

@ -26,7 +26,7 @@ pub mod output_selection;
pub mod serde_helpers;
use crate::{
artifacts::output_selection::{ContractOutputSelection, OutputSelection},
cache::FilteredSources,
filter::FilteredSources,
};
pub use serde_helpers::{deserialize_bytes, deserialize_opt_bytes};

155
ethers-solc/src/cache.rs
View File

@ -1,8 +1,9 @@
//! Support for compiling contracts
use crate::{
artifacts::{output_selection::OutputSelection, Settings, Sources},
artifacts::Sources,
config::SolcConfig,
error::{Result, SolcError},
filter::{FilteredSource, FilteredSourceInfo, FilteredSources},
resolver::GraphEdges,
utils, ArtifactFile, ArtifactOutput, Artifacts, ArtifactsMap, Project, ProjectPathsConfig,
Source,
@ -14,7 +15,6 @@ use std::{
btree_map::{BTreeMap, Entry},
hash_map, BTreeSet, HashMap, HashSet,
},
fmt,
fs::{self},
path::{Path, PathBuf},
time::{Duration, UNIX_EPOCH},
@ -729,157 +729,6 @@ impl<'a, T: ArtifactOutput> ArtifactsCacheInner<'a, T> {
}
}
/// Container type for a set of [FilteredSource]
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct FilteredSources(pub BTreeMap<PathBuf, FilteredSource>);
impl FilteredSources {
pub fn is_empty(&self) -> bool {
self.0.is_empty()
}
pub fn len(&self) -> usize {
self.0.len()
}
/// Returns `true` if all files are dirty
pub fn all_dirty(&self) -> bool {
self.0.values().all(|s| s.is_dirty())
}
/// Returns all entries that are dirty
pub fn dirty(&self) -> impl Iterator<Item = (&PathBuf, &FilteredSource)> + '_ {
self.0.iter().filter(|(_, s)| s.is_dirty())
}
/// Returns all entries that are clean
pub fn clean(&self) -> impl Iterator<Item = (&PathBuf, &FilteredSource)> + '_ {
self.0.iter().filter(|(_, s)| !s.is_dirty())
}
/// Returns all dirty files
pub fn dirty_files(&self) -> impl Iterator<Item = &PathBuf> + fmt::Debug + '_ {
self.0.iter().filter_map(|(k, s)| s.is_dirty().then(|| k))
}
/// While solc needs all the files to compile the actual _dirty_ files, we can tell solc to
/// output everything for those dirty files as currently configured in the settings, but output
/// nothing for the other files that are _not_ dirty.
///
/// This will modify the [OutputSelection] of the [Settings] so that we explicitly select the
/// files' output based on their state.
pub fn into_sources(self, settings: &mut Settings) -> Sources {
if !self.all_dirty() {
// settings can be optimized
tracing::trace!(
"Optimizing output selection for {}/{} sources",
self.clean().count(),
self.len()
);
let selection = settings
.output_selection
.as_mut()
.remove("*")
.unwrap_or_else(OutputSelection::default_file_output_selection);
for (file, source) in self.0.iter() {
if source.is_dirty() {
settings
.output_selection
.as_mut()
.insert(format!("{}", file.display()), selection.clone());
} else {
tracing::trace!("Optimizing output for {}", file.display());
settings.output_selection.as_mut().insert(
format!("{}", file.display()),
OutputSelection::empty_file_output_select(),
);
}
}
}
self.into()
}
}
impl From<FilteredSources> for Sources {
fn from(sources: FilteredSources) -> Self {
sources.0.into_iter().map(|(k, v)| (k, v.into_source())).collect()
}
}
impl From<Sources> for FilteredSources {
fn from(s: Sources) -> Self {
FilteredSources(s.into_iter().map(|(key, val)| (key, FilteredSource::Dirty(val))).collect())
}
}
impl From<BTreeMap<PathBuf, FilteredSource>> for FilteredSources {
fn from(s: BTreeMap<PathBuf, FilteredSource>) -> Self {
FilteredSources(s)
}
}
impl AsRef<BTreeMap<PathBuf, FilteredSource>> for FilteredSources {
fn as_ref(&self) -> &BTreeMap<PathBuf, FilteredSource> {
&self.0
}
}
impl AsMut<BTreeMap<PathBuf, FilteredSource>> for FilteredSources {
fn as_mut(&mut self) -> &mut BTreeMap<PathBuf, FilteredSource> {
&mut self.0
}
}
/// Represents the state of a filtered [Source]
#[derive(Debug, Clone, Eq, PartialEq)]
pub enum FilteredSource {
/// A source that fits the _dirty_ criteria
Dirty(Source),
/// A source that does _not_ fit the _dirty_ criteria but is included in the filtered set
/// because a _dirty_ file pulls it in, either directly on indirectly.
Clean(Source),
}
impl FilteredSource {
/// Returns the underlying source
pub fn source(&self) -> &Source {
match self {
FilteredSource::Dirty(s) => s,
FilteredSource::Clean(s) => s,
}
}
/// Consumes the type and returns the underlying source
pub fn into_source(self) -> Source {
match self {
FilteredSource::Dirty(s) => s,
FilteredSource::Clean(s) => s,
}
}
/// Whether this file is actually dirt
pub fn is_dirty(&self) -> bool {
matches!(self, FilteredSource::Dirty(_))
}
}
/// Helper type that determines the state of a source file
struct FilteredSourceInfo {
/// path to the source file
file: PathBuf,
/// contents of the file
source: Source,
/// idx in the [GraphEdges]
idx: usize,
/// whether this file is actually dirty
///
/// See also [ArtifactsCacheInner::is_dirty()]
dirty: bool,
}
/// Abstraction over configured caching which can be either non-existent or an already loaded cache
#[allow(clippy::large_enum_variant)]
#[derive(Debug)]

39
ethers-solc/src/compile/project.rs
View File

@ -114,6 +114,7 @@ use crate::{
};
use rayon::prelude::*;
use crate::filter::SparseOutputFileFilter;
use std::{collections::btree_map::BTreeMap, path::PathBuf};
#[derive(Debug)]
@ -123,6 +124,8 @@ pub struct ProjectCompiler<'a, T: ArtifactOutput> {
project: &'a Project<T>,
/// how to compile all the sources
sources: CompilerSources,
/// How to select solc [CompilerOutput] for files
sparse_output: SparseOutputFileFilter,
}
impl<'a, T: ArtifactOutput> ProjectCompiler<'a, T> {
@ -162,7 +165,7 @@ impl<'a, T: ArtifactOutput> ProjectCompiler<'a, T> {
CompilerSources::Sequential(sources_by_version)
};
Ok(Self { edges, project, sources })
Ok(Self { edges, project, sources, sparse_output: Default::default() })
}
/// Compiles the sources with a pinned `Solc` instance
@ -176,7 +179,14 @@ impl<'a, T: ArtifactOutput> ProjectCompiler<'a, T> {
let sources_by_version = BTreeMap::from([(solc, (version, sources))]);
let sources = CompilerSources::Sequential(sources_by_version);
Ok(Self { edges, project, sources })
Ok(Self { edges, project, sources, sparse_output: Default::default() })
}
/// Applies the specified [SparseOutputFileFilter] to be applied when selecting solc output for
/// specific files to be compiled
pub fn with_sparse_output(mut self, sparse_output: impl Into<SparseOutputFileFilter>) -> Self {
self.sparse_output = sparse_output.into();
self
}
/// Compiles all the sources of the `Project` in the appropriate mode
@ -203,13 +213,13 @@ impl<'a, T: ArtifactOutput> ProjectCompiler<'a, T> {
/// - sets proper source unit names
/// - check cache
fn preprocess(self) -> Result<PreprocessedState<'a, T>> {
let Self { edges, project, sources } = self;
let Self { edges, project, sources, sparse_output } = self;
let mut cache = ArtifactsCache::new(project, edges)?;
// retain and compile only dirty sources and all their imports
let sources = sources.filtered(&mut cache);
Ok(PreprocessedState { sources, cache })
Ok(PreprocessedState { sources, cache, sparse_output })
}
}
@ -222,14 +232,18 @@ struct PreprocessedState<'a, T: ArtifactOutput> {
sources: FilteredCompilerSources,
/// cache that holds [CacheEntry] object if caching is enabled and the project is recompiled
cache: ArtifactsCache<'a, T>,
sparse_output: SparseOutputFileFilter,
}
impl<'a, T: ArtifactOutput> PreprocessedState<'a, T> {
/// advance to the next state by compiling all sources
fn compile(self) -> Result<CompiledState<'a, T>> {
let PreprocessedState { sources, cache } = self;
let output =
sources.compile(&cache.project().solc_config.settings, &cache.project().paths)?;
let PreprocessedState { sources, cache, sparse_output } = self;
let output = sources.compile(
&cache.project().solc_config.settings,
&cache.project().paths,
sparse_output,
)?;
Ok(CompiledState { output, cache })
}
@ -351,13 +365,14 @@ impl FilteredCompilerSources {
self,
settings: &Settings,
paths: &ProjectPathsConfig,
sparse_output: SparseOutputFileFilter,
) -> Result<AggregatedCompilerOutput> {
match self {
FilteredCompilerSources::Sequential(input) => {
compile_sequential(input, settings, paths)
compile_sequential(input, settings, paths, sparse_output)
}
FilteredCompilerSources::Parallel(input, j) => {
compile_parallel(input, j, settings, paths)
compile_parallel(input, j, settings, paths, sparse_output)
}
}
}
@ -377,6 +392,7 @@ fn compile_sequential(
input: VersionedFilteredSources,
settings: &Settings,
paths: &ProjectPathsConfig,
sparse_output: SparseOutputFileFilter,
) -> Result<AggregatedCompilerOutput> {
let mut aggregated = AggregatedCompilerOutput::default();
tracing::trace!("compiling {} jobs sequentially", input.len());
@ -402,7 +418,7 @@ fn compile_sequential(
// depending on the composition of the filtered sources, the output selection can be
// optimized
let mut opt_settings = settings.clone();
let sources = filtered_sources.into_sources(&mut opt_settings);
let sources = sparse_output.sparse_sources(filtered_sources, &mut opt_settings);
for input in CompilerInput::with_sources(sources) {
let actually_dirty = input
@ -450,6 +466,7 @@ fn compile_parallel(
num_jobs: usize,
settings: &Settings,
paths: &ProjectPathsConfig,
sparse_output: SparseOutputFileFilter,
) -> Result<AggregatedCompilerOutput> {
debug_assert!(num_jobs > 1);
tracing::trace!(
@ -475,7 +492,7 @@ fn compile_parallel(
// depending on the composition of the filtered sources, the output selection can be
// optimized
let mut opt_settings = settings.clone();
let sources = filtered_sources.into_sources(&mut opt_settings);
let sources = sparse_output.sparse_sources(filtered_sources, &mut opt_settings);
for input in CompilerInput::with_sources(sources) {
let actually_dirty = input

252
ethers-solc/src/filter.rs Normal file
View File

@ -0,0 +1,252 @@
//! Types to apply filter to input types
use crate::{
artifacts::{output_selection::OutputSelection, Settings},
Source, Sources,
};
use std::{
collections::BTreeMap,
fmt,
fmt::Formatter,
path::{Path, PathBuf},
};
/// A predicate property that determines whether a file satisfies a certain condition
pub trait FileFilter {
/// The predicate function that should return if the given `file` should be included.
fn is_match(&self, file: &Path) -> bool;
}
impl<F> FileFilter for F
where
F: Fn(&Path) -> bool,
{
fn is_match(&self, file: &Path) -> bool {
(self)(file)
}
}
/// An [InputFileFilter] that matches all solidity files that end with `.t.sol`
#[derive(Default)]
pub struct TestFileFilter {
_priv: (),
}
impl fmt::Debug for TestFileFilter {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
f.debug_struct("TestFileFilter").finish()
}
}
impl fmt::Display for TestFileFilter {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
f.write_str("TestFileFilter")
}
}
impl FileFilter for TestFileFilter {
fn is_match(&self, file: &Path) -> bool {
file.file_name().and_then(|s| s.to_str()).map(|s| s.ends_with(".t.sol")).unwrap_or_default()
}
}
/// A type that can apply a filter to a set of preprocessed [FilteredSources] in order to set sparse
/// output for specific files
pub enum SparseOutputFileFilter {
/// Sets the configured [OutputSelection] for dirty files only.
///
/// In other words, we request the output of solc only for files that have been detected as
/// _dirty_.
AllDirty,
/// Apply an additional filter to [FilteredSources] to
Custom(Box<dyn FileFilter>),
}
impl SparseOutputFileFilter {
/// While solc needs all the files to compile the actual _dirty_ files, we can tell solc to
/// output everything for those dirty files as currently configured in the settings, but output
/// nothing for the other files that are _not_ dirty.
///
/// This will modify the [OutputSelection] of the [Settings] so that we explicitly select the
/// files' output based on their state.
pub fn sparse_sources(&self, sources: FilteredSources, settings: &mut Settings) -> Sources {
fn apply(
sources: &FilteredSources,
settings: &mut Settings,
f: impl Fn(&PathBuf, &FilteredSource) -> bool,
) {
let selection = settings
.output_selection
.as_mut()
.remove("*")
.unwrap_or_else(OutputSelection::default_file_output_selection);
for (file, source) in sources.0.iter() {
if f(file, source) {
settings
.output_selection
.as_mut()
.insert(format!("{}", file.display()), selection.clone());
} else {
tracing::trace!("using pruned output selection for {}", file.display());
settings.output_selection.as_mut().insert(
format!("{}", file.display()),
OutputSelection::empty_file_output_select(),
);
}
}
}
match self {
SparseOutputFileFilter::AllDirty => {
if !sources.all_dirty() {
// settings can be optimized
tracing::trace!(
"optimizing output selection for {}/{} sources",
sources.clean().count(),
sources.len()
);
apply(&sources, settings, |_, source| source.is_dirty())
}
}
SparseOutputFileFilter::Custom(f) => {
tracing::trace!("optimizing output selection with custom filter",);
apply(&sources, settings, |p, source| source.is_dirty() && f.is_match(p));
}
};
sources.into()
}
}
impl From<Box<dyn FileFilter>> for SparseOutputFileFilter {
fn from(f: Box<dyn FileFilter>) -> Self {
SparseOutputFileFilter::Custom(f)
}
}
impl Default for SparseOutputFileFilter {
fn default() -> Self {
SparseOutputFileFilter::AllDirty
}
}
impl fmt::Debug for SparseOutputFileFilter {
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
match self {
SparseOutputFileFilter::AllDirty => f.write_str("AllDirty"),
SparseOutputFileFilter::Custom(_) => f.write_str("Custom"),
}
}
}
/// Container type for a set of [FilteredSource]
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct FilteredSources(pub BTreeMap<PathBuf, FilteredSource>);
impl FilteredSources {
pub fn is_empty(&self) -> bool {
self.0.is_empty()
}
pub fn len(&self) -> usize {
self.0.len()
}
/// Returns `true` if all files are dirty
pub fn all_dirty(&self) -> bool {
self.0.values().all(|s| s.is_dirty())
}
/// Returns all entries that are dirty
pub fn dirty(&self) -> impl Iterator<Item = (&PathBuf, &FilteredSource)> + '_ {
self.0.iter().filter(|(_, s)| s.is_dirty())
}
/// Returns all entries that are clean
pub fn clean(&self) -> impl Iterator<Item = (&PathBuf, &FilteredSource)> + '_ {
self.0.iter().filter(|(_, s)| !s.is_dirty())
}
/// Returns all dirty files
pub fn dirty_files(&self) -> impl Iterator<Item = &PathBuf> + fmt::Debug + '_ {
self.0.iter().filter_map(|(k, s)| s.is_dirty().then(|| k))
}
}
impl From<FilteredSources> for Sources {
fn from(sources: FilteredSources) -> Self {
sources.0.into_iter().map(|(k, v)| (k, v.into_source())).collect()
}
}
impl From<Sources> for FilteredSources {
fn from(s: Sources) -> Self {
FilteredSources(s.into_iter().map(|(key, val)| (key, FilteredSource::Dirty(val))).collect())
}
}
impl From<BTreeMap<PathBuf, FilteredSource>> for FilteredSources {
fn from(s: BTreeMap<PathBuf, FilteredSource>) -> Self {
FilteredSources(s)
}
}
impl AsRef<BTreeMap<PathBuf, FilteredSource>> for FilteredSources {
fn as_ref(&self) -> &BTreeMap<PathBuf, FilteredSource> {
&self.0
}
}
impl AsMut<BTreeMap<PathBuf, FilteredSource>> for FilteredSources {
fn as_mut(&mut self) -> &mut BTreeMap<PathBuf, FilteredSource> {
&mut self.0
}
}
/// Represents the state of a filtered [Source]
#[derive(Debug, Clone, Eq, PartialEq)]
pub enum FilteredSource {
/// A source that fits the _dirty_ criteria
Dirty(Source),
/// A source that does _not_ fit the _dirty_ criteria but is included in the filtered set
/// because a _dirty_ file pulls it in, either directly on indirectly.
Clean(Source),
}
impl FilteredSource {
/// Returns the underlying source
pub fn source(&self) -> &Source {
match self {
FilteredSource::Dirty(s) => s,
FilteredSource::Clean(s) => s,
}
}
/// Consumes the type and returns the underlying source
pub fn into_source(self) -> Source {
match self {
FilteredSource::Dirty(s) => s,
FilteredSource::Clean(s) => s,
}
}
/// Whether this file is actually dirt
pub fn is_dirty(&self) -> bool {
matches!(self, FilteredSource::Dirty(_))
}
}
/// Helper type that determines the state of a source file
#[derive(Debug)]
pub struct FilteredSourceInfo {
/// path to the source file
pub file: PathBuf,
/// contents of the file
pub source: Source,
/// idx in the [GraphEdges]
pub idx: usize,
/// whether this file is actually dirty
///
/// See also [ArtifactsCacheInner::is_dirty()]
pub dirty: bool,
}

44
ethers-solc/src/lib.rs
View File

@ -26,8 +26,10 @@ pub mod remappings;
use crate::artifacts::Source;
pub mod error;
mod filter;
pub mod report;
pub mod utils;
pub use filter::{FileFilter, TestFileFilter};
use crate::{
artifacts::{Contract, Sources},
@ -287,6 +289,48 @@ impl<T: ArtifactOutput> Project<T> {
project::ProjectCompiler::with_sources(self, Source::read_all(files)?)?.compile()
}
/// Convenience function to compile only (re)compile files that match the provided [FileFilter].
/// Same as [`Self::svm_compile()`] but with only with those files as input that match
/// [FileFilter::is_match()].
///
/// # Example - Only compile Test files
///
/// ```
/// use ethers_solc::{Project, TestFileFilter};
/// # fn demo(project: Project) {
/// let project = Project::builder().build().unwrap();
/// let output = project
/// .compile_sparse(
/// TestFileFilter::default()
/// ).unwrap();
/// # }
/// ```
///
/// # Example - Apply a custom filter
///
/// ```
/// use std::path::Path;
/// use ethers_solc::Project;
/// # fn demo(project: Project) {
/// let project = Project::builder().build().unwrap();
/// let output = project
/// .compile_sparse(
/// |path: &Path| path.ends_with("Greeter.sol")
/// ).unwrap();
/// # }
/// ```
#[cfg(all(feature = "svm", feature = "async"))]
pub fn compile_sparse<F: FileFilter + 'static>(
&self,
filter: F,
) -> Result<ProjectCompileOutput<T>> {
let sources =
Source::read_all(self.paths.input_files().into_iter().filter(|p| filter.is_match(p)))?;
let filter: Box<dyn FileFilter> = Box::new(filter);
project::ProjectCompiler::with_sources(self, sources)?.with_sparse_output(filter).compile()
}
/// Compiles the given source files with the exact `Solc` executable
///
/// First all libraries for the sources are resolved by scanning all their imports.