feat: improved configs (#64)

* migrate config to figment

* support env variables

* add better error messages

* fix tests

* refactor

Cargo.lock

@@ -97,6 +97,15 @@ dependencies = [
  "rustc_version",
 ]
 
+[[package]]
+name = "atomic"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b88d82667eca772c4aa12f0f1348b3ae643424c8876448f3f7bd5787032e234c"
+dependencies = [
+ "autocfg",
+]
+
 [[package]]
 name = "atty"
 version = "0.2.14"
@@ -524,10 +533,11 @@ dependencies = [
  "common",
  "ethers",
  "eyre",
+ "figment",
  "hex",
  "serde",
  "ssz-rs",
- "toml",
+ "thiserror",
 ]
 
 [[package]]
@@ -1108,6 +1118,20 @@ dependencies = [
  "subtle",
 ]
 
+[[package]]
+name = "figment"
+version = "0.10.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e3bd154d9ae2f1bb0ada5b7eebd56529513efa5de7d2fc8c6adf33bc43260cf"
+dependencies = [
+ "atomic",
+ "pear",
+ "serde",
+ "toml",
+ "uncased",
+ "version_check",
+]
+
 [[package]]
 name = "fixed-hash"
 version = "0.5.2"
@@ -1678,6 +1702,12 @@ dependencies = [
  "hashbrown",
 ]
 
+[[package]]
+name = "inlinable_string"
+version = "0.1.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb"
+
 [[package]]
 name = "inout"
 version = "0.1.3"
@@ -2339,6 +2369,29 @@ dependencies = [
  "sha2 0.10.5",
 ]
 
+[[package]]
+name = "pear"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "15e44241c5e4c868e3eaa78b7c1848cadd6344ed4f54d029832d32b415a58702"
+dependencies = [
+ "inlinable_string",
+ "pear_codegen",
+ "yansi",
+]
+
+[[package]]
+name = "pear_codegen"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "82a5ca643c2303ecb740d506539deba189e16f2754040a42901cd8105d0282d0"
+dependencies = [
+ "proc-macro2",
+ "proc-macro2-diagnostics",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "percent-encoding"
 version = "2.1.0"
@@ -2488,6 +2541,19 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "proc-macro2-diagnostics"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4bf29726d67464d49fa6224a1d07936a8c08bb3fba727c7493f6cf1616fdaada"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+ "version_check",
+ "yansi",
+]
+
 [[package]]
 name = "quote"
 version = "1.0.21"
@@ -3551,6 +3617,15 @@ dependencies = [
  "static_assertions",
 ]
 
+[[package]]
+name = "uncased"
+version = "0.9.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09b01702b0fd0b3fadcf98e098780badda8742d4f4a7676615cad90e8ac73622"
+dependencies = [
+ "version_check",
+]
+
 [[package]]
 name = "unicase"
 version = "2.6.0"
@@ -3888,6 +3963,12 @@ dependencies = [
  "tap",
 ]
 
+[[package]]
+name = "yansi"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec"
+
 [[package]]
 name = "zeroize"
 version = "1.5.7"

cli/Cargo.toml

@@ -5,7 +5,7 @@ edition = "2021"
 
 [dependencies]
 tokio = { version = "1", features = ["full"] }
-clap = { version = "3.2.18", features = ["derive"] }
+clap = { version = "3.2.18", features = ["derive", "env"] }
 eyre = "0.6.8"
 dirs = "4.0.0"
 env_logger = "0.9.0"

cli/src/main.rs

@@ -12,7 +12,7 @@ use env_logger::Env;
 use eyre::Result;
 
 use client::{database::FileDB, Client};
-use config::{networks, Config};
+use config::{CliConfig, Config};
 use futures::executor::block_on;
 use log::info;
 
@@ -62,49 +62,48 @@ fn register_shutdown_handler(client: Client<FileDB>) {
 
 fn get_config() -> Config {
     let cli = Cli::parse();
-    let mut config = match cli.network.as_str() {
+
-        "mainnet" => networks::mainnet(),
+    let config_path = home_dir().unwrap().join(".lightclient/lightclient.toml");
-        "goerli" => networks::goerli(),
+
-        _ => {
+    let cli_config = cli.as_cli_config();
-            let home = home_dir().unwrap();
+
-            let config_path = home.join(format!(".lightclient/configs/{}.toml", cli.network));
+    Config::from_file(&config_path, &cli.network, &cli_config)
-            Config::from_file(&config_path).expect("could not read network config")
 }
+
+#[derive(Parser)]
+struct Cli {
+    #[clap(short, long, default_value = "mainnet")]
+    network: String,
+    #[clap(short, long, env)]
+    port: Option<u16>,
+    #[clap(short = 'w', long, env)]
+    checkpoint: Option<String>,
+    #[clap(short, long, env)]
+    execution_rpc: Option<String>,
+    #[clap(short, long, env)]
+    consensus_rpc: Option<String>,
+}
+
+impl Cli {
+    fn as_cli_config(&self) -> CliConfig {
+        let checkpoint = match &self.checkpoint {
+            Some(checkpoint) => Some(hex_str_to_bytes(&checkpoint).expect("invalid checkpoint")),
+            None => self.get_cached_checkpoint(),
         };
 
-    let data_dir = get_data_dir(&cli);
+        CliConfig {
-
+            checkpoint,
-    config.general.checkpoint = match cli.checkpoint {
+            execution_rpc: self.execution_rpc.clone(),
-        Some(checkpoint) => hex_str_to_bytes(&checkpoint).expect("invalid checkpoint"),
+            consensus_rpc: self.consensus_rpc.clone(),
-        None => get_cached_checkpoint(&data_dir).unwrap_or(config.general.checkpoint),
+            data_dir: self.get_data_dir(),
-    };
+            port: self.port,
-
-    config.general.execution_rpc = Some(cli.execution_rpc);
-
-    if let Some(port) = cli.port {
-        config.general.rpc_port = Some(port);
-    }
-
-    if let Some(consensus_rpc) = cli.consensus_rpc {
-        config.general.consensus_rpc = consensus_rpc;
-    }
-
-    config.machine.data_dir = Some(data_dir);
-
-    config
-}
-
-fn get_data_dir(cli: &Cli) -> PathBuf {
-    match &cli.data_dir {
-        Some(dir) => PathBuf::from(dir),
-        None => home_dir()
-            .unwrap()
-            .join(format!(".lightclient/data/{}", cli.network)),
         }
     }
 
-fn get_cached_checkpoint(data_dir: &PathBuf) -> Option<Vec<u8>> {
+    fn get_cached_checkpoint(&self) -> Option<Vec<u8>> {
+        let data_dir = self.get_data_dir();
         let checkpoint_file = data_dir.join("checkpoint");
+
         if checkpoint_file.exists() {
             let checkpoint_res = fs::read(checkpoint_file);
             match checkpoint_res {
@@ -116,18 +115,9 @@ fn get_cached_checkpoint(data_dir: &PathBuf) -> Option<Vec<u8>> {
         }
     }
 
-#[derive(Parser)]
+    fn get_data_dir(&self) -> PathBuf {
-struct Cli {
+        home_dir()
-    #[clap(short, long, default_value = "mainnet")]
+            .unwrap()
-    network: String,
+            .join(format!(".lightclient/data/{}", self.network))
-    #[clap(short, long)]
+    }
-    port: Option<u16>,
-    #[clap(short = 'w', long)]
-    checkpoint: Option<String>,
-    #[clap(short, long)]
-    execution_rpc: String,
-    #[clap(short, long)]
-    consensus_rpc: Option<String>,
-    #[clap(long)]
-    data_dir: Option<String>,
 }

client/src/client.rs

@@ -29,13 +29,13 @@ impl Client<FileDB> {
         let node = Node::new(config.clone()).await?;
         let node = Arc::new(RwLock::new(node));
 
-        let rpc = if let Some(port) = config.general.rpc_port {
+        let rpc = if let Some(port) = config.rpc_port {
             Some(Rpc::new(node.clone(), port))
         } else {
             None
         };
 
-        let data_dir = config.machine.data_dir.clone();
+        let data_dir = config.data_dir.clone();
         let db = FileDB::new(data_dir.ok_or(eyre!("data dir not found"))?);
 
         Ok(Client { node, rpc, db })

client/src/node.rs

@@ -28,13 +28,13 @@ pub struct Node {
 
 impl Node {
     pub async fn new(config: Arc<Config>) -> Result<Self> {
-        let consensus_rpc = &config.general.consensus_rpc;
+        let consensus_rpc = &config.consensus_rpc;
-        let checkpoint_hash = &config.general.checkpoint;
+        let checkpoint_hash = &config.checkpoint;
-        let execution_rpc = &config.general.execution_rpc;
+        let execution_rpc = &config.execution_rpc;
 
         let consensus =
             ConsensusClient::new(consensus_rpc, checkpoint_hash, config.clone()).await?;
-        let execution = ExecutionClient::new(execution_rpc.as_ref().unwrap())?;
+        let execution = ExecutionClient::new(execution_rpc)?;
 
         let payloads = BTreeMap::new();
         let finalized_payloads = BTreeMap::new();
@@ -216,7 +216,7 @@ impl Node {
     }
 
     pub fn chain_id(&self) -> u64 {
-        self.config.general.chain_id
+        self.config.chain.chain_id
     }
 
     pub fn get_header(&self) -> Result<Header> {

config/Cargo.toml

@@ -12,6 +12,7 @@ serde = { version = "1.0.143", features = ["derive"] }
 hex = "0.4.3"
 ssz-rs = { git = "https://github.com/ralexstokes/ssz-rs" }
 ethers = "0.17.0"
-toml = "0.5.9"
+figment = { version = "0.10.7", features = ["toml", "env"] }
+thiserror = "1.0.37"
 
 common = { path = "../common" }

config/configs/goerli.toml

@@ -1,22 +0,0 @@
-[general]
-chain_id = 5
-genesis_time = 1616508000
-genesis_root = "0x043db0d9a83813551ee2f33450d23797757d430911a9320530ad8a0eabc43efb"
-checkpoint = "0x172128eadf1da46467f4d6a822206698e2d3f957af117dd650954780d680dc99"
-consensus_rpc = "http://testing.prater.beacon-api.nimbus.team"
-execution_rpc = "https://eth-goerli.g.alchemy.com:443/v2/o_8Qa9kgwDPf9G8sroyQ-uQtyhyWa3ao"
-rpc_port = 8545
-
-[forks]
-
-[forks.genesis]
-epoch = 0
-fork_version = "0x00001020"
-
-[forks.altair]
-epoch = 36660
-fork_version = "0x01001020"
-
-[forks.bellatrix]
-epoch = 112260
-fork_version = "0x02001020"

config/configs/mainnet.toml

@@ -1,22 +0,0 @@
-[general]
-chain_id = 1
-genesis_time = 1606824023
-genesis_root = "0x4b363db94e286120d76eb905340fdd4e54bfe9f06bf33ff6cf5ad27f511bfe95"
-checkpoint = "0x03e315e11b3f88cd63dfb62c74a313c4a65949ce9e37599e0ee66533ceceadfd"
-consensus_rpc = "http://testing.mainnet.beacon-api.nimbus.team"
-execution_rpc = "https://eth-mainnet.g.alchemy.com/v2/Q0BqQPbTQfSMzrCNl4x80XS_PLLB1RNf"
-rpc_port = 8545
-
-[forks]
-
-[forks.genesis]
-epoch = 0
-fork_version = "0x00000000"
-
-[forks.altair]
-epoch = 74240
-fork_version = "0x01000000"
-
-[forks.bellatrix]
-epoch = 144896
-fork_version = "0x02000000"

config/src/lib.rs

@@ -1,58 +1,73 @@
 pub mod networks;
 
-use std::{
+use std::{collections::HashMap, path::PathBuf, process::exit};
-    fs,
-    path::{Path, PathBuf},
-};
 
 use eyre::Result;
-use serde::Deserialize;
+use figment::{
+    providers::{Format, Serialized, Toml},
+    value::Value,
+    Figment,
+};
+use networks::BaseConfig;
+use serde::{Deserialize, Serialize};
 
 use common::utils::hex_str_to_bytes;
 
-#[derive(Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Default)]
 pub struct Config {
-    pub general: General,
-    pub forks: Forks,
-    pub machine: Machine,
-}
-
-#[derive(Deserialize, Debug)]
-pub struct General {
-    pub chain_id: u64,
-    pub genesis_time: u64,
-    #[serde(deserialize_with = "bytes_deserialize")]
-    pub genesis_root: Vec<u8>,
-    #[serde(deserialize_with = "bytes_deserialize")]
-    pub checkpoint: Vec<u8>,
     pub consensus_rpc: String,
-    pub execution_rpc: Option<String>,
+    pub execution_rpc: String,
     pub rpc_port: Option<u16>,
-}
+    #[serde(
-
+        deserialize_with = "bytes_deserialize",
-#[derive(Deserialize, Debug)]
+        serialize_with = "bytes_serialize"
-pub struct Forks {
+    )]
-    pub genesis: Fork,
+    pub checkpoint: Vec<u8>,
-    pub altair: Fork,
-    pub bellatrix: Fork,
-}
-
-#[derive(Deserialize, Debug)]
-pub struct Fork {
-    pub epoch: u64,
-    #[serde(deserialize_with = "bytes_deserialize")]
-    pub fork_version: Vec<u8>,
-}
-
-#[derive(Deserialize, Debug)]
-pub struct Machine {
     pub data_dir: Option<PathBuf>,
+    pub chain: ChainConfig,
+    pub forks: Forks,
 }
 
 impl Config {
-    pub fn from_file(path: &Path) -> Result<Self> {
+    pub fn from_file(config_path: &PathBuf, network: &str, cli_config: &CliConfig) -> Self {
-        let contents = fs::read_to_string(path)?;
+        let base_config = match network {
-        Ok(toml::from_str(&contents)?)
+            "mainnet" => networks::mainnet(),
+            "goerli" => networks::goerli(),
+            _ => BaseConfig::default(),
+        };
+
+        let base_provider = Serialized::from(base_config, network);
+        let toml_provider = Toml::file(config_path).nested();
+        let user_provider = cli_config.as_provider(network);
+
+        let config_res = Figment::new()
+            .merge(base_provider)
+            .merge(toml_provider)
+            .merge(user_provider)
+            .select(network)
+            .extract();
+
+        match config_res {
+            Ok(config) => config,
+            Err(err) => {
+                match err.kind {
+                    figment::error::Kind::MissingField(field) => {
+                        println!(
+                            "\x1b[91merror\x1b[0m: missing configuration field: {}",
+                            field
+                        );
+                        println!(
+                            "\n\ttry supplying the propoper command line argument: --{}",
+                            field
+                        );
+                        println!("\talternatively, you can add the field to your lightclient.toml file or as an environment variable");
+                        println!("\nfor more information, check the github README");
+                    }
+                    _ => println!("cannot parse configuration: {}", err),
+                }
+                exit(1);
+            }
+        }
     }
 
     pub fn fork_version(&self, slot: u64) -> Vec<u8> {
@@ -68,6 +83,69 @@ impl Config {
     }
 }
 
+#[derive(Serialize)]
+pub struct CliConfig {
+    pub execution_rpc: Option<String>,
+    pub consensus_rpc: Option<String>,
+    pub checkpoint: Option<Vec<u8>>,
+    pub port: Option<u16>,
+    pub data_dir: PathBuf,
+}
+
+impl CliConfig {
+    fn as_provider(&self, network: &str) -> Serialized<HashMap<&str, Value>> {
+        let mut user_dict = HashMap::new();
+
+        if let Some(rpc) = &self.execution_rpc {
+            user_dict.insert("execution_rpc", Value::from(rpc.clone()));
+        }
+
+        if let Some(rpc) = &self.consensus_rpc {
+            user_dict.insert("consensus_rpc", Value::from(rpc.clone()));
+        }
+
+        if let Some(checkpoint) = &self.checkpoint {
+            user_dict.insert("checkpoint", Value::from(hex::encode(checkpoint)));
+        }
+
+        if let Some(port) = self.port {
+            user_dict.insert("port", Value::from(port));
+        }
+
+        user_dict.insert("data_dir", Value::from(self.data_dir.to_str().unwrap()));
+
+        Serialized::from(user_dict, network)
+    }
+}
+
+#[derive(Serialize, Deserialize, Debug, Default)]
+pub struct ChainConfig {
+    pub chain_id: u64,
+    pub genesis_time: u64,
+    #[serde(
+        deserialize_with = "bytes_deserialize",
+        serialize_with = "bytes_serialize"
+    )]
+    pub genesis_root: Vec<u8>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Default)]
+pub struct Forks {
+    pub genesis: Fork,
+    pub altair: Fork,
+    pub bellatrix: Fork,
+}
+
+#[derive(Serialize, Deserialize, Debug, Default)]
+pub struct Fork {
+    pub epoch: u64,
+    #[serde(
+        deserialize_with = "bytes_deserialize",
+        serialize_with = "bytes_serialize"
+    )]
+    pub fork_version: Vec<u8>,
+}
+
 fn bytes_deserialize<'de, D>(deserializer: D) -> Result<Vec<u8>, D::Error>
 where
     D: serde::Deserializer<'de>,
@@ -75,3 +153,11 @@ where
     let bytes: String = serde::Deserialize::deserialize(deserializer)?;
     Ok(hex_str_to_bytes(&bytes).unwrap())
 }
+
+fn bytes_serialize<S>(bytes: &Vec<u8>, serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: serde::Serializer,
+{
+    let bytes_string = hex::encode(bytes);
+    serializer.serialize_str(&bytes_string)
+}

config/src/networks.rs

@@ -1,23 +1,34 @@
+use serde::Serialize;
+
+use crate::{bytes_serialize, ChainConfig, Fork, Forks};
 use common::utils::hex_str_to_bytes;
 
-use crate::{Config, Fork, Forks, General, Machine};
+#[derive(Serialize, Default)]
+pub struct BaseConfig {
+    rpc_port: u16,
+    #[serde(
+        deserialize_with = "bytes_deserialize",
+        serialize_with = "bytes_serialize"
+    )]
+    pub checkpoint: Vec<u8>,
+    pub chain: ChainConfig,
+    pub forks: Forks,
+}
 
-pub fn mainnet() -> Config {
+pub fn mainnet() -> BaseConfig {
-    Config {
+    BaseConfig {
-        general: General {
+        checkpoint: hex_str_to_bytes(
+            "0x5ca31c7c795d8f2de2e844718cdb08835639c644365427b9f20f82083e7dac9a",
+        )
+        .unwrap(),
+        rpc_port: 8545,
+        chain: ChainConfig {
             chain_id: 1,
             genesis_time: 1606824023,
             genesis_root: hex_str_to_bytes(
                 "0x4b363db94e286120d76eb905340fdd4e54bfe9f06bf33ff6cf5ad27f511bfe95",
             )
             .unwrap(),
-            checkpoint: hex_str_to_bytes(
-                "0x5ca31c7c795d8f2de2e844718cdb08835639c644365427b9f20f82083e7dac9a",
-            )
-            .unwrap(),
-            consensus_rpc: "http://testing.mainnet.beacon-api.nimbus.team".to_string(),
-            execution_rpc: None,
-            rpc_port: Some(8545),
         },
         forks: Forks {
             genesis: Fork {
@@ -33,26 +44,23 @@ pub fn mainnet() -> Config {
                 fork_version: hex_str_to_bytes("0x02000000").unwrap(),
             },
         },
-        machine: Machine { data_dir: None },
     }
 }
 
-pub fn goerli() -> Config {
+pub fn goerli() -> BaseConfig {
-    Config {
+    BaseConfig {
-        general: General {
+        checkpoint: hex_str_to_bytes(
+            "0x1e591af1e90f2db918b2a132991c7c2ee9a4ab26da496bd6e71e4f0bd65ea870",
+        )
+        .unwrap(),
+        rpc_port: 8545,
+        chain: ChainConfig {
             chain_id: 5,
             genesis_time: 1616508000,
             genesis_root: hex_str_to_bytes(
                 "0x043db0d9a83813551ee2f33450d23797757d430911a9320530ad8a0eabc43efb",
             )
             .unwrap(),
-            checkpoint: hex_str_to_bytes(
-                "0x1e591af1e90f2db918b2a132991c7c2ee9a4ab26da496bd6e71e4f0bd65ea870",
-            )
-            .unwrap(),
-            consensus_rpc: "http://34.207.158.131:5052".to_string(),
-            execution_rpc: None,
-            rpc_port: Some(8545),
         },
         forks: Forks {
             genesis: Fork {
@@ -68,6 +76,5 @@ pub fn goerli() -> Config {
                 fork_version: hex_str_to_bytes("0x02001020").unwrap(),
             },
         },
-        machine: Machine { data_dir: None },
     }
 }

consensus/src/consensus.rs

@@ -397,13 +397,7 @@ impl<R: Rpc> ConsensusClient<R> {
     }
 
     fn compute_committee_sign_root(&self, header: Bytes32, slot: u64) -> Result<Node> {
-        let genesis_root = self
+        let genesis_root = self.config.chain.genesis_root.to_vec().try_into().unwrap();
-            .config
-            .general
-            .genesis_root
-            .to_vec()
-            .try_into()
-            .unwrap();
 
         let domain_type = &hex::decode("07000000")?[..];
         let fork_version = Vector::from_iter(self.config.fork_version(slot));
@@ -425,14 +419,14 @@ impl<R: Rpc> ConsensusClient<R> {
             .duration_since(UNIX_EPOCH)
             .unwrap();
 
-        let genesis_time = self.config.general.genesis_time;
+        let genesis_time = self.config.chain.genesis_time;
         let since_genesis = now - std::time::Duration::from_secs(genesis_time);
 
         since_genesis.as_secs() / 12
     }
 
     fn slot_timestamp(&self, slot: u64) -> u64 {
-        slot * 12 + self.config.general.genesis_time
+        slot * 12 + self.config.chain.genesis_time
     }
 
     /// Gets the duration until the next update
@@ -646,14 +640,19 @@ mod tests {
         types::Header,
         ConsensusClient,
     };
-    use config::networks;
+    use config::{networks, Config};
 
     async fn get_client() -> ConsensusClient<MockRpc> {
-        ConsensusClient::new(
+        let base_config = networks::goerli();
-            "testdata/",
+        let config = Config {
-            &networks::goerli().general.checkpoint,
+            consensus_rpc: String::new(),
-            Arc::new(networks::goerli()),
+            execution_rpc: String::new(),
-        )
+            chain: base_config.chain,
+            forks: base_config.forks,
+            ..Default::default()
+        };
+
+        ConsensusClient::new("testdata/", &base_config.checkpoint, Arc::new(config))
             .await
             .unwrap()
     }

consensus/tests/sync.rs

@@ -1,14 +1,19 @@
 use std::sync::Arc;
 
-use config::networks;
+use config::{networks, Config};
 use consensus::{rpc::mock_rpc::MockRpc, ConsensusClient};
 
 async fn setup() -> ConsensusClient<MockRpc> {
-    ConsensusClient::new(
+    let base_config = networks::goerli();
-        "testdata/",
+    let config = Config {
-        &networks::goerli().general.checkpoint,
+        consensus_rpc: String::new(),
-        Arc::new(networks::goerli()),
+        execution_rpc: String::new(),
-    )
+        chain: base_config.chain,
+        forks: base_config.forks,
+        ..Default::default()
+    };
+
+    ConsensusClient::new("testdata/", &base_config.checkpoint, Arc::new(config))
         .await
         .unwrap()
 }