LumeWeb
/
helios
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: improved configs (#64) 

* migrate config to figment

* support env variables

* add better error messages

* fix tests

* refactor
This commit is contained in:
Noah Citron 2022-10-05 13:52:07 -04:00 committed by GitHub
parent 5d1f4a6344
commit 8844f921e8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 325 additions and 200 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
Cargo.lock generated
View File

@ -97,6 +97,15 @@ dependencies = [
"rustc_version",
]
[[package]]
name = "atomic"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b88d82667eca772c4aa12f0f1348b3ae643424c8876448f3f7bd5787032e234c"
dependencies = [
"autocfg",
]
[[package]]
name = "atty"
version = "0.2.14"
@ -524,10 +533,11 @@ dependencies = [
"common",
"ethers",
"eyre",
"figment",
"hex",
"serde",
"ssz-rs",
"toml",
"thiserror",
]
[[package]]
@ -1108,6 +1118,20 @@ dependencies = [
"subtle",
]
[[package]]
name = "figment"
version = "0.10.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e3bd154d9ae2f1bb0ada5b7eebd56529513efa5de7d2fc8c6adf33bc43260cf"
dependencies = [
"atomic",
"pear",
"serde",
"toml",
"uncased",
"version_check",
]
[[package]]
name = "fixed-hash"
version = "0.5.2"
@ -1678,6 +1702,12 @@ dependencies = [
"hashbrown",
]
[[package]]
name = "inlinable_string"
version = "0.1.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb"
[[package]]
name = "inout"
version = "0.1.3"
@ -2339,6 +2369,29 @@ dependencies = [
"sha2 0.10.5",
]
[[package]]
name = "pear"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "15e44241c5e4c868e3eaa78b7c1848cadd6344ed4f54d029832d32b415a58702"
dependencies = [
"inlinable_string",
"pear_codegen",
"yansi",
]
[[package]]
name = "pear_codegen"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "82a5ca643c2303ecb740d506539deba189e16f2754040a42901cd8105d0282d0"
dependencies = [
"proc-macro2",
"proc-macro2-diagnostics",
"quote",
"syn",
]
[[package]]
name = "percent-encoding"
version = "2.1.0"
@ -2488,6 +2541,19 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "proc-macro2-diagnostics"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4bf29726d67464d49fa6224a1d07936a8c08bb3fba727c7493f6cf1616fdaada"
dependencies = [
"proc-macro2",
"quote",
"syn",
"version_check",
"yansi",
]
[[package]]
name = "quote"
version = "1.0.21"
@ -3551,6 +3617,15 @@ dependencies = [
"static_assertions",
]
[[package]]
name = "uncased"
version = "0.9.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09b01702b0fd0b3fadcf98e098780badda8742d4f4a7676615cad90e8ac73622"
dependencies = [
"version_check",
]
[[package]]
name = "unicase"
version = "2.6.0"
@ -3888,6 +3963,12 @@ dependencies = [
"tap",
]
[[package]]
name = "yansi"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec"
[[package]]
name = "zeroize"
version = "1.5.7"

2
cli/Cargo.toml
View File

@ -5,7 +5,7 @@ edition = "2021"
[dependencies]
tokio = { version = "1", features = ["full"] }
clap = { version = "3.2.18", features = ["derive"] }
clap = { version = "3.2.18", features = ["derive", "env"] }
eyre = "0.6.8"
dirs = "4.0.0"
env_logger = "0.9.0"

92
cli/src/main.rs
View File

@ -12,7 +12,7 @@ use env_logger::Env;
use eyre::Result;
use client::{database::FileDB, Client};
use config::{networks, Config};
use config::{CliConfig, Config};
use futures::executor::block_on;
use log::info;
@ -62,49 +62,48 @@ fn register_shutdown_handler(client: Client<FileDB>) {
fn get_config() -> Config {
let cli = Cli::parse();
let mut config = match cli.network.as_str() {
"mainnet" => networks::mainnet(),
"goerli" => networks::goerli(),
_ => {
let home = home_dir().unwrap();
let config_path = home.join(format!(".lightclient/configs/{}.toml", cli.network));
Config::from_file(&config_path).expect("could not read network config")
}
};
let data_dir = get_data_dir(&cli);
let config_path = home_dir().unwrap().join(".lightclient/lightclient.toml");
config.general.checkpoint = match cli.checkpoint {
Some(checkpoint) => hex_str_to_bytes(&checkpoint).expect("invalid checkpoint"),
None => get_cached_checkpoint(&data_dir).unwrap_or(config.general.checkpoint),
};
let cli_config = cli.as_cli_config();
config.general.execution_rpc = Some(cli.execution_rpc);
if let Some(port) = cli.port {
config.general.rpc_port = Some(port);
}
if let Some(consensus_rpc) = cli.consensus_rpc {
config.general.consensus_rpc = consensus_rpc;
}
config.machine.data_dir = Some(data_dir);
config
Config::from_file(&config_path, &cli.network, &cli_config)
}
fn get_data_dir(cli: &Cli) -> PathBuf {
match &cli.data_dir {
Some(dir) => PathBuf::from(dir),
None => home_dir()
.unwrap()
.join(format!(".lightclient/data/{}", cli.network)),
}
#[derive(Parser)]
struct Cli {
#[clap(short, long, default_value = "mainnet")]
network: String,
#[clap(short, long, env)]
port: Option<u16>,
#[clap(short = 'w', long, env)]
checkpoint: Option<String>,
#[clap(short, long, env)]
execution_rpc: Option<String>,
#[clap(short, long, env)]
consensus_rpc: Option<String>,
}
fn get_cached_checkpoint(data_dir: &PathBuf) -> Option<Vec<u8>> {
impl Cli {
fn as_cli_config(&self) -> CliConfig {
let checkpoint = match &self.checkpoint {
Some(checkpoint) => Some(hex_str_to_bytes(&checkpoint).expect("invalid checkpoint")),
None => self.get_cached_checkpoint(),
};
CliConfig {
checkpoint,
execution_rpc: self.execution_rpc.clone(),
consensus_rpc: self.consensus_rpc.clone(),
data_dir: self.get_data_dir(),
port: self.port,
}
}
fn get_cached_checkpoint(&self) -> Option<Vec<u8>> {
let data_dir = self.get_data_dir();
let checkpoint_file = data_dir.join("checkpoint");
if checkpoint_file.exists() {
let checkpoint_res = fs::read(checkpoint_file);
match checkpoint_res {
@ -114,20 +113,11 @@ fn get_cached_checkpoint(data_dir: &PathBuf) -> Option<Vec<u8>> {
} else {
None
}
}
}
#[derive(Parser)]
struct Cli {
#[clap(short, long, default_value = "mainnet")]
network: String,
#[clap(short, long)]
port: Option<u16>,
#[clap(short = 'w', long)]
checkpoint: Option<String>,
#[clap(short, long)]
execution_rpc: String,
#[clap(short, long)]
consensus_rpc: Option<String>,
#[clap(long)]
data_dir: Option<String>,
fn get_data_dir(&self) -> PathBuf {
home_dir()
.unwrap()
.join(format!(".lightclient/data/{}", self.network))
}
}

4
client/src/client.rs
View File

@ -29,13 +29,13 @@ impl Client<FileDB> {
let node = Node::new(config.clone()).await?;
let node = Arc::new(RwLock::new(node));
let rpc = if let Some(port) = config.general.rpc_port {
let rpc = if let Some(port) = config.rpc_port {
Some(Rpc::new(node.clone(), port))
} else {
None
};
let data_dir = config.machine.data_dir.clone();
let data_dir = config.data_dir.clone();
let db = FileDB::new(data_dir.ok_or(eyre!("data dir not found"))?);
Ok(Client { node, rpc, db })

10
client/src/node.rs
View File

@ -28,13 +28,13 @@ pub struct Node {
impl Node {
pub async fn new(config: Arc<Config>) -> Result<Self> {
let consensus_rpc = &config.general.consensus_rpc;
let checkpoint_hash = &config.general.checkpoint;
let execution_rpc = &config.general.execution_rpc;
let consensus_rpc = &config.consensus_rpc;
let checkpoint_hash = &config.checkpoint;
let execution_rpc = &config.execution_rpc;
let consensus =
ConsensusClient::new(consensus_rpc, checkpoint_hash, config.clone()).await?;
let execution = ExecutionClient::new(execution_rpc.as_ref().unwrap())?;
let execution = ExecutionClient::new(execution_rpc)?;
let payloads = BTreeMap::new();
let finalized_payloads = BTreeMap::new();
@ -216,7 +216,7 @@ impl Node {
}
pub fn chain_id(&self) -> u64 {
self.config.general.chain_id
self.config.chain.chain_id
}
pub fn get_header(&self) -> Result<Header> {

3
config/Cargo.toml
View File

@ -12,6 +12,7 @@ serde = { version = "1.0.143", features = ["derive"] }
hex = "0.4.3"
ssz-rs = { git = "https://github.com/ralexstokes/ssz-rs" }
ethers = "0.17.0"
toml = "0.5.9"
figment = { version = "0.10.7", features = ["toml", "env"] }
thiserror = "1.0.37"
common = { path = "../common" }

22
config/configs/goerli.toml
View File

@ -1,22 +0,0 @@
[general]
chain_id = 5
genesis_time = 1616508000
genesis_root = "0x043db0d9a83813551ee2f33450d23797757d430911a9320530ad8a0eabc43efb"
checkpoint = "0x172128eadf1da46467f4d6a822206698e2d3f957af117dd650954780d680dc99"
consensus_rpc = "http://testing.prater.beacon-api.nimbus.team"
execution_rpc = "https://eth-goerli.g.alchemy.com:443/v2/o_8Qa9kgwDPf9G8sroyQ-uQtyhyWa3ao"
rpc_port = 8545
[forks]
[forks.genesis]
epoch = 0
fork_version = "0x00001020"
[forks.altair]
epoch = 36660
fork_version = "0x01001020"
[forks.bellatrix]
epoch = 112260
fork_version = "0x02001020"

22
config/configs/mainnet.toml
View File

@ -1,22 +0,0 @@
[general]
chain_id = 1
genesis_time = 1606824023
genesis_root = "0x4b363db94e286120d76eb905340fdd4e54bfe9f06bf33ff6cf5ad27f511bfe95"
checkpoint = "0x03e315e11b3f88cd63dfb62c74a313c4a65949ce9e37599e0ee66533ceceadfd"
consensus_rpc = "http://testing.mainnet.beacon-api.nimbus.team"
execution_rpc = "https://eth-mainnet.g.alchemy.com/v2/Q0BqQPbTQfSMzrCNl4x80XS_PLLB1RNf"
rpc_port = 8545
[forks]
[forks.genesis]
epoch = 0
fork_version = "0x00000000"
[forks.altair]
epoch = 74240
fork_version = "0x01000000"
[forks.bellatrix]
epoch = 144896
fork_version = "0x02000000"

168
config/src/lib.rs
View File

@ -1,58 +1,73 @@
pub mod networks;
use std::{
fs,
path::{Path, PathBuf},
};
use std::{collections::HashMap, path::PathBuf, process::exit};
use eyre::Result;
use serde::Deserialize;
use figment::{
providers::{Format, Serialized, Toml},
value::Value,
Figment,
};
use networks::BaseConfig;
use serde::{Deserialize, Serialize};
use common::utils::hex_str_to_bytes;
#[derive(Deserialize, Debug)]
#[derive(Serialize, Deserialize, Debug, Default)]
pub struct Config {
pub general: General,
pub forks: Forks,
pub machine: Machine,
}
#[derive(Deserialize, Debug)]
pub struct General {
pub chain_id: u64,
pub genesis_time: u64,
#[serde(deserialize_with = "bytes_deserialize")]
pub genesis_root: Vec<u8>,
#[serde(deserialize_with = "bytes_deserialize")]
pub checkpoint: Vec<u8>,
pub consensus_rpc: String,
pub execution_rpc: Option<String>,
pub execution_rpc: String,
pub rpc_port: Option<u16>,
}
#[derive(Deserialize, Debug)]
pub struct Forks {
pub genesis: Fork,
pub altair: Fork,
pub bellatrix: Fork,
}
#[derive(Deserialize, Debug)]
pub struct Fork {
pub epoch: u64,
#[serde(deserialize_with = "bytes_deserialize")]
pub fork_version: Vec<u8>,
}
#[derive(Deserialize, Debug)]
pub struct Machine {
#[serde(
deserialize_with = "bytes_deserialize",
serialize_with = "bytes_serialize"
)]
pub checkpoint: Vec<u8>,
pub data_dir: Option<PathBuf>,
pub chain: ChainConfig,
pub forks: Forks,
}
impl Config {
pub fn from_file(path: &Path) -> Result<Self> {
let contents = fs::read_to_string(path)?;
Ok(toml::from_str(&contents)?)
pub fn from_file(config_path: &PathBuf, network: &str, cli_config: &CliConfig) -> Self {
let base_config = match network {
"mainnet" => networks::mainnet(),
"goerli" => networks::goerli(),
_ => BaseConfig::default(),
};
let base_provider = Serialized::from(base_config, network);
let toml_provider = Toml::file(config_path).nested();
let user_provider = cli_config.as_provider(network);
let config_res = Figment::new()
.merge(base_provider)
.merge(toml_provider)
.merge(user_provider)
.select(network)
.extract();
match config_res {
Ok(config) => config,
Err(err) => {
match err.kind {
figment::error::Kind::MissingField(field) => {
println!(
"\x1b[91merror\x1b[0m: missing configuration field: {}",
field
);
println!(
"\n\ttry supplying the propoper command line argument: --{}",
field
);
println!("\talternatively, you can add the field to your lightclient.toml file or as an environment variable");
println!("\nfor more information, check the github README");
}
_ => println!("cannot parse configuration: {}", err),
}
exit(1);
}
}
}
pub fn fork_version(&self, slot: u64) -> Vec<u8> {
@ -68,6 +83,69 @@ impl Config {
}
}
#[derive(Serialize)]
pub struct CliConfig {
pub execution_rpc: Option<String>,
pub consensus_rpc: Option<String>,
pub checkpoint: Option<Vec<u8>>,
pub port: Option<u16>,
pub data_dir: PathBuf,
}
impl CliConfig {
fn as_provider(&self, network: &str) -> Serialized<HashMap<&str, Value>> {
let mut user_dict = HashMap::new();
if let Some(rpc) = &self.execution_rpc {
user_dict.insert("execution_rpc", Value::from(rpc.clone()));
}
if let Some(rpc) = &self.consensus_rpc {
user_dict.insert("consensus_rpc", Value::from(rpc.clone()));
}
if let Some(checkpoint) = &self.checkpoint {
user_dict.insert("checkpoint", Value::from(hex::encode(checkpoint)));
}
if let Some(port) = self.port {
user_dict.insert("port", Value::from(port));
}
user_dict.insert("data_dir", Value::from(self.data_dir.to_str().unwrap()));
Serialized::from(user_dict, network)
}
}
#[derive(Serialize, Deserialize, Debug, Default)]
pub struct ChainConfig {
pub chain_id: u64,
pub genesis_time: u64,
#[serde(
deserialize_with = "bytes_deserialize",
serialize_with = "bytes_serialize"
)]
pub genesis_root: Vec<u8>,
}
#[derive(Serialize, Deserialize, Debug, Default)]
pub struct Forks {
pub genesis: Fork,
pub altair: Fork,
pub bellatrix: Fork,
}
#[derive(Serialize, Deserialize, Debug, Default)]
pub struct Fork {
pub epoch: u64,
#[serde(
deserialize_with = "bytes_deserialize",
serialize_with = "bytes_serialize"
)]
pub fork_version: Vec<u8>,
}
fn bytes_deserialize<'de, D>(deserializer: D) -> Result<Vec<u8>, D::Error>
where
D: serde::Deserializer<'de>,
@ -75,3 +153,11 @@ where
let bytes: String = serde::Deserialize::deserialize(deserializer)?;
Ok(hex_str_to_bytes(&bytes).unwrap())
}
fn bytes_serialize<S>(bytes: &Vec<u8>, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
let bytes_string = hex::encode(bytes);
serializer.serialize_str(&bytes_string)
}

53
config/src/networks.rs
View File

@ -1,23 +1,34 @@
use serde::Serialize;
use crate::{bytes_serialize, ChainConfig, Fork, Forks};
use common::utils::hex_str_to_bytes;
use crate::{Config, Fork, Forks, General, Machine};
#[derive(Serialize, Default)]
pub struct BaseConfig {
rpc_port: u16,
#[serde(
deserialize_with = "bytes_deserialize",
serialize_with = "bytes_serialize"
)]
pub checkpoint: Vec<u8>,
pub chain: ChainConfig,
pub forks: Forks,
}
pub fn mainnet() -> Config {
Config {
general: General {
pub fn mainnet() -> BaseConfig {
BaseConfig {
checkpoint: hex_str_to_bytes(
"0x5ca31c7c795d8f2de2e844718cdb08835639c644365427b9f20f82083e7dac9a",
)
.unwrap(),
rpc_port: 8545,
chain: ChainConfig {
chain_id: 1,
genesis_time: 1606824023,
genesis_root: hex_str_to_bytes(
"0x4b363db94e286120d76eb905340fdd4e54bfe9f06bf33ff6cf5ad27f511bfe95",
)
.unwrap(),
checkpoint: hex_str_to_bytes(
"0x5ca31c7c795d8f2de2e844718cdb08835639c644365427b9f20f82083e7dac9a",
)
.unwrap(),
consensus_rpc: "http://testing.mainnet.beacon-api.nimbus.team".to_string(),
execution_rpc: None,
rpc_port: Some(8545),
},
forks: Forks {
genesis: Fork {
@ -33,26 +44,23 @@ pub fn mainnet() -> Config {
fork_version: hex_str_to_bytes("0x02000000").unwrap(),
},
},
machine: Machine { data_dir: None },
}
}
pub fn goerli() -> Config {
Config {
general: General {
pub fn goerli() -> BaseConfig {
BaseConfig {
checkpoint: hex_str_to_bytes(
"0x1e591af1e90f2db918b2a132991c7c2ee9a4ab26da496bd6e71e4f0bd65ea870",
)
.unwrap(),
rpc_port: 8545,
chain: ChainConfig {
chain_id: 5,
genesis_time: 1616508000,
genesis_root: hex_str_to_bytes(
"0x043db0d9a83813551ee2f33450d23797757d430911a9320530ad8a0eabc43efb",
)
.unwrap(),
checkpoint: hex_str_to_bytes(
"0x1e591af1e90f2db918b2a132991c7c2ee9a4ab26da496bd6e71e4f0bd65ea870",
)
.unwrap(),
consensus_rpc: "http://34.207.158.131:5052".to_string(),
execution_rpc: None,
rpc_port: Some(8545),
},
forks: Forks {
genesis: Fork {
@ -68,6 +76,5 @@ pub fn goerli() -> Config {
fork_version: hex_str_to_bytes("0x02001020").unwrap(),
},
},
machine: Machine { data_dir: None },
}
}

29
consensus/src/consensus.rs
View File

@ -397,13 +397,7 @@ impl<R: Rpc> ConsensusClient<R> {
}
fn compute_committee_sign_root(&self, header: Bytes32, slot: u64) -> Result<Node> {
let genesis_root = self
.config
.general
.genesis_root
.to_vec()
.try_into()
.unwrap();
let genesis_root = self.config.chain.genesis_root.to_vec().try_into().unwrap();
let domain_type = &hex::decode("07000000")?[..];
let fork_version = Vector::from_iter(self.config.fork_version(slot));
@ -425,14 +419,14 @@ impl<R: Rpc> ConsensusClient<R> {
.duration_since(UNIX_EPOCH)
.unwrap();
let genesis_time = self.config.general.genesis_time;
let genesis_time = self.config.chain.genesis_time;
let since_genesis = now - std::time::Duration::from_secs(genesis_time);
since_genesis.as_secs() / 12
}
fn slot_timestamp(&self, slot: u64) -> u64 {
slot * 12 + self.config.general.genesis_time
slot * 12 + self.config.chain.genesis_time
}
/// Gets the duration until the next update
@ -646,14 +640,19 @@ mod tests {
types::Header,
ConsensusClient,
};
use config::networks;
use config::{networks, Config};
async fn get_client() -> ConsensusClient<MockRpc> {
ConsensusClient::new(
"testdata/",
&networks::goerli().general.checkpoint,
Arc::new(networks::goerli()),
)
let base_config = networks::goerli();
let config = Config {
consensus_rpc: String::new(),
execution_rpc: String::new(),
chain: base_config.chain,
forks: base_config.forks,
..Default::default()
};
ConsensusClient::new("testdata/", &base_config.checkpoint, Arc::new(config))
.await
.unwrap()
}

17
consensus/tests/sync.rs
View File

@ -1,14 +1,19 @@
use std::sync::Arc;
use config::networks;
use config::{networks, Config};
use consensus::{rpc::mock_rpc::MockRpc, ConsensusClient};
async fn setup() -> ConsensusClient<MockRpc> {
ConsensusClient::new(
"testdata/",
&networks::goerli().general.checkpoint,
Arc::new(networks::goerli()),
)
let base_config = networks::goerli();
let config = Config {
consensus_rpc: String::new(),
execution_rpc: String::new(),
chain: base_config.chain,
forks: base_config.forks,
..Default::default()
};
ConsensusClient::new("testdata/", &base_config.checkpoint, Arc::new(config))
.await
.unwrap()
}