feat(validator): Web3Signer remote signing and the keymanager /eth/v1/remotekeys route

CLAUDE.md

@@ -849,7 +849,27 @@ transitions are in `ethlambda-types`, per the section above. Nothing above
   keeps EIP-3076 minimal-watermark slashing protection in
   `--validators-dir/slashing_protection.json` (`crate::slashing_protection`),
   durable before every block and attestation signature; PTC votes have an
-  in-memory `(validator, slot)` dedup only, and envelopes are unguarded.
+  in-memory `(validator, slot)` dedup only, and envelopes are unguarded. A key can live
+  in a Web3Signer instead of the process (`SigningMethod::Web3Signer`,
+  `keys/web3signer.rs`): every `SigningContext::sign_*` is async, resolves the
+  signer under a read guard scoped to the lookup and signs after releasing it,
+  and sends the remote signer the message in full so its own slashing
+  protection applies on top of this client's record, which a remote key is
+  registered in and checked against like a local one; the signature the
+  signer returns is verified against the pubkey and root before it is used,
+  on the blocking pool (`SigningContext::verified`), never inline on the duty
+  task. Each duty signs its batch at once through
+  `signing::sign_all`, every signature bounded by `SIGNING_BUDGET`, so one
+  hung signer costs only its own keys; one signer is sent at most
+  `MAX_CONCURRENT_REQUESTS` at a time. The epoch's subnet subscriptions, whose
+  selection proofs are round trips for remote keys, are sent from a spawned
+  task after the slot's proposal, not from `refresh_epoch`. Remote keys are
+  `type: web3signer`
+  entries in `validator_definitions.yml` or `POST /eth/v1/remotekeys` (default signer
+  `--web3signer-url`); `/keystores` and `/remotekeys` partition the store, and
+  both deletes go through `keystores::delete_keys` (one read and one save of
+  the definitions file per request, the store's write lock held only for the
+  in-memory removals).
   `get_head_node` is one bottom-up walk for every fork, with
   the current slot's fork selecting the payload rules and the boost gate, so a
   follower on any network that schedules gloas (Sepolia does) pays work

bin/ethlambda/src/validator.rs

@@ -8,6 +8,7 @@ use std::path::PathBuf;
 use ethlambda_types::beacon::primitives::{Bytes32, ExecutionAddress, H160};
 use ethlambda_validator::ValidatorConfig;
 use ethlambda_validator::proposer_settings::{GRAFFITI_BYTES, graffiti_from_text};
+use eyre::WrapErr as _;
 
 /// An execution address is exactly this wide.
 const ADDRESS_BYTES: usize = 20;
@@ -79,6 +80,14 @@ pub(crate) struct ValidatorOptions {
     #[arg(long, default_value = "false")]
     pub(crate) enable_keymanager: bool,
 
+    /// Base URL of a Web3Signer remote signer (e.g. https://signer:9000),
+    /// the default for remote keys imported through the keymanager API's
+    /// `POST /eth/v1/remotekeys` without a `url` of their own.
+    ///
+    /// Optional: a remote key may name its own signer in that request or in
+    /// `validator_definitions.yml`, and local keystores never need one.
+    #[arg(long)]
+    pub(crate) web3signer_url: Option<String>,
     /// Register every loaded key that has no slashing-protection record,
     /// instead of refusing to start.
     ///
@@ -155,6 +164,13 @@ impl ValidatorOptions {
 #[tokio::main]
 pub(crate) async fn run(options: ValidatorOptions) -> eyre::Result<()> {
     options.validate_ports()?;
+    // The same raise `run_node` makes, and for a like reason: with remote
+    // keys every signing request in flight is a socket (capped per signer,
+    // see `Web3Signer`), next to the beacon nodes', the metrics server's and
+    // the keymanager's. On a default soft limit `connect` starts failing
+    // with `EMFILE` once several signers are busy at once, and every
+    // signature those requests were for fails with it.
+    crate::fd_limit::raise_fd_limit().wrap_err("failed to raise RLIMIT_NOFILE")?;
     let graffiti = options.graffiti()?;
     let suggested_fee_recipient = options.suggested_fee_recipient()?;
     ethlambda_validator::run(ValidatorConfig {
@@ -167,6 +183,7 @@ pub(crate) async fn run(options: ValidatorOptions) -> eyre::Result<()> {
         keymanager: options
             .enable_keymanager
             .then(|| std::net::SocketAddr::new(options.http_address, options.keymanager_port)),
+        web3signer_url: options.web3signer_url,
         init_slashing_protection: options.init_slashing_protection,
     })
     .await
@@ -188,6 +205,7 @@ mod tests {
             graffiti: graffiti.to_string(),
             suggested_fee_recipient: None,
             enable_keymanager: false,
+            web3signer_url: None,
             init_slashing_protection: false,
         }
     }

crates/net/rpc/src/beacon/validator_client_tests.rs

@@ -238,7 +238,7 @@ async fn the_validator_client_can_exit_a_validator_through_this_node() {
         store
             .insert_secret("test", &secret_key_for(0).to_bytes())
             .expect("valid key");
-        store
+        RwLock::new(store)
     };
     let options = ExitOptions {
         no_confirmation: true,
@@ -286,7 +286,7 @@ async fn the_validator_client_can_exit_a_validator_through_this_node() {
         store
             .insert_secret("test", &secret_key_for(1).to_bytes())
             .expect("valid key");
-        store
+        RwLock::new(store)
     };
     let options = ExitOptions {
         epoch: Some(0),
@@ -329,7 +329,7 @@ async fn the_validator_client_follows_this_nodes_shard_committee_period() {
         store
             .insert_secret("test", &secret_key_for(0).to_bytes())
             .expect("valid key");
-        store
+        RwLock::new(store)
     };
     let options = ExitOptions {
         no_confirmation: true,
@@ -1033,10 +1033,7 @@ async fn serve_fulu_pinned_under(
         .spec()
         .await
         .expect("the client reads this node's spec");
-    let context = Arc::new(SigningContext {
-        config: spec,
-        genesis_validators_root: genesis.genesis_validators_root,
-    });
+    let context = Arc::new(SigningContext::new(spec, genesis.genesis_validators_root));
     (Arc::new(client), state, network, context)
 }
 
@@ -1333,10 +1330,7 @@ mod gloas {
             .spec()
             .await
             .expect("the client reads this node's spec");
-        let context = Arc::new(SigningContext {
-            config: spec,
-            genesis_validators_root: genesis.genesis_validators_root,
-        });
+        let context = Arc::new(SigningContext::new(spec, genesis.genesis_validators_root));
         Served {
             client,
             state,

crates/validator/Cargo.toml

@@ -32,6 +32,9 @@ tracing.workspace = true
 # so this crate still builds if it is ever compiled on its own.
 tokio = { workspace = true, features = ["time", "macros", "rt"] }
 reqwest = { workspace = true, features = ["json"] }
+# `join_all`, so a slot's attestations are signed concurrently: each one is a
+# round trip when a remote signer holds the key.
+futures = "0.3"
 
 # BLS12-381. Same version and backend the state transition verifies with, so a
 # signature produced here and one checked there come from one implementation.