From e7f89204631cc09a0de3b9d8895663c5502c006d Mon Sep 17 00:00:00 2001 From: panos-xyz Date: Mon, 13 Jul 2026 21:48:22 +0800 Subject: [PATCH 01/12] refactor: migrate engine validation to reth main hooks --- Cargo.lock | 1624 ++++++------ Cargo.toml | 170 +- crates/chainspec/src/hardfork.rs | 11 +- crates/consensus/src/validation.rs | 5 +- crates/engine-api/src/builder.rs | 3 +- crates/engine-tree-ext/Cargo.toml | 80 - crates/engine-tree-ext/src/gate.rs | 67 - crates/engine-tree-ext/src/lib.rs | 13 - .../engine-tree-ext/src/payload_validator.rs | 2193 ----------------- crates/engine-tree-ext/src/trie_updates.rs | 357 --- crates/engine-tree-ext/tests/jade_boundary.rs | 212 -- crates/evm/Cargo.toml | 1 + crates/evm/src/block/mod.rs | 16 +- crates/evm/src/context.rs | 3 +- crates/evm/src/evm.rs | 5 +- crates/node/Cargo.toml | 6 +- crates/node/build.rs | 15 +- crates/node/src/node.rs | 1 + crates/node/src/test_utils.rs | 4 +- crates/node/src/validator.rs | 502 +++- crates/node/tests/it/engine.rs | 40 +- crates/node/tests/it/helpers.rs | 11 +- crates/payload/builder/src/builder.rs | 33 +- crates/payload/types/src/attributes.rs | 1 + crates/payload/types/src/lib.rs | 9 +- crates/revm/src/evm.rs | 73 +- crates/revm/src/precompiles.rs | 5 +- crates/rpc/src/eth/call.rs | 4 + crates/rpc/src/eth/mod.rs | 25 +- 29 files changed, 1494 insertions(+), 3995 deletions(-) delete mode 100644 crates/engine-tree-ext/Cargo.toml delete mode 100644 crates/engine-tree-ext/src/gate.rs delete mode 100644 crates/engine-tree-ext/src/lib.rs delete mode 100644 crates/engine-tree-ext/src/payload_validator.rs delete mode 100644 crates/engine-tree-ext/src/trie_updates.rs delete mode 100644 crates/engine-tree-ext/tests/jade_boundary.rs diff --git a/Cargo.lock b/Cargo.lock index 5e07217b..52398dd5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,7 +14,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "generic-array", ] @@ -101,14 +101,14 @@ dependencies = [ [[package]] name = "alloy-consensus" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d64da86c616b5092ea64eea648f311bbd58630a0b384c42d699175d6f9122b" +checksum = "a6ff0c4adba2abdcd9fb5829ae5f4394c06f8585ed283a9ba79aa33763c802e1" dependencies = [ - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rlp", - "alloy-serde 2.0.4", + "alloy-serde", "alloy-trie", "alloy-tx-macros", "arbitrary", @@ -129,24 +129,24 @@ dependencies = [ [[package]] name = "alloy-consensus-any" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd98696ca3617d3a9ba1a6f2011880cbfd5618228dab6400c9f8bca457859a8" +checksum = "7cdf48932b1db3216175e19a2b476d89d53076e004850ee7983c6807ba0fde74" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rlp", - "alloy-serde 2.0.4", + "alloy-serde", "arbitrary", "serde", ] [[package]] name = "alloy-dyn-abi" -version = "1.5.7" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc2db5c583aaef0255aa63a4fe827f826090142528bba48d1bf4119b62780cad" +checksum = "a475bb02d9cef2dbb99065c1664ab3fe1f9352e21d6d5ed3f02cdbfc06ed1abc" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -156,7 +156,7 @@ dependencies = [ "itoa", "serde", "serde_json", - "winnow 0.7.15", + "winnow 1.0.2", ] [[package]] @@ -207,9 +207,9 @@ dependencies = [ [[package]] name = "alloy-eip7928" -version = "0.3.6" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72bc95fd73591644f0022065bef3c027d1a5ed875a53cb19a898d71a00818c1d" +checksum = "b3b12337f74cbfa451cb04dac173974814a6ff463079e1793aa09600ba8813ab" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -222,32 +222,9 @@ dependencies = [ [[package]] name = "alloy-eips" -version = "1.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6ef28c9fdad22d4eec52d894f5f2673a0895f1e5ef196734568e68c0f6caca8" -dependencies = [ - "alloy-eip2124", - "alloy-eip2930", - "alloy-eip7702", - "alloy-eip7928", - "alloy-primitives", - "alloy-rlp", - "alloy-serde 1.8.3", - "auto_impl", - "borsh", - "c-kzg", - "derive_more", - "either", - "serde", - "serde_with", - "sha2", -] - -[[package]] -name = "alloy-eips" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64c0456f5f7a4497e9342d20f528e30f5288ddfa0d6a012bd5044afee46cd8a0" +checksum = "ea4c0453065b9206acc0f869a258dc8dcbbd595e144b4446f2c493a24a814d1f" dependencies = [ "alloy-eip2124", "alloy-eip2930", @@ -255,7 +232,7 @@ dependencies = [ "alloy-eip7928", "alloy-primitives", "alloy-rlp", - "alloy-serde 2.0.4", + "alloy-serde", "arbitrary", "auto_impl", "borsh", @@ -271,12 +248,12 @@ dependencies = [ [[package]] name = "alloy-evm" -version = "0.34.0" +version = "0.37.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1ceeea6dcbbcd4e546b27700763a6f6c3b3fee30054209884f521078b6fda4f" +checksum = "acde665074b478c97047dc2a461b4916cac77922d690e5b7415d1941ae7ff7bf" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-hardforks", "alloy-primitives", "alloy-rpc-types-engine", @@ -291,13 +268,13 @@ dependencies = [ [[package]] name = "alloy-genesis" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a71ff8b55d2b8aa05259f474cae7dea0e4991724dc18936b81cb23ec492a0c2a" +checksum = "027ba57264c5d05e4ff52e6090b3592e7526f5d5f5a844add6bbdd17c071c911" dependencies = [ - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", - "alloy-serde 2.0.4", + "alloy-serde", "alloy-trie", "borsh", "serde", @@ -320,9 +297,9 @@ dependencies = [ [[package]] name = "alloy-json-abi" -version = "1.5.7" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9dbe713da0c737d9e5e387b0ba790eb98b14dd207fe53eef50e19a5a8ec3dac" +checksum = "7c36c9d7f9021601b04bfef14a4b64849f6d73116a4e91e071d7fbfe10247901" dependencies = [ "alloy-primitives", "alloy-sol-type-parser", @@ -332,9 +309,9 @@ dependencies = [ [[package]] name = "alloy-json-rpc" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19e352478b756bad5d7203148e4b461861282ea2ded3da406ba24868b52cd098" +checksum = "c4691c60de5d628533752cd07e102d17c47874c06c04c91af33960fd94c484f4" dependencies = [ "alloy-primitives", "alloy-sol-types", @@ -347,19 +324,19 @@ dependencies = [ [[package]] name = "alloy-network" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed08ae169869e08370ed121612e0d3dadac33d1a256e9f2465926b23f0bd7d95" +checksum = "9d912bb639bf4ac31e83095afe9e907c8b9774ce0c405966228368309fcfc45f" dependencies = [ "alloy-consensus", "alloy-consensus-any", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-json-rpc", "alloy-network-primitives", "alloy-primitives", "alloy-rpc-types-any", "alloy-rpc-types-eth", - "alloy-serde 2.0.4", + "alloy-serde", "alloy-signer", "alloy-sol-types", "async-trait", @@ -373,22 +350,22 @@ dependencies = [ [[package]] name = "alloy-network-primitives" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e6c7ad28afe348a9a9c5624b67ee5b3607b8de98d5816b3056ecdfa6fa2697" +checksum = "d875a11bd98595f57f73890f2e36f4a1cca0fa670623e59c9fb08b2389979c36" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", - "alloy-serde 2.0.4", + "alloy-serde", "serde", ] [[package]] name = "alloy-primitives" -version = "1.5.7" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3b431b4e72cd8bd0ec7a50b4be18e73dab74de0dba180eef171055e5d5926e" +checksum = "4885c1409b6936c4898e646ef58baf6ec54edaf6d8179f79df805a7b85b7cf3e" dependencies = [ "alloy-rlp", "arbitrary", @@ -398,31 +375,32 @@ dependencies = [ "derive_more", "foldhash 0.2.0", "getrandom 0.4.2", - "hashbrown 0.16.1", + "hashbrown 0.17.0", "indexmap 2.14.0", "itoa", "k256", "keccak-asm", "paste", "proptest", - "proptest-derive", + "proptest-derive 0.8.0", "rand 0.9.4", "rapidhash", "ruint", "rustc-hash", + "secp256k1 0.31.1", "serde", - "sha3", + "sha3 0.11.0", ] [[package]] name = "alloy-provider" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93a7c17472b55482d4734154c2f5ed13f72e03f6752cebb927f6a2d8b52e646c" +checksum = "0e7d526d184d392a8fbcf4293e457789b307bb70646e4f8b902989a246529450" dependencies = [ "alloy-chains", "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-json-rpc", "alloy-network", "alloy-network-primitives", @@ -448,7 +426,7 @@ dependencies = [ "lru", "parking_lot", "pin-project", - "reqwest 0.13.3", + "reqwest", "serde", "serde_json", "thiserror 2.0.18", @@ -460,9 +438,9 @@ dependencies = [ [[package]] name = "alloy-pubsub" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8d86958b02bca85103d64fa60d7b364a8b017c6e40f2b02c3f50ca22964a738" +checksum = "6fa5976a77e32a63152e937fa90d0dae1f8142b41a9a99a6386d6ea58934cfa6" dependencies = [ "alloy-json-rpc", "alloy-primitives", @@ -482,9 +460,9 @@ dependencies = [ [[package]] name = "alloy-rlp" -version = "0.3.15" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc90b1e703d3c03f4ff7f48e82dd0bc1c8211ab7d079cd836a06fcfeb06651cb" +checksum = "24671b1f62edcf0f9b62994c7bf72cd621a04a4b99f5020ece1a647b40e2f103" dependencies = [ "alloy-rlp-derive", "arrayvec", @@ -493,9 +471,9 @@ dependencies = [ [[package]] name = "alloy-rlp-derive" -version = "0.3.15" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f36834a5c0a2fa56e171bf256c34d70fca07d0c0031583edea1c4946b7889c9e" +checksum = "9d4311c03125e8a18296504560b9de3d75ecbd0dcda7f71e6cf2a196d57e6fba" dependencies = [ "proc-macro2", "quote", @@ -504,9 +482,9 @@ dependencies = [ [[package]] name = "alloy-rpc-client" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5beb5c2fe6b960c8e8b038e69fd502a90a2e930afa4770efb748b163b0767729" +checksum = "755447dc13a04c6fa6db8dedb5d32ab5edfa4dd7aa34487ff192a3d873e0e8cf" dependencies = [ "alloy-json-rpc", "alloy-primitives", @@ -517,7 +495,7 @@ dependencies = [ "alloy-transport-ws", "futures", "pin-project", - "reqwest 0.13.3", + "reqwest", "serde", "serde_json", "tokio", @@ -530,22 +508,22 @@ dependencies = [ [[package]] name = "alloy-rpc-types" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ee1257a278f6d293e05c5162c5940a1561b1aa85ded0028b464c81de37ebfa5" +checksum = "96deb317fe224e98a8ae17a7ed7ea63478867385bf50b7abd53642b24cfd811a" dependencies = [ "alloy-primitives", "alloy-rpc-types-engine", "alloy-rpc-types-eth", - "alloy-serde 2.0.4", + "alloy-serde", "serde", ] [[package]] name = "alloy-rpc-types-admin" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2144d5b2866e651796eac0a997d3b5a056449c12e0d91be3184129e0c722885" +checksum = "3279cb55f7069c2fb1dbcd9ab2c6e79a02d63c92fd0b850addea0a3715d6b05f" dependencies = [ "alloy-genesis", "alloy-primitives", @@ -555,38 +533,38 @@ dependencies = [ [[package]] name = "alloy-rpc-types-anvil" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df32156f085e74eac942b6103744be49b817c302341aaa8cb0c1c88dc29228d9" +checksum = "9f89d27756bf3effdac25d07692724e840ce90ca1ff956a6b47dd2ae1612f597" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", - "alloy-serde 2.0.4", + "alloy-serde", "serde", ] [[package]] name = "alloy-rpc-types-any" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a234bfbdf7a76c3d13808f729af5321852de3dedcaa6fc6d5f54787aaf54c6a" +checksum = "911a513723ef0b90c3b072f65eebf54d6ebc8b651d06734e252d024bd89b88ac" dependencies = [ "alloy-consensus-any", "alloy-network-primitives", "alloy-primitives", "alloy-rpc-types-eth", - "alloy-serde 2.0.4", + "alloy-serde", "serde", "serde_json", ] [[package]] name = "alloy-rpc-types-beacon" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "296450f5e76bece0116c939b9437b0421a5da9c5d40031bf4cf9b38d3d94e475" +checksum = "81133671b8da58169589aac996f3c4da23bbdee85b13ecee7098065f3eae718a" dependencies = [ - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rpc-types-engine", "derive_more", @@ -602,9 +580,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-debug" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ab075ac1c25bcf697f133b7cd92e2fb26afe213e872ef79fdf77f0d7bcb3793" +checksum = "b9e9c1f9ac81c56b2d96901201db7eb95c4e9fc3c21237a4690f8c652aa62089" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -615,15 +593,15 @@ dependencies = [ [[package]] name = "alloy-rpc-types-engine" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73b12366c96f4013e1aeebc96c6b56e5f33f07853c42ea2f485045c0c157a4a1" +checksum = "5b1f682c4881aa7000ac7cc86d7aeeb3b09836a77a90b329820140233651aeea" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rlp", - "alloy-serde 2.0.4", + "alloy-serde", "derive_more", "ethereum_ssz", "ethereum_ssz_derive", @@ -635,17 +613,17 @@ dependencies = [ [[package]] name = "alloy-rpc-types-eth" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56a282daf869eeb7383d3d5c2deb35b0b3fb45ecb329513af4090fc61245ee18" +checksum = "4e1bd19904581ee2075b61faabab1a4cefa8b96d8f2c85343adeae8ecf615e2b" dependencies = [ "alloy-consensus", "alloy-consensus-any", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-network-primitives", "alloy-primitives", "alloy-rlp", - "alloy-serde 2.0.4", + "alloy-serde", "alloy-sol-types", "arbitrary", "itertools 0.14.0", @@ -657,28 +635,28 @@ dependencies = [ [[package]] name = "alloy-rpc-types-mev" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7adc1243d55744a66b3a6cbbbba96436e8df5d248f2ee8186bef4238ef704ec7" +checksum = "54c635f0c45a871b55c276653e3838d1687d86282eeaf3d83a1914e02563b3f2" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rpc-types-eth", - "alloy-serde 2.0.4", + "alloy-serde", "serde", "serde_json", ] [[package]] name = "alloy-rpc-types-trace" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6184b5d14152b68b0bb8beb621339d94f0b761a37958bb365fbf7c00922125c2" +checksum = "796729a4e1783904c6040f11a503c4d55ddb16f69dc199ad15e50c4ad039f371" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", - "alloy-serde 2.0.4", + "alloy-serde", "serde", "serde_json", "thiserror 2.0.18", @@ -686,32 +664,21 @@ dependencies = [ [[package]] name = "alloy-rpc-types-txpool" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f00b631c361e7c7baaf4f1f5a9877730f3507fed2acb9d4b34841b8184b2ec28" +checksum = "8533c450895de79eb581875bfc317d1a239ae71aba79e20ee158fa153268f163" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", - "alloy-serde 2.0.4", + "alloy-serde", "serde", ] [[package]] name = "alloy-serde" -version = "1.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11ece63b89294b8614ab3f483560c08d016930f842bf36da56bf0b764a15c11e" -dependencies = [ - "alloy-primitives", - "serde", - "serde_json", -] - -[[package]] -name = "alloy-serde" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0eada2558e921b39dfcead33c487364df9b31374f5733c1c9d2c891c4529933" +checksum = "c1e97b3e0b9f816b25083045dcfa69431bd059a078e828e4d82d296d1949b96c" dependencies = [ "alloy-primitives", "arbitrary", @@ -721,9 +688,9 @@ dependencies = [ [[package]] name = "alloy-signer" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41eb29f7a8adcd8941fbb8e134022a133e6f8dfd345f2e3b7109599f8a7dca08" +checksum = "6913d06ccc0d9c6ab67e2f82e2fbe292706da1f91442f30cded2d04b56bf0d58" dependencies = [ "alloy-primitives", "async-trait", @@ -736,9 +703,9 @@ dependencies = [ [[package]] name = "alloy-signer-local" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef839e7ce9b59aa60fa9a175e97986c6145c888d643b0f1fb0a3e7b8e56a2e2" +checksum = "c880813cd26bd1ddf8c2236e31295896efd1fcc5cc761d8894ae894503aeea53" dependencies = [ "alloy-consensus", "alloy-network", @@ -755,9 +722,9 @@ dependencies = [ [[package]] name = "alloy-sol-macro" -version = "1.5.7" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab81bab693da9bb79f7a95b64b394718259fdd7e41dceeced4cad57cb71c4f6a" +checksum = "840128ed2b2971d6d4668a553fe403a82683d3acc646c73e75887e7157408033" dependencies = [ "alloy-sol-macro-expander", "alloy-sol-macro-input", @@ -769,9 +736,9 @@ dependencies = [ [[package]] name = "alloy-sol-macro-expander" -version = "1.5.7" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "489f1620bb7e2483fb5819ed01ab6edc1d2f93939dce35a5695085a1afd1d699" +checksum = "63ec265e5d65d725175f6ca7711c970824c90ef9c0d1f1973711d4150ee612dd" dependencies = [ "alloy-sol-macro-input", "const-hex", @@ -780,16 +747,16 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "sha3", + "sha3 0.11.0", "syn 2.0.117", "syn-solidity", ] [[package]] name = "alloy-sol-macro-input" -version = "1.5.7" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56cef806ad22d4392c5fc83cf8f2089f988eb99c7067b4e0c6f1971fc1cca318" +checksum = "89bf01077f18650876cfa682eb1f949967b5cde03f1a51c955c469d2c9b4aa67" dependencies = [ "const-hex", "dunce", @@ -803,19 +770,19 @@ dependencies = [ [[package]] name = "alloy-sol-type-parser" -version = "1.5.7" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6df77fea9d6a2a75c0ef8d2acbdfd92286cc599983d3175ccdc170d3433d249" +checksum = "857b470ecdd2ed38beaf82ad1a38c516a8ff75266750f38b9eeed001d575241b" dependencies = [ "serde", - "winnow 0.7.15", + "winnow 1.0.2", ] [[package]] name = "alloy-sol-types" -version = "1.5.7" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64612d29379782a5dde6f4b6570d9c756d734d760c0c94c254d361e678a6591f" +checksum = "384cf252de0db2dec52821eac037a7f57e2aa33fe5b900ce6fe39973402341f1" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -825,9 +792,9 @@ dependencies = [ [[package]] name = "alloy-transport" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ac7a80c0bac3e44559d53d002e34c461dc2f23262b42cafec019bc70551abbe" +checksum = "999adfe5c91035c6bf4c4210e0eb8d0caed79d76fbf5e1b70d78721f6097ac04" dependencies = [ "alloy-json-rpc", "auto_impl", @@ -848,14 +815,14 @@ dependencies = [ [[package]] name = "alloy-transport-http" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eed3ed3300a998f88639ed619fdbbd88bd82865e00c6a8ecb796c99eb12358f6" +checksum = "0e79a2c1793afc61eed9ca0963da370a988b27d2bbfa67c78013e262d47424c4" dependencies = [ "alloy-json-rpc", "alloy-transport", "itertools 0.14.0", - "reqwest 0.13.3", + "reqwest", "serde_json", "tower", "tracing", @@ -864,9 +831,9 @@ dependencies = [ [[package]] name = "alloy-transport-ipc" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1075d9d30fd4d71e50000fd4afb19ed2664ceab20c2a29f3889a6e988329e02d" +checksum = "c24422d5159996688b0c094babf1969cb0197d453902da483711c92c1624fea2" dependencies = [ "alloy-json-rpc", "alloy-pubsub", @@ -884,9 +851,9 @@ dependencies = [ [[package]] name = "alloy-transport-ws" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e3bff84b2b2a46eb34cc522dc3f889a2867c70be90a377421429b662b3ec4ce" +checksum = "1bea36621cd4e4f06c1243e556b32fdc9c4db7b9b09b72a95a87383c0ffcc625" dependencies = [ "alloy-pubsub", "alloy-transport", @@ -914,7 +881,7 @@ dependencies = [ "derive_more", "nybbles", "proptest", - "proptest-derive", + "proptest-derive 0.7.0", "serde", "smallvec", "thiserror 2.0.18", @@ -923,11 +890,11 @@ dependencies = [ [[package]] name = "alloy-tx-macros" -version = "2.0.4" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99fce0350197dcd4ba4e9a7dd43915d908c0eb0e7352755791709a705e1c76b6" +checksum = "406bc1183f6843e0aba09f7b3365e828b597213d60793ba5cb41befc863e3a78" dependencies = [ - "darling 0.23.0", + "darling", "proc-macro2", "quote", "syn 2.0.117", @@ -978,7 +945,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -989,7 +956,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -1622,12 +1589,12 @@ dependencies = [ ] [[package]] -name = "block-padding" -version = "0.3.3" +name = "block-buffer" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" dependencies = [ - "generic-array", + "hybrid-array", ] [[package]] @@ -1642,6 +1609,31 @@ dependencies = [ "zeroize", ] +[[package]] +name = "bon" +version = "3.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a602c73c7b0148ec6d12af6fd5cc7a46e2eacc8878271a999abac56eed12f561" +dependencies = [ + "bon-macros", + "rustversion", +] + +[[package]] +name = "bon-macros" +version = "3.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f" +dependencies = [ + "darling", + "ident_case", + "prettyplease", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.117", +] + [[package]] name = "borsh" version = "1.6.1" @@ -1765,39 +1757,6 @@ dependencies = [ "serde", ] -[[package]] -name = "camino" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e629a66d692cb9ff1a1c664e41771b3dcaf961985a9774c0eb0bd1b51cf60a48" -dependencies = [ - "serde_core", -] - -[[package]] -name = "cargo-platform" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd0061da739915fae12ea00e16397555ed4371a6bb285431aab930f61b0aa4ba" -dependencies = [ - "serde", - "serde_core", -] - -[[package]] -name = "cargo_metadata" -version = "0.23.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef987d17b0a113becdd19d3d0022d04d7ef41f9efe4f3fb63ac44ba61df3ade9" -dependencies = [ - "camino", - "cargo-platform", - "semver 1.0.28", - "serde", - "serde_json", - "thiserror 2.0.18", -] - [[package]] name = "castaway" version = "0.2.4" @@ -1846,6 +1805,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + [[package]] name = "chrono" version = "0.4.44" @@ -1866,7 +1836,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "inout", ] @@ -1977,7 +1947,7 @@ dependencies = [ "ripemd", "serde", "sha2", - "sha3", + "sha3 0.10.9", "thiserror 1.0.69", ] @@ -2111,6 +2081,16 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -2274,6 +2254,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "ctr" version = "0.9.2" @@ -2310,38 +2299,14 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "darling" -version = "0.20.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" -dependencies = [ - "darling_core 0.20.11", - "darling_macro 0.20.11", -] - [[package]] name = "darling" version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" dependencies = [ - "darling_core 0.23.0", - "darling_macro 0.23.0", -] - -[[package]] -name = "darling_core" -version = "0.20.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim", - "syn 2.0.117", + "darling_core", + "darling_macro", ] [[package]] @@ -2358,33 +2323,22 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "darling_macro" -version = "0.20.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" -dependencies = [ - "darling_core 0.20.11", - "quote", - "syn 2.0.117", -] - [[package]] name = "darling_macro" version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ - "darling_core 0.23.0", + "darling_core", "quote", "syn 2.0.117", ] [[package]] name = "dashmap" -version = "6.1.0" +version = "6.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" dependencies = [ "arbitrary", "cfg-if", @@ -2419,7 +2373,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccc2776f0c61eca1ca32528f85548abd1a4be8fb53d1b21c013e4f18da1e7090" dependencies = [ "data-encoding", - "syn 1.0.109", + "syn 2.0.117", ] [[package]] @@ -2455,7 +2409,6 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" dependencies = [ - "powerfmt", "serde_core", ] @@ -2492,37 +2445,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "derive_builder" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" -dependencies = [ - "derive_builder_macro", -] - -[[package]] -name = "derive_builder_core" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" -dependencies = [ - "darling 0.20.11", - "proc-macro2", - "quote", - "syn 2.0.117", -] - -[[package]] -name = "derive_builder_macro" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" -dependencies = [ - "derive_builder_core", - "syn 2.0.117", -] - [[package]] name = "derive_more" version = "2.1.1" @@ -2567,12 +2489,22 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", + "block-buffer 0.10.4", "const-oid", - "crypto-common", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "crypto-common 0.2.2", +] + [[package]] name = "dirs-next" version = "2.0.0" @@ -2717,9 +2649,9 @@ dependencies = [ [[package]] name = "either" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" dependencies = [ "serde", ] @@ -2760,22 +2692,10 @@ dependencies = [ "rand 0.8.6", "secp256k1 0.30.0", "serde", - "sha3", + "sha3 0.10.9", "zeroize", ] -[[package]] -name = "enum-as-inner" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "enum-ordinalize" version = "4.3.2" @@ -2809,7 +2729,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2838,9 +2758,9 @@ dependencies = [ [[package]] name = "ethereum_ssz" -version = "0.10.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368a4a4e4273b0135111fe9464e35465067766a8f664615b5a86338b73864407" +checksum = "e462875ad8693755ea8913d6e905715c76ea4836e2254e18c9cf0f7a8f8c2a13" dependencies = [ "alloy-primitives", "ethereum_serde_utils", @@ -2853,11 +2773,11 @@ dependencies = [ [[package]] name = "ethereum_ssz_derive" -version = "0.10.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2cd82c68120c89361e1a457245cf212f7d9f541bffaffed530c8f2d54a160b2" +checksum = "daf022360bdbe9456eda5f35718a50476d5b2a0d51a97ed4eae27420737a6fba" dependencies = [ - "darling 0.23.0", + "darling", "proc-macro2", "quote", "syn 2.0.117", @@ -2957,9 +2877,9 @@ checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "fixed-cache" -version = "0.1.8" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c41c7aa69c00ebccf06c3fa7ffe2a6cf26a58b5fe4deabfe646285ff48136a8f" +checksum = "2fe63500644ef0269fe6b744e7e5dc5c20b5eebf3d881bc2be53f194636f6583" dependencies = [ "equivalent", "rapidhash", @@ -3172,8 +3092,8 @@ dependencies = [ "libc", "log", "rustversion", - "windows-link 0.1.3", - "windows-result 0.3.4", + "windows-link 0.2.1", + "windows-result 0.4.1", ] [[package]] @@ -3223,6 +3143,7 @@ dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", "wasip2", "wasip3", ] @@ -3239,15 +3160,14 @@ dependencies = [ [[package]] name = "git2" -version = "0.20.4" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b88256088d75a56f8ecfa070513a775dd9107f6530ef14919dac831af9cfe2b" +checksum = "ddddbf932745a6be37109b6112d3ee09696106f848449069d3a57bba937ab82e" dependencies = [ "bitflags", "libc", "libgit2-sys", "log", - "url", ] [[package]] @@ -3381,8 +3301,6 @@ dependencies = [ "allocator-api2", "equivalent", "foldhash 0.2.0", - "serde", - "serde_core", ] [[package]] @@ -3390,6 +3308,11 @@ name = "hashbrown" version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" +dependencies = [ + "foldhash 0.2.0", + "serde", + "serde_core", +] [[package]] name = "hashlink" @@ -3438,48 +3361,72 @@ dependencies = [ ] [[package]] -name = "hickory-proto" -version = "0.25.2" +name = "hickory-net" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8a6fe56c0038198998a6f217ca4e7ef3a5e51f46163bd6dd60b5c71ca6c6502" +checksum = "e2295ed2f9c31e471e1428a8f88a3f0e1f4b27c15049592138d1eebe9c35b183" dependencies = [ "async-trait", "cfg-if", "data-encoding", - "enum-as-inner", "futures-channel", "futures-io", "futures-util", + "hickory-proto", + "idna", + "ipnet", + "jni 0.22.4", + "rand 0.10.2", + "thiserror 2.0.18", + "tinyvec", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "hickory-proto" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bab31817bfb44672a252e97fe81cd0c18d1b2cf892108922f6818820df8c643" +dependencies = [ + "data-encoding", "idna", "ipnet", + "jni 0.22.4", "once_cell", - "rand 0.9.4", + "prefix-trie", + "rand 0.10.2", "ring", "serde", "thiserror 2.0.18", "tinyvec", - "tokio", "tracing", "url", ] [[package]] name = "hickory-resolver" -version = "0.25.2" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc62a9a99b0bfb44d2ab95a7208ac952d31060efc16241c87eaf36406fecf87a" +checksum = "f0d58d28879ceecde6607729660c2667a081ccdc082e082675042793960f178c" dependencies = [ "cfg-if", "futures-util", + "hickory-net", "hickory-proto", "ipconfig", + "ipnet", + "jni 0.22.4", "moka", + "ndk-context", "once_cell", "parking_lot", - "rand 0.9.4", + "rand 0.10.2", "resolv-conf", "serde", "smallvec", + "system-configuration", "thiserror 2.0.18", "tokio", "tracing", @@ -3576,6 +3523,15 @@ dependencies = [ "serde", ] +[[package]] +name = "hybrid-array" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.9.0" @@ -3609,7 +3565,6 @@ dependencies = [ "hyper-util", "log", "rustls", - "rustls-native-certs", "tokio", "tokio-rustls", "tower-service", @@ -3663,7 +3618,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.61.2", + "windows-core 0.62.2", ] [[package]] @@ -3929,7 +3884,6 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" dependencies = [ - "block-padding", "generic-array", ] @@ -3939,7 +3893,7 @@ version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5eb2d60ef19920a3a9193c3e371f726ec1dafc045dac788d0fb3704272458971" dependencies = [ - "darling 0.23.0", + "darling", "indoc", "proc-macro2", "quote", @@ -3979,6 +3933,9 @@ name = "ipnet" version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +dependencies = [ + "serde", +] [[package]] name = "is_terminal_polyfill" @@ -4356,6 +4313,16 @@ dependencies = [ "cpufeatures 0.2.17", ] +[[package]] +name = "keccak" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e24a010dd405bd7ed803e5253182815b41bf2e6a80cc3bfc066658e03a198aa" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", +] + [[package]] name = "keccak-asm" version = "0.1.6" @@ -4656,9 +4623,9 @@ checksum = "dae608c151f68243f2b000364e1f7b186d9c29845f7d2d85bd31b9ad77ad552b" [[package]] name = "macro-string" -version = "0.1.4" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b27834086c65ec3f9387b096d66e99f221cf081c2b738042aa252bcd41204e3" +checksum = "59a9dbbfc75d2688ed057456ce8a3ee3f48d12eec09229f560f3643b9f275653" dependencies = [ "proc-macro2", "quote", @@ -4706,9 +4673,9 @@ checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" [[package]] name = "memmap2" -version = "0.9.10" +version = "0.9.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" dependencies = [ "libc", ] @@ -4876,12 +4843,12 @@ version = "1.0.0" dependencies = [ "alloy-chains", "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-evm", "alloy-genesis", "alloy-hardforks", "alloy-primitives", - "alloy-serde 2.0.4", + "alloy-serde", "auto_impl", "eyre", "morph-primitives", @@ -4916,7 +4883,7 @@ name = "morph-engine-api" version = "1.0.0" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-genesis", "alloy-primitives", "alloy-rpc-types-engine", @@ -4939,52 +4906,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "morph-engine-tree-ext" -version = "1.0.0" -dependencies = [ - "alloy-consensus", - "alloy-eip7928", - "alloy-eips 2.0.4", - "alloy-evm", - "alloy-genesis", - "alloy-primitives", - "alloy-rlp", - "alloy-rpc-types-engine", - "alloy-rpc-types-eth", - "crossbeam-channel", - "derive_more", - "eyre", - "morph-chainspec", - "morph-node", - "morph-payload-types", - "morph-primitives", - "reth-chain-state", - "reth-consensus", - "reth-db", - "reth-engine-primitives", - "reth-engine-tree", - "reth-errors", - "reth-evm", - "reth-execution-cache", - "reth-node-api", - "reth-payload-builder", - "reth-payload-primitives", - "reth-primitives-traits", - "reth-provider", - "reth-revm", - "reth-tasks", - "reth-tracing", - "reth-trie", - "reth-trie-db", - "reth-trie-parallel", - "reth-trie-sparse", - "revm-primitives", - "serde_json", - "tokio", - "tracing", -] - [[package]] name = "morph-evm" version = "1.0.0" @@ -4993,6 +4914,7 @@ dependencies = [ "alloy-evm", "alloy-genesis", "alloy-primitives", + "alloy-rpc-types-eth", "derive_more", "morph-chainspec", "morph-payload-types", @@ -5016,7 +4938,7 @@ name = "morph-node" version = "1.0.0" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-genesis", "alloy-hardforks", "alloy-primitives", @@ -5033,7 +4955,6 @@ dependencies = [ "morph-chainspec", "morph-consensus", "morph-engine-api", - "morph-engine-tree-ext", "morph-evm", "morph-payload-builder", "morph-payload-types", @@ -5042,6 +4963,7 @@ dependencies = [ "morph-rpc", "morph-txpool", "parking_lot", + "reth-chain-state", "reth-chainspec", "reth-db", "reth-db-api", @@ -5049,6 +4971,8 @@ dependencies = [ "reth-engine-local", "reth-engine-tree", "reth-errors", + "reth-evm", + "reth-execution-cache", "reth-exex", "reth-node-api", "reth-node-builder", @@ -5081,7 +5005,7 @@ name = "morph-payload-builder" version = "1.0.0" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rlp", "metrics", @@ -5113,11 +5037,11 @@ name = "morph-payload-types" version = "1.0.0" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rlp", "alloy-rpc-types-engine", - "alloy-serde 2.0.4", + "alloy-serde", "morph-primitives", "rand 0.8.6", "reth-ethereum-primitives", @@ -5133,10 +5057,10 @@ name = "morph-primitives" version = "1.0.0" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rlp", - "alloy-serde 2.0.4", + "alloy-serde", "bytes", "modular-bitfield", "reth-codecs", @@ -5199,7 +5123,7 @@ name = "morph-revm" version = "1.0.0" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-evm", "alloy-primitives", "alloy-sol-types", @@ -5222,12 +5146,12 @@ name = "morph-rpc" version = "1.0.0" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-evm", "alloy-network", "alloy-primitives", "alloy-rpc-types-eth", - "alloy-serde 2.0.4", + "alloy-serde", "derive_more", "eyre", "jsonrpsee", @@ -5284,7 +5208,7 @@ name = "morph-txpool" version = "1.0.0" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-evm", "alloy-primitives", "c-kzg", @@ -5346,6 +5270,12 @@ dependencies = [ "unsigned-varint", ] +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + [[package]] name = "nom" version = "7.1.3" @@ -5356,6 +5286,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nonmax" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "610a5acd306ec67f907abe5567859a3c693fb9886eb1f012ab8f2a47bef3db51" +dependencies = [ + "serde", +] + [[package]] name = "notify" version = "8.2.0" @@ -5398,7 +5337,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -5436,9 +5375,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-integer" @@ -5507,7 +5446,6 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" dependencies = [ - "proc-macro-crate", "proc-macro2", "quote", "syn 2.0.117", @@ -5586,9 +5524,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "opentelemetry" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b84bcd6ae87133e903af7ef497404dda70c60d0ea14895fc8a5e6722754fc2a0" +checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682" dependencies = [ "futures-core", "futures-sink", @@ -5600,22 +5538,22 @@ dependencies = [ [[package]] name = "opentelemetry-http" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7a6d09a73194e6b66df7c8f1b680f156d916a1a942abf2de06823dd02b7855d" +checksum = "5683015d09e2df236ef005b17f6f196f0d5f6313c4fa43a7b6a53b52776e4331" dependencies = [ "async-trait", "bytes", "http", "opentelemetry", - "reqwest 0.12.28", + "reqwest", ] [[package]] name = "opentelemetry-otlp" -version = "0.31.1" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f69cd6acbb9af919df949cd1ec9e5e7fdc2ef15d234b6b795aaa525cc02f71f" +checksum = "9966929966d17620d7c316c643ba62631826e10021409357772d5eea84f62c35" dependencies = [ "http", "opentelemetry", @@ -5623,18 +5561,18 @@ dependencies = [ "opentelemetry-proto", "opentelemetry_sdk", "prost", - "reqwest 0.12.28", + "reqwest", "thiserror 2.0.18", "tokio", "tonic", - "tracing", + "tonic-types", ] [[package]] name = "opentelemetry-proto" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f" +checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638" dependencies = [ "opentelemetry", "opentelemetry_sdk", @@ -5645,21 +5583,22 @@ dependencies = [ [[package]] name = "opentelemetry-semantic-conventions" -version = "0.31.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e62e29dfe041afb8ed2a6c9737ab57db4907285d999ef8ad3a59092a36bdc846" +checksum = "c913ac17a6c451661ee255f4625d143e51647ae78ebd969b75e41c4442f4fe47" [[package]] name = "opentelemetry_sdk" -version = "0.31.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ae4f5991976fd48df6d843de219ca6d31b01daaab2dad5af2badeded372bd" +checksum = "9b59f80e1ac4d5ff7a2db8fb6c80badb7f0f3f858211fba08dd9aaec750894f9" dependencies = [ "futures-channel", "futures-executor", "futures-util", "opentelemetry", "percent-encoding", + "portable-atomic", "rand 0.9.4", "thiserror 2.0.18", ] @@ -5951,6 +5890,17 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "prefix-trie" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cf6e3177f0684016a5c209b00882e15f8bdd3f3bb48f0491df10cd102d0c6e7" +dependencies = [ + "either", + "ipnet", + "num-traits", +] + [[package]] name = "pretty_assertions" version = "1.4.1" @@ -6095,6 +6045,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "proptest-derive" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c57924a81864dddafba92e1bf92f9bf82f97096c44489548a60e888e1547549b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "prost" version = "0.14.3" @@ -6118,6 +6079,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "prost-types" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +dependencies = [ + "prost", +] + [[package]] name = "quanta" version = "0.12.6" @@ -6254,6 +6224,17 @@ dependencies = [ "serde", ] +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand_core 0.10.1", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -6293,6 +6274,12 @@ dependencies = [ "serde", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "rand_xorshift" version = "0.4.0" @@ -6497,46 +6484,6 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" -[[package]] -name = "reqwest" -version = "0.12.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" -dependencies = [ - "base64 0.22.1", - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-rustls", - "hyper-util", - "js-sys", - "log", - "percent-encoding", - "pin-project-lite", - "quinn", - "rustls", - "rustls-native-certs", - "rustls-pki-types", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper", - "tokio", - "tokio-rustls", - "tower", - "tower-http", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - [[package]] name = "reqwest" version = "0.13.3" @@ -6587,11 +6534,11 @@ checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" [[package]] name = "reth-basic-payload-builder" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "futures-core", "futures-util", @@ -6614,11 +6561,11 @@ dependencies = [ [[package]] name = "reth-chain-state" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-signer", "alloy-signer-local", @@ -6635,9 +6582,10 @@ dependencies = [ "reth-metrics", "reth-primitives-traits", "reth-storage-api", + "reth-tasks", "reth-trie", - "revm-database", - "revm-state", + "reth-trie-sparse", + "revm", "serde", "tokio", "tokio-stream", @@ -6646,12 +6594,12 @@ dependencies = [ [[package]] name = "reth-chainspec" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-chains", "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-evm", "alloy-genesis", "alloy-primitives", @@ -6666,8 +6614,8 @@ dependencies = [ [[package]] name = "reth-cli" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-genesis", "clap", @@ -6679,12 +6627,12 @@ dependencies = [ [[package]] name = "reth-cli-commands" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-chains", "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rlp", "backon", @@ -6703,7 +6651,7 @@ dependencies = [ "parking_lot", "ratatui", "rayon", - "reqwest 0.13.3", + "reqwest", "reth-chainspec", "reth-cli", "reth-cli-runner", @@ -6751,6 +6699,7 @@ dependencies = [ "secp256k1 0.30.0", "serde", "serde_json", + "socket2", "tar", "tokio", "tokio-stream", @@ -6762,8 +6711,8 @@ dependencies = [ [[package]] name = "reth-cli-runner" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "reth-tasks", "tokio", @@ -6772,10 +6721,10 @@ dependencies = [ [[package]] name = "reth-cli-util" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "cfg-if", "eyre", @@ -6791,12 +6740,12 @@ dependencies = [ [[package]] name = "reth-codecs" -version = "0.3.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fce542a96bf888f31854803e80b3340bc233927743aa580838014e8a88fe0d66" +checksum = "eb43f4858fef4e3b42b80954d2edf1ff67c8ad7498347083cdc7d0f6eff2f081" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-genesis", "alloy-primitives", "alloy-trie", @@ -6812,9 +6761,9 @@ dependencies = [ [[package]] name = "reth-codecs-derive" -version = "0.3.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634c90f1cc0f9887680ca785b0b21aa961070b9465917bf65afaec56a6d005bb" +checksum = "5798ae4d6b264764bc0d742468bd32c7fb515e774645d4930f95ff6311f3ae14" dependencies = [ "proc-macro2", "quote", @@ -6823,8 +6772,8 @@ dependencies = [ [[package]] name = "reth-config" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "eyre", "humantime-serde", @@ -6839,10 +6788,11 @@ dependencies = [ [[package]] name = "reth-consensus" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", + "alloy-eip7928", "alloy-primitives", "auto_impl", "reth-execution-types", @@ -6852,11 +6802,11 @@ dependencies = [ [[package]] name = "reth-consensus-common" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "reth-chainspec", "reth-consensus", @@ -6865,11 +6815,11 @@ dependencies = [ [[package]] name = "reth-consensus-debug-client" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-json-rpc", "alloy-primitives", "alloy-provider", @@ -6879,9 +6829,8 @@ dependencies = [ "derive_more", "eyre", "futures", - "reqwest 0.13.3", + "reqwest", "reth-node-api", - "reth-primitives-traits", "reth-tracing", "ringbuffer", "serde", @@ -6891,8 +6840,8 @@ dependencies = [ [[package]] name = "reth-db" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-primitives", "derive_more", @@ -6920,8 +6869,8 @@ dependencies = [ [[package]] name = "reth-db-api" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -6946,8 +6895,8 @@ dependencies = [ [[package]] name = "reth-db-common" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", "alloy-genesis", @@ -6976,10 +6925,10 @@ dependencies = [ [[package]] name = "reth-db-models" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "arbitrary", "bytes", @@ -6991,8 +6940,8 @@ dependencies = [ [[package]] name = "reth-discv4" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -7016,8 +6965,8 @@ dependencies = [ [[package]] name = "reth-discv5" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -7040,8 +6989,8 @@ dependencies = [ [[package]] name = "reth-dns-discovery" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-primitives", "dashmap", @@ -7064,11 +7013,11 @@ dependencies = [ [[package]] name = "reth-downloaders" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rlp", "async-compression", @@ -7099,11 +7048,11 @@ dependencies = [ [[package]] name = "reth-e2e-test-utils" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-network", "alloy-primitives", "alloy-provider", @@ -7156,13 +7105,12 @@ dependencies = [ [[package]] name = "reth-ecies" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "aes", "alloy-primitives", "alloy-rlp", - "block-padding", "byteorder", "cipher", "concat-kdf", @@ -7184,8 +7132,8 @@ dependencies = [ [[package]] name = "reth-engine-local" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -7207,11 +7155,11 @@ dependencies = [ [[package]] name = "reth-engine-primitives" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rpc-types-engine", "auto_impl", @@ -7232,12 +7180,12 @@ dependencies = [ [[package]] name = "reth-engine-tree" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", "alloy-eip7928", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-evm", "alloy-primitives", "alloy-rlp", @@ -7248,7 +7196,6 @@ dependencies = [ "indexmap 2.14.0", "metrics", "moka", - "parking_lot", "rayon", "reth-chain-state", "reth-chainspec", @@ -7280,7 +7227,6 @@ dependencies = [ "reth-trie-parallel", "reth-trie-sparse", "revm", - "revm-primitives", "schnellru", "thiserror 2.0.18", "tokio", @@ -7289,10 +7235,12 @@ dependencies = [ [[package]] name = "reth-engine-util" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", + "alloy-primitives", + "alloy-rlp", "alloy-rpc-types-engine", "eyre", "futures", @@ -7317,13 +7265,15 @@ dependencies = [ [[package]] name = "reth-era" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rlp", + "alloy-rpc-types-beacon", + "alloy-rpc-types-engine", "ethereum_ssz", "ethereum_ssz_derive", "sha2", @@ -7333,14 +7283,14 @@ dependencies = [ [[package]] name = "reth-era-downloader" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-primitives", "bytes", "eyre", "futures-util", - "reqwest 0.13.3", + "reqwest", "reth-era", "reth-fs-util", "sha2", @@ -7349,11 +7299,12 @@ dependencies = [ [[package]] name = "reth-era-utils" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", "alloy-primitives", + "alloy-rlp", "eyre", "futures-util", "reth-db-api", @@ -7371,8 +7322,8 @@ dependencies = [ [[package]] name = "reth-errors" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "reth-consensus", "reth-execution-errors", @@ -7382,8 +7333,8 @@ dependencies = [ [[package]] name = "reth-eth-wire" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-chains", "alloy-primitives", @@ -7410,13 +7361,13 @@ dependencies = [ [[package]] name = "reth-eth-wire-types" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-chains", "alloy-consensus", "alloy-eip7928", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-hardforks", "alloy-primitives", "alloy-rlp", @@ -7432,8 +7383,8 @@ dependencies = [ [[package]] name = "reth-ethereum-cli" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "clap", "eyre", @@ -7455,11 +7406,11 @@ dependencies = [ [[package]] name = "reth-ethereum-consensus" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "reth-chainspec", "reth-consensus", @@ -7471,10 +7422,10 @@ dependencies = [ [[package]] name = "reth-ethereum-engine-primitives" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rpc-types-engine", "reth-engine-primitives", @@ -7487,8 +7438,8 @@ dependencies = [ [[package]] name = "reth-ethereum-forks" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-eip2124", "alloy-hardforks", @@ -7500,11 +7451,11 @@ dependencies = [ [[package]] name = "reth-ethereum-payload-builder" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rlp", "alloy-rpc-types-engine", @@ -7530,11 +7481,11 @@ dependencies = [ [[package]] name = "reth-ethereum-primitives" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rpc-types-eth", "reth-codecs", @@ -7544,8 +7495,8 @@ dependencies = [ [[package]] name = "reth-etl" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "rayon", "reth-db-api", @@ -7554,11 +7505,12 @@ dependencies = [ [[package]] name = "reth-evm" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eip7928", + "alloy-eips", "alloy-evm", "alloy-primitives", "auto_impl", @@ -7578,11 +7530,11 @@ dependencies = [ [[package]] name = "reth-evm-ethereum" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-evm", "alloy-primitives", "alloy-rpc-types-engine", @@ -7598,8 +7550,8 @@ dependencies = [ [[package]] name = "reth-execution-cache" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-primitives", "fixed-cache", @@ -7616,8 +7568,8 @@ dependencies = [ [[package]] name = "reth-execution-errors" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-evm", "alloy-primitives", @@ -7629,11 +7581,11 @@ dependencies = [ [[package]] name = "reth-execution-types" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-evm", "alloy-primitives", "alloy-rlp", @@ -7648,11 +7600,11 @@ dependencies = [ [[package]] name = "reth-exex" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "eyre", "futures", @@ -7686,10 +7638,10 @@ dependencies = [ [[package]] name = "reth-exex-types" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "reth-chain-state", "reth-execution-types", @@ -7700,8 +7652,8 @@ dependencies = [ [[package]] name = "reth-fs-util" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "serde", "serde_json", @@ -7710,8 +7662,8 @@ dependencies = [ [[package]] name = "reth-invalid-block-hooks" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -7730,16 +7682,14 @@ dependencies = [ "reth-tracing", "reth-trie", "revm", - "revm-bytecode", - "revm-database", "serde", "serde_json", ] [[package]] name = "reth-ipc" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "bytes", "futures", @@ -7758,8 +7708,8 @@ dependencies = [ [[package]] name = "reth-libmdbx" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "bitflags", "byteorder", @@ -7775,8 +7725,8 @@ dependencies = [ [[package]] name = "reth-mdbx-sys" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "bindgen", "cc", @@ -7784,8 +7734,8 @@ dependencies = [ [[package]] name = "reth-metrics" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "futures", "metrics", @@ -7797,8 +7747,8 @@ dependencies = [ [[package]] name = "reth-net-banlist" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-primitives", "ipnet", @@ -7806,12 +7756,12 @@ dependencies = [ [[package]] name = "reth-net-nat" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "futures-util", "if-addrs", - "reqwest 0.13.3", + "reqwest", "serde_with", "thiserror 2.0.18", "tokio", @@ -7820,11 +7770,11 @@ dependencies = [ [[package]] name = "reth-network" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rlp", "aquamarine", @@ -7878,8 +7828,8 @@ dependencies = [ [[package]] name = "reth-network-api" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -7903,11 +7853,12 @@ dependencies = [ [[package]] name = "reth-network-p2p" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eip7928", + "alloy-eips", "alloy-primitives", "auto_impl", "derive_more", @@ -7926,8 +7877,8 @@ dependencies = [ [[package]] name = "reth-network-peers" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -7941,8 +7892,8 @@ dependencies = [ [[package]] name = "reth-network-types" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-eip2124", "humantime-serde", @@ -7955,8 +7906,8 @@ dependencies = [ [[package]] name = "reth-nippy-jar" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "anyhow", "bincode", @@ -7972,8 +7923,8 @@ dependencies = [ [[package]] name = "reth-node-api" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-rpc-types-engine", "eyre", @@ -7996,11 +7947,11 @@ dependencies = [ [[package]] name = "reth-node-builder" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-provider", "alloy-rpc-types", @@ -8064,11 +8015,11 @@ dependencies = [ [[package]] name = "reth-node-core" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rpc-types-engine", "clap", @@ -8119,14 +8070,20 @@ dependencies = [ [[package]] name = "reth-node-ethereum" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ - "alloy-eips 2.0.4", + "alloy-consensus", + "alloy-eips", "alloy-network", + "alloy-primitives", "alloy-rpc-types-engine", "alloy-rpc-types-eth", + "ethereum_ssz", + "ethereum_ssz_derive", "eyre", + "http-body-util", + "jsonrpsee", "reth-chainspec", "reth-engine-local", "reth-engine-primitives", @@ -8139,6 +8096,7 @@ dependencies = [ "reth-network", "reth-node-api", "reth-node-builder", + "reth-node-core", "reth-payload-primitives", "reth-primitives-traits", "reth-provider", @@ -8152,13 +8110,16 @@ dependencies = [ "reth-tracing", "reth-transaction-pool", "revm", + "serde", + "serde_json", "tokio", + "tower", ] [[package]] name = "reth-node-ethstats" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8181,11 +8142,11 @@ dependencies = [ [[package]] name = "reth-node-events" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rpc-types-engine", "derive_more", @@ -8205,8 +8166,8 @@ dependencies = [ [[package]] name = "reth-node-metrics" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "bytes", "eyre", @@ -8221,7 +8182,7 @@ dependencies = [ "metrics-util", "pprof_util", "procfs", - "reqwest 0.13.3", + "reqwest", "reth-fs-util", "reth-metrics", "reth-tasks", @@ -8234,8 +8195,8 @@ dependencies = [ [[package]] name = "reth-node-types" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "reth-chainspec", "reth-db-api", @@ -8246,8 +8207,8 @@ dependencies = [ [[package]] name = "reth-payload-builder" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8270,8 +8231,8 @@ dependencies = [ [[package]] name = "reth-payload-builder-primitives" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "pin-project", "reth-payload-primitives", @@ -8282,17 +8243,16 @@ dependencies = [ [[package]] name = "reth-payload-primitives" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rlp", "alloy-rpc-types-engine", "auto_impl", "either", - "reth-chain-state", "reth-chainspec", "reth-errors", "reth-execution-types", @@ -8306,8 +8266,8 @@ dependencies = [ [[package]] name = "reth-payload-util" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8316,8 +8276,8 @@ dependencies = [ [[package]] name = "reth-payload-validator" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", "alloy-rpc-types-engine", @@ -8326,12 +8286,12 @@ dependencies = [ [[package]] name = "reth-primitives-traits" -version = "0.3.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ee12e304adbacbb32248c9806ebafbe1e2811fbfefe53c5e5b710a8438b7ec0" +checksum = "f59c928b2865af5bcdbce94800270b7f7798f27a22ca0aabdd2b7d3e6587c9ce" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-genesis", "alloy-primitives", "alloy-rlp", @@ -8359,11 +8319,12 @@ dependencies = [ [[package]] name = "reth-provider" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eip7928", + "alloy-eips", "alloy-genesis", "alloy-primitives", "alloy-rpc-types-engine", @@ -8393,11 +8354,12 @@ dependencies = [ "reth-storage-api", "reth-storage-errors", "reth-tasks", + "reth-tokio-util", "reth-trie", "reth-trie-db", - "revm-database", - "revm-state", + "revm", "rocksdb", + "smallvec", "strum", "tokio", "tracing", @@ -8405,15 +8367,13 @@ dependencies = [ [[package]] name = "reth-prune" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", "alloy-primitives", "itertools 0.14.0", "metrics", - "rayon", "reth-config", "reth-db-api", "reth-errors", @@ -8434,8 +8394,8 @@ dependencies = [ [[package]] name = "reth-prune-types" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-primitives", "arbitrary", @@ -8450,8 +8410,8 @@ dependencies = [ [[package]] name = "reth-revm" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -8465,12 +8425,13 @@ dependencies = [ [[package]] name = "reth-rpc" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", "alloy-dyn-abi", - "alloy-eips 2.0.4", + "alloy-eip7928", + "alloy-eips", "alloy-evm", "alloy-genesis", "alloy-network", @@ -8486,7 +8447,7 @@ dependencies = [ "alloy-rpc-types-mev", "alloy-rpc-types-trace", "alloy-rpc-types-txpool", - "alloy-serde 2.0.4", + "alloy-serde", "alloy-signer", "alloy-signer-local", "async-trait", @@ -8514,6 +8475,7 @@ dependencies = [ "reth-network-peers", "reth-network-types", "reth-node-api", + "reth-payload-primitives", "reth-primitives-traits", "reth-revm", "reth-rpc-api", @@ -8524,12 +8486,10 @@ dependencies = [ "reth-rpc-server-types", "reth-storage-api", "reth-tasks", - "reth-tracing", "reth-transaction-pool", "reth-trie-common", "revm", "revm-inspectors", - "revm-primitives", "serde", "serde_json", "sha2", @@ -8542,10 +8502,10 @@ dependencies = [ [[package]] name = "reth-rpc-api" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ - "alloy-eips 2.0.4", + "alloy-eips", "alloy-genesis", "alloy-json-rpc", "alloy-primitives", @@ -8559,7 +8519,7 @@ dependencies = [ "alloy-rpc-types-mev", "alloy-rpc-types-trace", "alloy-rpc-types-txpool", - "alloy-serde 2.0.4", + "alloy-serde", "jsonrpsee", "reth-chain-state", "reth-engine-primitives", @@ -8572,8 +8532,8 @@ dependencies = [ [[package]] name = "reth-rpc-builder" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-network", "alloy-provider", @@ -8615,8 +8575,8 @@ dependencies = [ [[package]] name = "reth-rpc-convert" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", "alloy-evm", @@ -8635,10 +8595,10 @@ dependencies = [ [[package]] name = "reth-rpc-engine-api" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rlp", "alloy-rpc-types-engine", @@ -8666,13 +8626,13 @@ dependencies = [ [[package]] name = "reth-rpc-eth-api" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", "alloy-dyn-abi", "alloy-eip7928", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-evm", "alloy-json-rpc", "alloy-network", @@ -8680,7 +8640,7 @@ dependencies = [ "alloy-rlp", "alloy-rpc-types-eth", "alloy-rpc-types-mev", - "alloy-serde 2.0.4", + "alloy-serde", "async-trait", "auto_impl", "dyn-clone", @@ -8695,6 +8655,7 @@ dependencies = [ "reth-network-api", "reth-node-api", "reth-primitives-traits", + "reth-prune-types", "reth-revm", "reth-rpc-convert", "reth-rpc-eth-types", @@ -8712,16 +8673,19 @@ dependencies = [ [[package]] name = "reth-rpc-eth-types" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ + "alloy-chains", "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eip7928", + "alloy-eips", "alloy-evm", "alloy-network", "alloy-primitives", "alloy-rpc-client", "alloy-rpc-types-eth", + "alloy-serde", "alloy-sol-types", "alloy-transport", "derive_more", @@ -8731,7 +8695,7 @@ dependencies = [ "jsonrpsee-types", "metrics", "rand 0.9.4", - "reqwest 0.13.3", + "reqwest", "reth-chain-state", "reth-chainspec", "reth-errors", @@ -8760,8 +8724,8 @@ dependencies = [ [[package]] name = "reth-rpc-layer" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-rpc-types-engine", "http", @@ -8774,10 +8738,10 @@ dependencies = [ [[package]] name = "reth-rpc-server-types" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rpc-types-engine", "jsonrpsee-core", @@ -8790,9 +8754,9 @@ dependencies = [ [[package]] name = "reth-rpc-traits" -version = "0.3.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "860fe223501a76ff14aa3bf164f739f31008c2a2905ac85708bfd88f042e6151" +checksum = "ace77dbcdd59c014fda4565ebfec76cd1fe6f5d98584b4655e8d22a947b9eee3" dependencies = [ "alloy-consensus", "alloy-network", @@ -8805,11 +8769,10 @@ dependencies = [ [[package]] name = "reth-stages" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", "alloy-primitives", "alloy-rlp", "eyre", @@ -8818,7 +8781,7 @@ dependencies = [ "num-traits", "page_size", "rayon", - "reqwest 0.13.3", + "reqwest", "reth-chainspec", "reth-codecs", "reth-config", @@ -8857,10 +8820,10 @@ dependencies = [ [[package]] name = "reth-stages-api" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "aquamarine", "auto_impl", @@ -8885,8 +8848,8 @@ dependencies = [ [[package]] name = "reth-stages-types" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-primitives", "arbitrary", @@ -8899,8 +8862,8 @@ dependencies = [ [[package]] name = "reth-static-file" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-primitives", "parking_lot", @@ -8919,8 +8882,8 @@ dependencies = [ [[package]] name = "reth-static-file-types" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-primitives", "clap", @@ -8934,11 +8897,12 @@ dependencies = [ [[package]] name = "reth-storage-api" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eip7928", + "alloy-eips", "alloy-primitives", "alloy-rpc-types-engine", "auto_impl", @@ -8951,17 +8915,18 @@ dependencies = [ "reth-prune-types", "reth-stages-types", "reth-storage-errors", + "reth-tokio-util", "reth-trie-common", - "revm-database", + "revm", "serde_json", ] [[package]] name = "reth-storage-errors" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rlp", "derive_more", @@ -8969,15 +8934,14 @@ dependencies = [ "reth-primitives-traits", "reth-prune-types", "reth-static-file-types", - "revm-database-interface", - "revm-state", + "revm", "thiserror 2.0.18", ] [[package]] name = "reth-tasks" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "crossbeam-utils", "dashmap", @@ -8997,11 +8961,11 @@ dependencies = [ [[package]] name = "reth-testing-utils" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-genesis", "alloy-primitives", "rand 0.8.6", @@ -9013,8 +8977,8 @@ dependencies = [ [[package]] name = "reth-tokio-util" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "tokio", "tokio-stream", @@ -9023,8 +8987,8 @@ dependencies = [ [[package]] name = "reth-tracing" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "clap", "eyre", @@ -9032,6 +8996,7 @@ dependencies = [ "rolling-file", "tracing", "tracing-appender", + "tracing-chrome", "tracing-journald", "tracing-logfmt", "tracing-samply", @@ -9040,9 +9005,10 @@ dependencies = [ [[package]] name = "reth-tracing-otlp" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ + "base64 0.22.1", "clap", "eyre", "opentelemetry", @@ -9057,11 +9023,11 @@ dependencies = [ [[package]] name = "reth-transaction-pool" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rlp", "aquamarine", @@ -9087,8 +9053,6 @@ dependencies = [ "reth-storage-api", "reth-tasks", "revm", - "revm-interpreter", - "revm-primitives", "rustc-hash", "schnellru", "serde", @@ -9102,11 +9066,11 @@ dependencies = [ [[package]] name = "reth-trie" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", - "alloy-eips 2.0.4", + "alloy-eips", "alloy-primitives", "alloy-rlp", "alloy-trie", @@ -9121,21 +9085,20 @@ dependencies = [ "reth-storage-errors", "reth-trie-common", "reth-trie-sparse", - "revm-database", "tracing", "triehash", ] [[package]] name = "reth-trie-common" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-consensus", "alloy-primitives", "alloy-rlp", "alloy-rpc-types-eth", - "alloy-serde 2.0.4", + "alloy-serde", "alloy-trie", "arbitrary", "arrayvec", @@ -9148,15 +9111,15 @@ dependencies = [ "rayon", "reth-codecs", "reth-primitives-traits", - "revm-database", + "revm", "serde", "serde_with", ] [[package]] name = "reth-trie-db" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-primitives", "metrics", @@ -9165,7 +9128,6 @@ dependencies = [ "reth-execution-errors", "reth-metrics", "reth-primitives-traits", - "reth-stages-types", "reth-storage-api", "reth-storage-errors", "reth-trie", @@ -9175,19 +9137,15 @@ dependencies = [ [[package]] name = "reth-trie-parallel" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ - "alloy-eip7928", "alloy-evm", "alloy-primitives", "alloy-rlp", "crossbeam-channel", "crossbeam-utils", - "derive_more", - "itertools 0.14.0", "metrics", - "rayon", "reth-execution-errors", "reth-metrics", "reth-primitives-traits", @@ -9196,19 +9154,19 @@ dependencies = [ "reth-tasks", "reth-trie", "reth-trie-sparse", - "revm-state", + "revm", "thiserror 2.0.18", "tracing", ] [[package]] name = "reth-trie-sparse" -version = "2.2.0" -source = "git+https://github.com/paradigmxyz/reth?tag=v2.2.0#88505c7fcbfdebfd3b56d88c86b62e950043c6c4" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "alloy-primitives", - "alloy-rlp", "alloy-trie", + "either", "metrics", "rayon", "reth-execution-errors", @@ -9219,23 +9177,24 @@ dependencies = [ "serde_json", "slotmap", "smallvec", + "strum", "tracing", ] [[package]] name = "reth-zstd-compressors" -version = "0.3.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c12fafa33d2f420a9d39249a3e0357b1928d09429f30758b85280409092873b2" +checksum = "e1bd4ae4f54a2ba813eef082910bc646bd31cab8ed134308fa71f1068331fb84" dependencies = [ "zstd", ] [[package]] name = "revm" -version = "38.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91202d39dbe8e8d10e9e8f2b76c30da68ecd1d25be69ba6d853ad0d03a3a398a" +checksum = "9d51c254839fb36357d0b02d02e46ba57e683d7b017e2da33b47ae74685a9e59" dependencies = [ "revm-bytecode", "revm-context", @@ -9252,9 +9211,9 @@ dependencies = [ [[package]] name = "revm-bytecode" -version = "10.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdbb3a3d735efa94c91f2ef6bf20a35f99a77bc78f3e25bd758336901bdf9661" +checksum = "e33493cf6d996e9242db4f2002caef48ec9aa8c4f3fff3b18aed10ef3942eec7" dependencies = [ "bitvec", "phf", @@ -9264,9 +9223,9 @@ dependencies = [ [[package]] name = "revm-context" -version = "16.0.1" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f68d928d8b228e0faeb1c6ed75c4fde7d124f1ddf9119b67e7a0ad4041237d" +checksum = "86e48fc736d9542c082ea5305be44b6a14b53a615f0147ad57f62189fd813068" dependencies = [ "bitvec", "cfg-if", @@ -9281,9 +9240,9 @@ dependencies = [ [[package]] name = "revm-context-interface" -version = "17.0.1" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3758e6167c4ba7a59a689c519a047edaefcd4c37d74f279b93ed87bc8aece4" +checksum = "ac68282a454318246817486835a446519291dd0a8167d09de14c7e96f2147696" dependencies = [ "alloy-eip2930", "alloy-eip7702", @@ -9297,11 +9256,13 @@ dependencies = [ [[package]] name = "revm-database" -version = "13.0.1" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c281a1f11d3bcb8c0bba1199ed6bcb001d1aeb3d4fb366819e14f88723989a4e" +checksum = "d761681a43e48408ea23f7f66ff56ff7b6128cadb8f9135d1b94787d0c0fc6b4" dependencies = [ - "alloy-eips 1.8.3", + "alloy-eips", + "derive_more", + "either", "revm-bytecode", "revm-database-interface", "revm-primitives", @@ -9311,9 +9272,9 @@ dependencies = [ [[package]] name = "revm-database-interface" -version = "11.0.1" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d89efb9832a4e3742bb4ded5f7fe5bf905e8860e69427d4dfec153484fc6d304" +checksum = "b96e37d62fa60b45987b408486c84ebef4af4e7ddf1b3b96b4955a7263fd4e92" dependencies = [ "auto_impl", "either", @@ -9325,9 +9286,9 @@ dependencies = [ [[package]] name = "revm-handler" -version = "18.1.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "783e903d6922b7f5f9a940d1bb229530502d2924b1aed9d5ca5a94ebf065d460" +checksum = "f5e043ebfc0899c435e44475796285898e4f822e637bd67856da79cc170bd9df" dependencies = [ "auto_impl", "derive-where", @@ -9344,9 +9305,9 @@ dependencies = [ [[package]] name = "revm-inspector" -version = "19.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8216ad58422090d0daa9eb430e0a081f7ad07e7fd30681dee71f8420c99624e0" +checksum = "2e470b2a5e177918944d8cd26174455b61b8fbfbbe7fe411d28b4097e410d11e" dependencies = [ "auto_impl", "either", @@ -9362,9 +9323,9 @@ dependencies = [ [[package]] name = "revm-inspectors" -version = "0.39.0" +version = "0.41.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "731b682530a732ef9c189ef831589128e2ce34d4a306c956322ae2dffe009715" +checksum = "54e22e1aa328f78e8c3dea6a3a860a3a3c3a77c4a7e775e3fdd3dcbb24a152ac" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -9380,9 +9341,9 @@ dependencies = [ [[package]] name = "revm-interpreter" -version = "35.0.1" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ece9f41b69658c15d748288a4dbdfc06a63f3ce93d983af440de3f1631dce6a" +checksum = "fa5547f653f9e4c47b4f9a9075a0b7c0faea5fa29b7873f392733a943837825d" dependencies = [ "revm-bytecode", "revm-context-interface", @@ -9393,9 +9354,9 @@ dependencies = [ [[package]] name = "revm-precompile" -version = "34.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a346a8cc6c8c39bd65306641c692191299c0a7b63d38810e39e8fe9b92378660" +checksum = "6d785931e4b8750cf9d79c808c22f05979c8d191baafc8badd4651db82584c1c" dependencies = [ "ark-bls12-381", "ark-bn254", @@ -9419,24 +9380,24 @@ dependencies = [ [[package]] name = "revm-primitives" -version = "23.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c99bda77d9661521ba0b4bc04558c6692074f01e65dd420fa3b893033d9b8a2" +checksum = "66f39151a31afe93d97f7ac894dd9dcc989368d5ccba19ce079bfc1d73b77452" dependencies = [ "alloy-primitives", - "num_enum", "once_cell", "serde", ] [[package]] name = "revm-state" -version = "11.0.1" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c32490ed687dba31c3c882beb8c20408bdd30ef96690d8f145b0ee9a87040bfe" +checksum = "73ef2fffc28acc41e8ef7525f2b1d16288a10f80579a1b33e7e62abc9892314a" dependencies = [ "alloy-eip7928", "bitflags", + "nonmax", "revm-bytecode", "revm-primitives", "serde", @@ -9444,9 +9405,9 @@ dependencies = [ [[package]] name = "revm-statetest-types" -version = "17.0.1" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94743de1e54a812077b79dd3a87b4059175a804650b21582fb6feae96ef2be9c" +checksum = "d9ced634104a9857185f02a7a3a5dd4a503590fababafbb295ad213df6dbf302" dependencies = [ "alloy-eip7928", "k256", @@ -9650,7 +9611,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -9697,7 +9658,7 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19787cda76408ec5404443dc8b31795c87cd8fec49762dc75fa727740d34acc1" dependencies = [ - "core-foundation", + "core-foundation 0.10.1", "core-foundation-sys", "jni 0.21.1", "log", @@ -9718,7 +9679,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" dependencies = [ - "core-foundation", + "core-foundation 0.10.1", "core-foundation-sys", "jni 0.22.4", "log", @@ -9730,7 +9691,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs 1.0.7", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -9912,7 +9873,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ "bitflags", - "core-foundation", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -9942,10 +9903,6 @@ name = "semver" version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" -dependencies = [ - "serde", - "serde_core", -] [[package]] name = "semver-parser" @@ -10058,7 +10015,7 @@ version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf2ebbe86054f9b45bc3881e865683ccfaccce97b9b4cb53f3039d67f355a334" dependencies = [ - "darling 0.23.0", + "darling", "proc-macro2", "quote", "syn 2.0.117", @@ -10103,7 +10060,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77fd7028345d415a4034cf8777cd4f8ab1851274233b45f84e3d955502d93874" dependencies = [ "digest 0.10.7", - "keccak", + "keccak 0.1.6", +] + +[[package]] +name = "sha3" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be176f1a57ce4e3d31c1a166222d9768de5954f811601fb7ca06fc8203905ce1" +dependencies = [ + "digest 0.11.3", + "keccak 0.2.0", ] [[package]] @@ -10256,7 +10223,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -10360,9 +10327,9 @@ dependencies = [ [[package]] name = "syn-solidity" -version = "1.5.7" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53f425ae0b12e2f5ae65542e00898d500d4d318b4baf09f40fd0d410454e9947" +checksum = "ec005042c7d952febc1a3ef5b0f6674e9054aa836877a31c90b20e25b3d31744" dependencies = [ "paste", "proc-macro2", @@ -10404,6 +10371,27 @@ dependencies = [ "windows 0.62.2", ] +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tagptr" version = "0.2.0" @@ -10437,7 +10425,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -10545,12 +10533,11 @@ dependencies = [ [[package]] name = "time" -version = "0.3.47" +version = "0.3.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +checksum = "18dfaaeddcb932337b5e7866ee7d0ce9b76d2fd092997146f187ec09b4558a50" dependencies = [ "deranged", - "itoa", "libc", "num-conv", "num_threads", @@ -10562,15 +10549,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.27" +version = "0.2.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +checksum = "c431b87111666e491a90baa837f914fb45cd5dc3c268591b0220ff5057f2085f" dependencies = [ "num-conv", "time-core", @@ -10603,9 +10590,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.52.2" +version = "1.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "110a78583f19d5cdb2c5ccf321d1290344e71313c6c37d43520d386027d18386" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ "bytes", "libc", @@ -10780,6 +10767,17 @@ dependencies = [ "tonic", ] +[[package]] +name = "tonic-types" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ab1b02061f83d519bba3caa167f88f261ef05720ab8ebc954ade70de3348e8" +dependencies = [ + "prost", + "prost-types", + "tonic", +] + [[package]] name = "tower" version = "0.5.3" @@ -10879,6 +10877,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "tracing-chrome" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf0a738ed5d6450a9fb96e86a23ad808de2b727fd1394585da5cdd6788ffe724" +dependencies = [ + "serde_json", + "tracing-core", + "tracing-subscriber 0.3.23", +] + [[package]] name = "tracing-core" version = "0.1.36" @@ -10935,9 +10944,9 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.32.1" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac28f2d093c6c477eaa76b23525478f38de514fa9aeb1285738d4b97a9552fc" +checksum = "adbc64cba7137545b8044cb1fe9814f7aacf3c6b5f9b45be8bb5db538befdb26" dependencies = [ "js-sys", "opentelemetry", @@ -11024,7 +11033,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8840ad4d852e325d3afa7fde8a50b2412f89dce47d7eb291c0cc7f87cd040f38" dependencies = [ - "darling 0.23.0", + "darling", "proc-macro2", "quote", "syn 2.0.117", @@ -11160,7 +11169,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "subtle", ] @@ -11238,14 +11247,12 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "vergen" -version = "9.1.0" +version = "10.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b849a1f6d8639e8de261e81ee0fc881e3e3620db1af9f2e0da015d4382ceaf75" +checksum = "b5574dd2f922b1a46a06a4b1dc11193a4012108fd54cf725e1816cb8183d8778" dependencies = [ "anyhow", - "cargo_metadata", - "derive_builder", - "regex", + "bon", "rustversion", "time", "vergen-lib", @@ -11253,12 +11260,12 @@ dependencies = [ [[package]] name = "vergen-git2" -version = "9.1.0" +version = "10.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d51ab55ddf1188c8d679f349775362b0fa9e90bd7a4ac69838b2a087623f0d57" +checksum = "410e2f72acce10471037150cd9c585ee7b54560f348bf70cd5fc8e87d3433e45" dependencies = [ "anyhow", - "derive_builder", + "bon", "git2", "rustversion", "time", @@ -11268,12 +11275,12 @@ dependencies = [ [[package]] name = "vergen-lib" -version = "9.1.0" +version = "10.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b34a29ba7e9c59e62f229ae1932fb1b8fb8a6fdcc99215a641913f5f5a59a569" +checksum = "8cd42fd155c2c2971f6d00face12ec245fbb604fce011ccaf2306d014c2e97ca" dependencies = [ "anyhow", - "derive_builder", + "bon", "rustversion", ] @@ -11556,7 +11563,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -12011,9 +12018,6 @@ name = "winnow" version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" -dependencies = [ - "memchr", -] [[package]] name = "winnow" diff --git a/Cargo.toml b/Cargo.toml index a9b4e0f3..f30e05fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [workspace.package] version = "1.0.0" edition = "2024" -rust-version = "1.93" +rust-version = "1.95" license = "MIT OR Apache-2.0" publish = false @@ -13,7 +13,6 @@ members = [ "crates/chainspec", "crates/consensus", "crates/engine-api", - "crates/engine-tree-ext", "crates/evm", "crates/node", "crates/rpc", @@ -90,7 +89,6 @@ codegen-units = 1 morph-chainspec = { path = "crates/chainspec", default-features = false } morph-consensus = { path = "crates/consensus", default-features = false } morph-engine-api = { path = "crates/engine-api", default-features = false } -morph-engine-tree-ext = { path = "crates/engine-tree-ext", default-features = false } morph-evm = { path = "crates/evm", default-features = false } morph-node = { path = "crates/node"} morph-payload-builder = { path = "crates/payload/builder", default-features = false } @@ -101,98 +99,98 @@ morph-rpc = { path = "crates/rpc" } morph-revm = { path = "crates/revm", default-features = false } morph-txpool = { path = "crates/txpool", default-features = false } -reth-basic-payload-builder = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-chain-state = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-chainspec = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-cli = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-cli-commands = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-cli-util = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-codecs = { version = "0.3.1", default-features = false } -reth-codecs-derive = "0.3.1" -reth-consensus = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-consensus-common = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-db = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-db-api = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-e2e-test-utils = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-engine-local = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-engine-primitives = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-engine-tree = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-errors = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-eth-wire-types = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-exex = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-exex-test-utils = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-exex-types = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-ethereum = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-ethereum-cli = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-ethereum-consensus = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-ethereum-engine-primitives = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-ethereum-primitives = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0", default-features = false } -reth-evm = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-evm-ethereum = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-execution-types = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-metrics = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-network-peers = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-node-api = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-node-builder = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-node-core = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-node-ethereum = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-node-metrics = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-payload-builder = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-payload-primitives = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-payload-util = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-primitives-traits = { version = "0.3.1", default-features = false } -reth-provider = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-rpc = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-rpc-api = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-rpc-builder = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-rpc-convert = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-rpc-eth-api = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-rpc-eth-types = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-rpc-server-types = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-storage-api = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-tasks = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-tracing = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-trie = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-transaction-pool = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-zstd-compressors = { version = "0.3.1", default-features = false } -reth-execution-cache = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-trie-db = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-trie-parallel = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0" } -reth-trie-sparse = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0", default-features = false } +reth-basic-payload-builder = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-chain-state = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-chainspec = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-cli = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-cli-commands = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-cli-util = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-codecs = { version = "0.5.0", default-features = false } +reth-codecs-derive = "0.5.0" +reth-consensus = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-consensus-common = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-db = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-db-api = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-e2e-test-utils = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-engine-local = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-engine-primitives = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-engine-tree = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-errors = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-eth-wire-types = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-exex = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-exex-test-utils = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-exex-types = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-ethereum = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-ethereum-cli = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-ethereum-consensus = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-ethereum-engine-primitives = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-ethereum-primitives = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105", default-features = false } +reth-evm = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-evm-ethereum = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-execution-types = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-metrics = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-network-peers = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-node-api = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-node-builder = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-node-core = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-node-ethereum = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-node-metrics = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-payload-builder = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-payload-primitives = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-payload-util = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-primitives-traits = { version = "0.5.0", default-features = false } +reth-provider = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-rpc = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-rpc-api = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-rpc-builder = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-rpc-convert = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-rpc-eth-api = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-rpc-eth-types = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-rpc-server-types = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-storage-api = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-tasks = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-tracing = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-trie = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-transaction-pool = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-zstd-compressors = { version = "0.5.0", default-features = false } +reth-execution-cache = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-trie-db = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-trie-parallel = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-trie-sparse = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105", default-features = false } -reth-revm = { git = "https://github.com/paradigmxyz/reth", tag = "v2.2.0", features = [ +reth-revm = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105", features = [ "std", "optional-checks", ] } -revm = { version = "38.0.0", features = [ +revm = { version = "41.0.0", features = [ "optional_fee_charge", "optional_eip7623", ], default-features = false } -alloy = { version = "2.0.4", default-features = false } -alloy-consensus = { version = "2.0.4", default-features = false } -alloy-contract = { version = "2.0.4", default-features = false } -alloy-eip7928 = { version = "0.3.4", default-features = false } -alloy-eips = { version = "2.0.4", default-features = false } -alloy-evm = { version = "0.34.0", default-features = false } -alloy-genesis = "2.0.4" +alloy = { version = "2.1.1", default-features = false } +alloy-consensus = { version = "2.1.1", default-features = false } +alloy-contract = { version = "2.1.1", default-features = false } +alloy-eip7928 = { version = "0.4.5", default-features = false } +alloy-eips = { version = "2.1.1", default-features = false } +alloy-evm = { version = "0.37.0", default-features = false } +alloy-genesis = "2.1.1" alloy-hardforks = "0.4.7" -alloy-network = { version = "2.0.4", default-features = false } -alloy-primitives = { version = "1.5.6", default-features = false } -alloy-provider = { version = "2.0.4", default-features = false } -alloy-rlp = "0.3.13" -alloy-trie = "0.9.5" -alloy-rpc-types-engine = "2.0.4" -alloy-rpc-types-eth = { version = "2.0.4" } -alloy-serde = "2.0.4" -alloy-signer = "2.0.4" -alloy-signer-local = "2.0.4" -alloy-sol-types = { version = "1.5.6", default-features = false } -alloy-transport = "2.0.4" +alloy-network = { version = "2.1.1", default-features = false } +alloy-primitives = { version = "1.6.0", default-features = false } +alloy-provider = { version = "2.1.1", default-features = false } +alloy-rlp = "0.3.16" +alloy-trie = "0.9.4" +alloy-rpc-types-engine = "2.1.1" +alloy-rpc-types-eth = { version = "2.1.1" } +alloy-serde = "2.1.1" +alloy-signer = "2.1.1" +alloy-signer-local = "2.1.1" +alloy-sol-types = { version = "1.6.0", default-features = false } +alloy-transport = "2.1.1" alloy-chains = { version = "0.2.33", default-features = false } crossbeam-channel = "0.5.13" -revm-primitives = { version = "23.0.0", default-features = false } -revm-statetest-types = { version = "17.0.1", default-features = false } +revm-primitives = { version = "41.0.0", default-features = false } +revm-statetest-types = { version = "41.0.0", default-features = false } arbitrary = { version = "1.3", features = ["derive"] } async-lock = "3.4.1" async-trait = "0.1" @@ -234,8 +232,8 @@ tokio-stream = "0.1.17" tokio-util = "0.7.16" tracing = "0.1.41" tracing-subscriber = "0.3.22" -vergen = "9.1.0" -vergen-git2 = "9.1.0" +vergen = "10.0.1" +vergen-git2 = "10.0.1" criterion = "0.7.0" test-case = "3" pyroscope = "0.5.8" diff --git a/crates/chainspec/src/hardfork.rs b/crates/chainspec/src/hardfork.rs index 42d75185..63b005bf 100644 --- a/crates/chainspec/src/hardfork.rs +++ b/crates/chainspec/src/hardfork.rs @@ -234,7 +234,7 @@ mod tests { let params = GasParams::new_spec(spec); assert_eq!( - params.tx_eip7702_per_auth_state_gas(), + params.tx_eip7702_state_gas(), 0, "MorphHardfork {fork:?} must not enable EIP-8037 state gas" ); @@ -245,15 +245,14 @@ mod tests { fn test_eip7702_refund_stays_regular_for_morph_specs() { for spec in [SpecId::CANCUN, SpecId::PRAGUE, SpecId::OSAKA] { let params = GasParams::new_spec(spec); - let total_refund = 25_000; - let (state_refund, regular_refund) = params.split_eip7702_refund(total_refund); - assert_eq!( - state_refund, 0, + params.tx_eip7702_state_refund(2, 2), + 0, "spec={spec:?}: Morph must not route EIP-7702 refunds to state gas" ); assert_eq!( - regular_refund, total_refund, + params.tx_eip7702_auth_refund_regular(), + params.tx_eip7702_auth_refund(), "spec={spec:?}: Morph EIP-7702 refunds must remain subject to the regular refund cap" ); } diff --git a/crates/consensus/src/validation.rs b/crates/consensus/src/validation.rs index 3b32297a..cd12183c 100644 --- a/crates/consensus/src/validation.rs +++ b/crates/consensus/src/validation.rs @@ -89,7 +89,7 @@ const GAS_LIMIT_BOUND_DIVISOR: u64 = 1024; /// start, have sequential queue indices, and are consistent with `header.next_l1_msg_index`. /// 2. **Cross-block monotonicity** (`validate_header_against_parent`): `header.next_l1_msg_index` /// is monotonically non-decreasing relative to the parent. -/// 3. **Parent-aware exactness** (`MorphBasicEngineValidator`): once the engine tree has +/// 3. **Parent-aware exactness** (`MorphTreeEngineValidator`): once the engine tree has /// both parent header and block body, Jade blocks must exactly match the value derived /// from `parent.next_l1_msg_index` and the block's leading L1 messages. /// @@ -356,6 +356,7 @@ impl FullConsensus for MorphConsensus { block: &RecoveredBlock, result: &BlockExecutionResult, receipt_root_bloom: Option, + _block_access_list_hash: Option, ) -> Result<(), ConsensusError> { // Verify the block gas used let cumulative_gas_used = result @@ -2070,7 +2071,7 @@ mod tests { let recovered = reth_primitives_traits::RecoveredBlock::new_unhashed(block, vec![Address::ZERO]); - let post_result = consensus.validate_block_post_execution(&recovered, &result, None); + let post_result = consensus.validate_block_post_execution(&recovered, &result, None, None); assert!(matches!( post_result, Err(ConsensusError::BlockGasUsed { .. }) diff --git a/crates/engine-api/src/builder.rs b/crates/engine-api/src/builder.rs index 390e5a10..26aa0e3b 100644 --- a/crates/engine-api/src/builder.rs +++ b/crates/engine-api/src/builder.rs @@ -725,6 +725,7 @@ impl RealMorphL2EngineApi { parent_beacon_block_root: None, // Morph L2 has no PoS slot semantics; introduced in alloy 2.0. slot_number: None, + target_gas_limit: None, }, transactions: Some(params.transactions), gas_limit: gas_limit_override, @@ -737,7 +738,7 @@ impl RealMorphL2EngineApi { attributes: rpc_attributes, parent_hash, cache: None, - trie_handle: None, + state_root_handle: None, }; let _ = self .payload_builder diff --git a/crates/engine-tree-ext/Cargo.toml b/crates/engine-tree-ext/Cargo.toml deleted file mode 100644 index 255e6c88..00000000 --- a/crates/engine-tree-ext/Cargo.toml +++ /dev/null @@ -1,80 +0,0 @@ -[package] -name = "morph-engine-tree-ext" -version.workspace = true -edition.workspace = true -rust-version.workspace = true -license.workspace = true -publish.workspace = true - -[lints] -workspace = true - -[dependencies] -# morph local crates -morph-chainspec = { workspace = true } -# morph-primitives brought in transitively via morph-chainspec; direct dep here -# enables the reth-codec feature so cargo test -p morph-engine-tree-ext satisfies -# NodePrimitives::Receipt: FullReceipt (requires Compact impl). -morph-primitives = { workspace = true, features = ["reth-codec"] } - -# reth (workspace = true after Task 4 flips root Cargo.toml to paradigmxyz/reth v2.0.0) -reth-chain-state = { workspace = true } -reth-consensus = { workspace = true } -reth-db = { workspace = true } -reth-engine-primitives = { workspace = true } -reth-engine-tree = { workspace = true } -reth-errors = { workspace = true } -reth-evm = { workspace = true } -reth-execution-cache = { workspace = true } -reth-payload-primitives = { workspace = true } -reth-primitives-traits = { workspace = true } -reth-provider = { workspace = true } -reth-revm = { workspace = true } -reth-tasks = { workspace = true } -reth-trie = { workspace = true } -reth-trie-db = { workspace = true } -reth-trie-parallel = { workspace = true } - -# alloy / revm — workspace versions from reth v2.0.0 -alloy-consensus = { workspace = true } -alloy-eip7928 = { workspace = true } -alloy-eips = { workspace = true } -alloy-evm = { workspace = true } -alloy-primitives = { workspace = true } -alloy-rlp = { workspace = true } -revm-primitives = { workspace = true } - -# utilities -crossbeam-channel = { workspace = true } -derive_more = { workspace = true } -tokio = { workspace = true } -tracing = { workspace = true } - -[features] -default = ["std"] -std = [] -trie-debug = ["dep:reth-trie-sparse"] -# Forwards `morph-node`'s `test-utils` feature, which provides `MorphTestNode`, -# `TestNodeBuilder`, `HardforkSchedule`, etc. used by the Jade-boundary -# integration test. -test-utils = ["morph-node/test-utils"] - -[dependencies.reth-trie-sparse] -workspace = true -default-features = false -optional = true - -[dev-dependencies] -alloy-genesis = { workspace = true } -serde_json = { workspace = true } -# Jade boundary integration test — exercises MorphBasicEngineValidator through -# the real MorphNode stack via morph-node's test-utils feature. -morph-node = { workspace = true, features = ["test-utils"] } -morph-payload-types = { workspace = true } -reth-node-api = { workspace = true } -reth-payload-builder = { workspace = true } -reth-tracing = { workspace = true } -alloy-rpc-types-engine = { workspace = true } -alloy-rpc-types-eth = { workspace = true } -eyre = { workspace = true } -tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/crates/engine-tree-ext/src/gate.rs b/crates/engine-tree-ext/src/gate.rs deleted file mode 100644 index e3dbd9db..00000000 --- a/crates/engine-tree-ext/src/gate.rs +++ /dev/null @@ -1,67 +0,0 @@ -//! Decides whether strict state-root equality must be enforced for a block. -//! -//! Pre-Jade Morph blocks store ZK-trie roots in `header.state_root`, while reth -//! computes MPT roots. Skipping the strict check pre-Jade is safe because the -//! first post-Jade block's successful strict check retroactively anchors the -//! MPT state accumulated across the pre-Jade range (transitivity of EVM -//! execution; see the design spec). - -use morph_chainspec::{MorphChainSpec, MorphHardforks}; - -/// Returns `true` iff reth must enforce the strict -/// `computed_state_root == header.state_root()` check at the given block -/// timestamp. Post-Jade: true. Pre-Jade: false. -pub fn state_root_enforced_at(chain_spec: &MorphChainSpec, timestamp: u64) -> bool { - chain_spec.is_jade_active_at_timestamp(timestamp) -} - -#[cfg(test)] -mod tests { - use super::*; - use alloy_genesis::Genesis; - use serde_json::json; - use std::sync::Arc; - - fn chain_spec_with_jade_at(jade_time: u64) -> Arc { - let genesis: Genesis = serde_json::from_value(json!({ - "config": { - "chainId": 1337, - "bernoulliBlock": 0, - "curieBlock": 0, - "morph": {}, - "jadeForkTime": jade_time - }, - "alloc": {} - })) - .expect("valid test genesis"); - Arc::new(MorphChainSpec::from(genesis)) - } - - #[test] - fn skipped_before_jade() { - let cs = chain_spec_with_jade_at(1_000); - assert!(!state_root_enforced_at(&cs, 0)); - assert!(!state_root_enforced_at(&cs, 500)); - assert!(!state_root_enforced_at(&cs, 999)); - } - - #[test] - fn strict_at_and_after_jade() { - let cs = chain_spec_with_jade_at(1_000); - assert!(state_root_enforced_at(&cs, 1_000)); - assert!(state_root_enforced_at(&cs, 1_001)); - assert!(state_root_enforced_at(&cs, u64::MAX)); - } - - #[test] - fn skipped_when_jade_unset() { - let genesis: Genesis = serde_json::from_value(json!({ - "config": { "chainId": 1337, "bernoulliBlock": 0, "curieBlock": 0, "morph": {} }, - "alloc": {} - })) - .unwrap(); - let cs = Arc::new(MorphChainSpec::from(genesis)); - assert!(!state_root_enforced_at(&cs, 0)); - assert!(!state_root_enforced_at(&cs, 1_000_000)); - } -} diff --git a/crates/engine-tree-ext/src/lib.rs b/crates/engine-tree-ext/src/lib.rs deleted file mode 100644 index fc277227..00000000 --- a/crates/engine-tree-ext/src/lib.rs +++ /dev/null @@ -1,13 +0,0 @@ -//! Morph extension of reth's engine tree. -//! -//! Verbatim copy of `reth_engine_tree::tree::payload_validator` from reth v2.0.0, -//! used as the foundation for a Jade-gated state-root skip. -//! -//! `trie_updates` is a verbatim copy of the private sibling module from the same -//! source tree, required by `payload_validator::compare_trie_updates_with_serial`. - -pub mod gate; -pub mod payload_validator; -pub(crate) mod trie_updates; - -pub use payload_validator::MorphBasicEngineValidator; diff --git a/crates/engine-tree-ext/src/payload_validator.rs b/crates/engine-tree-ext/src/payload_validator.rs deleted file mode 100644 index 5d52e082..00000000 --- a/crates/engine-tree-ext/src/payload_validator.rs +++ /dev/null @@ -1,2193 +0,0 @@ -//! Types and traits for validating blocks and payloads. -//! -//! # Validation pipeline -//! -//! When the engine processes a new payload (`engine_newPayload`), validation happens in phases: -//! -//! ## Phase 1 – Payload conversion -//! [`PayloadValidator::convert_payload_to_block`] decodes the execution payload (RLP, hashing) -//! into a [`SealedBlock`]. This runs on a background thread concurrently with state setup. -//! -//! ## Phase 2 – Pre-execution consensus -//! - [`HeaderValidator::validate_header`] — standalone header checks (hash, gas, base fee, -//! fork-specific fields) -//! - [`Consensus::validate_block_pre_execution`] — body vs header (tx root, ommer hash, withdrawals -//! root) -//! - [`HeaderValidator::validate_header_against_parent`] — sequential checks (block number, -//! timestamp, gas limit, base fee vs parent) -//! -//! ## Phase 3 – Execution -//! Block transactions are executed via the EVM. Receipt roots are computed incrementally. -//! -//! ## Phase 4 – Post-execution consensus -//! - [`FullConsensus::validate_block_post_execution`] — gas used, receipt root, logs bloom, -//! requests hash -//! - [`PayloadValidator::validate_block_post_execution_with_hashed_state`] — network-specific -//! (no-op on L1, used by OP Stack) -//! -//! ## Payload attributes validation (`engine_forkchoiceUpdated`) -//! When the CL provides payload attributes to start building a block: -//! - [`PayloadValidator::validate_payload_attributes_against_header`] — ensures timestamp ordering -//! -//! If validation passes, a payload build job is started. If it fails, -//! `INVALID_PAYLOAD_ATTRIBUTES` is returned without rolling back the forkchoice update. -//! -//! [`HeaderValidator::validate_header`]: reth_consensus::HeaderValidator::validate_header -//! [`Consensus::validate_block_pre_execution`]: reth_consensus::Consensus::validate_block_pre_execution -//! [`HeaderValidator::validate_header_against_parent`]: reth_consensus::HeaderValidator::validate_header_against_parent -//! [`FullConsensus::validate_block_post_execution`]: reth_consensus::FullConsensus::validate_block_post_execution -//! [`SealedBlock`]: reth_primitives_traits::SealedBlock - -use alloy_consensus::transaction::{Either, TxHashRef}; -use alloy_eip7928::BlockAccessList; -use alloy_eips::{NumHash, eip1898::BlockWithParent, eip4895::Withdrawal}; -use alloy_evm::Evm; -use alloy_primitives::{B256, map::B256Set}; -use reth_engine_tree::tree::{ - CacheWaitDurations, CachedStateProvider, EngineApiMetrics, EngineApiTreeState, EngineValidator, - ExecutionEnv, PayloadHandle, StateProviderBuilder, TreeConfig, WaitForCaches, - error::{InsertBlockError, InsertBlockErrorKind, InsertPayloadError}, - instrumented_state::{InstrumentedStateProvider, StateProviderStats}, - payload_processor::{ - PayloadProcessor, - multiproof::{StateRootComputeOutcome, StateRootHandle}, - }, - payload_validator::TreeCtx, - precompile_cache::{CachedPrecompile, CachedPrecompileMetrics, PrecompileCacheMap}, -}; -use reth_revm::database::StateProviderDatabase; -#[cfg(feature = "trie-debug")] -use reth_trie_sparse::debug_recorder::TrieDebugRecorder; - -use crate::gate::state_root_enforced_at; -use morph_chainspec::{MorphChainSpec, MorphHardforks}; -use morph_primitives::{MorphHeader, MorphTxEnvelope}; -use reth_chain_state::{DeferredTrieData, ExecutedBlock, ExecutionTimingStats, LazyOverlay}; -use reth_consensus::{ConsensusError, FullConsensus, ReceiptRootBloom}; -use reth_engine_primitives::{ - ConfigureEngineEvm, ExecutableTxIterator, ExecutionPayload, InvalidBlockHook, PayloadValidator, -}; -use reth_engine_tree::tree::payload_processor::receipt_root_task::{ - IndexedReceipt, ReceiptRootTaskHandle, -}; -use reth_errors::{BlockExecutionError, ProviderResult}; -use reth_evm::{ - ConfigureEvm, EvmEnvFor, ExecutionCtxFor, OnStateHook, SpecFor, block::BlockExecutor, - execute::ExecutableTxFor, -}; -use reth_execution_cache::{CacheStats, SavedCache}; -use reth_payload_primitives::{ - BuiltPayload, InvalidPayloadAttributesError, NewPayloadError, PayloadTypes, -}; -use reth_primitives_traits::{ - AlloyBlockHeader, BlockBody, BlockTy, FastInstant as Instant, GotExpected, NodePrimitives, - RecoveredBlock, SealedBlock, SealedHeader, SignerRecoverable, -}; -use reth_provider::{ - BlockExecutionOutput, BlockNumReader, BlockReader, ChangeSetReader, DatabaseProviderFactory, - DatabaseProviderROFactory, HashedPostStateProvider, ProviderError, PruneCheckpointReader, - StageCheckpointReader, StateProvider, StateProviderFactory, StateReader, - StorageChangeSetReader, StorageSettingsCache, - providers::{OverlayBuilder, OverlayStateProviderFactory}, -}; -use reth_revm::db::{BundleAccount, State, states::bundle_state::BundleRetention}; -use reth_trie::{HashedPostState, StateRoot, trie_cursor::TrieCursorFactory, updates::TrieUpdates}; -use reth_trie_db::ChangesetCache; -use reth_trie_parallel::root::{ParallelStateRoot, ParallelStateRootError}; -use revm_primitives::{Address, KECCAK_EMPTY}; -use std::{ - collections::HashMap, - panic::{self, AssertUnwindSafe}, - sync::{ - Arc, - atomic::{AtomicUsize, Ordering}, - mpsc::RecvTimeoutError, - }, - time::Duration, -}; -use tracing::{Span, debug, debug_span, error, info, instrument, trace, warn}; - -/// Output of block or payload validation. -pub type ValidationOutcome>> = - Result<(ExecutedBlock, Option>), E>; - -/// Handle to a [`HashedPostState`] computed on a background thread. -type LazyHashedPostState = reth_tasks::LazyHandle; - -/// Result type for block validation with optional timing stats. -type InsertPayloadResult = Result< - (ExecutedBlock, Option>), - InsertPayloadError<::Block>, ->; - -/// Context providing access to tree state during validation. -/// -/// A helper type that provides reusable payload validation logic for network-specific validators. -/// -/// This type satisfies [`EngineValidator`] and is responsible for executing blocks/payloads. -/// -/// This type contains common validation, execution, and state root computation logic that can be -/// used by network-specific payload validators (e.g., Ethereum, Optimism). It is not meant to be -/// used as a standalone component, but rather as a building block for concrete implementations. -#[derive(derive_more::Debug)] -pub struct MorphBasicEngineValidator -where - Evm: ConfigureEvm, -{ - /// Provider for database access. - provider: P, - /// Consensus implementation for validation. - consensus: Arc>, - /// EVM configuration. - evm_config: Evm, - /// Configuration for the tree. - config: TreeConfig, - /// Payload processor for state root computation. - payload_processor: PayloadProcessor, - /// Precompile cache map. - precompile_cache_map: PrecompileCacheMap>, - /// Precompile cache metrics. - precompile_cache_metrics: HashMap, - /// Hook to call when invalid blocks are encountered. - #[debug(skip)] - invalid_block_hook: Box>, - /// Metrics for the engine api. - metrics: EngineApiMetrics, - /// Validator for the payload. - validator: V, - /// Changeset cache for in-memory trie changesets - changeset_cache: ChangesetCache, - /// Task runtime for spawning parallel work. - runtime: reth_tasks::Runtime, - /// Chain spec, used to gate pre-Jade state-root skipping. - chain_spec: Arc, -} - -impl MorphBasicEngineValidator -where - N: NodePrimitives< - Block = morph_primitives::Block, - BlockHeader = MorphHeader, - SignedTx = MorphTxEnvelope, - >, - P: DatabaseProviderFactory< - Provider: BlockReader - + StageCheckpointReader - + PruneCheckpointReader - + ChangeSetReader - + StorageChangeSetReader - + BlockNumReader - + StorageSettingsCache, - > + BlockReader
- + ChangeSetReader - + BlockNumReader - + StateProviderFactory - + StateReader - + HashedPostStateProvider - + Clone - + 'static, - Evm: ConfigureEvm + 'static, -{ - /// Creates a new `TreePayloadValidator`. - #[expect(clippy::too_many_arguments)] - pub fn new( - provider: P, - consensus: Arc>, - evm_config: Evm, - validator: V, - config: TreeConfig, - invalid_block_hook: Box>, - changeset_cache: ChangesetCache, - runtime: reth_tasks::Runtime, - chain_spec: Arc, - ) -> Self { - let precompile_cache_map = PrecompileCacheMap::default(); - let payload_processor = PayloadProcessor::new( - runtime.clone(), - evm_config.clone(), - &config, - precompile_cache_map.clone(), - ); - Self { - provider, - consensus, - evm_config, - payload_processor, - precompile_cache_map, - precompile_cache_metrics: HashMap::new(), - config, - invalid_block_hook, - metrics: EngineApiMetrics::default(), - validator, - changeset_cache, - runtime, - chain_spec, - } - } - - /// Converts a [`BlockOrPayload`] to a recovered block. - #[instrument(level = "debug", target = "engine::tree::payload_validator", skip_all)] - pub fn convert_to_block>>( - &self, - input: BlockOrPayload, - ) -> Result, NewPayloadError> - where - V: PayloadValidator, - { - match input { - BlockOrPayload::Payload(payload) => self.validator.convert_payload_to_block(payload), - BlockOrPayload::Block(block) => Ok(block), - } - } - - /// Returns EVM environment for the given payload or block. - pub fn evm_env_for>>( - &self, - input: &BlockOrPayload, - ) -> Result, Evm::Error> - where - V: PayloadValidator, - Evm: ConfigureEngineEvm, - { - match input { - BlockOrPayload::Payload(payload) => Ok(self.evm_config.evm_env_for_payload(payload)?), - BlockOrPayload::Block(block) => Ok(self.evm_config.evm_env(block.header())?), - } - } - - /// Returns [`ExecutableTxIterator`] for the given payload or block. - pub fn tx_iterator_for<'a, T: PayloadTypes>>( - &'a self, - input: &'a BlockOrPayload, - ) -> Result, NewPayloadError> - where - V: PayloadValidator, - Evm: ConfigureEngineEvm, - { - Ok(match input { - BlockOrPayload::Payload(payload) => { - let iter = self - .evm_config - .tx_iterator_for_payload(payload) - .map_err(NewPayloadError::other)?; - Either::Left(iter) - } - BlockOrPayload::Block(block) => { - let txs = block.body().clone_transactions(); - let convert = |tx: N::SignedTx| tx.try_into_recovered(); - Either::Right((txs, convert)) - } - }) - } - - /// Returns a [`ExecutionCtxFor`] for the given payload or block. - pub fn execution_ctx_for<'a, T: PayloadTypes>>( - &self, - input: &'a BlockOrPayload, - ) -> Result, Evm::Error> - where - V: PayloadValidator, - Evm: ConfigureEngineEvm, - { - match input { - BlockOrPayload::Payload(payload) => Ok(self.evm_config.context_for_payload(payload)?), - BlockOrPayload::Block(block) => Ok(self.evm_config.context_for_block(block)?), - } - } - - /// Handles execution errors by checking if header validation errors should take precedence. - /// - /// When an execution error occurs, this function checks if there are any header validation - /// errors that should be reported instead, as header validation errors have higher priority. - fn handle_execution_error>>( - &self, - input: BlockOrPayload, - execution_err: InsertBlockErrorKind, - parent_block: &SealedHeader, - ) -> InsertPayloadResult - where - V: PayloadValidator, - { - debug!( - target: "engine::tree::payload_validator", - ?execution_err, - block = ?input.num_hash(), - "Block execution failed, checking for header validation errors" - ); - - // If execution failed, we should first check if there are any header validation - // errors that take precedence over the execution error - let block = self.convert_to_block(input)?; - - // Validate block consensus rules which includes header validation - if let Err(consensus_err) = self.validate_block_inner(&block, None) { - // Header validation error takes precedence over execution error - return Err(InsertBlockError::new(block, consensus_err.into()).into()); - } - - // Also validate against the parent - if let Err(consensus_err) = self - .consensus - .validate_header_against_parent(block.sealed_header(), parent_block) - { - // Parent validation error takes precedence over execution error - return Err(InsertBlockError::new(block, consensus_err.into()).into()); - } - - // No header validation errors, return the original execution error - Err(InsertBlockError::new(block, execution_err).into()) - } - - /// Validates a block that has already been converted from a payload. - /// - /// This method performs: - /// - Consensus validation - /// - Block execution - /// - State root computation - /// - Fork detection - #[instrument( - level = "debug", - target = "engine::tree::payload_validator", - skip_all, - fields( - parent = ?input.parent_hash(), - type_name = ?input.type_name(), - ) - )] - pub fn validate_block_with_state>>( - &mut self, - input: BlockOrPayload, - mut ctx: TreeCtx<'_, N>, - ) -> InsertPayloadResult - where - V: PayloadValidator + Clone, - Evm: ConfigureEngineEvm, - { - // Spawn payload conversion on a background thread so it runs concurrently with the - // rest of the function (setup + execution). For payloads this overlaps the cost of - // RLP decoding + header hashing. - let is_payload = matches!(&input, BlockOrPayload::Payload(_)); - let convert_to_block = match &input { - BlockOrPayload::Payload(_) => { - let payload_clone = input.clone(); - let validator = self.validator.clone(); - let handle = self.payload_processor.executor().spawn_blocking_named( - "payload-convert", - move || { - let BlockOrPayload::Payload(payload) = payload_clone else { - unreachable!() - }; - validator.convert_payload_to_block(payload) - }, - ); - Either::Left(handle) - } - BlockOrPayload::Block(_) => Either::Right(()), - }; - - // Returns the sealed block, either by awaiting the background conversion task (for - // payloads) or by extracting the already-converted block directly. - let convert_to_block = - move |input: BlockOrPayload| -> Result, NewPayloadError> { - match convert_to_block { - Either::Left(handle) => handle.try_into_inner().expect("sole handle"), - Either::Right(()) => { - let BlockOrPayload::Block(block) = input else { - unreachable!() - }; - Ok(block) - } - } - }; - - /// A helper macro that returns the block in case there was an error - /// This macro is used for early returns before block conversion - macro_rules! ensure_ok { - ($expr:expr) => { - match $expr { - Ok(val) => val, - Err(e) => { - let block = convert_to_block(input)?; - return Err(InsertBlockError::new(block, e.into()).into()); - } - } - }; - } - - /// A helper macro for handling errors after the input has been converted to a block - macro_rules! ensure_ok_post_block { - ($expr:expr, $block:expr) => { - match $expr { - Ok(val) => val, - Err(e) => { - return Err( - InsertBlockError::new($block.into_sealed_block(), e.into()).into() - ) - } - } - }; - } - - let parent_hash = input.parent_hash(); - - trace!(target: "engine::tree::payload_validator", "Fetching block state provider"); - let _enter = - debug_span!(target: "engine::tree::payload_validator", "state_provider").entered(); - let Some(provider_builder) = - ensure_ok!(self.state_provider_builder(parent_hash, ctx.state())) - else { - // this is pre-validated in the tree - return Err(InsertBlockError::new( - convert_to_block(input)?, - ProviderError::HeaderNotFound(parent_hash.into()).into(), - ) - .into()); - }; - let mut state_provider = ensure_ok!(provider_builder.build()); - drop(_enter); - - // Fetch parent block. This goes to memory most of the time unless the parent block is - // beyond the in-memory buffer. - let Some(parent_block) = ensure_ok!(self.sealed_header_by_hash(parent_hash, ctx.state())) - else { - return Err(InsertBlockError::new( - convert_to_block(input)?, - ProviderError::HeaderNotFound(parent_hash.into()).into(), - ) - .into()); - }; - - let evm_env = debug_span!(target: "engine::tree::payload_validator", "evm_env") - .in_scope(|| self.evm_env_for(&input)) - .map_err(NewPayloadError::other)?; - - let env = ExecutionEnv { - evm_env, - hash: input.hash(), - parent_hash: input.parent_hash(), - parent_state_root: parent_block.state_root(), - transaction_count: input.transaction_count(), - gas_used: input.gas_used(), - withdrawals: input.withdrawals().map(|w| w.to_vec()), - // Morph blocks (pre-Amsterdam) carry no BAL. Skip BAL-driven prewarming - // and let the payload processor install its own state hook for prewarm - // updates instead. - decoded_bal: None, - }; - - // Plan the strategy used for state root computation. - let strategy = self.plan_state_root_computation(); - - debug!( - target: "engine::tree::payload_validator", - ?strategy, - "Decided which state root algorithm to run" - ); - - // Get an iterator over the transactions in the payload - let txs = self.tx_iterator_for(&input)?; - - // Morph blocks (pre-Amsterdam) carry no BAL, so we deliberately drop the - // upstream BAL extraction. The header validator returns `None` for - // `block_access_list_hash()`, so any BAL bytes carried on the wire are - // ignored. - - // Create lazy overlay from ancestors - this doesn't block, allowing execution to start - // before the trie data is ready. The overlay will be computed on first access. - let (lazy_overlay, anchor_hash) = Self::get_parent_lazy_overlay(parent_hash, ctx.state()); - - // Create overlay factory for payload processor (StateRootTask path needs it for - // multiproofs). v2.2.0 moved `with_block_hash` / `with_lazy_overlay` onto - // `OverlayBuilder`, so we configure the builder first and pass it to the factory. - let overlay_builder = OverlayBuilder::::new(anchor_hash, self.changeset_cache.clone()) - .with_lazy_overlay(lazy_overlay); - let overlay_factory = - OverlayStateProviderFactory::new(self.provider.clone(), overlay_builder); - - // Spawn the appropriate processor based on strategy - let mut handle = ensure_ok!(self.spawn_payload_processor( - env.clone(), - txs, - provider_builder, - overlay_factory.clone(), - strategy, - )); - - // Create optional cache stats for detailed block logging - let slow_block_enabled = self.config.slow_block_threshold().is_some(); - let cache_stats = slow_block_enabled.then(|| Arc::new(CacheStats::default())); - - // Use cached state provider before executing, used in execution after prewarming threads - // complete - if let Some((caches, cache_metrics)) = handle.caches().zip(handle.cache_metrics()) { - state_provider = Box::new( - CachedStateProvider::new(state_provider, caches, cache_metrics) - .with_cache_stats(cache_stats.clone()), - ); - }; - - let state_provider_stats = if slow_block_enabled || self.config.state_provider_metrics() { - let instrumented_state_provider = - InstrumentedStateProvider::new(state_provider, "engine"); - let stats = slow_block_enabled.then(|| instrumented_state_provider.stats()); - state_provider = Box::new(instrumented_state_provider); - stats - } else { - None - }; - - // Execute the block and handle any execution errors. - // The receipt root task is spawned before execution and receives receipts incrementally - // as transactions complete, allowing parallel computation during execution. - let execute_block_start = Instant::now(); - let (output, senders, receipt_root_rx) = - match self.execute_block(state_provider, env, &input, &mut handle) { - Ok(output) => output, - Err(err) => return self.handle_execution_error(input, err, &parent_block), - }; - let execution_duration = execute_block_start.elapsed(); - - // After executing the block we can stop prewarming transactions - handle.stop_prewarming_execution(); - - // Create ExecutionOutcome early so we can terminate caching before validation and state - // root computation. Using Arc allows sharing with both the caching task and the deferred - // trie task without cloning the expensive BundleState. - let output = Arc::new(output); - - // Terminate caching task early since execution is complete and caching is no longer - // needed. This frees up resources while state root computation continues. - let valid_block_tx = handle.terminate_caching(Some(output.clone())); - - // Spawn hashed post state computation in background so it runs concurrently with - // block conversion and receipt root computation. This is a pure CPU-bound task - // (keccak256 hashing of all changed addresses and storage slots). - let hashed_state_output = output.clone(); - let hashed_state_provider = self.provider.clone(); - let hashed_state: LazyHashedPostState = self - .payload_processor - .executor() - .spawn_blocking_named("hash-post-state", move || { - let _span = debug_span!( - target: "engine::tree::payload_validator", - "hashed_post_state", - ) - .entered(); - hashed_state_provider.hashed_post_state(&hashed_state_output.state) - }); - - let block = convert_to_block(input)?; - let transaction_root = is_payload.then(|| { - let body = block.body().clone(); - let parent_span = Span::current(); - let num_hash = block.num_hash(); - self.payload_processor.executor().spawn_blocking_named("payload-tx-root", move || { - let _span = - debug_span!(target: "engine::tree::payload_validator", parent: parent_span, "payload_tx_root", block = ?num_hash) - .entered(); - body.calculate_tx_root() - }) - }); - let block = block.with_senders(senders); - - // Wait for the receipt root computation to complete. - let receipt_root_bloom = { - let _enter = debug_span!( - target: "engine::tree::payload_validator", - "wait_receipt_root", - ) - .entered(); - - receipt_root_rx - .blocking_recv() - .inspect_err(|_| { - tracing::error!( - target: "engine::tree::payload_validator", - "Receipt root task dropped sender without result, receipt root calculation likely aborted" - ); - }) - .ok() - }; - let transaction_root = transaction_root.map(|handle| { - let _span = - debug_span!(target: "engine::tree::payload_validator", "wait_payload_tx_root") - .entered(); - handle.try_into_inner().expect("sole handle") - }); - - let hashed_state = ensure_ok_post_block!( - self.validate_post_execution( - &block, - &parent_block, - &output, - &mut ctx, - transaction_root, - receipt_root_bloom, - hashed_state, - ), - block - ); - - let root_time = Instant::now(); - let mut maybe_state_root = None; - let mut state_root_task_failed = false; - #[cfg(feature = "trie-debug")] - let mut trie_debug_recorders = Vec::new(); - - match strategy { - StateRootStrategy::StateRootTask => { - debug!(target: "engine::tree::payload_validator", "Using sparse trie state root algorithm"); - - let task_result = ensure_ok_post_block!( - self.await_state_root_with_timeout( - &mut handle, - overlay_factory.clone(), - &hashed_state, - ), - block - ); - - match task_result { - Ok(StateRootComputeOutcome { - state_root, - trie_updates, - #[cfg(feature = "trie-debug")] - debug_recorders, - }) => { - let elapsed = root_time.elapsed(); - info!(target: "engine::tree::payload_validator", ?state_root, ?elapsed, "State root task finished"); - - #[cfg(feature = "trie-debug")] - { - trie_debug_recorders = debug_recorders; - } - - // Compare trie updates with serial computation if configured - if self.config.always_compare_trie_updates() { - let _has_diff = self.compare_trie_updates_with_serial( - overlay_factory.clone(), - &hashed_state, - trie_updates.as_ref().clone(), - ); - #[cfg(feature = "trie-debug")] - if _has_diff { - Self::write_trie_debug_recorders( - block.header().number(), - &trie_debug_recorders, - ); - } - } - - // Pre-Jade Morph blocks store a ZK-trie root in header.state_root; - // reth computes an MPT root which will never match. The strict check - // is only enforced post-Jade. See gate::state_root_enforced_at and - // the design spec for transitivity of EVM execution. - let strict_enforced = - state_root_enforced_at(&self.chain_spec, block.header().timestamp()); - - // we double check the state root here for good measure - if !strict_enforced || state_root == block.header().state_root() { - maybe_state_root = Some((state_root, trie_updates, elapsed)) - } else { - warn!( - target: "engine::tree::payload_validator", - ?state_root, - block_state_root = ?block.header().state_root(), - "State root task returned incorrect state root" - ); - #[cfg(feature = "trie-debug")] - Self::write_trie_debug_recorders( - block.header().number(), - &trie_debug_recorders, - ); - state_root_task_failed = true; - } - } - Err(error) => { - debug!(target: "engine::tree::payload_validator", %error, "State root task failed"); - state_root_task_failed = true; - } - } - } - StateRootStrategy::Parallel => { - debug!(target: "engine::tree::payload_validator", "Using parallel state root algorithm"); - match self.compute_state_root_parallel(overlay_factory.clone(), &hashed_state) { - Ok(result) => { - let elapsed = root_time.elapsed(); - info!( - target: "engine::tree::payload_validator", - regular_state_root = ?result.0, - ?elapsed, - "Regular root task finished" - ); - maybe_state_root = Some((result.0, Arc::new(result.1), elapsed)); - } - Err(error) => { - debug!(target: "engine::tree::payload_validator", %error, "Parallel state root computation failed"); - } - } - } - StateRootStrategy::Synchronous => {} - } - - // Determine the state root. - // If the state root was computed in parallel, we use it. - // Otherwise, we fall back to computing it synchronously. - let (state_root, trie_output, root_elapsed) = if let Some(maybe_state_root) = - maybe_state_root - { - maybe_state_root - } else { - // fallback is to compute the state root regularly in sync - if self.config.state_root_fallback() { - debug!(target: "engine::tree::payload_validator", "Using state root fallback for testing"); - } else { - warn!(target: "engine::tree::payload_validator", "Failed to compute state root in parallel"); - self.metrics - .block_validation - .state_root_parallel_fallback_total - .increment(1); - } - - let (root, updates) = ensure_ok_post_block!( - Self::compute_state_root_serial(overlay_factory.clone(), &hashed_state), - block - ); - - if state_root_task_failed { - self.metrics - .block_validation - .state_root_task_fallback_success_total - .increment(1); - } - - (root, Arc::new(updates), root_time.elapsed()) - }; - - self.metrics - .block_validation - .record_state_root(&trie_output, root_elapsed.as_secs_f64()); - self.metrics - .record_state_root_gas_bucket(block.header().gas_used(), root_elapsed.as_secs_f64()); - debug!(target: "engine::tree::payload_validator", ?root_elapsed, "Calculated state root"); - - // ensure state root matches — post-Jade only. Pre-Jade blocks carry a - // ZK-trie root in the header which will never equal reth's MPT root. - // Correctness is proven transitively by the first post-Jade block's - // successful strict check (see design spec). - let strict_enforced = state_root_enforced_at(&self.chain_spec, block.header().timestamp()); - - if strict_enforced && state_root != block.header().state_root() { - #[cfg(feature = "trie-debug")] - Self::write_trie_debug_recorders(block.header().number(), &trie_debug_recorders); - - // call post-block hook - self.on_invalid_block( - &parent_block, - &block, - &output, - Some((&trie_output, state_root)), - ctx.state_mut(), - ); - let block_state_root = block.header().state_root(); - return Err(InsertBlockError::new( - block.into_sealed_block(), - ConsensusError::BodyStateRootDiff( - GotExpected { - got: state_root, - expected: block_state_root, - } - .into(), - ) - .into(), - ) - .into()); - } - - let timing_stats = state_provider_stats.map(|stats| { - self.calculate_timing_stats( - &block, - stats, - cache_stats, - &output, - execution_duration, - root_elapsed, - ) - }); - - if let Some(valid_block_tx) = valid_block_tx { - let _ = valid_block_tx.send(()); - } - - // Create the overlay provider NOW, while we're on the engine loop thread and trie changeset - // eviction cannot race with us. If we deferred this to the background task, persistence - // could advance and evict changeset cache entries between factory creation and the task - // actually running, causing expensive DB fallback computations when building the overlay. - let changeset_provider = - ensure_ok_post_block!(overlay_factory.database_provider_ro(), block); - - let executed_block = self.spawn_deferred_trie_task( - block, - output, - &ctx, - hashed_state, - trie_output, - changeset_provider, - ); - Ok((executed_block, timing_stats)) - } - - /// Return sealed block header from database or in-memory state by hash. - fn sealed_header_by_hash( - &self, - hash: B256, - state: &EngineApiTreeState, - ) -> ProviderResult>> { - // check memory first - let header = state.tree_state().sealed_header_by_hash(&hash); - - if header.is_some() { - Ok(header) - } else { - self.provider.sealed_header_by_hash(hash) - } - } - - /// Validate if block is correct and satisfies all the consensus rules that concern the header - /// and block body itself. - #[instrument(level = "debug", target = "engine::tree::payload_validator", skip_all)] - fn validate_block_inner( - &self, - block: &SealedBlock, - transaction_root: Option, - ) -> Result<(), ConsensusError> { - if let Err(e) = self.consensus.validate_header(block.sealed_header()) { - error!(target: "engine::tree::payload_validator", ?block, "Failed to validate header {}: {e}", block.hash()); - return Err(e); - } - - if let Err(e) = self - .consensus - .validate_block_pre_execution_with_tx_root(block, transaction_root) - { - error!(target: "engine::tree::payload_validator", ?block, "Failed to validate block {}: {e}", block.hash()); - return Err(e); - } - - Ok(()) - } - - /// Validates Morph's parent-aware L1 message index invariant. - /// - /// go-ethereum derives `NextL1MsgIndex` from the parent queue index plus the - /// L1 messages processed by this block. Since `next_l1_msg_index` is not part - /// of the block hash, Jade and later must reject any mismatch before accepting - /// the payload. - fn validate_next_l1_msg_index_against_parent( - &self, - block: &RecoveredBlock, - parent_block: &SealedHeader, - ) -> Result<(), ConsensusError> { - if !self - .chain_spec - .is_jade_active_at_timestamp(block.header().timestamp()) - { - return Ok(()); - } - - let mut expected = parent_block.next_l1_msg_index; - for tx in block.body().transactions() { - if !tx.is_l1_msg() { - break; - } - - let queue_index = tx.queue_index().ok_or_else(|| { - ConsensusError::msg("L1 message transaction is missing queue index") - })?; - // Match geth's `NumL1MessagesProcessed(parent_queue_index)`: forward skips - // are counted as processed, so the derived next index is last_queue + 1. - expected = queue_index.checked_add(1).ok_or_else(|| { - ConsensusError::msg(format!( - "invalid block.NextL1MsgIndex: expected {}, got {}", - u64::MAX, - block.header().next_l1_msg_index - )) - })?; - } - - let actual = block.header().next_l1_msg_index; - if actual != expected { - return Err(ConsensusError::msg(format!( - "invalid block.NextL1MsgIndex: expected {expected}, got {actual}" - ))); - } - - Ok(()) - } - - /// Executes a block with the given state provider. - /// - /// This method orchestrates block execution: - /// 1. Sets up the EVM with state database and precompile caching - /// 2. Spawns a background task for incremental receipt root computation - /// 3. Executes transactions with metrics collection via state hooks - /// 4. Merges state transitions and records execution metrics - #[instrument(level = "debug", target = "engine::tree::payload_validator", skip_all)] - #[expect(clippy::type_complexity)] - fn execute_block( - &mut self, - state_provider: S, - env: ExecutionEnv, - input: &BlockOrPayload, - handle: &mut PayloadHandle, Err, N::Receipt>, - ) -> Result< - ( - BlockExecutionOutput, - Vec
, - tokio::sync::oneshot::Receiver<(B256, alloy_primitives::Bloom)>, - ), - InsertBlockErrorKind, - > - where - S: StateProvider + Send, - Err: core::error::Error + Send + Sync + 'static, - V: PayloadValidator, - T: PayloadTypes>, - Evm: ConfigureEngineEvm, - { - debug!(target: "engine::tree::payload_validator", "Executing block"); - - let mut db = debug_span!(target: "engine::tree", "build_state_db").in_scope(|| { - State::builder() - .with_database(StateProviderDatabase::new(state_provider)) - .with_bundle_update() - .build() - }); - - let (spec_id, mut executor) = { - let _span = debug_span!(target: "engine::tree", "create_evm").entered(); - let spec_id = *env.evm_env.spec_id(); - let evm = self.evm_config.evm_with_env(&mut db, env.evm_env); - let ctx = self - .execution_ctx_for(input) - .map_err(|e| InsertBlockErrorKind::Other(Box::new(e)))?; - let executor = self.evm_config.create_executor(evm, ctx); - (spec_id, executor) - }; - - if !self.config.precompile_cache_disabled() { - let _span = debug_span!(target: "engine::tree", "setup_precompile_cache").entered(); - executor - .evm_mut() - .precompiles_mut() - .map_cacheable_precompiles(|address, precompile| { - let metrics = self - .precompile_cache_metrics - .entry(*address) - .or_insert_with(|| CachedPrecompileMetrics::new_with_address(*address)) - .clone(); - CachedPrecompile::wrap( - precompile, - self.precompile_cache_map.cache_for_address(*address), - spec_id, - Some(metrics), - ) - }); - } - - // Spawn background task to compute receipt root and logs bloom incrementally. - // Unbounded channel is used since tx count bounds capacity anyway (max ~30k txs per block). - let receipts_len = input.transaction_count(); - let (receipt_tx, receipt_rx) = crossbeam_channel::unbounded(); - let (result_tx, result_rx) = tokio::sync::oneshot::channel(); - let task_handle = ReceiptRootTaskHandle::new(receipt_rx, result_tx); - self.payload_processor - .executor() - .spawn_blocking_named("receipt-root", move || task_handle.run(receipts_len)); - - let transaction_count = input.transaction_count(); - let executed_tx_index = Arc::clone(handle.executed_tx_index()); - let executor = executor.with_state_hook( - handle - .state_hook() - .map(|hook| Box::new(hook) as Box), - ); - - let execution_start = Instant::now(); - - // Execute all transactions and finalize - let (executor, senders) = self.execute_transactions( - executor, - transaction_count, - handle.iter_transactions(), - &receipt_tx, - &executed_tx_index, - )?; - drop(receipt_tx); - - // Finish execution and get the result - let post_exec_start = Instant::now(); - let (_evm, result) = debug_span!(target: "engine::tree", "BlockExecutor::finish") - .in_scope(|| executor.finish()) - .map(|(evm, result)| (evm.into_db(), result))?; - self.metrics - .record_post_execution(post_exec_start.elapsed()); - - // Merge transitions into bundle state - debug_span!(target: "engine::tree", "merge_transitions") - .in_scope(|| db.merge_transitions(BundleRetention::Reverts)); - - let output = BlockExecutionOutput { - result, - state: db.take_bundle(), - }; - - let execution_duration = execution_start.elapsed(); - self.metrics - .record_block_execution(&output, execution_duration); - self.metrics - .record_block_execution_gas_bucket(output.result.gas_used, execution_duration); - debug!(target: "engine::tree::payload_validator", elapsed = ?execution_duration, "Executed block"); - - Ok((output, senders, result_rx)) - } - - /// Executes transactions and collects senders, streaming receipts to a background task. - /// - /// This method handles: - /// - Applying pre-execution changes (e.g., beacon root updates) - /// - Executing each transaction with timing metrics - /// - Streaming receipts to the receipt root computation task - /// - Collecting transaction senders for later use - /// - /// Returns the executor (for finalization) and the collected senders. - fn execute_transactions( - &self, - mut executor: E, - transaction_count: usize, - transactions: impl Iterator>, - receipt_tx: &crossbeam_channel::Sender>, - executed_tx_index: &AtomicUsize, - ) -> Result<(E, Vec
), BlockExecutionError> - where - E: BlockExecutor, - Tx: alloy_evm::block::ExecutableTx + alloy_evm::RecoveredTx, - InnerTx: TxHashRef, - Err: core::error::Error + Send + Sync + 'static, - { - let mut senders = Vec::with_capacity(transaction_count); - - // Apply pre-execution changes (e.g., beacon root update) - let pre_exec_start = Instant::now(); - debug_span!(target: "engine::tree", "pre_execution") - .in_scope(|| executor.apply_pre_execution_changes())?; - self.metrics.record_pre_execution(pre_exec_start.elapsed()); - - // Execute transactions - let exec_span = debug_span!(target: "engine::tree", "execution").entered(); - let mut transactions = transactions.into_iter(); - // Some executors may execute transactions that do not append receipts during the - // main loop (e.g., system transactions whose receipts are added during finalization). - // In that case, invoking the callback on every transaction would resend the previous - // receipt with the same index and can panic the ordered root builder. - let mut last_sent_len = 0usize; - loop { - // Measure time spent waiting for next transaction from iterator - // (e.g., parallel signature recovery) - let wait_start = Instant::now(); - let Some(tx_result) = transactions.next() else { - break; - }; - self.metrics.record_transaction_wait(wait_start.elapsed()); - - let tx = tx_result.map_err(BlockExecutionError::other)?; - let tx_signer = *>::signer(&tx); - - senders.push(tx_signer); - - let _enter = debug_span!( - target: "engine::tree", - "execute tx", - tx_index = senders.len() - 1, - ) - .entered(); - trace!(target: "engine::tree", "Executing transaction"); - - let tx_start = Instant::now(); - executor.execute_transaction(tx)?; - self.metrics - .record_transaction_execution(tx_start.elapsed()); - - // advance the shared counter so prewarm workers skip already-executed txs - executed_tx_index.store(senders.len(), Ordering::Relaxed); - - let current_len = executor.receipts().len(); - if current_len > last_sent_len { - last_sent_len = current_len; - // Send the latest receipt to the background task for incremental root computation. - if let Some(receipt) = executor.receipts().last() { - let tx_index = current_len - 1; - let _ = receipt_tx.send(IndexedReceipt::new(tx_index, receipt.clone())); - } - } - } - drop(exec_span); - - Ok((executor, senders)) - } - - /// Compute state root for the given hashed post state in parallel. - /// - /// Uses an overlay factory which provides the state of the parent block, along with the - /// [`HashedPostState`] containing the changes of this block, to compute the state root and - /// trie updates for this block. - /// - /// # Returns - /// - /// Returns `Ok(_)` if computed successfully. - /// Returns `Err(_)` if error was encountered during computation. - #[instrument(level = "debug", target = "engine::tree::payload_validator", skip_all)] - fn compute_state_root_parallel( - &self, - overlay_factory: OverlayStateProviderFactory, - hashed_state: &LazyHashedPostState, - ) -> Result<(B256, TrieUpdates), ParallelStateRootError> { - let hashed_state = hashed_state.get(); - // The `hashed_state` argument will be taken into account as part of the overlay, but we - // need to use the prefix sets which were generated from it to indicate to the - // ParallelStateRoot which parts of the trie need to be recomputed. - let prefix_sets = hashed_state.construct_prefix_sets().freeze(); - let overlay_factory = - overlay_factory.with_extended_hashed_state_overlay(hashed_state.clone_into_sorted()); - ParallelStateRoot::new(overlay_factory, prefix_sets, self.runtime.clone()) - .incremental_root_with_updates() - } - - /// Compute state root for the given hashed post state in serial. - /// - /// Uses an overlay factory which provides the state of the parent block, along with the - /// [`HashedPostState`] containing the changes of this block, to compute the state root and - /// trie updates for this block. - fn compute_state_root_serial( - overlay_factory: OverlayStateProviderFactory, - hashed_state: &LazyHashedPostState, - ) -> ProviderResult<(B256, TrieUpdates)> { - let hashed_state = hashed_state.get(); - // The `hashed_state` argument will be taken into account as part of the overlay, but we - // need to use the prefix sets which were generated from it to indicate to the - // StateRoot which parts of the trie need to be recomputed. - let prefix_sets = hashed_state.construct_prefix_sets().freeze(); - let overlay_factory = - overlay_factory.with_extended_hashed_state_overlay(hashed_state.clone_into_sorted()); - - let provider = overlay_factory.database_provider_ro()?; - - Ok(StateRoot::new(&provider, &provider) - .with_prefix_sets(prefix_sets) - .root_with_updates()?) - } - - /// Awaits the state root from the background task, with an optional timeout fallback. - /// - /// If a timeout is configured (`state_root_task_timeout`), this method first waits for the - /// state root task up to the timeout duration. If the task doesn't complete in time, a - /// sequential state root computation is spawned via `spawn_blocking`. Both computations - /// then race: the main thread polls the task receiver and the sequential result channel - /// in a loop, returning whichever finishes first. - /// - /// If no timeout is configured, this simply awaits the state root task without any fallback. - /// - /// Returns `ProviderResult>` where the outer `ProviderResult` captures - /// unrecoverable errors from the sequential fallback (e.g. DB errors), while the inner - /// `Result` captures parallel state root task errors that can still fall back to serial. - #[instrument( - level = "debug", - target = "engine::tree::payload_validator", - name = "await_state_root", - skip_all - )] - fn await_state_root_with_timeout( - &self, - handle: &mut PayloadHandle, - overlay_factory: OverlayStateProviderFactory, - hashed_state: &LazyHashedPostState, - ) -> ProviderResult> { - let Some(timeout) = self.config.state_root_task_timeout() else { - return Ok(handle.state_root()); - }; - - let task_rx = handle.take_state_root_rx(); - - match task_rx.recv_timeout(timeout) { - Ok(result) => Ok(result), - Err(RecvTimeoutError::Disconnected) => Ok(Err(ParallelStateRootError::Other( - "sparse trie task dropped".to_string(), - ))), - Err(RecvTimeoutError::Timeout) => { - warn!( - target: "engine::tree::payload_validator", - ?timeout, - "State root task timed out, spawning sequential fallback" - ); - self.metrics - .block_validation - .state_root_task_timeout_total - .increment(1); - - let (seq_tx, seq_rx) = - std::sync::mpsc::channel::>(); - - let seq_overlay = overlay_factory; - let seq_hashed_state = hashed_state.clone(); - self.payload_processor - .executor() - .spawn_blocking_named("serial-root", move || { - let result = - Self::compute_state_root_serial(seq_overlay, &seq_hashed_state); - let _ = seq_tx.send(result); - }); - - const POLL_INTERVAL: std::time::Duration = std::time::Duration::from_millis(10); - - loop { - match task_rx.recv_timeout(POLL_INTERVAL) { - Ok(result) => { - debug!( - target: "engine::tree::payload_validator", - source = "task", - "State root timeout race won" - ); - return Ok(result); - } - Err(RecvTimeoutError::Disconnected) => { - debug!( - target: "engine::tree::payload_validator", - "State root task dropped, waiting for sequential fallback" - ); - let result = seq_rx.recv().map_err(|_| { - ProviderError::other(std::io::Error::other( - "both state root computations failed", - )) - })?; - let (state_root, trie_updates) = result?; - return Ok(Ok(StateRootComputeOutcome { - state_root, - trie_updates: Arc::new(trie_updates), - #[cfg(feature = "trie-debug")] - debug_recorders: Vec::new(), - })); - } - Err(RecvTimeoutError::Timeout) => {} - } - - if let Ok(result) = seq_rx.try_recv() { - debug!( - target: "engine::tree::payload_validator", - source = "sequential", - "State root timeout race won" - ); - let (state_root, trie_updates) = result?; - return Ok(Ok(StateRootComputeOutcome { - state_root, - trie_updates: Arc::new(trie_updates), - #[cfg(feature = "trie-debug")] - debug_recorders: Vec::new(), - })); - } - } - } - } - } - - /// Compares trie updates from the state root task with serial state root computation. - /// - /// This is used for debugging and validating the correctness of the parallel state root - /// task implementation. When enabled via `--engine.state-root-task-compare-updates`, this - /// method runs a separate serial state root computation and compares the resulting trie - /// updates. - fn compare_trie_updates_with_serial( - &self, - overlay_factory: OverlayStateProviderFactory, - hashed_state: &LazyHashedPostState, - task_trie_updates: TrieUpdates, - ) -> bool { - debug!(target: "engine::tree::payload_validator", "Comparing trie updates with serial computation"); - - match Self::compute_state_root_serial(overlay_factory.clone(), hashed_state) { - Ok((serial_root, serial_trie_updates)) => { - debug!( - target: "engine::tree::payload_validator", - ?serial_root, - "Serial state root computation finished for comparison" - ); - - // Get a database provider to use as trie cursor factory - match overlay_factory.database_provider_ro() { - Ok(provider) => { - match crate::trie_updates::compare_trie_updates( - &provider, - task_trie_updates, - serial_trie_updates, - ) { - Ok(has_diff) => return has_diff, - Err(err) => { - warn!( - target: "engine::tree::payload_validator", - %err, - "Error comparing trie updates" - ); - return true; - } - } - } - Err(err) => { - warn!( - target: "engine::tree::payload_validator", - %err, - "Failed to get database provider for trie update comparison" - ); - } - } - } - Err(err) => { - warn!( - target: "engine::tree::payload_validator", - %err, - "Failed to compute serial state root for comparison" - ); - } - } - false - } - - /// Writes trie debug recorders to a JSON file for the given block number. - /// - /// The file is written to the current working directory as - /// `trie_debug_block_{block_number}.json`. - #[cfg(feature = "trie-debug")] - fn write_trie_debug_recorders( - block_number: u64, - recorders: &[(Option, TrieDebugRecorder)], - ) { - let path = format!("trie_debug_block_{block_number}.json"); - match serde_json::to_string_pretty(recorders) { - Ok(json) => match std::fs::write(&path, json) { - Ok(()) => { - warn!( - target: "engine::tree::payload_validator", - %path, - "Wrote trie debug recorders to file" - ); - } - Err(err) => { - warn!( - target: "engine::tree::payload_validator", - %err, - %path, - "Failed to write trie debug recorders" - ); - } - }, - Err(err) => { - warn!( - target: "engine::tree::payload_validator", - %err, - "Failed to serialize trie debug recorders" - ); - } - } - } - - /// Validates the block after execution. - /// - /// This performs: - /// - parent header validation - /// - post-execution consensus validation - /// - state-root based post-execution validation - /// - /// If `receipt_root_bloom` is provided, it will be used instead of computing the receipt root - /// and logs bloom from the receipts. - /// - /// The `hashed_state` handle wraps the background hashed post state computation. - #[instrument(level = "debug", target = "engine::tree::payload_validator", skip_all)] - #[expect(clippy::too_many_arguments)] - fn validate_post_execution>>( - &self, - block: &RecoveredBlock, - parent_block: &SealedHeader, - output: &BlockExecutionOutput, - ctx: &mut TreeCtx<'_, N>, - transaction_root: Option, - receipt_root_bloom: Option, - hashed_state: LazyHashedPostState, - ) -> Result - where - V: PayloadValidator, - { - let start = Instant::now(); - - trace!(target: "engine::tree::payload_validator", block=?block.num_hash(), "Validating block consensus"); - // validate block consensus rules - if let Err(e) = self.validate_block_inner(block, transaction_root) { - return Err(e.into()); - } - - // now validate against the parent - let _enter = debug_span!(target: "engine::tree::payload_validator", "validate_header_against_parent").entered(); - if let Err(e) = self - .consensus - .validate_header_against_parent(block.sealed_header(), parent_block) - { - warn!(target: "engine::tree::payload_validator", ?block, "Failed to validate header {} against parent: {e}", block.hash()); - return Err(e.into()); - } - drop(_enter); - - if let Err(e) = self.validate_next_l1_msg_index_against_parent(block, parent_block) { - warn!( - target: "engine::tree::payload_validator", - ?block, - "Failed to validate NextL1MsgIndex against parent for block {}: {e}", - block.hash() - ); - return Err(e.into()); - } - - // Validate block post-execution rules - let _enter = - debug_span!(target: "engine::tree::payload_validator", "validate_block_post_execution") - .entered(); - if let Err(err) = - self.consensus - .validate_block_post_execution(block, output, receipt_root_bloom) - { - // call post-block hook - self.on_invalid_block(parent_block, block, output, None, ctx.state_mut()); - return Err(err.into()); - } - drop(_enter); - - // Wait for the background keccak256 hashing task to complete. This blocks until - // all changed addresses and storage slots have been hashed. - let hashed_state_ref = - debug_span!(target: "engine::tree::payload_validator", "wait_hashed_post_state") - .in_scope(|| hashed_state.get()); - - let _enter = debug_span!(target: "engine::tree::payload_validator", "validate_block_post_execution_with_hashed_state").entered(); - if let Err(err) = self - .validator - .validate_block_post_execution_with_hashed_state(hashed_state_ref, block) - { - // call post-block hook - self.on_invalid_block(parent_block, block, output, None, ctx.state_mut()); - return Err(err.into()); - } - - // record post-execution validation duration - self.metrics - .block_validation - .post_execution_validation_duration - .record(start.elapsed().as_secs_f64()); - - Ok(hashed_state) - } - - /// Spawns a payload processor task based on the state root strategy. - /// - /// This method determines how to execute the block and compute its state root based on - /// the selected strategy: - /// - `StateRootTask`: Uses a dedicated task for state root computation with proof generation - /// - `Parallel`: Computes state root in parallel with block execution - /// - `Synchronous`: Falls back to sequential execution and state root computation - /// - /// The method handles strategy fallbacks if the preferred approach fails, ensuring - /// block execution always completes with a valid state root. - /// - /// # Arguments - /// - /// * `overlay_factory` - Pre-computed overlay factory for multiproof generation - /// (`StateRootTask`) - #[instrument( - level = "debug", - target = "engine::tree::payload_validator", - skip_all, - fields(?strategy) - )] - fn spawn_payload_processor>( - &mut self, - env: ExecutionEnv, - txs: T, - provider_builder: StateProviderBuilder, - overlay_factory: OverlayStateProviderFactory, - strategy: StateRootStrategy, - ) -> Result< - PayloadHandle< - impl ExecutableTxFor + use, - impl core::error::Error + Send + Sync + 'static + use, - N::Receipt, - >, - InsertBlockErrorKind, - > { - match strategy { - StateRootStrategy::StateRootTask => { - let spawn_start = Instant::now(); - - let handle = self.payload_processor.spawn( - env, - txs, - provider_builder, - overlay_factory, - &self.config, - ); - - // record prewarming initialization duration - self.metrics - .block_validation - .spawn_payload_processor - .record(spawn_start.elapsed().as_secs_f64()); - - Ok(handle) - } - StateRootStrategy::Parallel | StateRootStrategy::Synchronous => { - let start = Instant::now(); - let handle = - self.payload_processor - .spawn_cache_exclusive(env, txs, provider_builder); - - // Record prewarming initialization duration - self.metrics - .block_validation - .spawn_payload_processor - .record(start.elapsed().as_secs_f64()); - - Ok(handle) - } - } - } - - /// Creates a `StateProviderBuilder` for the given parent hash. - /// - /// This method checks if the parent is in the tree state (in-memory) or persisted to disk, - /// and creates the appropriate provider builder. - fn state_provider_builder( - &self, - hash: B256, - state: &EngineApiTreeState, - ) -> ProviderResult>> { - if let Some((historical, blocks)) = state.tree_state().blocks_by_hash(hash) { - debug!(target: "engine::tree::payload_validator", %hash, %historical, "found canonical state for block in memory, creating provider builder"); - // the block leads back to the canonical chain - return Ok(Some(StateProviderBuilder::new( - self.provider.clone(), - historical, - Some(blocks), - ))); - } - - // Check if the block is persisted - if let Some(header) = self.provider.header(hash)? { - debug!(target: "engine::tree::payload_validator", %hash, number = %header.number(), "found canonical state for block in database, creating provider builder"); - // For persisted blocks, we create a builder that will fetch state directly from the - // database - return Ok(Some(StateProviderBuilder::new( - self.provider.clone(), - hash, - None, - ))); - } - - debug!(target: "engine::tree::payload_validator", %hash, "no canonical state found for block"); - Ok(None) - } - - /// Determines the state root computation strategy based on configuration. - /// - /// Note: Use state root task only if prefix sets are empty, otherwise proof generation is - /// too expensive because it requires walking all paths in every proof. - const fn plan_state_root_computation(&self) -> StateRootStrategy { - if self.config.state_root_fallback() { - StateRootStrategy::Synchronous - } else if self.config.use_state_root_task() { - StateRootStrategy::StateRootTask - } else { - StateRootStrategy::Parallel - } - } - - /// Called when an invalid block is encountered during validation. - fn on_invalid_block( - &self, - parent_header: &SealedHeader, - block: &RecoveredBlock, - output: &BlockExecutionOutput, - trie_updates: Option<(&TrieUpdates, B256)>, - state: &mut EngineApiTreeState, - ) { - if state.has_invalid_header(&block.hash()) { - // we already marked this block as invalid - return; - } - self.invalid_block_hook - .on_invalid_block(parent_header, block, output, trie_updates); - } - - /// Creates a [`LazyOverlay`] for the parent block without blocking. - /// - /// Returns a lazy overlay that will compute the trie input on first access, and the anchor - /// block hash (the highest persisted ancestor). This allows execution to start immediately - /// while the trie input computation is deferred until the overlay is actually needed. - /// - /// If parent is on disk (no in-memory blocks), returns `None` for the lazy overlay. - /// - /// Uses a cached overlay if available for the canonical head (the common case). - fn get_parent_lazy_overlay( - parent_hash: B256, - state: &EngineApiTreeState, - ) -> (Option>, B256) { - // Get blocks leading to the parent to determine the anchor - let (anchor_hash, blocks) = state - .tree_state() - .blocks_by_hash(parent_hash) - .unwrap_or_else(|| (parent_hash, vec![])); - - if blocks.is_empty() { - debug!(target: "engine::tree::payload_validator", "Parent found on disk, no lazy overlay needed"); - return (None, anchor_hash); - } - - // Try to use the cached overlay if it matches both parent hash and anchor - if let Some(cached) = state - .tree_state() - .get_cached_overlay(parent_hash, anchor_hash) - { - debug!( - target: "engine::tree::payload_validator", - %parent_hash, - %anchor_hash, - "Using cached canonical overlay" - ); - return (Some(cached.overlay.clone()), cached.anchor_hash); - } - - debug!( - target: "engine::tree::payload_validator", - %anchor_hash, - num_blocks = blocks.len(), - "Creating lazy overlay for in-memory blocks" - ); - - // v2.2.0 ingests the executed blocks themselves; per-block - // `DeferredTrieData` extraction is no longer required. - (Some(LazyOverlay::new(blocks)), anchor_hash) - } - - /// Spawns a background task to compute and sort trie data for the executed block. - /// - /// This function creates a [`DeferredTrieData`] handle with fallback inputs and spawns a - /// blocking task that calls `wait_cloned()` to: - /// 1. Sort the block's hashed state and trie updates - /// 2. Merge ancestor overlays and extend with the sorted data - /// 3. Create an [`AnchoredTrieInput`](reth_chain_state::AnchoredTrieInput) for efficient future - /// trie computations - /// 4. Cache the result so subsequent calls return immediately - /// - /// If the background task hasn't completed when `trie_data()` is called, `wait_cloned()` - /// computes from the stored inputs, eliminating deadlock risk and duplicate computation. - /// - /// The validation hot path can return immediately after state root verification, - /// while consumers (DB writes, overlay providers, proofs) get trie data either - /// from the completed task or via fallback computation. - fn spawn_deferred_trie_task( - &self, - block: RecoveredBlock, - execution_outcome: Arc>, - ctx: &TreeCtx<'_, N>, - hashed_state: LazyHashedPostState, - trie_output: Arc, - changeset_provider: impl TrieCursorFactory + Send + 'static, - ) -> ExecutedBlock { - // Capture parent hash and ancestor overlays for deferred trie input construction. - let (anchor_hash, overlay_blocks) = ctx - .state() - .tree_state() - .blocks_by_hash(block.parent_hash()) - .unwrap_or_else(|| (block.parent_hash(), Vec::new())); - - // Collect lightweight ancestor trie data handles. We don't call trie_data() here; - // the merge and any fallback sorting happens in the compute_trie_input_task. - let ancestors: Vec = overlay_blocks - .iter() - .rev() - .map(|b| b.trie_data_handle()) - .collect(); - - // Create deferred handle with fallback inputs in case the background task hasn't completed. - // Resolve the lazy handle into Arc. By this point the hashed state has - // already been computed and used for state root verification, so .get() returns instantly. - let hashed_state = match hashed_state.try_into_inner() { - Ok(state) => Arc::new(state), - Err(handle) => Arc::new(handle.get().clone()), - }; - let deferred_trie_data = - DeferredTrieData::pending(hashed_state, trie_output, anchor_hash, ancestors); - let deferred_handle_task = deferred_trie_data.clone(); - let block_validation_metrics = self.metrics.block_validation.clone(); - - // Capture block info and cache handle for changeset computation - let block_hash = block.hash(); - let block_number = block.number(); - - // Register a pending changeset entry so that concurrent readers will wait for - // this computation to finish rather than falling back to the expensive DB path. - // The guard ensures the pending entry is cancelled if the task panics. - let pending_changeset_guard = self.changeset_cache.register_pending(block_hash); - - // Spawn background task to compute trie data. Calling `wait_cloned` will compute from - // the stored inputs and cache the result, so subsequent calls return immediately. - let compute_trie_input_task = move || { - let _span = debug_span!( - target: "engine::tree::payload_validator", - "compute_trie_input_task", - block_number - ) - .entered(); - - let result = panic::catch_unwind(AssertUnwindSafe(|| { - let compute_start = Instant::now(); - let computed = deferred_handle_task.wait_cloned(); - block_validation_metrics - .deferred_trie_compute_duration - .record(compute_start.elapsed().as_secs_f64()); - - // Record sizes of the computed trie data - block_validation_metrics - .hashed_post_state_size - .record(computed.hashed_state.total_len() as f64); - block_validation_metrics - .trie_updates_sorted_size - .record(computed.trie_updates.total_len() as f64); - if let Some(anchored) = &computed.anchored_trie_input { - block_validation_metrics - .anchored_overlay_trie_updates_size - .record(anchored.trie_input.nodes.total_len() as f64); - block_validation_metrics - .anchored_overlay_hashed_state_size - .record(anchored.trie_input.state.total_len() as f64); - } - - // Compute and cache changesets using the computed trie_updates. - // Use the pre-created provider to avoid races with changeset cache - // eviction that can happen between task spawn and execution. - let changeset_start = Instant::now(); - - match reth_trie::changesets::compute_trie_changesets( - &changeset_provider, - &computed.trie_updates, - ) { - Ok(changesets) => { - debug!( - target: "engine::tree::changeset", - ?block_number, - elapsed = ?changeset_start.elapsed(), - "Computed and caching changesets" - ); - - pending_changeset_guard.resolve(block_number, Arc::new(changesets)); - } - Err(e) => { - warn!( - target: "engine::tree::changeset", - ?block_number, - ?e, - "Failed to compute changesets in deferred trie task" - ); - } - } - })); - - if result.is_err() { - error!( - target: "engine::tree::payload_validator", - "Deferred trie task panicked; fallback computation will be used when trie data is accessed" - ); - } - }; - - // Spawn task that computes trie data asynchronously. - self.payload_processor - .executor() - .spawn_blocking_named("trie-input", compute_trie_input_task); - - ExecutedBlock::with_deferred_trie_data( - Arc::new(block), - execution_outcome, - deferred_trie_data, - ) - } - - fn calculate_timing_stats( - &self, - block: &RecoveredBlock, - provider_stats: Arc, - cache_stats: Option>, - output: &BlockExecutionOutput, - execution_duration: Duration, - state_hash_duration: Duration, - ) -> Box { - let accounts_read = provider_stats.total_account_fetches(); - let storage_read = provider_stats.total_storage_fetches(); - let code_read = provider_stats.total_code_fetches(); - let code_bytes_read = provider_stats.total_code_fetched_bytes(); - - // Write stats from BundleState (final state changes) - let accounts_changed = output.state.state.len(); - let accounts_deleted = output - .state - .state - .values() - .filter(|acc| acc.was_destroyed()) - .count(); - let storage_slots_changed = output - .state - .state - .values() - .map(|account| account.storage.len()) - .sum::(); - let storage_slots_deleted = output - .state - .state - .values() - .flat_map(|account| account.storage.values()) - .filter(|slot| { - slot.present_value.is_zero() && !slot.previous_or_original_value.is_zero() - }) - .count(); - - // Helper: check if account represents a new contract deployment - let is_new_deployment = |acc: &BundleAccount| -> bool { - let has_code_now = acc - .info - .as_ref() - .is_some_and(|info| info.code_hash != KECCAK_EMPTY); - let had_no_code_before = acc - .original_info - .as_ref() - .map(|info| info.code_hash == KECCAK_EMPTY) - .unwrap_or(true); - has_code_now && had_no_code_before - }; - - let bytecodes_changed = output - .state - .state - .values() - .filter(|acc| is_new_deployment(acc)) - .count(); - - // Unique new code hashes to count actual bytes persisted (deduplicated) - let unique_new_code_hashes: B256Set = output - .state - .state - .values() - .filter(|acc| is_new_deployment(acc)) - .filter_map(|acc| acc.info.as_ref().map(|info| info.code_hash)) - .collect(); - let code_bytes_written: usize = unique_new_code_hashes - .iter() - .filter_map(|hash| { - output - .state - .contracts - .get(hash) - .map(|bytecode| bytecode.original_bytes().len()) - }) - .sum(); - - // Total time spent fetching state during execution - let state_read_duration = provider_stats.total_account_fetch_latency() - + provider_stats.total_storage_fetch_latency() - + provider_stats.total_code_fetch_latency(); - - // EIP-7702 delegation tracking from bytecode changes - // Count new EIP-7702 bytecodes as delegations set - let eip7702_delegations_set = output - .state - .contracts - .values() - .filter(|bytecode| bytecode.is_eip7702()) - .count(); - // Delegations cleared: accounts where bytecode changed FROM EIP-7702 TO empty - // This detects when an EIP-7702 delegation is removed by setting code to empty - // Note: Clearing a delegation does NOT destroy the account - it just empties the - // bytecode - let eip7702_delegations_cleared = output - .state - .state - .values() - .filter(|acc| { - // Check if original bytecode was EIP-7702 - let original_was_eip7702 = acc - .original_info - .as_ref() - .and_then(|info| info.code.as_ref()) - .map(|bytecode| bytecode.is_eip7702()) - .unwrap_or(false); - - // Check if current code is empty (delegation cleared) - let code_now_empty = acc - .info - .as_ref() - .map(|info| info.code_hash == KECCAK_EMPTY) - .unwrap_or(false); - - original_was_eip7702 && code_now_empty - }) - .count(); - - // Get cache statistics for detailed block logging - let (account_cache_hits, account_cache_misses) = cache_stats - .as_ref() - .map(|s| (s.account_hits(), s.account_misses())) - .unwrap_or_default(); - let (storage_cache_hits, storage_cache_misses) = cache_stats - .as_ref() - .map(|s| (s.storage_hits(), s.storage_misses())) - .unwrap_or_default(); - let (code_cache_hits, code_cache_misses) = cache_stats - .as_ref() - .map(|s| (s.code_hits(), s.code_misses())) - .unwrap_or_default(); - - // Build execution timing stats for detailed block logging - Box::new(ExecutionTimingStats { - block_number: block.number(), - block_hash: block.hash(), - gas_used: output.result.gas_used, - tx_count: block.transaction_count(), - execution_duration, - state_read_duration, - state_hash_duration, - accounts_read, - storage_read, - code_read, - code_bytes_read, - accounts_changed, - accounts_deleted, - storage_slots_changed, - storage_slots_deleted, - bytecodes_changed, - code_bytes_written, - eip7702_delegations_set, - eip7702_delegations_cleared, - account_cache_hits, - account_cache_misses, - storage_cache_hits, - storage_cache_misses, - code_cache_hits, - code_cache_misses, - }) - } -} - -/// Strategy describing how to compute the state root. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum StateRootStrategy { - /// Use the state root task (background sparse trie computation). - StateRootTask, - /// Run the parallel state root computation on the calling thread. - Parallel, - /// Fall back to synchronous computation via the state provider. - Synchronous, -} - -impl EngineValidator for MorphBasicEngineValidator -where - P: DatabaseProviderFactory< - Provider: BlockReader - + StageCheckpointReader - + PruneCheckpointReader - + ChangeSetReader - + StorageChangeSetReader - + BlockNumReader - + StorageSettingsCache, - > + BlockReader
- + StateProviderFactory - + StateReader - + ChangeSetReader - + BlockNumReader - + HashedPostStateProvider - + Clone - + 'static, - N: NodePrimitives< - Block = morph_primitives::Block, - BlockHeader = MorphHeader, - SignedTx = MorphTxEnvelope, - >, - V: PayloadValidator + Clone, - Evm: ConfigureEngineEvm + 'static, - Types: PayloadTypes>, -{ - fn validate_payload_attributes_against_header( - &self, - attr: &Types::PayloadAttributes, - header: &N::BlockHeader, - ) -> Result<(), InvalidPayloadAttributesError> { - self.validator - .validate_payload_attributes_against_header(attr, header) - } - - fn convert_payload_to_block( - &self, - payload: Types::ExecutionData, - ) -> Result, NewPayloadError> { - let block = self.validator.convert_payload_to_block(payload)?; - Ok(block) - } - - fn validate_payload( - &mut self, - payload: Types::ExecutionData, - ctx: TreeCtx<'_, N>, - ) -> ValidationOutcome { - self.validate_block_with_state(BlockOrPayload::Payload(payload), ctx) - } - - fn validate_block( - &mut self, - block: SealedBlock, - ctx: TreeCtx<'_, N>, - ) -> ValidationOutcome { - self.validate_block_with_state(BlockOrPayload::Block(block), ctx) - } - - fn on_inserted_executed_block(&self, block: ExecutedBlock) { - self.payload_processor.on_inserted_executed_block( - block.recovered_block.block_with_parent(), - &block.execution_output.state, - ); - } - - fn cache_for(&self, block_hash: B256) -> Option { - Some(self.payload_processor.cache_for(block_hash)) - } - - fn sparse_trie_handle_for( - &self, - parent_hash: B256, - parent_state_root: B256, - state: &EngineApiTreeState, - ) -> Option { - let (lazy_overlay, anchor_hash) = Self::get_parent_lazy_overlay(parent_hash, state); - let overlay_builder = OverlayBuilder::::new(anchor_hash, self.changeset_cache.clone()) - .with_lazy_overlay(lazy_overlay); - let overlay_factory = - OverlayStateProviderFactory::new(self.provider.clone(), overlay_builder); - - Some(self.payload_processor.spawn_state_root( - overlay_factory, - parent_state_root, - // Full proof workers — tx count unknown at FCU time (block built incrementally) - false, - &self.config, - )) - } -} - -impl WaitForCaches for MorphBasicEngineValidator -where - Evm: ConfigureEvm, -{ - fn wait_for_caches(&self) -> CacheWaitDurations { - self.payload_processor.wait_for_caches() - } -} - -/// Enum representing either block or payload being validated. -#[derive(Debug, Clone)] -pub enum BlockOrPayload { - /// Payload. - Payload(T::ExecutionData), - /// Block. - Block(SealedBlock::Primitives>>), -} - -impl BlockOrPayload { - /// Returns the hash of the block. - pub fn hash(&self) -> B256 { - match self { - Self::Payload(payload) => payload.block_hash(), - Self::Block(block) => block.hash(), - } - } - - /// Returns the number and hash of the block. - pub fn num_hash(&self) -> NumHash { - match self { - Self::Payload(payload) => payload.num_hash(), - Self::Block(block) => block.num_hash(), - } - } - - /// Returns the parent hash of the block. - pub fn parent_hash(&self) -> B256 { - match self { - Self::Payload(payload) => payload.parent_hash(), - Self::Block(block) => block.parent_hash(), - } - } - - /// Returns [`BlockWithParent`] for the block. - pub fn block_with_parent(&self) -> BlockWithParent { - match self { - Self::Payload(payload) => payload.block_with_parent(), - Self::Block(block) => block.block_with_parent(), - } - } - - /// Returns a string showing whether or not this is a block or payload. - pub const fn type_name(&self) -> &'static str { - match self { - Self::Payload(_) => "payload", - Self::Block(_) => "block", - } - } - - /// Returns the block access list if available. - pub const fn block_access_list(&self) -> Option> { - // TODO decode and return `BlockAccessList` - None - } - - /// Returns the number of transactions in the payload or block. - pub fn transaction_count(&self) -> usize - where - T::ExecutionData: ExecutionPayload, - { - match self { - Self::Payload(payload) => payload.transaction_count(), - Self::Block(block) => block.transaction_count(), - } - } - - /// Returns the withdrawals from the payload or block. - pub fn withdrawals(&self) -> Option<&[Withdrawal]> - where - T::ExecutionData: ExecutionPayload, - { - match self { - Self::Payload(payload) => payload.withdrawals().map(|w| w.as_slice()), - Self::Block(block) => block.body().withdrawals().map(|w| w.as_slice()), - } - } - - /// Returns the total gas used by the block. - pub fn gas_used(&self) -> u64 - where - T::ExecutionData: ExecutionPayload, - { - match self { - Self::Payload(payload) => payload.gas_used(), - Self::Block(block) => block.gas_used(), - } - } -} diff --git a/crates/engine-tree-ext/src/trie_updates.rs b/crates/engine-tree-ext/src/trie_updates.rs deleted file mode 100644 index 3ac7b523..00000000 --- a/crates/engine-tree-ext/src/trie_updates.rs +++ /dev/null @@ -1,357 +0,0 @@ -use alloy_primitives::{ - B256, - map::{B256Map, HashMap}, -}; -use reth_db::DatabaseError; -use reth_trie::{ - BranchNodeCompact, Nibbles, - trie_cursor::{TrieCursor, TrieCursorFactory}, - updates::{StorageTrieUpdates, TrieUpdates}, -}; -use std::collections::BTreeSet; -use tracing::warn; - -#[derive(Debug)] -struct EntryDiff { - task: T, - regular: T, - database: T, -} - -#[derive(Debug, Default)] -struct TrieUpdatesDiff { - account_nodes: HashMap>>, - removed_nodes: HashMap>, - storage_tries: B256Map, -} - -impl TrieUpdatesDiff { - fn has_differences(&self) -> bool { - !self.account_nodes.is_empty() - || !self.removed_nodes.is_empty() - || !self.storage_tries.is_empty() - } - - pub(super) fn log_differences(mut self) { - if self.has_differences() { - for ( - path, - EntryDiff { - task, - regular, - database, - }, - ) in &mut self.account_nodes - { - warn!(target: "engine::tree", ?path, ?task, ?regular, ?database, "Difference in account trie updates"); - } - - for ( - path, - EntryDiff { - task: task_removed, - regular: regular_removed, - database: database_not_exists, - }, - ) in &self.removed_nodes - { - warn!(target: "engine::tree", ?path, ?task_removed, ?regular_removed, ?database_not_exists, "Difference in removed account trie nodes"); - } - - for (address, storage_diff) in self.storage_tries { - storage_diff.log_differences(address); - } - } - } -} - -#[derive(Debug, Default)] -struct StorageTrieUpdatesDiff { - is_deleted: Option>, - storage_nodes: HashMap>>, - removed_nodes: HashMap>, -} - -impl StorageTrieUpdatesDiff { - fn has_differences(&self) -> bool { - self.is_deleted.is_some() - || !self.storage_nodes.is_empty() - || !self.removed_nodes.is_empty() - } - - fn log_differences(&self, address: B256) { - if let Some(EntryDiff { - task: task_deleted, - regular: regular_deleted, - database: database_not_exists, - }) = self.is_deleted - { - warn!(target: "engine::tree", ?address, ?task_deleted, ?regular_deleted, ?database_not_exists, "Difference in storage trie deletion"); - } - - for ( - path, - EntryDiff { - task, - regular, - database, - }, - ) in &self.storage_nodes - { - warn!(target: "engine::tree", ?address, ?path, ?task, ?regular, ?database, "Difference in storage trie updates"); - } - - for ( - path, - EntryDiff { - task: task_removed, - regular: regular_removed, - database: database_not_exists, - }, - ) in &self.removed_nodes - { - warn!(target: "engine::tree", ?address, ?path, ?task_removed, ?regular_removed, ?database_not_exists, "Difference in removed storage trie nodes"); - } - } -} - -/// Compares the trie updates from state root task, regular state root calculation and database, -/// and logs the differences if there's any. -/// -/// Returns `true` if there are differences. -pub(crate) fn compare_trie_updates( - trie_cursor_factory: impl TrieCursorFactory, - task: TrieUpdates, - regular: TrieUpdates, -) -> Result { - let mut task = adjust_trie_updates(task); - let mut regular = adjust_trie_updates(regular); - - let mut diff = TrieUpdatesDiff::default(); - - // compare account nodes - let mut account_trie_cursor = trie_cursor_factory.account_trie_cursor()?; - for key in task - .account_nodes - .keys() - .chain(regular.account_nodes.keys()) - .copied() - .collect::>() - { - let (task, regular) = ( - task.account_nodes.remove(&key), - regular.account_nodes.remove(&key), - ); - let database = account_trie_cursor.seek_exact(key)?.map(|x| x.1); - - if !branch_nodes_equal(task.as_ref(), regular.as_ref(), database.as_ref())? { - diff.account_nodes.insert( - key, - EntryDiff { - task, - regular, - database, - }, - ); - } - } - - // compare removed nodes - let mut account_trie_cursor = trie_cursor_factory.account_trie_cursor()?; - for key in task - .removed_nodes - .iter() - .chain(regular.removed_nodes.iter()) - .copied() - .collect::>() - { - let (task_removed, regular_removed) = ( - task.removed_nodes.contains(&key), - regular.removed_nodes.contains(&key), - ); - let database_not_exists = account_trie_cursor.seek_exact(key)?.is_none(); - // If the deletion is a no-op, meaning that the entry is not in the - // database, do not add it to the diff. - if task_removed != regular_removed && !database_not_exists { - diff.removed_nodes.insert( - key, - EntryDiff { - task: task_removed, - regular: regular_removed, - database: database_not_exists, - }, - ); - } - } - - // compare storage tries - for key in task - .storage_tries - .keys() - .chain(regular.storage_tries.keys()) - .copied() - .collect::>() - { - let (mut task, mut regular) = ( - task.storage_tries.remove(&key), - regular.storage_tries.remove(&key), - ); - if task != regular { - #[expect(clippy::or_fun_call)] - let storage_diff = compare_storage_trie_updates( - || trie_cursor_factory.storage_trie_cursor(key), - // Compare non-existent storage tries as empty. - task.as_mut().unwrap_or(&mut Default::default()), - regular.as_mut().unwrap_or(&mut Default::default()), - )?; - if storage_diff.has_differences() { - diff.storage_tries.insert(key, storage_diff); - } - } - } - - // log differences - let has_differences = diff.has_differences(); - diff.log_differences(); - - Ok(has_differences) -} - -fn compare_storage_trie_updates( - trie_cursor: impl Fn() -> Result, - task: &mut StorageTrieUpdates, - regular: &mut StorageTrieUpdates, -) -> Result { - // Check if the storage trie exists by seeking to the first entry - let database_not_exists = trie_cursor()?.seek(Nibbles::default())?.is_none(); - let mut diff = StorageTrieUpdatesDiff { - // If the deletion is a no-op, meaning that the entry is not in the - // database, do not add it to the diff. - is_deleted: (task.is_deleted != regular.is_deleted && !database_not_exists).then_some( - EntryDiff { - task: task.is_deleted, - regular: regular.is_deleted, - database: database_not_exists, - }, - ), - ..Default::default() - }; - - // compare storage nodes - let mut storage_trie_cursor = trie_cursor()?; - for key in task - .storage_nodes - .keys() - .chain(regular.storage_nodes.keys()) - .copied() - .collect::>() - { - let (task, regular) = ( - task.storage_nodes.remove(&key), - regular.storage_nodes.remove(&key), - ); - let database = storage_trie_cursor.seek_exact(key)?.map(|x| x.1); - if !branch_nodes_equal(task.as_ref(), regular.as_ref(), database.as_ref())? { - diff.storage_nodes.insert( - key, - EntryDiff { - task, - regular, - database, - }, - ); - } - } - - // compare removed nodes - let mut storage_trie_cursor = trie_cursor()?; - for key in task - .removed_nodes - .iter() - .chain(regular.removed_nodes.iter()) - .collect::>() - { - let (task_removed, regular_removed) = ( - task.removed_nodes.contains(key), - regular.removed_nodes.contains(key), - ); - if task_removed == regular_removed { - continue; - } - let database_not_exists = storage_trie_cursor.seek_exact(*key)?.map(|x| x.1).is_none(); - // If the deletion is a no-op, meaning that the entry is not in the - // database, do not add it to the diff. - if !database_not_exists { - diff.removed_nodes.insert( - *key, - EntryDiff { - task: task_removed, - regular: regular_removed, - database: database_not_exists, - }, - ); - } - } - - Ok(diff) -} - -/// Filters the removed nodes of both account trie updates and storage trie updates, so that they -/// don't include those nodes that were also updated. -fn adjust_trie_updates(trie_updates: TrieUpdates) -> TrieUpdates { - TrieUpdates { - removed_nodes: trie_updates - .removed_nodes - .into_iter() - .filter(|key| !trie_updates.account_nodes.contains_key(key)) - .collect(), - storage_tries: trie_updates - .storage_tries - .into_iter() - .map(|(address, updates)| { - ( - address, - StorageTrieUpdates { - removed_nodes: updates - .removed_nodes - .into_iter() - .filter(|key| !updates.storage_nodes.contains_key(key)) - .collect(), - ..updates - }, - ) - }) - .collect(), - ..trie_updates - } -} - -/// Compares the branch nodes from state root task and regular state root calculation. -/// -/// If one of the branch nodes is [`None`], it means it's not updated and the other is compared to -/// the branch node from the database. -/// -/// Returns `true` if they are equal. -fn branch_nodes_equal( - task: Option<&BranchNodeCompact>, - regular: Option<&BranchNodeCompact>, - database: Option<&BranchNodeCompact>, -) -> Result { - Ok(match (task, regular) { - (Some(task), Some(regular)) => { - task.state_mask == regular.state_mask - && task.tree_mask == regular.tree_mask - && task.hash_mask == regular.hash_mask - && task.hashes == regular.hashes - && task.root_hash == regular.root_hash - } - (None, None) => true, - _ => { - if task.is_some() { - task == database - } else { - regular == database - } - } - }) -} diff --git a/crates/engine-tree-ext/tests/jade_boundary.rs b/crates/engine-tree-ext/tests/jade_boundary.rs deleted file mode 100644 index 7f5d8469..00000000 --- a/crates/engine-tree-ext/tests/jade_boundary.rs +++ /dev/null @@ -1,212 +0,0 @@ -//! End-to-end verification of retroactive trust across the Jade hardfork -//! boundary. -//! -//! These tests exercise the `MorphBasicEngineValidator` through the real -//! `MorphNode` stack: they spin up an ephemeral node, build a valid block, -//! tamper with the header's `state_root`, and re-import via the Engine API. -//! -//! Retroactive-trust invariants: -//! * Pre-Jade: header state root is NOT compared against the MPT root the -//! validator computes — a mismatched `state_root` must still import. -//! * Post-Jade: the validator enforces strict MPT equality — the same -//! tampered `state_root` must be rejected as INVALID. -//! -//! Two existing sibling tests verify these invariants at the crate-of-use -//! level (`crates/node/tests/it/engine.rs::state_root_validation_skipped_pre_jade` -//! and `crates/node/tests/it/consensus.rs::post_jade_state_root_mismatch_is_rejected`). -//! The tests in this file pin the contract to the engine-tree-ext crate: if -//! someone tweaks `MorphBasicEngineValidator` in a way that breaks the -//! boundary, `cargo test -p morph-engine-tree-ext` is expected to catch it. - -#![cfg(feature = "test-utils")] - -use alloy_consensus::{BlockHeader, proofs}; -use alloy_primitives::{Address, B256}; -use alloy_rpc_types_engine::PayloadAttributes; -use morph_node::test_utils::{HardforkSchedule, MorphTestNode, TestNodeBuilder}; -use morph_payload_types::{MorphBuiltPayload, MorphPayloadAttributes, MorphPayloadTypes}; -use reth_node_api::PayloadTypes; -use reth_payload_builder::BuildNewPayload; -use reth_payload_primitives::BuiltPayload; -use reth_primitives_traits::SealedBlock; -use reth_provider::BlockReaderIdExt; - -/// Build an empty block through the payload builder without submitting it. -/// -/// `node.advance_block()` would time out waiting for a non-empty payload since -/// the pool is empty — instead, drive the builder directly with empty L1 -/// messages and resolve the payload synchronously via -/// `PayloadKind::WaitForPending`. This avoids fixed sleeps + polling, which -/// were flake-prone on loaded CI runners. -async fn build_candidate_block(node: &mut MorphTestNode) -> eyre::Result { - const BUILD_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); - - let head = node - .inner - .provider - .sealed_header_by_number_or_tag(alloy_rpc_types_eth::BlockNumberOrTag::Latest)?; - let (head_hash, head_ts) = head - .map(|h| (h.hash(), h.timestamp())) - .unwrap_or((B256::ZERO, 0)); - - let rpc_attrs = MorphPayloadAttributes { - inner: PayloadAttributes { - timestamp: head_ts + 1, - prev_randao: B256::ZERO, - suggested_fee_recipient: Address::ZERO, - withdrawals: Some(vec![]), - parent_beacon_block_root: Some(B256::ZERO), - }, - transactions: Some(vec![]), - gas_limit: None, - base_fee_per_gas: None, - }; - - let payload_id = node - .inner - .payload_builder_handle - .send_new_payload(BuildNewPayload { - attributes: rpc_attrs, - parent_hash: head_hash, - cache: None, - trie_handle: None, - }) - .await? - .map_err(|e| eyre::eyre!("payload build failed: {e}"))?; - - tokio::time::timeout( - BUILD_TIMEOUT, - node.inner - .payload_builder_handle - .resolve_kind(payload_id, reth_node_api::PayloadKind::WaitForPending), - ) - .await - .map_err(|_| eyre::eyre!("payload build timed out after {:?}", BUILD_TIMEOUT))? - .ok_or_else(|| eyre::eyre!("no payload response for id {payload_id:?}"))? - .map_err(|e| eyre::eyre!("payload build error: {e}")) -} - -/// Tamper with a payload's header and ask the engine to import the result. -/// -/// Returns `true` if the engine accepted the block (VALID), `false` otherwise. -async fn try_import_with_tampered_state_root( - node: &mut MorphTestNode, - base: &MorphBuiltPayload, - bogus_state_root: B256, -) -> eyre::Result { - let sealed = base.block(); - let morph_header: morph_primitives::MorphHeader = sealed.header().inner.clone().into(); - let body = sealed.body().clone(); - let mut block = morph_primitives::Block::new(morph_header, body); - - block.header.inner.state_root = bogus_state_root; - block.header.inner.transactions_root = - proofs::calculate_transaction_root(&block.body.transactions); - - let modified_sealed = SealedBlock::seal_slow(block); - let execution_data = MorphPayloadTypes::block_to_payload(modified_sealed); - let status = node - .inner - .add_ons_handle - .beacon_engine_handle - .new_payload(execution_data) - .await?; - - Ok(status.is_valid()) -} - -#[tokio::test(flavor = "multi_thread")] -async fn pre_jade_block_with_tampered_state_root_imports() -> eyre::Result<()> { - reth_tracing::init_test_tracing(); - - let (mut nodes, _wallet) = TestNodeBuilder::new() - .with_schedule(HardforkSchedule::PreJade) - .build() - .await?; - let mut node = nodes.pop().unwrap(); - - let base_payload = build_candidate_block(&mut node).await?; - let accepted = - try_import_with_tampered_state_root(&mut node, &base_payload, B256::from([0xFF; 32])) - .await?; - - assert!( - accepted, - "pre-Jade block with tampered state_root must be accepted — retroactive trust skips \ - state-root validation before Jade" - ); - Ok(()) -} - -#[tokio::test(flavor = "multi_thread")] -async fn post_jade_block_with_tampered_state_root_is_rejected() -> eyre::Result<()> { - reth_tracing::init_test_tracing(); - - let (mut nodes, _wallet) = TestNodeBuilder::new() - .with_schedule(HardforkSchedule::AllActive) - .build() - .await?; - let mut node = nodes.pop().unwrap(); - - let base_payload = build_candidate_block(&mut node).await?; - let accepted = - try_import_with_tampered_state_root(&mut node, &base_payload, B256::from([0xFF; 32])) - .await?; - - assert!( - !accepted, - "post-Jade block with tampered state_root must be rejected — MorphBasicEngineValidator \ - enforces strict MPT root equality after Jade" - ); - Ok(()) -} - -/// Regression test: P2P-downloaded blocks enter the engine tree via the -/// Block-input path (`insert_block`), which never invokes -/// `convert_payload_to_block` and therefore registers no withdraw-trie-root -/// expectation. The validator must treat the missing entry as -/// `SkipValidation` so the downloaded block is accepted; otherwise sync -/// stalls indefinitely with `"missing withdraw trie root expectation -/// cache entry"`. -/// -/// The test runs two interconnected nodes: node[0] builds and imports a -/// block via the Engine API (Payload path → expectation registered on -/// node[0] only), then node[1] points its forkchoice at the new head so -/// reth's downloader fetches the block from node[0] over P2P. The download -/// lands in node[1]'s engine tree as a `Block` input. -#[tokio::test(flavor = "multi_thread")] -async fn p2p_downloaded_block_imports_without_registered_expectation() -> eyre::Result<()> { - reth_tracing::init_test_tracing(); - - let (mut nodes, _wallet) = TestNodeBuilder::new() - .with_schedule(HardforkSchedule::AllActive) - .with_num_nodes(2) - .build() - .await?; - let node1 = nodes.pop().expect("two nodes requested"); - let mut node0 = nodes.pop().expect("two nodes requested"); - - let payload = build_candidate_block(&mut node0).await?; - let head_hash = payload.block().hash(); - - let exec_data = MorphPayloadTypes::block_to_payload(payload.block().clone()); - let status = node0 - .inner - .add_ons_handle - .beacon_engine_handle - .new_payload(exec_data) - .await?; - assert!( - status.is_valid(), - "node[0] must accept its own freshly-built block, got {status:?}" - ); - node0.update_forkchoice(head_hash, head_hash).await?; - - // sync_to keeps issuing FCU until node[1]'s head matches `head_hash`. - // Since node[1] does not have the block, reth's engine tree will trigger - // its block downloader against the connected peer (node[0]) — downloaded - // blocks enter via the Block input path, exercising the bug. - node1.sync_to(head_hash).await?; - - Ok(()) -} diff --git a/crates/evm/Cargo.toml b/crates/evm/Cargo.toml index a9f9cf1a..282f0989 100644 --- a/crates/evm/Cargo.toml +++ b/crates/evm/Cargo.toml @@ -27,6 +27,7 @@ reth-rpc-eth-api = { workspace = true, optional = true } alloy-evm.workspace = true alloy-consensus.workspace = true alloy-primitives.workspace = true +alloy-rpc-types-eth.workspace = true derive_more.workspace = true thiserror.workspace = true diff --git a/crates/evm/src/block/mod.rs b/crates/evm/src/block/mod.rs index 6292f88a..42ce3269 100644 --- a/crates/evm/src/block/mod.rs +++ b/crates/evm/src/block/mod.rs @@ -19,8 +19,7 @@ use alloy_consensus::transaction::TxHashRef; use alloy_evm::{ Database, Evm, RecoveredTx, block::{ - BlockExecutionError, BlockExecutionResult, BlockExecutor, ExecutableTx, GasOutput, - OnStateHook, StateChangeSource, TxResult, + BlockExecutionError, BlockExecutionResult, BlockExecutor, ExecutableTx, GasOutput, TxResult, }, }; use alloy_primitives::{Address, Log, U256}; @@ -96,9 +95,6 @@ pub struct MorphBlockExecutor { /// Cached hardfork for this block (constant across all transactions). /// Set in `apply_pre_execution_changes`, reused in `commit_transaction`. hardfork: MorphHardfork, - /// Hook for notifying the engine about per-transaction state changes. - /// Used by `StateRootTask` for incremental sparse trie computation. - state_hook: Option>, } impl MorphBlockExecutor @@ -124,7 +120,6 @@ where receipts: Vec::new(), gas_used: 0, hardfork: MorphHardfork::default(), - state_hook: None, } } @@ -275,11 +270,6 @@ where post_fee_logs, } = output; - // Notify the state hook (e.g. StateRootTask) BEFORE committing. - if let Some(hook) = &mut self.state_hook { - hook.on_state(StateChangeSource::Transaction(self.receipts.len()), &state); - } - // EIP-8037 separates regular and state gas; pre-Amsterdam morph treats // them as a single number, so use the unified `tx_gas_used` getter. let gas_used = result.gas().tx_gas_used(); @@ -334,10 +324,6 @@ where )) } - fn set_state_hook(&mut self, hook: Option>) { - self.state_hook = hook; - } - fn evm_mut(&mut self) -> &mut Self::Evm { &mut self.evm } diff --git a/crates/evm/src/context.rs b/crates/evm/src/context.rs index 083a0302..de01f683 100644 --- a/crates/evm/src/context.rs +++ b/crates/evm/src/context.rs @@ -17,9 +17,10 @@ impl reth_rpc_eth_api::helpers::pending_block::BuildPendingEnv, + block_overrides: Option<&alloy_rpc_types_eth::BlockOverrides>, ) -> Self { Self { - inner: NextBlockEnvAttributes::build_pending_env(parent), + inner: NextBlockEnvAttributes::build_pending_env(parent, block_overrides), base_fee_per_gas: None, } } diff --git a/crates/evm/src/evm.rs b/crates/evm/src/evm.rs index 96005018..e2454e4d 100644 --- a/crates/evm/src/evm.rs +++ b/crates/evm/src/evm.rs @@ -4,7 +4,7 @@ use alloy_evm::{ revm::{ Context, ExecuteEvm, InspectEvm, Inspector, SystemCallEvm, context::{ - CfgEnv, + CfgEnv, DBErrorMarker, result::{EVMError, ResultAndState}, }, inspector::NoOpInspector, @@ -26,8 +26,7 @@ impl EvmFactory for MorphEvmFactory { type Evm>> = MorphEvm; type Context = MorphContext; type Tx = MorphTxEnv; - type Error = - EVMError; + type Error = EVMError; type HaltReason = MorphHaltReason; type Spec = MorphHardfork; type BlockEnv = MorphBlockEnv; diff --git a/crates/node/Cargo.toml b/crates/node/Cargo.toml index 3c6eca37..53257bb2 100644 --- a/crates/node/Cargo.toml +++ b/crates/node/Cargo.toml @@ -14,7 +14,6 @@ workspace = true morph-chainspec.workspace = true morph-consensus.workspace = true morph-engine-api.workspace = true -morph-engine-tree-ext.workspace = true morph-evm.workspace = true morph-payload-builder.workspace = true morph-payload-types.workspace = true @@ -26,11 +25,14 @@ morph-txpool.workspace = true # Reth dependencies reth-db.workspace = true reth-db-api.workspace = true +reth-chain-state.workspace = true reth-node-core.workspace = true reth-chainspec.workspace = true reth-engine-local.workspace = true reth-engine-tree.workspace = true reth-errors.workspace = true +reth-evm.workspace = true +reth-execution-cache.workspace = true reth-exex.workspace = true reth-node-api.workspace = true reth-node-builder.workspace = true @@ -77,7 +79,7 @@ alloy-signer = { workspace = true, optional = true } alloy-signer-local = { workspace = true, optional = true } [build-dependencies] -vergen.workspace = true +vergen = { workspace = true, features = ["build", "cargo", "emit_and_set"] } vergen-git2.workspace = true [dev-dependencies] diff --git a/crates/node/build.rs b/crates/node/build.rs index d068046d..6a83410c 100644 --- a/crates/node/build.rs +++ b/crates/node/build.rs @@ -1,26 +1,23 @@ #![allow(missing_docs)] use std::{env, error::Error}; -use vergen::{BuildBuilder, CargoBuilder, Emitter}; -use vergen_git2::Git2Builder; +use vergen::{Build, Cargo, Emitter}; +use vergen_git2::Git2; fn main() -> Result<(), Box> { let mut emitter = Emitter::default(); - let build_builder = BuildBuilder::default().build_timestamp(true).build()?; + let build_builder = Build::builder().build_timestamp(true).build(); emitter.add_instructions(&build_builder)?; - let cargo_builder = CargoBuilder::default() - .features(true) - .target_triple(true) - .build()?; + let cargo_builder = Cargo::builder().features(true).target_triple(true).build(); emitter.add_instructions(&cargo_builder)?; - let git_builder = Git2Builder::default() + let git_builder = Git2::builder() .describe(false, true, None) .dirty(true) .sha(false) - .build()?; + .build(); emitter.add_instructions(&git_builder)?; emitter.emit_and_set()?; diff --git a/crates/node/src/node.rs b/crates/node/src/node.rs index 5256862c..347cef93 100644 --- a/crates/node/src/node.rs +++ b/crates/node/src/node.rs @@ -213,6 +213,7 @@ impl PayloadAttributesBuilder .then(B256::random), // Morph L2 has no PoS slot semantics; field added in alloy 2.0. slot_number: None, + target_gas_limit: None, }, // No L1 transactions in local mining mode transactions: None, diff --git a/crates/node/src/test_utils.rs b/crates/node/src/test_utils.rs index d8f50d00..d58cfe5e 100644 --- a/crates/node/src/test_utils.rs +++ b/crates/node/src/test_utils.rs @@ -354,6 +354,7 @@ pub async fn advance_empty_block(node: &mut MorphTestNode) -> eyre::Result eyre::Result morph_payload_types::MorphPay withdrawals: Some(vec![]), parent_beacon_block_root: Some(B256::ZERO), slot_number: None, + target_gas_limit: None, }, transactions: None, gas_limit: None, diff --git a/crates/node/src/validator.rs b/crates/node/src/validator.rs index b43e9c3d..977a4e6d 100644 --- a/crates/node/src/validator.rs +++ b/crates/node/src/validator.rs @@ -6,12 +6,26 @@ use alloy_primitives::{B256, keccak256}; use dashmap::DashMap; use morph_chainspec::{ L2_MESSAGE_QUEUE_ADDRESS, L2_MESSAGE_QUEUE_WITHDRAW_TRIE_ROOT_SLOT, MorphChainSpec, + MorphHardforks, }; use morph_payload_types::{MorphExecutionData, MorphPayloadTypes}; -use morph_primitives::MorphHeader; +use morph_primitives::{MorphHeader, MorphPrimitives}; use parking_lot::Mutex; +use reth_chain_state::{ExecutedBlock, StateTrieOverlayManager}; use reth_chainspec::EthChainSpec; -use reth_errors::ConsensusError; +use reth_engine_tree::tree::payload_validator::TreeCtx; +use reth_engine_tree::tree::{ + BasicEngineValidator, CacheWaitDurations, EngineApiTreeState, EngineValidator, WaitForCaches, + error::{InsertBlockError, InsertBlockErrorKind}, + state_root_strategy::{ + DefaultStateRootStrategy, LazyHashedPostState, PayloadStateRootHandle, + PayloadStateRootJobContext, PreparedStateRootJob, StateRootJob, StateRootJobContext, + StateRootJobOutcome, StateRootStrategy, + }, +}; +use reth_errors::{ConsensusError, ProviderResult}; +use reth_evm::{ConfigureEvm, revm::context::Block as _}; +use reth_execution_cache::SavedCache; use reth_node_api::{ AddOnsContext, FullNodeComponents, InvalidPayloadAttributesError, NewPayloadError, NodeTypes, PayloadAttributes, PayloadTypes, PayloadValidator, @@ -20,8 +34,12 @@ use reth_node_builder::{ invalid_block_hook::InvalidBlockHookExt, rpc::{EngineValidatorBuilder, PayloadValidatorBuilder}, }; +use reth_payload_primitives::BuiltPayloadExecutedBlock; use reth_primitives_traits::{RecoveredBlock, SealedBlock}; -use reth_provider::ChainSpecProvider; +use reth_provider::{ + BlockExecutionOutput, BlockReader, ChainSpecProvider, StateProviderFactory, StateReader, + StateRootProvider, +}; use reth_tracing::tracing; use std::{collections::VecDeque, sync::Arc}; @@ -80,20 +98,16 @@ where >, >, Node::Provider: ChainSpecProvider, - PVB: PayloadValidatorBuilder, - PVB::Validator: reth_node_api::PayloadValidator< - ::Payload, - Block = reth_node_api::BlockTy, - > + Clone, + PVB: PayloadValidatorBuilder, { - type EngineValidator = - morph_engine_tree_ext::MorphBasicEngineValidator; + type EngineValidator = MorphTreeEngineValidator; async fn build_tree_validator( self, ctx: &AddOnsContext<'_, Node>, tree_config: reth_node_api::TreeConfig, changeset_cache: reth_trie_db::ChangesetCache, + state_trie_overlays: StateTrieOverlayManager, ) -> eyre::Result { let validator = self.payload_validator_builder.build(ctx).await?; let data_dir = ctx @@ -103,17 +117,336 @@ where .resolve_datadir(ctx.config.chain.chain()); let invalid_block_hook = ctx.create_invalid_block_hook(&data_dir).await?; let chain_spec = ctx.node.provider().chain_spec(); - - Ok(morph_engine_tree_ext::MorphBasicEngineValidator::new( - ctx.node.provider().clone(), + let post_execution_validator = validator.clone(); + + let provider = ctx.node.provider().clone(); + let state_root_strategy: Arc< + dyn StateRootStrategy, + > = Arc::new(MorphStateRootStrategy::new(chain_spec.clone())); + let tree_config = strict_morph_tree_config(tree_config); + let validator = BasicEngineValidator::new( + provider.clone(), Arc::new(ctx.node.consensus().clone()), ctx.node.evm_config().clone(), validator, tree_config, invalid_block_hook, changeset_cache, + state_trie_overlays, ctx.node.task_executor().clone(), + ) + .with_state_root_strategy(state_root_strategy); + + Ok(MorphTreeEngineValidator::new( + validator, + provider, + chain_spec, + post_execution_validator, + )) + } +} + +/// Morph only relaxes the header-root equality check before Jade, through +/// [`MorphStateRootStrategy`]. The upstream debug skip also suppresses trie +/// updates, so allowing it here would make both pre-Jade persistence and +/// post-Jade validation unsafe. +fn strict_morph_tree_config(tree_config: reth_node_api::TreeConfig) -> reth_node_api::TreeConfig { + if tree_config.skip_state_root() { + tracing::warn!( + target: "reth::cli", + "ignoring --debug.skip-state-root: Morph requires trie updates before Jade and strict state-root validation from Jade onward" + ); + } + tree_config.with_skip_state_root(false) +} + +/// Upstream engine validator plus Morph-specific validation hooks. +/// +/// State execution, caching and trie maintenance remain entirely upstream. This +/// wrapper preserves the parent-aware L1 queue invariant and the optional +/// consensus-layer withdraw-trie-root cross-check. +pub struct MorphTreeEngineValidator +where + Evm: ConfigureEvm, +{ + inner: BasicEngineValidator, + provider: P, + chain_spec: Arc, + post_execution_validator: MorphEngineValidator, +} + +impl MorphTreeEngineValidator +where + Evm: ConfigureEvm, +{ + const fn new( + inner: BasicEngineValidator, + provider: P, + chain_spec: Arc, + post_execution_validator: MorphEngineValidator, + ) -> Self { + Self { + inner, + provider, + chain_spec, + post_execution_validator, + } + } +} + +impl MorphTreeEngineValidator +where + Evm: ConfigureEvm, + P: BlockReader
, +{ + fn validate_next_l1_msg_index( + &self, + block: &SealedBlock, + ctx: &TreeCtx<'_, MorphPrimitives>, + ) -> Result<(), InsertBlockErrorKind> { + if !self + .chain_spec + .is_jade_active_at_timestamp(block.header().timestamp()) + { + return Ok(()); + } + + let parent_hash = block.parent_hash(); + let parent = match ctx.state().tree_state().sealed_header_by_hash(&parent_hash) { + Some(parent) => Some(parent), + None => self.provider.sealed_header_by_hash(parent_hash)?, + }; + let Some(parent) = parent else { + // The upstream validator reports the missing parent with the + // canonical provider error before executing the block. + return Ok(()); + }; + + let mut expected = parent.next_l1_msg_index; + for tx in block.body().transactions() { + if !tx.is_l1_msg() { + break; + } + let queue_index = tx.queue_index().ok_or_else(|| { + ConsensusError::msg("L1 message transaction is missing queue index") + })?; + expected = queue_index.checked_add(1).ok_or_else(|| { + ConsensusError::msg(format!( + "invalid block.NextL1MsgIndex: expected {}, got {}", + u64::MAX, + block.header().next_l1_msg_index + )) + })?; + } + + let actual = block.header().next_l1_msg_index; + if actual != expected { + return Err(ConsensusError::msg(format!( + "invalid block.NextL1MsgIndex: expected {expected}, got {actual}" + )) + .into()); + } + Ok(()) + } + + fn validate_next_l1_msg_index_for_block( + &self, + block: &SealedBlock, + ctx: &TreeCtx<'_, MorphPrimitives>, + ) -> Result<(), reth_engine_tree::tree::error::InsertPayloadError> + { + self.validate_next_l1_msg_index(block, ctx) + .map_err(|kind| InsertBlockError::new(block.clone(), kind).into()) + } + + fn validate_withdraw_trie_root( + &self, + output: reth_engine_tree::tree::ValidationOutput, + ) -> reth_engine_tree::tree::ValidationOutcome { + let block = output.executed_block.recovered_block(); + if let Err(err) = self + .post_execution_validator + .validate_withdraw_trie_root_update(block.hash(), || { + output.executed_block.hashed_state() + }) + { + return Err(InsertBlockError::consensus_error( + err, + output.executed_block.sealed_block().clone(), + ) + .into()); + } + + Ok(output) + } +} + +impl EngineValidator + for MorphTreeEngineValidator +where + P: BlockReader
+ Send + Sync + 'static, + Evm: ConfigureEvm + 'static, + BasicEngineValidator: + EngineValidator, +{ + fn validate_payload_attributes_against_header( + &self, + attr: &::PayloadAttributes, + header: &MorphHeader, + ) -> Result<(), InvalidPayloadAttributesError> { + self.inner + .validate_payload_attributes_against_header(attr, header) + } + + fn convert_payload_to_block( + &self, + payload: MorphExecutionData, + ) -> Result, NewPayloadError> { + self.inner.convert_payload_to_block(payload) + } + + fn validate_payload( + &mut self, + payload: MorphExecutionData, + ctx: TreeCtx<'_, MorphPrimitives>, + ) -> reth_engine_tree::tree::ValidationOutcome { + let block_hash = payload.block.hash(); + self.validate_next_l1_msg_index_for_block(payload.block.as_ref(), &ctx)?; + match self.inner.validate_payload(payload, ctx) { + Ok(output) => self.validate_withdraw_trie_root(output), + Err(err) => { + self.post_execution_validator + .take_withdraw_trie_root_expectation(block_hash); + Err(err) + } + } + } + + fn validate_block( + &mut self, + block: SealedBlock, + ctx: TreeCtx<'_, MorphPrimitives>, + ) -> reth_engine_tree::tree::ValidationOutcome { + self.validate_next_l1_msg_index_for_block(&block, &ctx)?; + let output = self.inner.validate_block(block, ctx)?; + self.validate_withdraw_trie_root(output) + } + + fn on_inserted_executed_block( + &self, + block: BuiltPayloadExecutedBlock, + ) -> ProviderResult> { + self.inner.on_inserted_executed_block(block) + } + + fn cache_for(&self, block_hash: B256) -> Option { + self.inner.cache_for(block_hash) + } + + fn payload_state_root_handle_for( + &self, + parent_hash: B256, + parent_header: &MorphHeader, + timestamp: u64, + state: &mut EngineApiTreeState, + ) -> Option { + self.inner + .payload_state_root_handle_for(parent_hash, parent_header, timestamp, state) + } +} + +impl WaitForCaches for MorphTreeEngineValidator +where + Evm: ConfigureEvm, + BasicEngineValidator: WaitForCaches, +{ + fn wait_for_caches(&self) -> CacheWaitDurations { + self.inner.wait_for_caches() + } +} + +/// Uses upstream's optimized state-root path from Jade onward. Before Jade it +/// computes and persists the real MPT updates but trusts the historical header +/// root, which is a ZK-trie commitment and cannot be compared to the MPT root. +#[derive(Debug)] +struct MorphStateRootStrategy { + chain_spec: Arc, + default: DefaultStateRootStrategy, +} + +impl MorphStateRootStrategy { + fn new(chain_spec: Arc) -> Self { + Self { chain_spec, + default: DefaultStateRootStrategy::default(), + } + } +} + +impl StateRootStrategy for MorphStateRootStrategy +where + P: BlockReader
+ + StateProviderFactory + + StateReader + + Clone + + Send + + Sync + + 'static, + Evm: ConfigureEvm + 'static, + DefaultStateRootStrategy: StateRootStrategy, +{ + fn prepare( + &self, + ctx: StateRootJobContext<'_, MorphPrimitives, P, Evm>, + ) -> ProviderResult> { + let timestamp: u64 = ctx.env().evm_env.block_env.timestamp().saturating_to(); + if self.chain_spec.is_jade_active_at_timestamp(timestamp) { + return self.default.prepare(ctx); + } + + Ok(PreparedStateRootJob::new( + Box::new(PreJadeStateRootJob { + provider_builder: ctx.provider_builder(), + }), + None, + )) + } + + fn prepare_payload_builder( + &self, + ctx: PayloadStateRootJobContext<'_, MorphPrimitives, P>, + ) -> ProviderResult> { + if self.chain_spec.is_jade_active_at_timestamp(ctx.timestamp()) { + return self.default.prepare_payload_builder(ctx); + } + Ok(None) + } +} + +struct PreJadeStateRootJob

{ + provider_builder: reth_engine_tree::tree::StateProviderBuilder, +} + +impl

StateRootJob for PreJadeStateRootJob

+where + P: BlockReader + StateProviderFactory + StateReader + Clone + Send + Sync + 'static, +{ + fn name(&self) -> &'static str { + "morph-pre-jade-trusted-header" + } + + fn finish( + &mut self, + block: &RecoveredBlock, + _output: Arc>, + hashed_state: &LazyHashedPostState, + ) -> ProviderResult { + let provider = self.provider_builder.build()?; + let (_mpt_root, trie_updates) = + provider.state_root_with_updates(hashed_state.get().as_ref().clone())?; + Ok(StateRootJobOutcome::new( + block.header().state_root(), + Arc::new(trie_updates), )) } } @@ -184,8 +517,8 @@ impl MorphEngineValidator { removed } - fn updated_withdraw_trie_root_from_hashed_state( - state_updates: &reth_trie::HashedPostState, + fn updated_withdraw_trie_root_from_sorted_hashed_state( + state_updates: &reth_trie::HashedPostStateSorted, ) -> Option { let hashed_address = keccak256(L2_MESSAGE_QUEUE_ADDRESS); let hashed_slot = keccak256(B256::from(L2_MESSAGE_QUEUE_WITHDRAW_TRIE_ROOT_SLOT)); @@ -193,8 +526,44 @@ impl MorphEngineValidator { state_updates .storages .get(&hashed_address) - .and_then(|storage| storage.storage.get(&hashed_slot).copied()) - .map(B256::from) + .and_then(|storage| { + storage + .storage_slots + .binary_search_by_key(&hashed_slot, |(slot, _)| *slot) + .ok() + .map(|index| B256::from(storage.storage_slots[index].1)) + }) + } + + fn validate_withdraw_trie_root_update( + &self, + block_hash: B256, + state_updates: impl FnOnce() -> Arc, + ) -> Result<(), ConsensusError> { + let Some(expectation) = self.take_withdraw_trie_root_expectation(block_hash) else { + tracing::debug!( + target: "morph::engine_validator", + %block_hash, + "no withdraw trie root expectation registered; skipping CL cross-check" + ); + return Ok(()); + }; + let WithdrawTrieRootExpectation::Verify(expected) = expectation else { + return Ok(()); + }; + + let Some(actual) = + Self::updated_withdraw_trie_root_from_sorted_hashed_state(state_updates().as_ref()) + else { + // The slot was not touched, so its value is unchanged from the parent. + return Ok(()); + }; + if actual != expected { + return Err(ConsensusError::msg(format!( + "withdraw trie root mismatch: expected {expected}, got {actual}" + ))); + } + Ok(()) } } @@ -219,51 +588,6 @@ impl PayloadValidator for MorphEngineValidator { Ok(sealed_block) } - fn validate_block_post_execution_with_hashed_state( - &self, - state_updates: &reth_trie::HashedPostState, - block: &RecoveredBlock, - ) -> Result<(), ConsensusError> { - let Some(expectation) = self.take_withdraw_trie_root_expectation(block.hash()) else { - // No CL-supplied expectation. Reachable on the Block-input path - // (P2P-downloaded blocks, pipeline backfill, and buffered blocks - // whose expectation was evicted from the bounded LRU before - // reattach) — `convert_payload_to_block` was never invoked to - // register one. Treat as SkipValidation so sync isn't stalled; - // the strict post-Jade state-root check upstream still covers - // withdraw-trie consistency through state-root equality. - tracing::debug!( - target: "morph::engine_validator", - block_hash = %block.hash(), - "no withdraw trie root expectation registered; skipping CL cross-check" - ); - return Ok(()); - }; - let WithdrawTrieRootExpectation::Verify(expected_withdraw_trie_root) = expectation else { - return Ok(()); - }; - - // Only validate if the withdraw trie root slot was actually updated in this block. - // If the slot is absent from hashed_state, the root is unchanged from the parent — - // the consensus layer guarantees the expected value is correct in that case. - // Doing a DB read for the parent state here would be expensive (history_by_block_hash - // + storage lookup) and would occur while holding the execution cache write lock, - // causing lock contention with the next block's cache lookup. - let Some(actual_withdraw_trie_root) = - Self::updated_withdraw_trie_root_from_hashed_state(state_updates) - else { - return Ok(()); - }; - - if actual_withdraw_trie_root != expected_withdraw_trie_root { - return Err(ConsensusError::msg(format!( - "withdraw trie root mismatch: expected {expected_withdraw_trie_root}, got {actual_withdraw_trie_root}" - ))); - } - - Ok(()) - } - fn validate_payload_attributes_against_header( &self, attr: &::PayloadAttributes, @@ -283,6 +607,13 @@ mod tests { use alloy_primitives::U256; use reth_trie::{HashedPostState, HashedStorage}; + #[test] + fn morph_tree_config_never_uses_upstream_empty_update_state_root_skip() { + let config = reth_node_api::TreeConfig::default().with_skip_state_root(true); + + assert!(!strict_morph_tree_config(config).skip_state_root()); + } + #[test] fn test_extract_updated_withdraw_trie_root_from_hashed_state() { let expected = B256::from([0x11; 32]); @@ -295,7 +626,9 @@ mod tests { ); assert_eq!( - MorphEngineValidator::updated_withdraw_trie_root_from_hashed_state(&state), + MorphEngineValidator::updated_withdraw_trie_root_from_sorted_hashed_state( + &state.into_sorted(), + ), Some(expected) ); } @@ -304,7 +637,9 @@ mod tests { fn test_extract_updated_withdraw_trie_root_from_hashed_state_missing_slot() { let state = HashedPostState::default(); assert_eq!( - MorphEngineValidator::updated_withdraw_trie_root_from_hashed_state(&state), + MorphEngineValidator::updated_withdraw_trie_root_from_sorted_hashed_state( + &state.into_sorted(), + ), None ); } @@ -436,7 +771,10 @@ mod tests { HashedStorage::from_iter(false, [(hashed_slot, U256::from_be_bytes([0x11; 32]))]), ); assert!( - MorphEngineValidator::updated_withdraw_trie_root_from_hashed_state(&state).is_none() + MorphEngineValidator::updated_withdraw_trie_root_from_sorted_hashed_state( + &state.into_sorted(), + ) + .is_none() ); } @@ -450,7 +788,10 @@ mod tests { HashedStorage::from_iter(false, [(wrong_slot, U256::from_be_bytes([0x22; 32]))]), ); assert!( - MorphEngineValidator::updated_withdraw_trie_root_from_hashed_state(&state).is_none() + MorphEngineValidator::updated_withdraw_trie_root_from_sorted_hashed_state( + &state.into_sorted(), + ) + .is_none() ); } @@ -475,8 +816,10 @@ mod tests { let validator = MorphEngineValidator::new(); let block = empty_recovered_block_with_hash(B256::from([0xab; 32])); - let result = validator - .validate_block_post_execution_with_hashed_state(&HashedPostState::default(), &block); + let state = HashedPostState::default(); + let result = validator.validate_withdraw_trie_root_update(block.hash(), || { + Arc::new(state.clone().into_sorted()) + }); assert!( result.is_ok(), @@ -496,8 +839,10 @@ mod tests { ); let block = empty_recovered_block_with_hash(hash); - let result = validator - .validate_block_post_execution_with_hashed_state(&HashedPostState::default(), &block); + let state = HashedPostState::default(); + let result = validator.validate_withdraw_trie_root_update(block.hash(), || { + Arc::new(state.clone().into_sorted()) + }); assert!(result.is_ok()); // expectation must be consumed. @@ -522,8 +867,10 @@ mod tests { // empty hashed state → no withdraw-slot diff → skip per the doc-comment // explanation in the validator. - let result = validator - .validate_block_post_execution_with_hashed_state(&HashedPostState::default(), &block); + let state = HashedPostState::default(); + let result = validator.validate_withdraw_trie_root_update(block.hash(), || { + Arc::new(state.clone().into_sorted()) + }); assert!(result.is_ok()); } @@ -552,7 +899,9 @@ mod tests { HashedStorage::from_iter(false, [(hashed_slot, U256::from_be_bytes(actual.0))]), ); - let result = validator.validate_block_post_execution_with_hashed_state(&state, &block); + let result = validator.validate_withdraw_trie_root_update(block.hash(), || { + Arc::new(state.clone().into_sorted()) + }); assert!( result.is_ok(), @@ -582,7 +931,9 @@ mod tests { let block = empty_recovered_block_with_hash(hash); let err = validator - .validate_block_post_execution_with_hashed_state(&state, &block) + .validate_withdraw_trie_root_update(block.hash(), || { + Arc::new(state.clone().into_sorted()) + }) .expect_err("mismatched root must fail"); assert!( err.to_string().contains("withdraw trie root mismatch"), @@ -617,6 +968,7 @@ mod tests { withdrawals: None, parent_beacon_block_root: None, slot_number: None, + target_gas_limit: None, }, transactions: None, gas_limit: None, @@ -637,6 +989,7 @@ mod tests { withdrawals: None, parent_beacon_block_root: None, slot_number: None, + target_gas_limit: None, }, transactions: None, gas_limit: None, @@ -657,6 +1010,7 @@ mod tests { withdrawals: None, parent_beacon_block_root: None, slot_number: None, + target_gas_limit: None, }, transactions: None, gas_limit: None, diff --git a/crates/node/tests/it/engine.rs b/crates/node/tests/it/engine.rs index 7404758b..90be9bb0 100644 --- a/crates/node/tests/it/engine.rs +++ b/crates/node/tests/it/engine.rs @@ -10,9 +10,11 @@ use alloy_rpc_types_engine::PayloadAttributes; use jsonrpsee::core::client::ClientT; use morph_node::test_utils::{HardforkSchedule, TestNodeBuilder}; use morph_payload_types::{ - AssembleL2BlockParams, ExecutableL2Data, GenericResponse, MorphPayloadAttributes, SafeL2Data, + AssembleL2BlockParams, ExecutableL2Data, GenericResponse, MorphPayloadAttributes, + MorphPayloadTypes, SafeL2Data, }; use morph_primitives::MorphHeader; +use reth_node_api::PayloadTypes; use reth_payload_builder::BuildNewPayload; use reth_payload_primitives::BuiltPayload; use reth_provider::{BlockIdReader, BlockReaderIdExt}; @@ -55,6 +57,39 @@ async fn state_root_validation_skipped_pre_jade() -> eyre::Result<()> { Ok(()) } +/// P2P-downloaded blocks enter through the block-input path, where no +/// Engine-API withdraw-root expectation has been registered locally. +#[tokio::test(flavor = "multi_thread")] +async fn p2p_downloaded_block_imports_without_registered_expectation() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let (mut nodes, _wallet) = TestNodeBuilder::new() + .with_schedule(HardforkSchedule::AllActive) + .with_num_nodes(2) + .build() + .await?; + let node1 = nodes.pop().expect("two nodes requested"); + let mut node0 = nodes.pop().expect("two nodes requested"); + + let payload = build_block_no_submit(&mut node0, vec![]).await?; + let head_hash = payload.block().hash(); + let execution_data = MorphPayloadTypes::block_to_payload(payload.block().clone(), None); + let status = node0 + .inner + .add_ons_handle + .beacon_engine_handle + .new_payload(execution_data) + .await?; + assert!( + status.is_valid(), + "source node rejected its block: {status:?}" + ); + node0.update_forkchoice(head_hash, head_hash).await?; + + node1.sync_to(head_hash).await?; + Ok(()) +} + /// `engine_newL2Block` can import consecutive blocks assembled over the authenticated RPC. #[tokio::test(flavor = "multi_thread")] async fn new_l2_block_imports_consecutive_assembled_blocks_over_rpc() -> eyre::Result<()> { @@ -580,6 +615,7 @@ async fn payload_builder_hash_matches_block_hash_with_nonzero_prev_randao() -> e withdrawals: Some(vec![]), parent_beacon_block_root: Some(B256::ZERO), slot_number: None, + target_gas_limit: None, }, transactions: Some(vec![]), gas_limit: None, @@ -593,7 +629,7 @@ async fn payload_builder_hash_matches_block_hash_with_nonzero_prev_randao() -> e attributes: rpc_attrs, parent_hash: head_hash, cache: None, - trie_handle: None, + state_root_handle: None, }) .await? .map_err(|e| eyre::eyre!("payload build failed: {e}"))?; diff --git a/crates/node/tests/it/helpers.rs b/crates/node/tests/it/helpers.rs index 808e9ed2..b08a17b3 100644 --- a/crates/node/tests/it/helpers.rs +++ b/crates/node/tests/it/helpers.rs @@ -48,6 +48,7 @@ pub(crate) async fn advance_block_with_l1_messages( withdrawals: Some(vec![]), parent_beacon_block_root: Some(B256::ZERO), slot_number: None, + target_gas_limit: None, }, transactions: Some(l1_messages), gas_limit: None, @@ -61,7 +62,7 @@ pub(crate) async fn advance_block_with_l1_messages( attributes: rpc_attrs, parent_hash: head_hash, cache: None, - trie_handle: None, + state_root_handle: None, }) .await? .map_err(|e| eyre::eyre!("payload build failed: {e}"))?; @@ -122,6 +123,7 @@ pub(crate) async fn build_block_no_submit( withdrawals: Some(vec![]), parent_beacon_block_root: Some(B256::ZERO), slot_number: None, + target_gas_limit: None, }, transactions: Some(l1_messages), gas_limit: None, @@ -135,7 +137,7 @@ pub(crate) async fn build_block_no_submit( attributes: rpc_attrs, parent_hash: head_hash, cache: None, - trie_handle: None, + state_root_handle: None, }) .await? .map_err(|e| eyre::eyre!("payload build failed: {e}"))?; @@ -192,7 +194,7 @@ pub(crate) async fn craft_and_try_import_block( let modified_sealed = SealedBlock::seal_slow(block); // Convert to execution payload and try to import - let execution_data = MorphPayloadTypes::block_to_payload(modified_sealed); + let execution_data = MorphPayloadTypes::block_to_payload(modified_sealed, None); let status = node .inner .add_ons_handle @@ -230,6 +232,7 @@ pub(crate) async fn expect_payload_build_failure( withdrawals: Some(vec![]), parent_beacon_block_root: Some(B256::ZERO), slot_number: None, + target_gas_limit: None, }, transactions: Some(l1_messages), gas_limit: None, @@ -243,7 +246,7 @@ pub(crate) async fn expect_payload_build_failure( attributes: rpc_attrs, parent_hash: head_hash, cache: None, - trie_handle: None, + state_root_handle: None, }) .await? { diff --git a/crates/payload/builder/src/builder.rs b/crates/payload/builder/src/builder.rs index 2f35359d..e2dc38b4 100644 --- a/crates/payload/builder/src/builder.rs +++ b/crates/payload/builder/src/builder.rs @@ -21,7 +21,7 @@ use reth_chainspec::ChainSpecProvider; use reth_evm::{ ConfigureEvm, Database, Evm, NextBlockEnvAttributes, block::{BlockExecutionError, BlockValidationError}, - execute::{BlockBuilder, BlockBuilderOutcome, BlockExecutor}, + execute::{BlockBuilder, BlockBuilderOutcome}, }; use reth_execution_cache::{CachedStateMetrics, CachedStateMetricsSource, CachedStateProvider}; use reth_execution_types::BlockExecutionOutput; @@ -169,7 +169,7 @@ where let BuildArguments { mut cached_reads, execution_cache, - trie_handle, + state_root_handle, config, cancel, best_payload, @@ -179,6 +179,7 @@ where let parent_hash = config.parent_header.hash(); let payload_id = config.payload_id; let parent_header = config.parent_header.clone(); + let parent_block_info = config.parent_block_info; let builder_attrs = MorphPayloadBuilderAttributes::try_new( parent_hash, config.attributes, @@ -187,6 +188,7 @@ where .map_err(|e| PayloadBuilderError::Other(e.into()))?; let builder_config = PayloadConfig { parent_header, + parent_block_info, attributes: builder_attrs, payload_id, }; @@ -215,7 +217,9 @@ where state_provider = Box::new(CachedStateProvider::new( state_provider, execution_cache.cache().clone(), - CachedStateMetrics::zeroed(CachedStateMetricsSource::Builder), + Some(CachedStateMetrics::zeroed( + CachedStateMetricsSource::Builder, + )), )); } let state = StateProviderDatabase::new(state_provider.as_ref()); @@ -226,7 +230,7 @@ where state_provider.as_ref(), ctx, best, - trie_handle, + state_root_handle, ) .map(|out| out.with_cached_reads(cached_reads)) } @@ -268,7 +272,7 @@ where config, cached_reads: Default::default(), execution_cache: None, - trie_handle: None, + state_root_handle: None, cancel: Default::default(), best_payload: None, }; @@ -686,7 +690,7 @@ fn build_payload_inner<'a, DB, BestTxs>( state_provider: &(impl StateProvider + ?Sized), ctx: MorphPayloadBuilderCtx, best: impl FnOnce(BestTransactionsAttributes) -> BestTxs + Send + Sync + 'a, - trie_handle: Option, + mut state_root_handle: Option, ) -> Result, PayloadBuilderError> where DB: Database, @@ -734,10 +738,11 @@ where // state hook so per-tx state diffs stream to the background trie task // during execution. The final `state_root()` recv() at finish time will // return quickly since most work is done concurrently. - if let Some(ref handle) = trie_handle { + if let Some(handle) = state_root_handle.as_mut() { builder - .executor_mut() - .set_state_hook(Some(Box::new(handle.state_hook()))); + .evm_mut() + .db_mut() + .set_state_hook(Some(Box::new(handle.take_state_hook()))); } // 1. Apply pre-execution changes (system contracts, etc.) @@ -815,8 +820,9 @@ where hashed_state, trie_updates, mut block, - } = if let Some(mut handle) = trie_handle { - builder.executor_mut().set_state_hook(None); + block_access_list: _, + } = if let Some(mut handle) = state_root_handle { + builder.evm_mut().db_mut().set_state_hook(None); match handle.state_root() { Ok(outcome) => builder.finish( state_provider, @@ -890,8 +896,9 @@ where recovered_block: Arc::new(block), execution_output: Arc::new(execution_output), // Keep unsorted; conversion to sorted is deferred until required. - hashed_state: Err(Arc::new(hashed_state)).into(), - trie_updates: Err(Arc::new(trie_updates)).into(), + hashed_state: Arc::new(hashed_state), + trie_updates: Arc::new(trie_updates), + changed_paths: None, }; let payload = MorphBuiltPayload::new( diff --git a/crates/payload/types/src/attributes.rs b/crates/payload/types/src/attributes.rs index 3aaea3ae..86d72131 100644 --- a/crates/payload/types/src/attributes.rs +++ b/crates/payload/types/src/attributes.rs @@ -334,6 +334,7 @@ mod tests { parent_beacon_block_root: None, // Morph L2 has no PoS slot semantics; field added in alloy 2.0. slot_number: None, + target_gas_limit: None, }, transactions: None, gas_limit: None, diff --git a/crates/payload/types/src/lib.rs b/crates/payload/types/src/lib.rs index 9ded5aae..f940589b 100644 --- a/crates/payload/types/src/lib.rs +++ b/crates/payload/types/src/lib.rs @@ -83,6 +83,12 @@ impl MorphExecutionData { } } +impl From for MorphExecutionData { + fn from(value: MorphBuiltPayload) -> Self { + Self::new(value.block) + } +} + impl ExecutionPayload for MorphExecutionData { fn parent_hash(&self) -> B256 { self.block.parent_hash() @@ -140,6 +146,7 @@ impl PayloadTypes for MorphPayloadTypes { block: SealedBlock< <::Primitives as NodePrimitives>::Block, >, + _bal: Option, ) -> Self::ExecutionData { MorphExecutionData::new(Arc::new(block)) } @@ -237,7 +244,7 @@ mod tests { fn test_block_to_payload() { let block = create_test_block(); let hash = block.hash(); - let data = MorphPayloadTypes::block_to_payload(block); + let data = MorphPayloadTypes::block_to_payload(block, None); assert_eq!(data.block_hash(), hash); assert!(data.expected_withdraw_trie_root.is_none()); } diff --git a/crates/revm/src/evm.rs b/crates/revm/src/evm.rs index d8ba55ba..10d7fe5a 100644 --- a/crates/revm/src/evm.rs +++ b/crates/revm/src/evm.rs @@ -14,7 +14,7 @@ use revm::{ }, inspector::InspectorEvmTr, interpreter::{ - Host, Instruction, InstructionContext, InstructionResult, + Host, Instruction, InstructionContext, InstructionExecResult, InstructionResult, gas::{BLOCKHASH, WARM_STORAGE_READ_COST}, interpreter::EthInterpreter, interpreter_types::{RuntimeFlag, StackTr}, @@ -70,10 +70,9 @@ fn morph_blockhash_result(chain_id: u64, current_number: u64, requested_number: /// for numbers within the 256-block lookup window. fn blockhash_morph( context: InstructionContext<'_, MorphContext, EthInterpreter>, -) { +) -> InstructionExecResult { let Some(([], number)) = StackTr::popn_top::<0>(&mut context.interpreter.stack) else { - context.interpreter.halt_underflow(); - return; + return Err(InstructionResult::StackUnderflow); }; let requested_number_u64 = as_u64_saturated(*number); @@ -81,6 +80,7 @@ fn blockhash_morph( let chain_id_u64 = as_u64_saturated(context.host.chain_id()); *number = morph_blockhash_result(chain_id_u64, current_number_u64, requested_number_u64); + Ok(()) } /// Morph custom SLOAD opcode. @@ -93,10 +93,11 @@ fn blockhash_morph( /// instead of "dirty" (100 gas), causing a 2800 gas mismatch vs go-eth. /// /// The DB read hits the State cache (O(1)) and only triggers on cold SLOADs. -fn sload_morph(context: InstructionContext<'_, MorphContext, EthInterpreter>) { +fn sload_morph( + context: InstructionContext<'_, MorphContext, EthInterpreter>, +) -> InstructionExecResult { let Some(([], index)) = StackTr::popn_top::<0>(&mut context.interpreter.stack) else { - context.interpreter.halt_underflow(); - return; + return Err(InstructionResult::StackUnderflow); }; let target = context.interpreter.input.target_address; @@ -125,16 +126,16 @@ fn sload_morph(context: InstructionContext<'_, MorphContext, E .gas .record_regular_cost(additional_cold_cost) { - context.interpreter.halt_oog(); - return; + return Err(InstructionResult::OutOfGas); } } *index = storage.data; } - Err(LoadError::ColdLoadSkipped) => context.interpreter.halt_oog(), - Err(LoadError::DBError) => context.interpreter.halt_fatal(), + Err(LoadError::ColdLoadSkipped) => return Err(InstructionResult::OutOfGas), + Err(LoadError::DBError) => return Err(InstructionResult::FatalExternalError), } + Ok(()) } /// Morph custom SSTORE opcode. @@ -150,17 +151,15 @@ fn sload_morph(context: InstructionContext<'_, MorphContext, E /// same 2800-gas-per-write divergence vs go-eth that `sload_morph` fixes. /// /// Uses DB-direct lookup (no per-tx runtime map needed). -fn sstore_morph(context: InstructionContext<'_, MorphContext, EthInterpreter>) { +fn sstore_morph( + context: InstructionContext<'_, MorphContext, EthInterpreter>, +) -> InstructionExecResult { if context.interpreter.runtime_flag.is_static() { - context - .interpreter - .halt(InstructionResult::StateChangeDuringStaticCall); - return; + return Err(InstructionResult::StateChangeDuringStaticCall); } let Some([index, value]) = StackTr::popn::<2>(&mut context.interpreter.stack) else { - context.interpreter.halt_underflow(); - return; + return Err(InstructionResult::StackUnderflow); }; let target = context.interpreter.input.target_address; @@ -169,10 +168,7 @@ fn sstore_morph(context: InstructionContext<'_, MorphContext, if spec_id.is_enabled_in(ISTANBUL) && context.interpreter.gas.remaining() <= context.host.gas_params().call_stipend() { - context - .interpreter - .halt(InstructionResult::ReentrancySentryOOG); - return; + return Err(InstructionResult::ReentrancySentryOOG); } if !context @@ -180,8 +176,7 @@ fn sstore_morph(context: InstructionContext<'_, MorphContext, .gas .record_regular_cost(context.host.gas_params().sstore_static_gas()) { - context.interpreter.halt_oog(); - return; + return Err(InstructionResult::OutOfGas); } let mut state_load = if spec_id.is_enabled_in(BERLIN) { @@ -193,18 +188,15 @@ fn sstore_morph(context: InstructionContext<'_, MorphContext, { Ok(load) => load, Err(LoadError::ColdLoadSkipped) => { - context.interpreter.halt_oog(); - return; + return Err(InstructionResult::OutOfGas); } Err(LoadError::DBError) => { - context.interpreter.halt_fatal(); - return; + return Err(InstructionResult::FatalExternalError); } } } else { let Some(load) = context.host.sstore(target, index, value) else { - context.interpreter.halt_fatal(); - return; + return Err(InstructionResult::FatalExternalError); }; load }; @@ -232,8 +224,7 @@ fn sstore_morph(context: InstructionContext<'_, MorphContext, state_load.is_cold, ); if !context.interpreter.gas.record_regular_cost(dynamic_gas) { - context.interpreter.halt_oog(); - return; + return Err(InstructionResult::OutOfGas); } context.interpreter.gas.record_refund( @@ -242,6 +233,7 @@ fn sstore_morph(context: InstructionContext<'_, MorphContext, .gas_params() .sstore_refund(is_istanbul, &state_load.data), ); + Ok(()) } /// MorphEvm extends the Evm with Morph specific types and logic. @@ -291,22 +283,27 @@ impl MorphEvm { let mut instructions = EthInstructions::new_mainnet_with_spec(spec.into()); // Morph custom BLOCKHASH implementation (matches Morph geth). - instructions.insert_instruction(0x40, Instruction::new(blockhash_morph::, BLOCKHASH)); + instructions.insert_instruction( + 0x40, + Instruction::new(blockhash_morph::), + BLOCKHASH as u16, + ); // Morph custom SLOAD: fixes original_value corruption from token fee deduction. instructions.insert_instruction( 0x54, - Instruction::new(sload_morph::, WARM_STORAGE_READ_COST), + Instruction::new(sload_morph::), + WARM_STORAGE_READ_COST as u16, ); // Morph custom SSTORE: same original_value fix on the SSTORE cold path. // Static gas = 0 because sstore_morph manages all gas accounting itself // (static + dynamic + refund). - instructions.insert_instruction(0x55, Instruction::new(sstore_morph::, 0)); + instructions.insert_instruction(0x55, Instruction::new(sstore_morph::), 0); // SELFDESTRUCT is disabled in Morph - instructions.insert_instruction(0xff, Instruction::unknown()); + instructions.insert_instruction(0xff, Instruction::unknown(), 0); // BLOBHASH is disabled in Morph - instructions.insert_instruction(0x49, Instruction::unknown()); + instructions.insert_instruction(0x49, Instruction::unknown(), 0); // BLOBBASEFEE is disabled in Morph - instructions.insert_instruction(0x4a, Instruction::unknown()); + instructions.insert_instruction(0x4a, Instruction::unknown(), 0); Self::new_inner(Evm { ctx, inspector, diff --git a/crates/revm/src/precompiles.rs b/crates/revm/src/precompiles.rs index 02d0cf32..cce783fa 100644 --- a/crates/revm/src/precompiles.rs +++ b/crates/revm/src/precompiles.rs @@ -43,9 +43,8 @@ use revm::{ precompile::{ Precompile, PrecompileHalt, PrecompileId, PrecompileOutput, PrecompileResult, Precompiles, }, - primitives::{OnceLock, hardfork::SpecId}, + primitives::{AddressSet, OnceLock, hardfork::SpecId}, }; -use std::boxed::Box; use std::string::String; /// Standard precompile addresses @@ -395,7 +394,7 @@ where } #[inline] - fn warm_addresses(&self) -> Box> { + fn warm_addresses(&self) -> &AddressSet { self.inner.warm_addresses() } diff --git a/crates/rpc/src/eth/call.rs b/crates/rpc/src/eth/call.rs index f4a5e7f2..a1718bbf 100644 --- a/crates/rpc/src/eth/call.rs +++ b/crates/rpc/src/eth/call.rs @@ -57,6 +57,10 @@ where self.eth_api().max_simulate_blocks() } + fn compute_state_root_for_eth_simulate(&self) -> bool { + self.eth_api().compute_state_root_for_eth_simulate() + } + fn evm_memory_limit(&self) -> u64 { self.eth_api().evm_memory_limit() } diff --git a/crates/rpc/src/eth/mod.rs b/crates/rpc/src/eth/mod.rs index 96a08d78..cb7331a1 100644 --- a/crates/rpc/src/eth/mod.rs +++ b/crates/rpc/src/eth/mod.rs @@ -17,8 +17,8 @@ use reth_rpc_convert::{RpcConvert, RpcConverter, RpcTypes}; use reth_rpc_eth_api::{ EthApiTypes, RpcNodeCore, RpcNodeCoreExt, helpers::{ - EthApiSpec, EthBlocks, EthFees, EthState, EthTransactions, LoadBlock, LoadFee, - LoadPendingBlock, LoadState, LoadTransaction, SpawnBlocking, Trace, + EthApiSpec, EthBlocks, EthFees, EthState, EthSubscriptions, EthTransactions, LoadBlock, + LoadFee, LoadPendingBlock, LoadState, LoadTransaction, SpawnBlocking, Trace, bal::GetBlockAccessList, pending_block::PendingEnvBuilder, }, }; @@ -320,6 +320,20 @@ where self.inner.eth_api.send_raw_transaction_sync_timeout() } + async fn send_pool_transaction( + &self, + origin: reth_transaction_pool::TransactionOrigin, + tx: reth_primitives_traits::WithEncoded>, + ) -> Result { + reth_rpc_eth_api::helpers::EthTransactions::send_pool_transaction( + &self.inner.eth_api, + origin, + tx, + ) + .await + .map_err(Into::into) + } + async fn send_transaction( &self, origin: reth_transaction_pool::TransactionOrigin, @@ -337,6 +351,13 @@ where } } +impl EthSubscriptions for MorphEthApi +where + N: MorphNodeCore, + Rpc: RpcConvert, +{ +} + impl EthFees for MorphEthApi where N: MorphNodeCore, From 204c2ce1dff20bbee9f0ff81a54dd8072f9411ce Mon Sep 17 00:00:00 2001 From: panos Date: Tue, 14 Jul 2026 10:11:52 +0800 Subject: [PATCH 02/12] fix(node): clear withdraw expectation on L1 index pre-check failure validate_payload already registered an expectation in convert_payload_to_block; clearing it on the early L1-index reject path matches the inner-failure cleanup and avoids a stale cache entry until LRU eviction. --- crates/node/src/validator.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/crates/node/src/validator.rs b/crates/node/src/validator.rs index 977a4e6d..463824cc 100644 --- a/crates/node/src/validator.rs +++ b/crates/node/src/validator.rs @@ -311,7 +311,16 @@ where ctx: TreeCtx<'_, MorphPrimitives>, ) -> reth_engine_tree::tree::ValidationOutcome { let block_hash = payload.block.hash(); - self.validate_next_l1_msg_index_for_block(payload.block.as_ref(), &ctx)?; + // `convert_payload_to_block` may already have registered a withdraw-root + // expectation for this hash. Clear it on every early reject path so a + // later re-import of the same hash does not observe a stale entry. + if let Err(err) = + self.validate_next_l1_msg_index_for_block(payload.block.as_ref(), &ctx) + { + self.post_execution_validator + .take_withdraw_trie_root_expectation(block_hash); + return Err(err); + } match self.inner.validate_payload(payload, ctx) { Ok(output) => self.validate_withdraw_trie_root(output), Err(err) => { From b33856fff1c4f0cf0342a366fdabe86f0fca7ddd Mon Sep 17 00:00:00 2001 From: panos Date: Tue, 14 Jul 2026 10:18:32 +0800 Subject: [PATCH 03/12] chore: fix rustfmt and bump crossbeam-epoch for RUSTSEC-2026-0204 Format the L1-index cleanup path, upgrade crossbeam-epoch to 0.9.20, and drop advisory ignores that no longer match after the reth main bump. --- Cargo.lock | 4 ++-- crates/node/src/validator.rs | 4 +--- deny.toml | 14 -------------- 3 files changed, 3 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 52398dd5..da7e94b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2176,9 +2176,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] diff --git a/crates/node/src/validator.rs b/crates/node/src/validator.rs index 463824cc..33057571 100644 --- a/crates/node/src/validator.rs +++ b/crates/node/src/validator.rs @@ -314,9 +314,7 @@ where // `convert_payload_to_block` may already have registered a withdraw-root // expectation for this hash. Clear it on every early reject path so a // later re-import of the same hash does not observe a stale entry. - if let Err(err) = - self.validate_next_l1_msg_index_for_block(payload.block.as_ref(), &ctx) - { + if let Err(err) = self.validate_next_l1_msg_index_for_block(payload.block.as_ref(), &ctx) { self.post_execution_validator .take_withdraw_trie_root_expectation(block_hash); return Err(err); diff --git a/deny.toml b/deny.toml index 26053b0c..0b19c29f 100644 --- a/deny.toml +++ b/deny.toml @@ -8,20 +8,6 @@ ignore = [ "RUSTSEC-2024-0436", # https://rustsec.org/advisories/RUSTSEC-2025-0141 bincode is unmaintained "RUSTSEC-2025-0141", - # https://rustsec.org/advisories/RUSTSEC-2026-0097 rand unsound with custom logger - # pinned transitively via reth; we don't install a custom logger so this is - # a false positive in our usage, but cargo-deny can't see usage - "RUSTSEC-2026-0097", - # https://rustsec.org/advisories/RUSTSEC-2026-0098 rustls-webpki URI name constraints - "RUSTSEC-2026-0098", - # https://rustsec.org/advisories/RUSTSEC-2026-0099 rustls-webpki wildcard name constraints - "RUSTSEC-2026-0099", - # https://rustsec.org/advisories/RUSTSEC-2026-0118 hickory-proto NSEC3 validation loop - # pinned transitively via reth-dns-discovery -> hickory-resolver ^0.25; no 0.25 fix is available - "RUSTSEC-2026-0118", - # https://rustsec.org/advisories/RUSTSEC-2026-0119 hickory-proto O(n^2) name compression - # fixed in hickory-proto 0.26.1, but hickory-resolver ^0.25 cannot select it - "RUSTSEC-2026-0119", # https://rustsec.org/advisories/RUSTSEC-2026-0173 proc-macro-error2 is unmaintained # build-time proc-macro pulled transitively via alloy-sol-macro; no fix available "RUSTSEC-2026-0173", From c38f45fbe4934fbe2eee042fd503486d6d53afa4 Mon Sep 17 00:00:00 2001 From: panos-xyz Date: Tue, 14 Jul 2026 18:03:21 +0800 Subject: [PATCH 04/12] refactor(reference-index): replace ExEx with canonical cursor runtime --- Cargo.lock | 15 +- Cargo.toml | 3 - bin/morph-reth/Cargo.toml | 3 - bin/morph-reth/src/main.rs | 30 +- crates/node/Cargo.toml | 5 - crates/node/src/add_ons.rs | 133 ++-- crates/node/src/args.rs | 12 + crates/node/src/exex/mod.rs | 5 - crates/node/src/exex/reference_index.rs | 569 ----------------- crates/node/src/lib.rs | 1 - crates/node/tests/it/reference_index.rs | 151 ++--- crates/reference-index/Cargo.toml | 7 +- crates/reference-index/src/backfill.rs | 353 ++--------- crates/reference-index/src/db.rs | 271 ++++---- crates/reference-index/src/lib.rs | 37 +- crates/reference-index/src/metrics.rs | 33 + crates/reference-index/src/reader.rs | 344 +++++++++-- crates/reference-index/src/reconcile.rs | 142 +---- crates/reference-index/src/runtime.rs | 787 ++++++++++++++++++++++++ crates/reference-index/src/source.rs | 86 +++ crates/reference-index/src/tables.rs | 2 + crates/reference-index/src/types.rs | 70 ++- crates/reference-index/src/writer.rs | 83 +-- crates/rpc/Cargo.toml | 1 + crates/rpc/src/morph/handler.rs | 317 +++++++++- crates/rpc/src/morph/rpc.rs | 11 +- 26 files changed, 1995 insertions(+), 1476 deletions(-) delete mode 100644 crates/node/src/exex/mod.rs delete mode 100644 crates/node/src/exex/reference_index.rs create mode 100644 crates/reference-index/src/metrics.rs create mode 100644 crates/reference-index/src/runtime.rs create mode 100644 crates/reference-index/src/source.rs diff --git a/Cargo.lock b/Cargo.lock index da7e94b3..16307f55 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4966,14 +4966,12 @@ dependencies = [ "reth-chain-state", "reth-chainspec", "reth-db", - "reth-db-api", "reth-e2e-test-utils", "reth-engine-local", "reth-engine-tree", "reth-errors", "reth-evm", "reth-execution-cache", - "reth-exex", "reth-node-api", "reth-node-builder", "reth-node-core", @@ -4984,7 +4982,6 @@ dependencies = [ "reth-provider", "reth-rpc-builder", "reth-rpc-eth-api", - "reth-storage-api", "reth-tasks", "reth-tracing", "reth-transaction-pool", @@ -4992,7 +4989,6 @@ dependencies = [ "reth-trie-db", "serde", "serde_json", - "tempfile", "tokio", "tokio-stream", "tracing", @@ -5080,16 +5076,19 @@ dependencies = [ "alloy-primitives", "alloy-rlp", "eyre", - "morph-chainspec", + "metrics", "morph-primitives", - "reth-chainspec", + "parking_lot", + "reth-chain-state", "reth-codecs", "reth-db", "reth-db-api", "reth-errors", + "reth-metrics", "reth-primitives-traits", "reth-provider", "reth-storage-api", + "reth-tasks", "serde", "tempfile", "thiserror 2.0.18", @@ -5103,13 +5102,10 @@ version = "1.0.0" dependencies = [ "alloy-primitives", "clap", - "eyre", "morph-chainspec", "morph-consensus", "morph-evm", "morph-node", - "morph-reference-index", - "reth-chainspec", "reth-cli", "reth-cli-util", "reth-ethereum-cli", @@ -5178,6 +5174,7 @@ dependencies = [ "revm", "serde", "serde_json", + "tempfile", "thiserror 2.0.18", "tokio", "tracing", diff --git a/Cargo.toml b/Cargo.toml index f30e05fc..cd966912 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -117,9 +117,6 @@ reth-engine-primitives = { git = "https://github.com/paradigmxyz/reth", rev = "f reth-engine-tree = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-errors = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-eth-wire-types = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } -reth-exex = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } -reth-exex-test-utils = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } -reth-exex-types = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-ethereum = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-ethereum-cli = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-ethereum-consensus = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } diff --git a/bin/morph-reth/Cargo.toml b/bin/morph-reth/Cargo.toml index b953ca54..485617c2 100644 --- a/bin/morph-reth/Cargo.toml +++ b/bin/morph-reth/Cargo.toml @@ -20,10 +20,8 @@ morph-chainspec = { workspace = true, features = ["cli"] } morph-consensus.workspace = true morph-evm.workspace = true morph-node.workspace = true -morph-reference-index.workspace = true # Reth CLI -reth-chainspec.workspace = true reth-cli.workspace = true reth-cli-util.workspace = true reth-ethereum-cli.workspace = true @@ -33,7 +31,6 @@ reth-rpc-server-types.workspace = true # Other alloy-primitives.workspace = true clap.workspace = true -eyre.workspace = true tracing.workspace = true [features] diff --git a/bin/morph-reth/src/main.rs b/bin/morph-reth/src/main.rs index 45b7d24f..a39d8adf 100644 --- a/bin/morph-reth/src/main.rs +++ b/bin/morph-reth/src/main.rs @@ -20,12 +20,7 @@ use clap::Parser; use morph_chainspec::{MORPH_DEFAULT_PRIORITY_FEE, MorphChainSpec, MorphChainSpecParser}; use morph_consensus::MorphConsensus; use morph_evm::{MorphEvmConfig, evm::MorphEvmFactory}; -use morph_node::{ - MorphAddOns, MorphArgs, MorphNode, - exex::{ReferenceIndexControl, reference_index_exex}, -}; -use morph_reference_index::ReferenceIndexDb; -use reth_chainspec::EthChainSpec; +use morph_node::{MorphAddOns, MorphArgs, MorphNode}; use reth_cli_util::sigsegv_handler; use reth_ethereum_cli::{Cli, Commands}; use reth_node_builder::Node; @@ -80,33 +75,12 @@ fn main() { cli.run_with_components::(components, async move |builder, morph_args| { info!(target: "morph::cli", "Starting Morph-Reth node"); - // Open the reference index DB before launching the node so we - // can wire it into both the ExEx and the add-ons. - let chain_spec = builder.config().chain.clone(); - let datadir = builder.config().datadir(); - let reference_index_path = datadir.data_dir().join("morph").join("reference_index"); - let chain_id = chain_spec.chain().id(); - let genesis_hash = chain_spec.genesis_hash(); // from EthChainSpec trait - - info!( - target: "morph::reference_index", - path = %reference_index_path.display(), - chain_id, - "opening Morph reference index database" - ); - let db = ReferenceIndexDb::open(&reference_index_path, chain_id, genesis_hash)?; - let (control, startup_rx) = ReferenceIndexControl::new(db); - - let exex_control = control.clone(); let node = MorphNode::new(morph_args); let handle = builder .with_types::() .with_components(node.components_builder()) - .with_add_ons(MorphAddOns::new().with_reference_index(control)) - .install_exex("morph-reference-index", async move |ctx| { - Ok(reference_index_exex(ctx, exex_control, startup_rx)) - }) + .with_add_ons(MorphAddOns::new()) .launch_with_debug_capabilities() .await?; diff --git a/crates/node/Cargo.toml b/crates/node/Cargo.toml index 53257bb2..bb6427b6 100644 --- a/crates/node/Cargo.toml +++ b/crates/node/Cargo.toml @@ -24,7 +24,6 @@ morph-txpool.workspace = true # Reth dependencies reth-db.workspace = true -reth-db-api.workspace = true reth-chain-state.workspace = true reth-node-core.workspace = true reth-chainspec.workspace = true @@ -33,7 +32,6 @@ reth-engine-tree.workspace = true reth-errors.workspace = true reth-evm.workspace = true reth-execution-cache.workspace = true -reth-exex.workspace = true reth-node-api.workspace = true reth-node-builder.workspace = true reth-node-ethereum.workspace = true @@ -43,8 +41,6 @@ reth-primitives-traits.workspace = true reth-provider.workspace = true reth-rpc-builder.workspace = true reth-rpc-eth-api.workspace = true -reth-storage-api.workspace = true -reth-tasks.workspace = true reth-transaction-pool.workspace = true reth-tracing.workspace = true reth-trie.workspace = true @@ -99,7 +95,6 @@ morph-chainspec.workspace = true morph-payload-types.workspace = true morph-primitives.workspace = true serde_json.workspace = true -tempfile.workspace = true [[test]] name = "it" diff --git a/crates/node/src/add_ons.rs b/crates/node/src/add_ons.rs index 2a1583bf..65249d57 100644 --- a/crates/node/src/add_ons.rs +++ b/crates/node/src/add_ons.rs @@ -2,16 +2,19 @@ use crate::{ MorphNode, - exex::ReferenceIndexControl, validator::{MorphEngineValidatorBuilder, MorphTreeEngineValidatorBuilder}, }; +use alloy_hardforks::ForkCondition; +use morph_chainspec::{MorphHardfork, MorphHardforks}; use morph_evm::MorphEvmConfig; use morph_primitives::{Block, MorphHeader, MorphReceipt}; -use morph_reference_index::{DEFAULT_LAG_THRESHOLD, ReferenceIndexReader}; +use morph_reference_index::{ReferenceIndexConfig, ReferenceIndexRuntime}; use morph_rpc::{ MorphEthApiBuilder, MorphEthConfigApiServer, MorphEthConfigHandler, morph::{MorphRpc, MorphRpcHandler, MorphRpcServer}, }; +use reth_chain_state::CanonStateSubscriptions; +use reth_chainspec::EthChainSpec; use reth_node_api::{AddOnsContext, FullNodeComponents, FullNodeTypes, NodeAddOns, NodePrimitives}; use reth_node_builder::{ NodeAdapter, @@ -44,10 +47,6 @@ pub struct MorphAddOns< > { /// Inner RPC add-ons from reth. inner: RpcAddOns, - /// Optional reference-index control injected by `main.rs`. When present - /// the add-on spawns startup indexing on launch and registers the - /// `morph_` RPC namespace. - reference_index: Option, } impl MorphAddOns, MorphEthApiBuilder> @@ -69,16 +68,19 @@ where Identity::default(), Identity::default(), ), - reference_index: None, } } +} - /// Attach a reference index control so the add-on can spawn startup - /// indexing and register the `morph_` RPC namespace on launch. - pub fn with_reference_index(mut self, control: ReferenceIndexControl) -> Self { - self.reference_index = Some(control); - self +fn ensure_reference_index_pruning_compatible( + bodies_history_pruning_enabled: bool, +) -> eyre::Result<()> { + if bodies_history_pruning_enabled { + eyre::bail!( + "Morph reference index requires canonical block bodies from Jade onward; disable bodies-history pruning" + ) } + Ok(()) } impl Default for MorphAddOns, MorphEthApiBuilder> @@ -119,40 +121,56 @@ where let task_executor = ctx.node.task_executor().clone(); let block_tag_tracker = std::sync::Arc::new(morph_engine_api::BlockTagTracker::default()); + // The reference index backfills directly from canonical block bodies. Read the + // effective modes from the provider so both CLI arguments and reth.toml are covered. + let bodies_history_pruning_enabled = provider + .database_provider_ro()? + .prune_modes_ref() + .bodies_history + .is_some(); + ensure_reference_index_pruning_compatible(bodies_history_pruning_enabled)?; + + let canonical_notifications = provider.subscribe_to_canonical_state(); + let jade_timestamp = match chain_spec.morph_fork_activation(MorphHardfork::Jade) { + ForkCondition::Timestamp(timestamp) => timestamp, + ForkCondition::Never => u64::MAX, + condition => eyre::bail!( + "Morph reference index requires timestamp-based Jade activation, got {condition:?}" + ), + }; + let reference_index_path = ctx + .config + .datadir() + .data_dir() + .join("morph") + .join("reference_index"); + let reference_index_config = ReferenceIndexConfig::new( + &reference_index_path, + chain_spec.chain().id(), + chain_spec.genesis_hash(), + jade_timestamp, + ); + let (reference_index_runtime, reference_index_handle) = + ReferenceIndexRuntime::new(reference_index_config, provider.clone()); + let runtime_executor = task_executor.clone(); + task_executor.spawn_task(async move { + reference_index_runtime + .run(runtime_executor, canonical_notifications) + .await; + }); + + tracing::info!( + target: "morph::reference_index", + path = %reference_index_path.display(), + "Morph reference index background runtime started" + ); + // Create Morph eth_config handler (EIP-7910 + morph extension) let eth_config_handler = MorphEthConfigHandler::new(ctx.node.provider().clone(), ctx.node.evm_config().clone()); - // Spawn reference index startup indexing (Task A) if configured. - let reference_rpc_handler = if let Some(control) = self.reference_index { - let startup_control = control.clone(); - let startup_node = ctx.node.clone(); - // spawn_critical_task causes node shutdown on panic/error, matching the spec - // requirement that reference index startup failures are fatal. - task_executor.spawn_critical_task("morph reference index startup", async move { - let result = tokio::task::spawn_blocking(move || { - crate::exex::run_startup_indexing(&startup_node, &startup_control) - }) - .await - .unwrap_or_else(|e| Err(eyre::eyre!("reference index startup panicked: {e}"))); - - match result { - Ok(()) => {} - Err(err) => { - // Propagate to spawn_critical_task which will shut down the node. - panic!("reference index startup failed: {err:?}"); - } - } - }); - - let morph_rpc_ctx = MorphRpc::new( - ReferenceIndexReader::new(control.db, DEFAULT_LAG_THRESHOLD), - provider.clone(), - ); - Some(MorphRpcHandler::new(morph_rpc_ctx)) - } else { - None - }; + let morph_rpc_ctx = MorphRpc::new(reference_index_handle, provider.clone()); + let reference_rpc_handler = MorphRpcHandler::new(morph_rpc_ctx); // Use launch_add_ons_with to register custom Engine API and eth_config self.inner @@ -171,14 +189,13 @@ where .map_err(|e| eyre::eyre!("Failed to register eth_config handler: {}", e))?; tracing::info!(target: "morph::node", "Morph eth_config handler registered successfully"); - // Register morph_ RPC namespace (if reference index was configured). - if let Some(handler) = reference_rpc_handler { - tracing::debug!(target: "morph::node", "Registering morph_ RPC namespace"); - modules - .merge_configured(handler.into_rpc()) - .map_err(|e| eyre::eyre!("Failed to register morph_ RPC: {}", e))?; - tracing::info!(target: "morph::node", "morph_ RPC namespace registered"); - } + // The namespace remains registered while the index catches up; handlers return + // a structured unavailable/behind error until the durable cursor is live. + tracing::debug!(target: "morph::node", "Registering morph_ RPC namespace"); + modules + .merge_configured(reference_rpc_handler.into_rpc()) + .map_err(|e| eyre::eyre!("Failed to register morph_ RPC: {}", e))?; + tracing::info!(target: "morph::node", "morph_ RPC namespace registered"); // Create and register Morph L2 Engine API tracing::debug!(target: "morph::node", "Registering Morph L2 Engine API"); @@ -242,3 +259,19 @@ where self.inner.engine_validator_builder() } } + +#[cfg(test)] +mod tests { + use super::ensure_reference_index_pruning_compatible; + + #[test] + fn reference_index_accepts_retained_block_bodies() { + assert!(ensure_reference_index_pruning_compatible(false).is_ok()); + } + + #[test] + fn reference_index_rejects_bodies_history_pruning() { + let error = ensure_reference_index_pruning_compatible(true).unwrap_err(); + assert!(error.to_string().contains("bodies-history pruning")); + } +} diff --git a/crates/node/src/args.rs b/crates/node/src/args.rs index 333e77d3..7e5e9233 100644 --- a/crates/node/src/args.rs +++ b/crates/node/src/args.rs @@ -93,6 +93,18 @@ mod tests { assert_eq!(args.max_tx_per_block, Some(1000)); } + #[test] + fn reference_index_disable_flag_is_not_supported() { + assert!( + CommandParser::::try_parse_from(["test", "--morph.disable-reference-index"]) + .is_err() + ); + assert!( + CommandParser::::try_parse_from(["test", "--disable-reference-index"]) + .is_err() + ); + } + #[test] fn test_default_trait_impl() { let args = MorphArgs::default(); diff --git a/crates/node/src/exex/mod.rs b/crates/node/src/exex/mod.rs deleted file mode 100644 index b6e8c227..00000000 --- a/crates/node/src/exex/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -//! Morph-specific Execution Extensions (ExEx). - -pub mod reference_index; - -pub use reference_index::{ReferenceIndexControl, reference_index_exex, run_startup_indexing}; diff --git a/crates/node/src/exex/reference_index.rs b/crates/node/src/exex/reference_index.rs deleted file mode 100644 index dc25835b..00000000 --- a/crates/node/src/exex/reference_index.rs +++ /dev/null @@ -1,569 +0,0 @@ -//! Reference index ExEx: drains canonical chain notifications and keeps the -//! index incrementally up to date. -//! -//! ## Lifecycle -//! -//! **Task A** (`run_startup_indexing`): runs in a spawned task, executes -//! backfill → reconcile, then sets `is_ready = true` and signals the ExEx with -//! `FinishedHeight(indexed_to)`. -//! -//! **Task B** (`reference_index_exex`): registered via `install_exex` and -//! started by reth's framework at node launch. It drains all notifications -//! immediately to avoid backpressure; writes are gated behind `is_ready`. - -use alloy_consensus::BlockHeader; -use alloy_eips::BlockNumHash; -use futures::FutureExt; -use morph_chainspec::spec::MorphChainSpec; -use morph_primitives::MorphPrimitives; -use morph_reference_index::{ - DEFAULT_BACKFILL_BATCH_BLOCKS, DEFAULT_MAX_REORG_DEPTH, ReferenceIndexDb, - backfill::{maybe_reset_jade_sentinel, run_backfill}, - reconcile::run_startup_reconcile, - writer::{delete_block, update_indexed_to, write_block}, -}; -use reth_db_api::transaction::{DbTx, DbTxMut}; -use reth_exex::{ExExContext, ExExEvent, ExExNotification}; -use reth_node_api::{FullNodeComponents, NodeTypes}; -use reth_provider::{ - BlockHashReader, BlockNumReader, BlockReader, ChainSpecProvider, HeaderProvider, -}; -use reth_storage_api::TransactionVariant; -use tokio::sync::watch; -use tokio_stream::{Stream, StreamExt}; -use tracing::{debug, error, info}; - -const TARGET: &str = "morph::reference_index"; -const MAX_LIVE_NOTIFICATION_BATCH: usize = 1024; - -// ── shared control ──────────────────────────────────────────────────────────── - -/// Shared handle that connects Task A (startup indexing) with Task B (ExEx). -/// -/// Task A completes, sets `is_ready`, then sends the startup -/// `FinishedHeight` through the watch channel. Task B receives this and -/// forwards it to reth's ExEx event bus. -#[derive(Clone, Debug)] -pub struct ReferenceIndexControl { - pub db: ReferenceIndexDb, - startup_tx: watch::Sender>, -} - -impl ReferenceIndexControl { - /// Create a new control pair. - /// - /// Returns `(control, receiver)`. The receiver must be passed to - /// [`reference_index_exex`] so the ExEx knows when startup has finished. - pub fn new(db: ReferenceIndexDb) -> (Self, watch::Receiver>) { - let (startup_tx, startup_rx) = watch::channel(None); - (Self { db, startup_tx }, startup_rx) - } - - /// Called by Task A after backfill + reconcile complete. - pub fn mark_startup_finished(&self, block: BlockNumHash) -> eyre::Result<()> { - self.startup_tx.send(Some(block))?; - Ok(()) - } -} - -// ── Task B: ExEx ────────────────────────────────────────────────────────────── - -/// Main ExEx loop. -/// -/// Drains notifications from node launch to avoid backpressure. While -/// `is_ready = false` each notification is discarded. After `is_ready` -/// the first notification triggers a gap check before normal processing. -pub async fn reference_index_exex( - mut ctx: ExExContext, - control: ReferenceIndexControl, - mut startup_rx: watch::Receiver>, -) -> eyre::Result<()> -where - Node: FullNodeComponents< - Types: NodeTypes, - >, - Node::Provider: BlockReader - + BlockNumReader - + HeaderProvider

- + BlockHashReader, -{ - let mut first_ready = true; - // Track last sent FinishedHeight to keep progress monotonic. Without this, - // the `tokio::select!` arms can race: a live commit may send `FinishedHeight(H+k)` - // before the startup watch channel is observed, which would otherwise - // regress reth's pruning marker back to `H`. - let mut last_finished: Option = None; - - loop { - tokio::select! { - // Forward the startup FinishedHeight when Task A finishes. - changed = startup_rx.changed() => { - if changed.is_ok() && let Some(block) = *startup_rx.borrow_and_update() { - debug!( - target: TARGET, - block_number = block.number, - "startup complete; forwarding initial FinishedHeight" - ); - send_finished_height_monotonic(&ctx.events, block, &mut last_finished)?; - } - } - - maybe_notification = ctx.notifications.try_next() => { - let Some(notification) = maybe_notification? else { break; }; - - if !control.db.is_ready() { - // Drain without writing to avoid backpressure. - if let Some(chain) = notification.committed_chain() { - debug!( - target: TARGET, - tip = chain.tip().number(), - "drained notification while index initializing" - ); - } - continue; - } - - // On the first is_ready notification: fill any gap that opened - // between when Task A finished reconcile and when this notification - // arrived. The goal is that after this branch runs, the index - // covers exactly the canonical range that is not going to be - // touched by the upcoming handle_notification call. - // - // fill_gap_idempotent delete-then-writes so stale entries from - // possible mini-reorgs during the drain window are cleaned up. - if first_ready { - first_ready = false; - let indexed_to = control.db.indexed_to()?; - if let Some(chain) = notification.committed_chain() { - // ChainCommitted / ChainReorged: blocks to index start at - // chain.first(). Anything in (indexed_to, notif_start-1] - // must be backfilled from main DB so handle_notification - // can pick up from notif_start. - let notif_start = chain.first().number(); - if notif_start > indexed_to + 1 { - fill_gap_idempotent( - &control.db, - &ctx.components.provider().clone(), - indexed_to + 1, - notif_start - 1, - )?; - } - } else if let Some(old) = notification.reverted_chain() { - // ChainReverted: handle_notification will delete old.blocks - // and set indexed_to = parent (= revert_start - 1). - // If parent > indexed_to, the drain window committed and - // never wrote canonical blocks (indexed_to+1..=parent) that - // will SURVIVE this revert. We must fill them now so the - // post-revert indexed_to = parent is consistent. - let revert_start = old.first().number(); - let parent = revert_start.saturating_sub(1); - if parent > indexed_to { - fill_gap_idempotent( - &control.db, - &ctx.components.provider().clone(), - indexed_to + 1, - parent, - )?; - } - // parent <= indexed_to: revert overlaps already-indexed - // range; handle_notification's delete_block + rollback of - // indexed_to takes care of it, no backfill needed. - } - } - - let notifications = drain_ready_notifications(&mut ctx.notifications, notification)?; - match handle_notifications_batch(&ctx.events, &control.db, notifications, &mut last_finished) { - Ok(()) => {} - Err(e) => { - error!(target: TARGET, ?e, "error processing notification"); - return Err(e); - } - } - } - } - } - - Ok(()) -} - -/// Fill (or repair) index entries for blocks `[from, to]` using canonical main DB data. -/// -/// Uses delete-then-write per block to stay idempotent even if some entries -/// already exist (e.g. partial prior write or mini-reorg during drain window). -fn fill_gap_idempotent( - db: &ReferenceIndexDb, - provider: &Provider, - from: u64, - to: u64, -) -> eyre::Result<()> -where - Provider: BlockReader, -{ - if from > to { - return Ok(()); - } - info!( - target: TARGET, - from, to, - "idempotent gap fill between startup reconcile and first ExEx notification" - ); - let tx = db.tx_mut()?; - for number in from..=to { - // Delete any stale entries before writing canonical ones. - delete_block(&tx, number)?; - // `WithHash` is required: write_block stores tx hashes in the index keys. - let block = provider - .sealed_block_with_senders(number.into(), TransactionVariant::WithHash)? - .ok_or_else(|| eyre::eyre!("missing block {number} during gap fill"))?; - write_block( - &tx, - block.number(), - block.hash(), - block.timestamp(), - &block.body().transactions, - )?; - } - update_indexed_to(&tx, to)?; - tx.commit()?; - Ok(()) -} - -/// Drain already-ready notifications so fast catch-up can be indexed in one DB transaction. -fn drain_ready_notifications( - notifications: &mut S, - first: ExExNotification, -) -> eyre::Result>> -where - S: Stream>> + Unpin, -{ - let mut batch = Vec::with_capacity(32); - batch.push(first); - - while batch.len() < MAX_LIVE_NOTIFICATION_BATCH { - match notifications.try_next().now_or_never() { - Some(Ok(Some(notification))) => batch.push(notification), - Some(Ok(None)) | None => break, - Some(Err(err)) => return Err(err), - } - } - - Ok(batch) -} - -/// Process ready ExEx notifications in one atomic commit. -fn handle_notifications_batch( - events: &tokio::sync::mpsc::UnboundedSender, - db: &ReferenceIndexDb, - notifications: Vec>, - last_finished: &mut Option, -) -> eyre::Result<()> { - if notifications.is_empty() { - return Ok(()); - } - - let tx = db.tx_mut()?; - let mut finished_height = None; - let mut allow_regression = false; - for notification in notifications { - let effect = apply_notification_to_tx(&tx, notification)?; - allow_regression |= effect.allow_regression; - finished_height = effect.finished_height; - } - tx.commit()?; - - if let Some(block) = finished_height { - if allow_regression { - events.send(ExExEvent::FinishedHeight(block))?; - *last_finished = Some(block); - } else { - send_finished_height_monotonic(events, block, last_finished)?; - } - } - - Ok(()) -} - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -struct NotificationEffect { - finished_height: Option, - allow_regression: bool, -} - -fn apply_notification_to_tx( - tx: &Tx, - notification: ExExNotification, -) -> eyre::Result { - match notification { - ExExNotification::ChainCommitted { new } => { - for block in new.blocks_iter() { - write_block( - tx, - block.number(), - block.hash(), - block.timestamp(), - &block.body().transactions, - )?; - } - update_indexed_to(tx, new.tip().number())?; - Ok(NotificationEffect { - finished_height: Some(new.tip().num_hash()), - allow_regression: false, - }) - } - ExExNotification::ChainReverted { old } => { - let parent = old.first().number().saturating_sub(1); - for block in old.blocks_iter() { - delete_block(tx, block.number())?; - } - update_indexed_to(tx, parent)?; - // FinishedHeight not sent on revert per spec. - Ok(NotificationEffect { - finished_height: None, - allow_regression: true, - }) - } - ExExNotification::ChainReorged { old, new } => { - for block in old.blocks_iter() { - delete_block(tx, block.number())?; - } - for block in new.blocks_iter() { - write_block( - tx, - block.number(), - block.hash(), - block.timestamp(), - &block.body().transactions, - )?; - } - update_indexed_to(tx, new.tip().number())?; - // On reorg to a shorter chain the tip may go down; that is the one - // case we ALLOW FinishedHeight to regress (reth's ExExEvent doc - // explicitly permits "on reorgs, height may go down"). - Ok(NotificationEffect { - finished_height: Some(new.tip().num_hash()), - allow_regression: true, - }) - } - } -} - -/// Send a FinishedHeight only if it strictly advances `last_finished`. -/// -/// Used for ChainCommitted and the startup watch channel, where progress -/// must only go forward. ChainReorged has its own allow-regress path. -fn send_finished_height_monotonic( - events: &tokio::sync::mpsc::UnboundedSender, - block: BlockNumHash, - last_finished: &mut Option, -) -> eyre::Result<()> { - if last_finished.is_none_or(|last| block.number > last.number) { - events.send(ExExEvent::FinishedHeight(block))?; - *last_finished = Some(block); - } - Ok(()) -} - -// ── Task A: startup indexing ────────────────────────────────────────────────── - -/// Execute backfill → reconcile, set `is_ready = true`, then send the startup -/// `FinishedHeight` through `control`. -/// -/// Call once from a spawned task after the node's provider is available. -pub fn run_startup_indexing(node: &Node, control: &ReferenceIndexControl) -> eyre::Result<()> -where - Node: FullNodeComponents< - Types: NodeTypes, - >, - Node::Provider: BlockReader - + BlockNumReader - + HeaderProvider
- + BlockHashReader - + ChainSpecProvider, -{ - let provider = node.provider().clone(); - let chain_spec = provider.chain_spec(); - let head = provider.best_block_number()?; - - // Re-resolve jade sentinel if Jade has since activated. - maybe_reset_jade_sentinel(&control.db, &provider, chain_spec.as_ref(), head)?; - - // Run backfill (no-op if already Complete). - run_backfill( - &control.db, - &provider, - chain_spec.as_ref(), - head, - DEFAULT_BACKFILL_BATCH_BLOCKS, - )?; - - // Re-read head in case new blocks arrived during backfill. - let current_head = provider.best_block_number()?; - - // Startup reconcile: canonical hash check + suffix gap. - run_startup_reconcile( - &control.db, - &provider, - current_head, - DEFAULT_MAX_REORG_DEPTH, - )?; - - // Atomically mark ready and signal the ExEx. - control.db.set_ready(true); - - let indexed_to = control.db.indexed_to()?; - let hash = provider - .block_hash(indexed_to)? - .ok_or_else(|| eyre::eyre!("missing canonical hash for block {indexed_to}"))?; - - control.mark_startup_finished(BlockNumHash { - number: indexed_to, - hash, - })?; - - info!( - target: TARGET, - indexed_to, - "reference index ready" - ); - - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - use alloy_consensus::Header; - use alloy_primitives::B256; - use morph_primitives::{Block, BlockBody, MorphHeader, MorphPrimitives}; - use reth_primitives_traits::SealedBlock; - use reth_provider::{Chain, ExecutionOutcome}; - use std::{collections::BTreeMap, sync::Arc}; - - #[test] - fn batched_commits_emit_finished_height_once_at_tip() -> eyre::Result<()> { - let dir = tempfile::tempdir()?; - let db = ReferenceIndexDb::open(dir.path(), 2910, B256::ZERO)?; - let (events, mut events_rx) = tokio::sync::mpsc::unbounded_channel(); - let mut last_finished = None; - - let block1 = recovered_block(1, B256::ZERO); - let block1_hash = block1.hash(); - let block2 = recovered_block(2, block1_hash); - let block2_num_hash = block2.num_hash(); - - handle_notifications_batch( - &events, - &db, - vec![ - committed_notification(vec![block1]), - committed_notification(vec![block2]), - ], - &mut last_finished, - )?; - - assert_eq!(db.indexed_to()?, 2); - assert!(db.indexed_block_hash(1)?.is_some()); - assert!(db.indexed_block_hash(2)?.is_some()); - assert_eq!(last_finished, Some(block2_num_hash)); - assert_eq!( - events_rx.try_recv()?, - ExExEvent::FinishedHeight(block2_num_hash) - ); - assert!(events_rx.try_recv().is_err()); - - Ok(()) - } - - #[test] - fn batch_with_reorg_allows_final_finished_height_to_regress() -> eyre::Result<()> { - let dir = tempfile::tempdir()?; - let db = ReferenceIndexDb::open(dir.path(), 2910, B256::ZERO)?; - let (events, mut events_rx) = tokio::sync::mpsc::unbounded_channel(); - let mut last_finished = Some(BlockNumHash { - number: 12, - hash: B256::repeat_byte(0x12), - }); - - let old10 = recovered_block(10, B256::ZERO); - let old11 = recovered_block(11, old10.hash()); - let old12 = recovered_block(12, old11.hash()); - let new10 = recovered_block(10, B256::repeat_byte(0x99)); - let new11 = recovered_block(11, new10.hash()); - let new11_num_hash = new11.num_hash(); - - handle_notifications_batch( - &events, - &db, - vec![ - reorg_notification(vec![old10, old11, old12], vec![new10]), - committed_notification(vec![new11]), - ], - &mut last_finished, - )?; - - assert_eq!(db.indexed_to()?, 11); - assert_eq!(last_finished, Some(new11_num_hash)); - assert_eq!( - events_rx.try_recv()?, - ExExEvent::FinishedHeight(new11_num_hash) - ); - assert!(events_rx.try_recv().is_err()); - - Ok(()) - } - - fn committed_notification( - blocks: Vec>, - ) -> ExExNotification { - ExExNotification::ChainCommitted { - new: Arc::new(Chain::new( - blocks, - ExecutionOutcome::default(), - BTreeMap::new(), - )), - } - } - - fn reorg_notification( - old: Vec>, - new: Vec>, - ) -> ExExNotification { - ExExNotification::ChainReorged { - old: Arc::new(Chain::new( - old, - ExecutionOutcome::default(), - BTreeMap::new(), - )), - new: Arc::new(Chain::new( - new, - ExecutionOutcome::default(), - BTreeMap::new(), - )), - } - } - - fn recovered_block( - number: u64, - parent_hash: B256, - ) -> reth_primitives_traits::RecoveredBlock { - let header = MorphHeader { - inner: Header { - number, - parent_hash, - timestamp: 1_000 + number, - gas_limit: 30_000_000, - parent_beacon_block_root: Some(B256::ZERO), - ..Default::default() - }, - next_l1_msg_index: 0, - }; - let body = BlockBody { - transactions: vec![], - ommers: vec![], - withdrawals: None, - }; - SealedBlock::seal_slow(Block { header, body }) - .try_recover() - .expect("empty test block should recover") - } -} diff --git a/crates/node/src/lib.rs b/crates/node/src/lib.rs index 70a01590..399c8a57 100644 --- a/crates/node/src/lib.rs +++ b/crates/node/src/lib.rs @@ -22,7 +22,6 @@ pub mod add_ons; pub mod args; pub mod components; -pub mod exex; pub mod node; #[cfg(feature = "test-utils")] pub mod test_utils; diff --git a/crates/node/tests/it/reference_index.rs b/crates/node/tests/it/reference_index.rs index 5a19f94f..7d9afaa4 100644 --- a/crates/node/tests/it/reference_index.rs +++ b/crates/node/tests/it/reference_index.rs @@ -1,53 +1,81 @@ -//! Integration tests for `morph_getTransactionHashesByReference`. +//! End-to-end tests for `morph_getTransactionHashesByReference`. //! -//! These tests spin up a real Morph test node, produce blocks with reference- -//! carrying `MorphTx` transactions, then run the reference index backfill + -//! reconcile against the node's provider, and verify the query results. -//! -//! Because `reth_e2e_test_utils::setup_engine` doesn't support ExEx injection, -//! we test the storage layer (backfill/reconcile/query) directly rather than -//! through the live ExEx. The ExEx logic is separately covered by the unit -//! tests in `morph-reference-index`. +//! The node add-on owns the reference-index runtime. These tests only produce +//! canonical blocks and call the public RPC, so they cover notification wakeups, +//! provider-backed backfill, cursor readiness, and RPC registration together. use alloy_consensus::transaction::TxHashRef; use alloy_primitives::{B256, U64}; -use morph_node::test_utils::{MorphTxBuilder, TEST_TOKEN_ID, TestNodeBuilder}; -use morph_reference_index::{ - DEFAULT_LAG_THRESHOLD, DEFAULT_MAX_REORG_DEPTH, ReferenceIndexDb, ReferenceIndexReader, - ReferenceQuery, backfill::run_backfill, reconcile::run_startup_reconcile, +use jsonrpsee::core::client::ClientT; +use morph_node::test_utils::{ + HardforkSchedule, MorphTestNode, MorphTxBuilder, TEST_TOKEN_ID, TestNodeBuilder, }; +use morph_reference_index::ReferenceTransactionResult; +use morph_rpc::ReferenceQueryArgs; use reth_payload_primitives::BuiltPayload; -use reth_provider::BlockNumReader; -use tempfile::TempDir; - -// ── helpers ─────────────────────────────────────────────────────────────────── - -async fn open_and_backfill_index

(provider: &P, dir: &TempDir) -> ReferenceIndexDb -where - P: reth_provider::BlockReader - + reth_provider::HeaderProvider

- + BlockNumReader - + reth_provider::BlockHashReader - + reth_provider::ChainSpecProvider, -{ - let chain_spec = provider.chain_spec(); - let chain_id = reth_chainspec::EthChainSpec::chain(chain_spec.as_ref()).id(); - let genesis_hash = reth_chainspec::EthChainSpec::genesis_hash(chain_spec.as_ref()); - - let db = ReferenceIndexDb::open(dir.path(), chain_id, genesis_hash).unwrap(); - - let head = provider.best_block_number().unwrap(); - run_backfill(&db, provider, chain_spec.as_ref(), head, 256).unwrap(); - run_startup_reconcile(&db, provider, head, DEFAULT_MAX_REORG_DEPTH).unwrap(); - - db.set_ready(true); - db +use std::time::Duration; + +async fn wait_for_reference_query( + node: &MorphTestNode, + args: ReferenceQueryArgs, +) -> eyre::Result> { + let client = node + .rpc_client() + .ok_or_else(|| eyre::eyre!("HTTP RPC client not available"))?; + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + let mut last_error = None; + + while tokio::time::Instant::now() < deadline { + match client + .request("morph_getTransactionHashesByReference", (args.clone(),)) + .await + { + Ok(results) => return Ok(results), + Err(error) => { + last_error = Some(error.to_string()); + tokio::time::sleep(Duration::from_millis(25)).await; + } + } + } + + Err(eyre::eyre!( + "reference index did not become queryable before timeout: {}", + last_error.as_deref().unwrap_or("no RPC response") + )) } -// ── tests ───────────────────────────────────────────────────────────────────── +fn query_args(reference: B256, offset: Option, limit: Option) -> ReferenceQueryArgs { + ReferenceQueryArgs { + reference, + offset, + limit, + } +} + +/// Before Jade, the namespace is available and returns an empty result without +/// requiring per-block index writes. +#[tokio::test(flavor = "multi_thread")] +async fn reference_index_is_empty_before_jade() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let (mut nodes, _wallet) = TestNodeBuilder::new() + .with_schedule(HardforkSchedule::PreJade) + .build() + .await?; + let node = nodes.pop().unwrap(); + assert!( + !node.inner.data_dir.exex_wal().exists(), + "reference index must not create an ExEx WAL" + ); + let results = + wait_for_reference_query(&node, query_args(B256::with_last_byte(0x98), None, None)).await?; + + assert!(results.is_empty()); + Ok(()) +} -/// Produce one block with a reference-carrying MorphTx and verify the index -/// returns it for the correct reference. +/// Produce one block with a reference-carrying MorphTx and verify the live +/// background index returns it through the public RPC. #[tokio::test(flavor = "multi_thread")] async fn reference_index_finds_single_morph_tx() -> eyre::Result<()> { reth_tracing::init_test_tracing(); @@ -70,13 +98,7 @@ async fn reference_index_finds_single_morph_tx() -> eyre::Result<()> { .unwrap() .tx_hash(); - let dir = TempDir::new()?; - let db = open_and_backfill_index(&node.inner.provider, &dir).await; - - let reader = ReferenceIndexReader::new(db, DEFAULT_LAG_THRESHOLD); - let canonical_tip = node.inner.provider.best_block_number()?; - let query = ReferenceQuery::new(reference, None, None).unwrap(); - let results = reader.query(query, canonical_tip)?; + let results = wait_for_reference_query(&node, query_args(reference, None, None)).await?; assert_eq!(results.len(), 1, "should find exactly one transaction"); assert_eq!(results[0].transaction_hash, tx_hash); @@ -85,7 +107,7 @@ async fn reference_index_finds_single_morph_tx() -> eyre::Result<()> { Ok(()) } -/// Produce multiple blocks with the same reference and verify pagination. +/// Produce multiple blocks with the same reference and verify RPC pagination. #[tokio::test(flavor = "multi_thread")] async fn reference_index_pagination() -> eyre::Result<()> { reth_tracing::init_test_tracing(); @@ -117,33 +139,20 @@ async fn reference_index_pagination() -> eyre::Result<()> { ); } - let dir = TempDir::new()?; - let db = open_and_backfill_index(&node.inner.provider, &dir).await; - - let reader = ReferenceIndexReader::new(db, DEFAULT_LAG_THRESHOLD); - let canonical_tip = node.inner.provider.best_block_number()?; - - // Page 1: offset=0, limit=2 → first two entries. - let page1 = reader.query( - ReferenceQuery::new(reference, Some(0), Some(2)).unwrap(), - canonical_tip, - )?; + let page1 = wait_for_reference_query(&node, query_args(reference, Some(0), Some(2))).await?; assert_eq!(page1.len(), 2); assert_eq!(page1[0].transaction_hash, tx_hashes[0]); assert_eq!(page1[1].transaction_hash, tx_hashes[1]); - // Page 2: offset=2, limit=2 → last entry only. - let page2 = reader.query( - ReferenceQuery::new(reference, Some(2), Some(2)).unwrap(), - canonical_tip, - )?; + let page2 = wait_for_reference_query(&node, query_args(reference, Some(2), Some(2))).await?; assert_eq!(page2.len(), 1); assert_eq!(page2[0].transaction_hash, tx_hashes[2]); Ok(()) } -/// A different reference key returns no results. +/// A different reference key returns an empty successful result once the index +/// is exactly at the canonical head. #[tokio::test(flavor = "multi_thread")] async fn reference_index_no_results_for_unrelated_reference() -> eyre::Result<()> { reth_tracing::init_test_tracing(); @@ -160,15 +169,7 @@ async fn reference_index_no_results_for_unrelated_reference() -> eyre::Result<() node.rpc.inject_tx(raw_tx).await?; node.advance_block().await?; - let dir = TempDir::new()?; - let db = open_and_backfill_index(&node.inner.provider, &dir).await; - let reader = ReferenceIndexReader::new(db, DEFAULT_LAG_THRESHOLD); - let canonical_tip = node.inner.provider.best_block_number()?; - - let results = reader.query( - ReferenceQuery::new(other_reference, None, None).unwrap(), - canonical_tip, - )?; + let results = wait_for_reference_query(&node, query_args(other_reference, None, None)).await?; assert!( results.is_empty(), "unrelated reference should return nothing" diff --git a/crates/reference-index/Cargo.toml b/crates/reference-index/Cargo.toml index fd2cc786..6afde198 100644 --- a/crates/reference-index/Cargo.toml +++ b/crates/reference-index/Cargo.toml @@ -14,16 +14,19 @@ alloy-consensus.workspace = true alloy-primitives.workspace = true alloy-rlp.workspace = true eyre.workspace = true -morph-chainspec.workspace = true morph-primitives = { workspace = true, features = ["reth-codec"] } -reth-chainspec.workspace = true +metrics.workspace = true +parking_lot.workspace = true reth-codecs.workspace = true +reth-chain-state.workspace = true reth-db.workspace = true reth-db-api.workspace = true reth-errors.workspace = true +reth-metrics.workspace = true reth-provider.workspace = true reth-primitives-traits.workspace = true reth-storage-api.workspace = true +reth-tasks.workspace = true serde.workspace = true thiserror.workspace = true tokio.workspace = true diff --git a/crates/reference-index/src/backfill.rs b/crates/reference-index/src/backfill.rs index 099cc33f..e7ac5dae 100644 --- a/crates/reference-index/src/backfill.rs +++ b/crates/reference-index/src/backfill.rs @@ -1,319 +1,76 @@ -//! Historical reference index backfill. -//! -//! Runs once at startup to index all MorphTx transactions from -//! `jade_first_block_number` up to the chain head at startup time. +//! Bounded historical reference-index backfill helpers. use crate::{ - JADE_NOT_ACTIVE_SENTINEL, ReferenceIndexDb, - types::{BackfillState, ReferenceIndexError}, - writer::{ - set_backfill_state, set_jade_first_block_number, update_backfill_current, - update_indexed_from, update_indexed_to, write_block, - }, + db::ReferenceIndexDb, + source::{CanonicalBlock, CanonicalChain}, + types::ReferenceIndexError, + writer::{set_jade_first_block_number, update_indexed_to, write_block}, }; -use alloy_consensus::BlockHeader; -use morph_chainspec::hardfork::{MorphHardfork, MorphHardforks}; -use morph_primitives::MorphHeader; -use reth_chainspec::ForkCondition; use reth_db_api::transaction::DbTx; -use reth_provider::{BlockReader, HeaderProvider}; -use reth_storage_api::{BlockNumReader, TransactionVariant}; -use tracing::{debug, info}; -const TARGET: &str = "morph::reference_index"; - -/// Determine the first block number at which the Jade hardfork is active. -/// -/// Returns `JADE_NOT_ACTIVE_SENTINEL` (`u64::MAX`) when: -/// - The chain spec has no Jade timestamp condition, or -/// - The current canonical head has not yet reached Jade activation. -/// -/// The dual-condition check (`block(N).timestamp >= jade_ts` AND -/// `block(N-1).timestamp < jade_ts`) guarantees we return the **first** -/// Jade block, not just any block after activation. -pub fn resolve_jade_first_block( - provider: &P, - chain_spec: &CS, - head: u64, -) -> Result -where - P: HeaderProvider
, - CS: MorphHardforks, -{ - let jade_ts = match chain_spec.morph_fork_activation(MorphHardfork::Jade) { - ForkCondition::Timestamp(ts) => ts, - _ => return Ok(JADE_NOT_ACTIVE_SENTINEL), - }; - - let head_header = provider - .header_by_number(head)? - .ok_or_else(|| ReferenceIndexError::Other(eyre::eyre!("missing canonical head header")))?; - - if head_header.timestamp() < jade_ts { - return Ok(JADE_NOT_ACTIVE_SENTINEL); +/// Find the first canonical block whose timestamp activates Jade. +pub(crate) fn resolve_jade_first_block( + chain: &C, + head: crate::CanonicalTip, + jade_timestamp: u64, +) -> Result, ReferenceIndexError> { + if head.timestamp < jade_timestamp { + return Ok(None); } - // Binary-search over [0, head] to find the lowest block whose timestamp - // is >= jade_ts. We still verify the prev block to guard against unusual - // timestamp monotonicity violations. - let mut lo = 0u64; - let mut hi = head; - while lo < hi { - let mid = lo + (hi - lo) / 2; - let mid_header = provider - .header_by_number(mid)? - .ok_or_else(|| ReferenceIndexError::Other(eyre::eyre!("missing header at {mid}")))?; - if mid_header.timestamp() < jade_ts { - lo = mid + 1; - } else { - hi = mid; - } - } - - // `lo` is now the first block whose timestamp >= jade_ts. - // Verify that the previous block (if any) has timestamp < jade_ts. - if lo > 0 { - let prev = provider.header_by_number(lo - 1)?.ok_or_else(|| { - ReferenceIndexError::Other(eyre::eyre!("missing header at {}", lo - 1)) + let mut low = 0u64; + let mut high = head.number; + while low < high { + let middle = low + (high - low) / 2; + let timestamp = chain.block_timestamp(middle)?.ok_or_else(|| { + ReferenceIndexError::Other(eyre::eyre!( + "missing canonical header {middle} while resolving Jade" + )) })?; - if prev.timestamp() >= jade_ts { - // Shouldn't happen on a well-formed chain, but fall back to sentinel. - return Ok(JADE_NOT_ACTIVE_SENTINEL); + if timestamp < jade_timestamp { + low = middle + 1; + } else { + high = middle; } } - Ok(lo) + Ok(Some(low)) } -/// Run (or resume) the historical backfill. -/// -/// This function is synchronous and blocks the calling task until backfill is -/// complete. Batches are committed atomically so a crash mid-run can be -/// safely resumed from the last persisted checkpoint. -pub fn run_backfill( +/// Commit one already-validated contiguous canonical block batch. +pub(crate) fn commit_canonical_batch( db: &ReferenceIndexDb, - provider: &P, - chain_spec: &CS, - head_at_startup: u64, - batch_size: u64, -) -> Result<(), ReferenceIndexError> -where - P: BlockReader - + BlockNumReader - + HeaderProvider
, - CS: MorphHardforks, -{ - validate_backfill_batch_size(batch_size)?; - - let state = db.backfill_state()?; - - // `jade_first_block_number` is our canonical lower bound; it's written as the - // very first step on the NotStarted path. InProgress/Complete resumes read - // it back from IndexMeta. - let start = match state { - BackfillState::Complete => return Ok(()), - BackfillState::InProgress => { - // Resume from `max(indexed_to + 1, jade_first_block_number)`: if the - // crash happened between InProgress write and the first batch commit, - // `indexed_to` is still 0 and must not be used as-is. - let jade_first_stored = db - .jade_first_block_number()? - .unwrap_or(JADE_NOT_ACTIVE_SENTINEL); - if jade_first_stored == JADE_NOT_ACTIVE_SENTINEL { - // Previously sentinel; crash may have happened before the follow-up - // txn that would have marked Complete. Re-resolve against the current - // head: Jade might have activated in the meantime. - let new_jade = resolve_jade_first_block(provider, chain_spec, head_at_startup)?; - if new_jade == JADE_NOT_ACTIVE_SENTINEL { - // Still not active; safe to mark complete immediately. - let tx = db.tx_mut()?; - update_indexed_from(&tx, head_at_startup)?; - update_indexed_to(&tx, head_at_startup)?; - set_backfill_state(&tx, BackfillState::Complete)?; - tx.commit()?; - return Ok(()); - } - // Jade now active; persist the real first block and continue backfill. - let tx = db.tx_mut()?; - set_jade_first_block_number(&tx, new_jade)?; - tx.commit()?; - db.indexed_to()?.saturating_add(1).max(new_jade) - } else { - db.indexed_to()?.saturating_add(1).max(jade_first_stored) - } - } - BackfillState::NotStarted => { - let jade_first = resolve_jade_first_block(provider, chain_spec, head_at_startup)?; - - let tx = db.tx_mut()?; - set_jade_first_block_number(&tx, jade_first)?; - set_backfill_state(&tx, BackfillState::InProgress)?; - tx.commit()?; - - if jade_first == JADE_NOT_ACTIVE_SENTINEL || jade_first > head_at_startup { - // Nothing to backfill; mark complete immediately. - let tx = db.tx_mut()?; - update_indexed_from(&tx, head_at_startup)?; - update_indexed_to(&tx, head_at_startup)?; - set_backfill_state(&tx, BackfillState::Complete)?; - tx.commit()?; - return Ok(()); - } - - jade_first - } + jade_first_block: u64, + expected_start: u64, + blocks: &[CanonicalBlock], +) -> Result { + let Some(last) = blocks.last() else { + return Ok(0); }; - if start > head_at_startup { - // Already up to date. - let tx = db.tx_mut()?; - set_backfill_state(&tx, BackfillState::Complete)?; - tx.commit()?; - return Ok(()); - } - - let jade_first = db.jade_first_block_number()?.unwrap_or(start); - - info!( - target: TARGET, - start, head_at_startup, - "starting reference index backfill" - ); - - let mut current = start; - - while current <= head_at_startup { - let batch_end = current.saturating_add(batch_size - 1).min(head_at_startup); - let is_last_batch = batch_end == head_at_startup; - - let tx = db.tx_mut()?; - for number in current..=batch_end { - // `WithHash` is required because write_block stores transaction hashes in the - // reference index keys. `NoHash` would leave tx hashes uninitialised per - // reth's BlockReader contract (see blockchain_provider.rs:309). - let block = provider - .sealed_block_with_senders(number.into(), TransactionVariant::WithHash)? - .ok_or_else(|| { - ReferenceIndexError::Other(eyre::eyre!( - "missing block {number} during backfill" - )) - })?; - - write_block( - &tx, - block.number(), - block.hash(), - block.timestamp(), - &block.body().transactions, - )?; - } - - if is_last_batch { - // Atomic: data + Complete + indexed_from + indexed_to in one commit. - update_indexed_from(&tx, jade_first)?; - update_indexed_to(&tx, head_at_startup)?; - set_backfill_state(&tx, BackfillState::Complete)?; - } else { - update_backfill_current(&tx, batch_end)?; - update_indexed_to(&tx, batch_end)?; - set_backfill_state(&tx, BackfillState::InProgress)?; - } - tx.commit()?; - - debug!( - target: TARGET, - batch_start = current, - batch_end, - is_last_batch, - "backfill batch committed" - ); - - current = batch_end + 1; - } - - info!(target: TARGET, "reference index backfill complete"); - Ok(()) -} - -fn validate_backfill_batch_size(batch_size: u64) -> Result<(), ReferenceIndexError> { - if batch_size == 0 { - return Err(ReferenceIndexError::InvalidBackfillBatchSize); - } - Ok(()) -} - -/// Re-validate and (if needed) reset `jade_first_block_number` when the DB -/// was opened with a sentinel value (`u64::MAX`) from a previous run where -/// Jade had not yet activated. -/// -/// Call this once at startup before `run_backfill`. -pub fn maybe_reset_jade_sentinel( - db: &ReferenceIndexDb, - provider: &P, - chain_spec: &CS, - head: u64, -) -> Result<(), ReferenceIndexError> -where - P: HeaderProvider
, - CS: MorphHardforks, -{ - if db.backfill_state()? != BackfillState::Complete { - return Ok(()); - } - match db.jade_first_block_number()? { - Some(n) if n == JADE_NOT_ACTIVE_SENTINEL => { - // Jade was not active when last resolved. Re-try now. - let new = resolve_jade_first_block(provider, chain_spec, head)?; - if new != JADE_NOT_ACTIVE_SENTINEL { - // Jade has since activated; reset backfill state so the - // next startup picks it up. - let tx = db.tx_mut()?; - set_jade_first_block_number(&tx, new)?; - set_backfill_state(&tx, BackfillState::NotStarted)?; - tx.commit()?; - info!( - target: TARGET, - jade_first_block = new, - "Jade has activated; resetting backfill to index from first Jade block" - ); - } + for (offset, block) in blocks.iter().enumerate() { + let expected = expected_start.saturating_add(offset as u64); + if block.number != expected { + return Err(ReferenceIndexError::Other(eyre::eyre!( + "non-contiguous reference-index batch: expected block {expected}, got {}", + block.number + ))); } - _ => {} - } - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn jade_not_active_sentinel_is_u64_max() { - assert_eq!(JADE_NOT_ACTIVE_SENTINEL, u64::MAX); - } - - #[test] - fn backfill_state_try_from_roundtrip() { - assert_eq!( - BackfillState::try_from(0u8).unwrap(), - BackfillState::NotStarted - ); - assert_eq!( - BackfillState::try_from(1u8).unwrap(), - BackfillState::InProgress - ); - assert_eq!( - BackfillState::try_from(2u8).unwrap(), - BackfillState::Complete - ); - assert!(BackfillState::try_from(3u8).is_err()); } - #[test] - fn backfill_batch_size_zero_is_rejected() { - assert!(matches!( - validate_backfill_batch_size(0), - Err(ReferenceIndexError::InvalidBackfillBatchSize) - )); + let tx = db.tx_mut()?; + let mut references_written = 0u64; + for block in blocks { + references_written += write_block( + &tx, + block.number, + block.hash, + block.timestamp, + &block.transactions, + )?; } + set_jade_first_block_number(&tx, jade_first_block)?; + update_indexed_to(&tx, last.number)?; + tx.commit()?; + Ok(references_written) } diff --git a/crates/reference-index/src/db.rs b/crates/reference-index/src/db.rs index 3c37beea..3bf8ec64 100644 --- a/crates/reference-index/src/db.rs +++ b/crates/reference-index/src/db.rs @@ -1,8 +1,11 @@ //! Reference index database opening and metadata helpers. use crate::{ - tables::{IndexedBlockKey, IndexedBlocks, MetaKey, MetaValue, ReferenceIndexTables}, - types::{BackfillState, ReferenceIndexError, SCHEMA_VERSION}, + tables::{ + BlockReferenceIndex, IndexMeta, IndexedBlockKey, IndexedBlocks, MetaKey, MetaValue, + ReferenceIndex, ReferenceIndexTables, + }, + types::{ReferenceIndexError, SCHEMA_VERSION}, }; use alloy_primitives::B256; use reth_db::{DatabaseEnv, mdbx::DatabaseArguments}; @@ -10,96 +13,73 @@ use reth_db_api::{ Database, transaction::{DbTx, DbTxMut}, }; -use std::{ - path::Path, - sync::{ - Arc, - atomic::{AtomicBool, Ordering}, - }, -}; - -// ── meta key discriminants ──────────────────────────────────────────────────── +use std::{path::Path, sync::Arc}; /// Discriminant values for the `IndexMeta` table. #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[repr(u8)] -pub enum IndexMetaKey { - IndexedFrom = 1, - IndexedTo = 2, - BackfillState = 3, - BackfillCurrent = 4, - ChainId = 5, - GenesisHash = 6, - SchemaVersion = 7, - JadeFirstBlockNumber = 8, +pub(crate) enum IndexMetaKey { + IndexedTo = 1, + ChainId = 2, + GenesisHash = 3, + SchemaVersion = 4, + JadeFirstBlockNumber = 5, } impl From for MetaKey { - fn from(k: IndexMetaKey) -> Self { - Self(k as u8) + fn from(key: IndexMetaKey) -> Self { + Self(key as u8) } } -// ── codec helpers ───────────────────────────────────────────────────────────── - -pub fn encode_u64(value: u64) -> MetaValue { +pub(crate) fn encode_u64(value: u64) -> MetaValue { MetaValue(value.to_be_bytes().to_vec()) } -pub fn decode_u64(value: MetaValue) -> Result { +pub(crate) fn decode_u64(value: MetaValue) -> Result { let bytes: [u8; 8] = value .0 .as_slice() .try_into() - .map_err(|_| ReferenceIndexError::Other(eyre::eyre!("invalid u64 metadata length")))?; + .map_err(|_| ReferenceIndexError::CorruptMetadata("invalid u64 value"))?; Ok(u64::from_be_bytes(bytes)) } -pub fn encode_u32(value: u32) -> MetaValue { +pub(crate) fn encode_u32(value: u32) -> MetaValue { MetaValue(value.to_be_bytes().to_vec()) } -pub fn decode_u32(value: MetaValue) -> Result { +pub(crate) fn decode_u32(value: MetaValue) -> Result { let bytes: [u8; 4] = value .0 .as_slice() .try_into() - .map_err(|_| ReferenceIndexError::Other(eyre::eyre!("invalid u32 metadata length")))?; + .map_err(|_| ReferenceIndexError::CorruptMetadata("invalid u32 value"))?; Ok(u32::from_be_bytes(bytes)) } -pub fn encode_b256(value: B256) -> MetaValue { +pub(crate) fn encode_b256(value: B256) -> MetaValue { MetaValue(value.as_slice().to_vec()) } -pub fn decode_b256(value: MetaValue) -> Result { +pub(crate) fn decode_b256(value: MetaValue) -> Result { let bytes: [u8; 32] = value .0 .as_slice() .try_into() - .map_err(|_| ReferenceIndexError::Other(eyre::eyre!("invalid B256 metadata length")))?; + .map_err(|_| ReferenceIndexError::CorruptMetadata("invalid B256 value"))?; Ok(B256::new(bytes)) } -// ── database handle ─────────────────────────────────────────────────────────── - -/// Handle to the reference index MDBX database. -/// -/// Wraps the raw [`DatabaseEnv`] and exposes metadata read helpers and an -/// atomic readiness flag. All writes go through `tx_mut()`. +/// Cloneable handle to the reference-index MDBX environment. #[derive(Debug, Clone)] -pub struct ReferenceIndexDb { +pub(crate) struct ReferenceIndexDb { db: Arc, - ready: Arc, } impl ReferenceIndexDb { - /// Open (or create) the reference index at `path` and validate chain identity. - /// - /// On the first open (empty DB), writes `chain_id`, `genesis_hash`, and - /// `schema_version` into `IndexMeta`. On every subsequent open, re-reads - /// those values and returns an error if any mismatch is detected. - pub fn open( + /// Open (or create) the index and validate its complete chain identity. + pub(crate) fn open( path: impl AsRef, chain_id: u64, genesis_hash: B256, @@ -108,20 +88,14 @@ impl ReferenceIndexDb { path, DatabaseArguments::new(reth_db::models::ClientVersion::default()), ) - .map_err(|e| { - ReferenceIndexError::Other(eyre::eyre!("failed to open reference index DB: {e}")) + .map_err(|error| { + ReferenceIndexError::Other(eyre::eyre!("failed to open reference index DB: {error}")) })?; - - let this = Self { - db: Arc::new(db), - ready: Arc::new(AtomicBool::new(false)), - }; - + let this = Self { db: Arc::new(db) }; this.validate_or_init_chain_identity(chain_id, genesis_hash)?; Ok(this) } - /// Check (or initialise) the persisted chain identity in `IndexMeta`. fn validate_or_init_chain_identity( &self, chain_id: u64, @@ -129,125 +103,100 @@ impl ReferenceIndexDb { ) -> Result<(), ReferenceIndexError> { let tx = self.tx()?; let stored_chain_id = tx - .get::(IndexMetaKey::ChainId.into())? + .get::(IndexMetaKey::ChainId.into())? .map(decode_u64) .transpose()?; - - // First-ever open: write identity and return. - if stored_chain_id.is_none() { - drop(tx); - let tx = self.tx_mut()?; - tx.put::(IndexMetaKey::ChainId.into(), encode_u64(chain_id))?; - tx.put::( - IndexMetaKey::GenesisHash.into(), - encode_b256(genesis_hash), - )?; - tx.put::( - IndexMetaKey::SchemaVersion.into(), - encode_u32(SCHEMA_VERSION), - )?; - tx.commit()?; - return Ok(()); - } - - // Subsequent opens: validate every field. - if stored_chain_id != Some(chain_id) { - return Err(ReferenceIndexError::ChainIdentityMismatch("chain_id")); - } - let stored_genesis = tx - .get::(IndexMetaKey::GenesisHash.into())? + .get::(IndexMetaKey::GenesisHash.into())? .map(decode_b256) .transpose()?; - if stored_genesis != Some(genesis_hash) { - return Err(ReferenceIndexError::ChainIdentityMismatch("genesis_hash")); - } - let stored_schema = tx - .get::(IndexMetaKey::SchemaVersion.into())? + .get::(IndexMetaKey::SchemaVersion.into())? .map(decode_u32) .transpose()?; - match stored_schema { - Some(v) if v != SCHEMA_VERSION => { - return Err(ReferenceIndexError::SchemaMismatch { - expected: SCHEMA_VERSION, - actual: v, - }); + let has_runtime_meta = tx + .get::(IndexMetaKey::IndexedTo.into())? + .is_some() + || tx + .get::(IndexMetaKey::JadeFirstBlockNumber.into())? + .is_some(); + let has_index_data = tx.entries::()? != 0 + || tx.entries::()? != 0 + || tx.entries::()? != 0; + + match (stored_chain_id, stored_genesis, stored_schema) { + (None, None, None) if !has_runtime_meta && !has_index_data => { + drop(tx); + let tx = self.tx_mut()?; + tx.put::(IndexMetaKey::ChainId.into(), encode_u64(chain_id))?; + tx.put::(IndexMetaKey::GenesisHash.into(), encode_b256(genesis_hash))?; + tx.put::( + IndexMetaKey::SchemaVersion.into(), + encode_u32(SCHEMA_VERSION), + )?; + tx.commit()?; + return Ok(()); + } + (Some(stored_chain_id), Some(stored_genesis), Some(stored_schema)) => { + if stored_chain_id != chain_id { + return Err(ReferenceIndexError::ChainIdentityMismatch("chain_id")); + } + if stored_genesis != genesis_hash { + return Err(ReferenceIndexError::ChainIdentityMismatch("genesis_hash")); + } + if stored_schema != SCHEMA_VERSION { + return Err(ReferenceIndexError::SchemaMismatch { + expected: SCHEMA_VERSION, + actual: stored_schema, + }); + } + } + _ => { + return Err(ReferenceIndexError::CorruptMetadata( + "chain identity fields must be all present or all absent", + )); } - _ => {} } Ok(()) } - // ── readiness ───────────────────────────────────────────────────────────── - - pub fn is_ready(&self) -> bool { - self.ready.load(Ordering::Acquire) - } - - pub fn set_ready(&self, ready: bool) { - self.ready.store(ready, Ordering::Release); - } - - // ── transaction factory ─────────────────────────────────────────────────── - - pub fn tx(&self) -> Result<::TX, ReferenceIndexError> { + pub(crate) fn tx(&self) -> Result<::TX, ReferenceIndexError> { Ok(self.db.tx()?) } - pub fn tx_mut(&self) -> Result<::TXMut, ReferenceIndexError> { + pub(crate) fn tx_mut(&self) -> Result<::TXMut, ReferenceIndexError> { Ok(self.db.tx_mut()?) } - // ── metadata reads ──────────────────────────────────────────────────────── - - pub fn backfill_state(&self) -> Result { + pub(crate) fn indexed_to(&self) -> Result, ReferenceIndexError> { let tx = self.tx()?; - match tx.get::(IndexMetaKey::BackfillState.into())? { - Some(v) => BackfillState::try_from(*v.0.first().unwrap_or(&0)), - None => Ok(BackfillState::NotStarted), - } - } - - pub fn indexed_to(&self) -> Result { - let tx = self.tx()?; - tx.get::(IndexMetaKey::IndexedTo.into())? - .map(decode_u64) - .transpose() - .map(|v| v.unwrap_or(0)) - } - - pub fn indexed_from(&self) -> Result, ReferenceIndexError> { - let tx = self.tx()?; - tx.get::(IndexMetaKey::IndexedFrom.into())? + tx.get::(IndexMetaKey::IndexedTo.into())? .map(decode_u64) .transpose() } - pub fn jade_first_block_number(&self) -> Result, ReferenceIndexError> { + pub(crate) fn jade_first_block_number(&self) -> Result, ReferenceIndexError> { let tx = self.tx()?; - tx.get::(IndexMetaKey::JadeFirstBlockNumber.into())? + tx.get::(IndexMetaKey::JadeFirstBlockNumber.into())? .map(decode_u64) .transpose() } - /// Returns the canonical block hash stored in `IndexedBlocks` for `block_number`. - pub fn indexed_block_hash( + pub(crate) fn indexed_block_hash( &self, block_number: u64, ) -> Result, ReferenceIndexError> { let tx = self.tx()?; Ok(tx .get::(IndexedBlockKey { block_number })? - .map(|v| v.0)) + .map(|value| value.0)) } } #[cfg(test)] mod tests { use super::*; - use alloy_primitives::B256; use tempfile::TempDir; fn open_temp_db() -> (TempDir, ReferenceIndexDb) { @@ -257,20 +206,18 @@ mod tests { } #[test] - fn open_writes_chain_identity_on_first_open() { + fn new_database_has_no_durable_cursor() { let (_dir, db) = open_temp_db(); - assert_eq!(db.backfill_state().unwrap(), BackfillState::NotStarted); - assert_eq!(db.indexed_to().unwrap(), 0); + assert_eq!(db.indexed_to().unwrap(), None); } #[test] fn open_rejects_mismatched_chain_id() { let dir = TempDir::new().unwrap(); ReferenceIndexDb::open(dir.path(), 2818, B256::ZERO).unwrap(); - // Re-open with wrong chain_id. - let err = ReferenceIndexDb::open(dir.path(), 9999, B256::ZERO).unwrap_err(); + let error = ReferenceIndexDb::open(dir.path(), 9999, B256::ZERO).unwrap_err(); assert!(matches!( - err, + error, ReferenceIndexError::ChainIdentityMismatch("chain_id") )); } @@ -279,22 +226,56 @@ mod tests { fn open_rejects_mismatched_genesis_hash() { let dir = TempDir::new().unwrap(); ReferenceIndexDb::open(dir.path(), 2818, B256::ZERO).unwrap(); - let err = ReferenceIndexDb::open(dir.path(), 2818, B256::repeat_byte(0xff)).unwrap_err(); + let error = ReferenceIndexDb::open(dir.path(), 2818, B256::repeat_byte(0xff)).unwrap_err(); assert!(matches!( - err, + error, ReferenceIndexError::ChainIdentityMismatch("genesis_hash") )); } + #[test] + fn open_rejects_incomplete_identity_metadata() { + let dir = TempDir::new().unwrap(); + let db = ReferenceIndexDb::open(dir.path(), 2818, B256::ZERO).unwrap(); + let tx = db.tx_mut().unwrap(); + tx.delete::(IndexMetaKey::SchemaVersion.into(), None) + .unwrap(); + tx.commit().unwrap(); + drop(db); + + let error = ReferenceIndexDb::open(dir.path(), 2818, B256::ZERO).unwrap_err(); + assert!(matches!(error, ReferenceIndexError::CorruptMetadata(_))); + } + + #[test] + fn open_rejects_the_pre_release_schema() { + let dir = TempDir::new().unwrap(); + let db = ReferenceIndexDb::open(dir.path(), 2818, B256::ZERO).unwrap(); + let tx = db.tx_mut().unwrap(); + tx.put::(IndexMetaKey::SchemaVersion.into(), encode_u32(1)) + .unwrap(); + tx.commit().unwrap(); + drop(db); + + let error = ReferenceIndexDb::open(dir.path(), 2818, B256::ZERO).unwrap_err(); + assert!(matches!( + error, + ReferenceIndexError::SchemaMismatch { + expected: SCHEMA_VERSION, + actual: 1 + } + )); + } + #[test] fn encode_decode_u64_roundtrip() { - let v = 0xDEAD_BEEF_CAFE_1234u64; - assert_eq!(decode_u64(encode_u64(v)).unwrap(), v); + let value = 0xDEAD_BEEF_CAFE_1234u64; + assert_eq!(decode_u64(encode_u64(value)).unwrap(), value); } #[test] fn encode_decode_b256_roundtrip() { - let v = B256::repeat_byte(0xab); - assert_eq!(decode_b256(encode_b256(v)).unwrap(), v); + let value = B256::repeat_byte(0xab); + assert_eq!(decode_b256(encode_b256(value)).unwrap(), value); } } diff --git a/crates/reference-index/src/lib.rs b/crates/reference-index/src/lib.rs index 692cfce6..d6421c6d 100644 --- a/crates/reference-index/src/lib.rs +++ b/crates/reference-index/src/lib.rs @@ -4,7 +4,6 @@ use alloy_consensus as _; use alloy_rlp as _; -use morph_chainspec as _; use morph_primitives as _; use reth_codecs as _; use reth_primitives_traits as _; @@ -13,26 +12,26 @@ use reth_storage_api as _; use tokio as _; use tracing as _; -pub mod backfill; -pub mod db; -pub mod reader; -pub mod reconcile; -pub mod tables; -pub mod types; -pub mod writer; +mod backfill; +mod db; +mod metrics; +mod reader; +mod reconcile; +mod runtime; +mod source; +mod tables; +mod types; +mod writer; -pub use db::ReferenceIndexDb; -pub use reader::ReferenceIndexReader; +pub use reader::{ReferenceIndexHandle, ReferenceIndexPhase}; +pub use runtime::{ReferenceIndexConfig, ReferenceIndexRuntime}; +pub use source::{CanonicalBlock, CanonicalChain}; pub use types::{ - BackfillState, JADE_NOT_ACTIVE_SENTINEL, ReferenceIndexError, ReferenceQuery, - ReferenceTransactionResult, SCHEMA_VERSION, + CanonicalTip, ReferenceIndexError, ReferenceQuery, ReferenceTransactionResult, SCHEMA_VERSION, }; -/// Default number of canonical blocks the ExEx may lag before RPC returns an error. -pub const DEFAULT_LAG_THRESHOLD: u64 = 16; - -/// Default number of blocks checked during startup reconcile for offline reorgs. -pub const DEFAULT_MAX_REORG_DEPTH: u64 = 64; - /// Default backfill batch size. -pub const DEFAULT_BACKFILL_BATCH_BLOCKS: u64 = 256; +pub(crate) const DEFAULT_BACKFILL_BATCH_BLOCKS: u64 = 512; + +/// Operational live-lag service-level objective. RPC readiness remains lag zero. +pub(crate) const LIVE_LAG_SLO: u64 = 16; diff --git a/crates/reference-index/src/metrics.rs b/crates/reference-index/src/metrics.rs new file mode 100644 index 00000000..6f1d84d5 --- /dev/null +++ b/crates/reference-index/src/metrics.rs @@ -0,0 +1,33 @@ +//! Runtime observability for the derived reference index. + +use reth_metrics::{ + Metrics, + metrics::{Counter, Gauge, Histogram}, +}; + +#[derive(Metrics, Clone)] +#[metrics(scope = "morph.reference_index")] +pub(crate) struct ReferenceIndexMetrics { + /// Current runtime phase encoded by `ReferenceIndexPhase`. + pub(crate) phase: Gauge, + /// Current canonical target block. + pub(crate) target_block: Gauge, + /// Last durably indexed canonical block, or zero before activation. + pub(crate) indexed_block: Gauge, + /// Canonical block lag observed after the latest reconciliation turn. + pub(crate) lag_blocks: Gauge, + /// One only when the runtime phase is live; zero otherwise. + pub(crate) rpc_ready: Gauge, + /// Canonical blocks committed to the index. + pub(crate) indexed_blocks_total: Counter, + /// Durable index batches committed. + pub(crate) committed_batches_total: Counter, + /// Canonical suffix repairs performed. + pub(crate) reorgs_total: Counter, + /// Canonical notification receiver lag events. + pub(crate) lagged_notifications_total: Counter, + /// Failed reconciliation turns. + pub(crate) failures_total: Counter, + /// Wall-clock duration of a reconciliation turn. + pub(crate) batch_duration_seconds: Histogram, +} diff --git a/crates/reference-index/src/reader.rs b/crates/reference-index/src/reader.rs index efe6dc7a..f415e8da 100644 --- a/crates/reference-index/src/reader.rs +++ b/crates/reference-index/src/reader.rs @@ -1,48 +1,183 @@ //! Reference index read path helpers. use crate::{ - db::ReferenceIndexDb, - tables::{ReferenceIndex, ReferenceIndexKey}, - types::{ReferenceIndexError, ReferenceQuery, ReferenceTransactionResult}, + db::{IndexMetaKey, ReferenceIndexDb, decode_u64}, + metrics::ReferenceIndexMetrics, + tables::{IndexMeta, IndexedBlockKey, IndexedBlocks, ReferenceIndex, ReferenceIndexKey}, + types::{CanonicalTip, ReferenceIndexError, ReferenceQuery, ReferenceTransactionResult}, }; use alloy_primitives::{B256, U64}; +use parking_lot::RwLock; use reth_db_api::{cursor::DbCursorRO, transaction::DbTx}; +use std::sync::{ + Arc, + atomic::{AtomicU8, AtomicU64, Ordering}, +}; + +/// Runtime phase that controls whether reference queries can be served. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +pub enum ReferenceIndexPhase { + /// Runtime has not opened and validated its derived database yet. + Opening = 0, + /// Canonical head predates Jade, so the complete result is empty. + PreJade = 1, + /// Historical chain sync is active and index writes are intentionally paused. + Deferred = 2, + /// A bounded historical batch is being reconciled or committed. + Backfill = 3, + /// Durable cursor and hash matched the canonical head after the latest turn. + Live = 4, + /// A non-canonical indexed suffix is being removed. + Repairing = 5, + /// Queries cannot be served until retry or a manual rebuild succeeds. + Unavailable = 6, +} + +impl ReferenceIndexPhase { + fn from_u8(value: u8) -> Self { + match value { + 0 => Self::Opening, + 1 => Self::PreJade, + 2 => Self::Deferred, + 3 => Self::Backfill, + 4 => Self::Live, + 5 => Self::Repairing, + 6 => Self::Unavailable, + _ => unreachable!("reference index phase is only written by this crate"), + } + } +} + +#[derive(Debug)] +struct ReferenceIndexShared { + db: RwLock>, + phase: AtomicU8, + /// Even values are stable snapshots; odd values mean a transition is in progress. + generation: AtomicU64, + metrics: ReferenceIndexMetrics, +} -/// Read-only facade for reference index queries. +/// Cloneable query handle shared by the runtime and RPC layer. /// -/// `canonical_tip` must be the current chain head block number so the reader -/// can detect excessive lag between the index and the live chain. +/// The runtime installs the database and advances the phase. Queries only +/// succeed in [`ReferenceIndexPhase::Live`] when the durable cursor exactly +/// matches the supplied canonical tip number and hash. #[derive(Clone, Debug)] -pub struct ReferenceIndexReader { - db: ReferenceIndexDb, - lag_threshold: u64, +pub struct ReferenceIndexHandle { + shared: Arc, + jade_timestamp: u64, } -impl ReferenceIndexReader { - pub const fn new(db: ReferenceIndexDb, lag_threshold: u64) -> Self { - Self { db, lag_threshold } +impl ReferenceIndexHandle { + /// Create a handle before the background runtime has opened the database. + pub fn new(jade_timestamp: u64) -> Self { + Self { + shared: Arc::new(ReferenceIndexShared { + db: RwLock::new(None), + phase: AtomicU8::new(ReferenceIndexPhase::Opening as u8), + generation: AtomicU64::new(0), + metrics: ReferenceIndexMetrics::default(), + }), + jade_timestamp, + } + } + + /// Current in-memory runtime phase, exposed for logs and metrics. + pub fn phase(&self) -> ReferenceIndexPhase { + ReferenceIndexPhase::from_u8(self.shared.phase.load(Ordering::Acquire)) + } + + pub(crate) fn install_db(&self, db: ReferenceIndexDb) { + *self.shared.db.write() = Some(db); + } + + pub(crate) fn set_phase(&self, phase: ReferenceIndexPhase) { + if self.phase() == phase { + return; + } + self.shared.generation.fetch_add(1, Ordering::AcqRel); + self.shared.phase.store(phase as u8, Ordering::Release); + self.shared.generation.fetch_add(1, Ordering::Release); + self.shared.metrics.phase.set(phase as u8 as f64); + self.shared + .metrics + .rpc_ready + .set(f64::from(phase == ReferenceIndexPhase::Live)); + } + + pub(crate) fn db(&self) -> Option { + self.shared.db.read().clone() + } + + pub(crate) fn metrics(&self) -> &ReferenceIndexMetrics { + &self.shared.metrics + } + + #[cfg(test)] + pub(crate) fn generation(&self) -> u64 { + self.shared.generation.load(Ordering::Acquire) } - /// Execute a paginated reference query. + /// Execute a paginated query at an exact canonical chain snapshot. /// - /// `canonical_tip` is the current best block number, used to compute lag. - pub fn query( + /// Cursor validation and result iteration share one MDBX read transaction, + /// so callers never observe a cursor from one index commit and rows from + /// another. The RPC layer must additionally verify that its canonical tip + /// did not change while this method was running. + pub fn query_at( &self, query: ReferenceQuery, - canonical_tip: u64, + canonical_tip: CanonicalTip, ) -> Result, ReferenceIndexError> { - if !self.db.is_ready() { - return Err(ReferenceIndexError::Initializing); + let generation = loop { + let generation = self.shared.generation.load(Ordering::Acquire); + if generation.is_multiple_of(2) { + break generation; + } + std::hint::spin_loop(); + }; + match self.phase() { + ReferenceIndexPhase::Opening => return Err(ReferenceIndexError::Initializing), + ReferenceIndexPhase::PreJade => { + return if canonical_tip.timestamp < self.jade_timestamp { + Ok(Vec::new()) + } else { + Err(ReferenceIndexError::IndexBehind) + }; + } + ReferenceIndexPhase::Deferred | ReferenceIndexPhase::Backfill => { + return Err(ReferenceIndexError::IndexBehind); + } + ReferenceIndexPhase::Repairing | ReferenceIndexPhase::Unavailable => { + return Err(ReferenceIndexError::IndexUnavailable); + } + ReferenceIndexPhase::Live => {} } - let indexed_to = self.db.indexed_to()?; - if canonical_tip.saturating_sub(indexed_to) > self.lag_threshold { + let db = self.db().ok_or(ReferenceIndexError::IndexUnavailable)?; + let tx = db.tx()?; + let indexed_to = tx + .get::(IndexMetaKey::IndexedTo.into())? + .map(decode_u64) + .transpose()?; + let Some(indexed_to) = indexed_to else { + return Err(ReferenceIndexError::IndexUnavailable); + }; + + let indexed_hash = tx + .get::(IndexedBlockKey { + block_number: indexed_to, + })? + .map(|value| value.0); + let Some(indexed_hash) = indexed_hash else { + return Err(ReferenceIndexError::IndexUnavailable); + }; + if indexed_to != canonical_tip.number || indexed_hash != canonical_tip.hash { return Err(ReferenceIndexError::IndexBehind); } - let tx = self.db.tx()?; let mut cursor = tx.cursor_read::()?; - let seek_key = ReferenceIndexKey { reference: query.reference, block_number: 0, @@ -53,12 +188,8 @@ impl ReferenceIndexReader { let mut skipped = 0u64; let mut results = Vec::new(); let mut next = cursor.seek(seek_key)?; - while let Some((key, value)) = next { - if key.reference != query.reference { - break; - } - if results.len() as u64 >= query.limit { + if key.reference != query.reference || results.len() as u64 >= query.limit { break; } if skipped < query.offset { @@ -74,6 +205,12 @@ impl ReferenceIndexReader { next = cursor.next()?; } + if self.shared.generation.load(Ordering::Acquire) != generation + || self.phase() != ReferenceIndexPhase::Live + { + return Err(ReferenceIndexError::IndexUnavailable); + } + Ok(results) } } @@ -82,61 +219,164 @@ impl ReferenceIndexReader { mod tests { use super::*; use crate::{ - ReferenceIndexDb, + db::ReferenceIndexDb, writer::{update_indexed_to, write_block}, }; use alloy_primitives::B256; use tempfile::TempDir; - fn open_ready_db() -> (TempDir, ReferenceIndexDb) { + fn handle_with_phase( + db: ReferenceIndexDb, + jade_timestamp: u64, + phase: ReferenceIndexPhase, + ) -> ReferenceIndexHandle { + let handle = ReferenceIndexHandle::new(jade_timestamp); + handle.install_db(db); + handle.set_phase(phase); + handle + } + + #[test] + fn query_returns_initializing_while_runtime_is_opening() { + let handle = ReferenceIndexHandle::new(200); + let query = ReferenceQuery::new(B256::ZERO, None, None).unwrap(); + let tip = CanonicalTip { + number: 0, + hash: B256::ZERO, + timestamp: 0, + }; + + assert!(matches!( + handle.query_at(query, tip), + Err(ReferenceIndexError::Initializing) + )); + } + + #[test] + fn setting_the_same_phase_does_not_invalidate_readers() { + let handle = ReferenceIndexHandle::new(200); + let generation = handle.shared.generation.load(Ordering::Acquire); + + handle.set_phase(ReferenceIndexPhase::Opening); + + assert_eq!(handle.shared.generation.load(Ordering::Acquire), generation); + } + + #[test] + fn query_returns_index_behind_when_tip_is_one_block_ahead() { let dir = TempDir::new().unwrap(); let db = ReferenceIndexDb::open(dir.path(), 2818, B256::ZERO).unwrap(); - db.set_ready(true); - (dir, db) + let tx = db.tx_mut().unwrap(); + write_block(&tx, 10, B256::repeat_byte(0xaa), 100, &[]).unwrap(); + update_indexed_to(&tx, 10).unwrap(); + tx.commit().unwrap(); + let handle = handle_with_phase(db, 0, ReferenceIndexPhase::Live); + let query = ReferenceQuery::new(B256::with_last_byte(1), None, None).unwrap(); + let tip = CanonicalTip { + number: 11, + hash: B256::repeat_byte(0xbb), + timestamp: 101, + }; + + assert!(matches!( + handle.query_at(query, tip), + Err(ReferenceIndexError::IndexBehind) + )); } #[test] - fn query_returns_initializing_when_not_ready() { + fn query_at_rejects_a_non_canonical_cursor_hash() { let dir = TempDir::new().unwrap(); let db = ReferenceIndexDb::open(dir.path(), 2818, B256::ZERO).unwrap(); - // is_ready stays false - let reader = ReferenceIndexReader::new(db, 16); - let q = ReferenceQuery::new(B256::ZERO, None, None).unwrap(); + let tx = db.tx_mut().unwrap(); + write_block(&tx, 10, B256::repeat_byte(0xaa), 100, &[]).unwrap(); + update_indexed_to(&tx, 10).unwrap(); + tx.commit().unwrap(); + let handle = handle_with_phase(db, 0, ReferenceIndexPhase::Live); + let query = ReferenceQuery::new(B256::with_last_byte(1), None, None).unwrap(); + let tip = CanonicalTip { + number: 10, + hash: B256::repeat_byte(0xbb), + timestamp: 100, + }; + assert!(matches!( - reader.query(q, 100), - Err(ReferenceIndexError::Initializing) + handle.query_at(query, tip), + Err(ReferenceIndexError::IndexBehind) )); } #[test] - fn query_returns_index_behind_when_lag_exceeds_threshold() { - let (_dir, db) = open_ready_db(); + fn live_query_treats_a_missing_cursor_as_unavailable() { + let dir = TempDir::new().unwrap(); + let db = ReferenceIndexDb::open(dir.path(), 2818, B256::ZERO).unwrap(); + let handle = handle_with_phase(db, 0, ReferenceIndexPhase::Live); + let query = ReferenceQuery::new(B256::ZERO, None, None).unwrap(); + let tip = CanonicalTip { + number: 1, + hash: B256::repeat_byte(0x11), + timestamp: 100, + }; + + assert!(matches!( + handle.query_at(query, tip), + Err(ReferenceIndexError::IndexUnavailable) + )); + } + #[test] + fn live_query_treats_a_missing_cursor_hash_as_unavailable() { + let dir = TempDir::new().unwrap(); + let db = ReferenceIndexDb::open(dir.path(), 2818, B256::ZERO).unwrap(); let tx = db.tx_mut().unwrap(); - update_indexed_to(&tx, 10).unwrap(); + update_indexed_to(&tx, 1).unwrap(); tx.commit().unwrap(); + let handle = handle_with_phase(db, 0, ReferenceIndexPhase::Live); + let query = ReferenceQuery::new(B256::ZERO, None, None).unwrap(); + let tip = CanonicalTip { + number: 1, + hash: B256::repeat_byte(0x11), + timestamp: 100, + }; - let reader = ReferenceIndexReader::new(db, 16); - let q = ReferenceQuery::new(B256::with_last_byte(1), None, None).unwrap(); - // canonical_tip=27 → lag=17 > threshold=16 assert!(matches!( - reader.query(q, 27), - Err(ReferenceIndexError::IndexBehind) + handle.query_at(query, tip), + Err(ReferenceIndexError::IndexUnavailable) )); } #[test] - fn query_returns_empty_when_no_reference_txs() { - let (_dir, db) = open_ready_db(); + fn pre_jade_query_is_complete_without_a_cursor() { + let dir = TempDir::new().unwrap(); + let db = ReferenceIndexDb::open(dir.path(), 2818, B256::ZERO).unwrap(); + let handle = handle_with_phase(db, 200, ReferenceIndexPhase::PreJade); + let query = ReferenceQuery::new(B256::with_last_byte(1), None, None).unwrap(); + let tip = CanonicalTip { + number: 100, + hash: B256::repeat_byte(0xaa), + timestamp: 199, + }; + assert!(handle.query_at(query, tip).unwrap().is_empty()); + } + + #[test] + fn live_query_returns_empty_when_no_reference_txs_exist() { + let dir = TempDir::new().unwrap(); + let db = ReferenceIndexDb::open(dir.path(), 2818, B256::ZERO).unwrap(); + let block_hash = B256::repeat_byte(0x01); let tx = db.tx_mut().unwrap(); - write_block(&tx, 1, B256::repeat_byte(0x01), 100, &[]).unwrap(); + write_block(&tx, 1, block_hash, 100, &[]).unwrap(); update_indexed_to(&tx, 1).unwrap(); tx.commit().unwrap(); + let handle = handle_with_phase(db, 0, ReferenceIndexPhase::Live); + let query = ReferenceQuery::new(B256::with_last_byte(0x42), None, None).unwrap(); + let tip = CanonicalTip { + number: 1, + hash: block_hash, + timestamp: 100, + }; - let reader = ReferenceIndexReader::new(db, 16); - let q = ReferenceQuery::new(B256::with_last_byte(0x42), None, None).unwrap(); - let results = reader.query(q, 1).unwrap(); - assert!(results.is_empty()); + assert!(handle.query_at(query, tip).unwrap().is_empty()); } } diff --git a/crates/reference-index/src/reconcile.rs b/crates/reference-index/src/reconcile.rs index 204cce62..55591de5 100644 --- a/crates/reference-index/src/reconcile.rs +++ b/crates/reference-index/src/reconcile.rs @@ -1,132 +1,48 @@ -//! Startup canonical chain reconciliation. -//! -//! After backfill completes, reconcile checks the last `max_reorg_depth` -//! indexed blocks against the current canonical chain to detect any reorgs -//! that occurred while the node was offline, then fills any suffix gap between -//! `indexed_to` and `head_at_startup`. +//! Bounded rollback helpers for canonical-chain reconciliation. use crate::{ - ReferenceIndexDb, + db::ReferenceIndexDb, types::ReferenceIndexError, - writer::{delete_block, update_indexed_to, write_block}, + writer::{clear_indexed_to, delete_block, update_indexed_to}, }; -use alloy_consensus::BlockHeader; -use morph_primitives::MorphHeader; use reth_db_api::transaction::DbTx; -use reth_provider::{BlockHashReader, BlockReader, HeaderProvider}; -use reth_storage_api::TransactionVariant; -use tracing::{debug, info}; -const TARGET: &str = "morph::reference_index"; - -/// Run the startup reconciliation pass. -/// -/// Steps: -/// A. Canonical hash check over the last `max_reorg_depth` indexed blocks. -/// On mismatch, find the lowest diverging height, delete forward entries, -/// reset `indexed_to`, and continue as if filling a gap. -/// B. Suffix gap fill: write every block from `indexed_to + 1` to -/// `current_canonical_head` into the three index tables. -pub fn run_startup_reconcile

( +/// Delete the indexed suffix after `ancestor` using bounded write transactions. +pub(crate) fn rollback_suffix( db: &ReferenceIndexDb, - provider: &P, - current_head: u64, - max_reorg_depth: u64, -) -> Result<(), ReferenceIndexError> -where - P: BlockReader - + HeaderProvider

- + BlockHashReader, -{ - let indexed_to = db.indexed_to()?; - - // ── Step A: canonical hash check ──────────────────────────────────────── - // Use indexed_from as lower bound to avoid scanning blocks that were never - // indexed (e.g. pre-Jade blocks on the sentinel path where backfill sets - // indexed_to but writes no IndexedBlocks entries). - let indexed_from = db.indexed_from()?.unwrap_or(indexed_to); - let depth_start = indexed_to.saturating_sub(max_reorg_depth.saturating_sub(1)); - let check_start = indexed_from.max(depth_start); - let mut fork_height: Option = None; - - for number in check_start..=indexed_to { - let indexed_hash = db.indexed_block_hash(number)?; + ancestor: Option, + batch_size: u64, +) -> Result<(), ReferenceIndexError> { + if batch_size == 0 { + return Err(ReferenceIndexError::InvalidBackfillBatchSize); + } - // `None` means the block was never written to IndexedBlocks (e.g. the - // sentinel path where backfill marks indexed_to = head but writes no - // IndexedBlocks entries for pre-Jade blocks). Treat as "not indexed, - // skip" rather than a hash mismatch to avoid spurious reorg detection. - let Some(indexed) = indexed_hash else { - continue; + loop { + let Some(cursor) = db.indexed_to()? else { + return Ok(()); }; - - let canonical_hash = provider.block_hash(number)?; - if Some(indexed) != canonical_hash { - debug!( - target: TARGET, - number, - ?indexed, - ?canonical_hash, - "canonical hash mismatch during reconcile" - ); - fork_height = Some(number); - break; + if ancestor.is_some_and(|ancestor| cursor <= ancestor) { + return Ok(()); } - } - - // ── Step B: apply reorg if detected ────────────────────────────────────── - let rebuild_start = if let Some(fh) = fork_height { - info!( - target: TARGET, - fork_height = fh, - old_indexed_to = indexed_to, - "offline reorg detected; rolling back index" - ); + let lower_bound = ancestor.map_or(0, |ancestor| ancestor.saturating_add(1)); + let batch_start = cursor + .saturating_sub(batch_size.saturating_sub(1)) + .max(lower_bound); let tx = db.tx_mut()?; - for number in fh..=indexed_to { + for number in batch_start..=cursor { delete_block(&tx, number)?; } - let new_indexed_to = fh.saturating_sub(1); - update_indexed_to(&tx, new_indexed_to)?; - tx.commit()?; - - fh - } else { - indexed_to.saturating_add(1) - }; - - // ── Step C: suffix gap fill ─────────────────────────────────────────────── - if rebuild_start <= current_head { - info!( - target: TARGET, - rebuild_start, - current_head, - "filling reference index suffix gap" - ); - let tx = db.tx_mut()?; - for number in rebuild_start..=current_head { - // `WithHash` is required: we persist tx hashes as part of the index keys. - let block = provider - .sealed_block_with_senders(number.into(), TransactionVariant::WithHash)? - .ok_or_else(|| { - ReferenceIndexError::Other(eyre::eyre!( - "missing block {number} during reconcile" - )) - })?; - - write_block( - &tx, - block.number(), - block.hash(), - block.timestamp(), - &block.body().transactions, - )?; + if batch_start == lower_bound { + if let Some(ancestor) = ancestor { + update_indexed_to(&tx, ancestor)?; + } else { + clear_indexed_to(&tx)?; + } + } else { + update_indexed_to(&tx, batch_start - 1)?; } - update_indexed_to(&tx, current_head)?; tx.commit()?; } - - Ok(()) } diff --git a/crates/reference-index/src/runtime.rs b/crates/reference-index/src/runtime.rs new file mode 100644 index 00000000..112cc71a --- /dev/null +++ b/crates/reference-index/src/runtime.rs @@ -0,0 +1,787 @@ +//! Background canonical-cursor runtime for the Morph transaction reference index. + +use crate::{ + CanonicalChain, DEFAULT_BACKFILL_BATCH_BLOCKS, LIVE_LAG_SLO, ReferenceIndexError, + ReferenceIndexHandle, ReferenceIndexPhase, + backfill::{commit_canonical_batch, resolve_jade_first_block}, + db::ReferenceIndexDb, + reconcile::rollback_suffix, +}; +use alloy_primitives::B256; +use morph_primitives::MorphPrimitives; +use reth_chain_state::{CanonStateNotification, CanonStateNotifications}; +use reth_tasks::TaskExecutor; +use std::{ + collections::VecDeque, + path::{Path, PathBuf}, + time::{Duration, Instant}, +}; +use tokio::sync::broadcast::{ + Receiver, + error::{RecvError, TryRecvError}, +}; +use tokio::time::MissedTickBehavior; +use tracing::{debug, info, warn}; + +const RAPID_SYNC_BLOCK_THRESHOLD: u64 = 64; +const RAPID_SYNC_WINDOW: Duration = Duration::from_secs(2); +const RAPID_SYNC_QUIET_PERIOD: Duration = Duration::from_secs(5); +const RECONCILE_POLL_INTERVAL: Duration = Duration::from_secs(1); +const MAX_RETRY_BACKOFF: Duration = Duration::from_secs(30); +const TARGET: &str = "morph::reference_index"; + +#[derive(Debug)] +struct RapidSyncDetector { + observations: VecDeque<(Instant, u64)>, + blocks_in_window: u64, + defer_until: Option, +} + +impl RapidSyncDetector { + fn new() -> Self { + Self { + observations: VecDeque::new(), + blocks_in_window: 0, + defer_until: None, + } + } + + fn record_blocks(&mut self, now: Instant, blocks: usize) { + if blocks == 0 { + return; + } + while self.observations.front().is_some_and(|(observed_at, _)| { + now.saturating_duration_since(*observed_at) > RAPID_SYNC_WINDOW + }) { + let (_, expired_blocks) = self.observations.pop_front().expect("front was present"); + self.blocks_in_window = self.blocks_in_window.saturating_sub(expired_blocks); + } + let already_deferred = self.should_defer(now); + self.observations.push_back((now, blocks as u64)); + self.blocks_in_window = self.blocks_in_window.saturating_add(blocks as u64); + if already_deferred || self.blocks_in_window > RAPID_SYNC_BLOCK_THRESHOLD { + self.defer_until = Some(now + RAPID_SYNC_QUIET_PERIOD); + } + } + + fn record_lagged(&mut self, now: Instant) { + self.defer_until = Some(now + RAPID_SYNC_QUIET_PERIOD); + } + + fn should_defer(&self, now: Instant) -> bool { + self.defer_until.is_some_and(|until| now < until) + } +} + +#[derive(Debug)] +enum PendingBroadcastEvent { + Item(T), + Lagged(u64), +} + +/// Drain notifications already queued before starting another blocking batch. +fn drain_ready_broadcast( + receiver: &mut Receiver, + mut on_event: impl FnMut(PendingBroadcastEvent), +) -> bool { + loop { + match receiver.try_recv() { + Ok(item) => on_event(PendingBroadcastEvent::Item(item)), + Err(TryRecvError::Lagged(skipped)) => { + on_event(PendingBroadcastEvent::Lagged(skipped)); + } + Err(TryRecvError::Empty) => return true, + Err(TryRecvError::Closed) => return false, + } + } +} + +fn observe_canonical_notification( + notification: CanonStateNotification, + detector: &mut RapidSyncDetector, +) { + let changed_blocks = + notification.committed().len() + notification.reverted().map_or(0, |chain| chain.len()); + detector.record_blocks(Instant::now(), changed_blocks); + debug!(target: TARGET, changed_blocks, "canonical notification woke reference index"); +} + +fn observe_lagged_notification( + skipped: u64, + detector: &mut RapidSyncDetector, + handle: &ReferenceIndexHandle, +) { + handle.metrics().lagged_notifications_total.increment(1); + detector.record_lagged(Instant::now()); + warn!(target: TARGET, skipped, "reference index lagged canonical notifications; reconciling from provider"); +} + +/// Immutable runtime configuration derived from the node configuration. +#[derive(Debug, Clone)] +pub struct ReferenceIndexConfig { + path: PathBuf, + chain_id: u64, + genesis_hash: B256, + jade_timestamp: u64, +} + +impl ReferenceIndexConfig { + /// Create the immutable identity and activation configuration for one index DB. + pub fn new( + path: impl AsRef, + chain_id: u64, + genesis_hash: B256, + jade_timestamp: u64, + ) -> Self { + Self { + path: path.as_ref().to_path_buf(), + chain_id, + genesis_hash, + jade_timestamp, + } + } +} + +/// Single-writer runtime that reconciles a durable cursor with the canonical chain. +pub struct ReferenceIndexRuntime { + config: ReferenceIndexConfig, + chain: C, + handle: ReferenceIndexHandle, +} + +impl ReferenceIndexRuntime { + /// Create a single-writer runtime and its cloneable RPC query handle. + pub fn new(config: ReferenceIndexConfig, chain: C) -> (Self, ReferenceIndexHandle) { + let handle = ReferenceIndexHandle::new(config.jade_timestamp); + ( + Self { + config, + chain, + handle: handle.clone(), + }, + handle, + ) + } + + fn open_db(&self) -> Result { + if let Some(db) = self.handle.db() { + return Ok(db); + } + let db = ReferenceIndexDb::open( + &self.config.path, + self.config.chain_id, + self.config.genesis_hash, + )?; + self.handle.install_db(db.clone()); + Ok(db) + } + + fn reconcile_cursor( + &self, + db: &ReferenceIndexDb, + jade_first_block: u64, + head: crate::CanonicalTip, + ) -> Result<(), ReferenceIndexError> { + let Some(cursor) = db.indexed_to()? else { + return Ok(()); + }; + + let candidate = cursor.min(head.number); + let indexed_hash = db.indexed_block_hash(candidate)?.ok_or_else(|| { + ReferenceIndexError::CorruptMetadata( + "durable cursor range is missing an IndexedBlocks hash", + ) + })?; + if cursor == head.number && indexed_hash == head.hash { + return Ok(()); + } + if cursor < head.number && self.chain.canonical_hash(cursor)? == Some(indexed_hash) { + return Ok(()); + } + + self.handle.set_phase(ReferenceIndexPhase::Repairing); + self.handle.metrics().reorgs_total.increment(1); + let mut ancestor = None; + let mut number = candidate; + loop { + let indexed_hash = db.indexed_block_hash(number)?.ok_or_else(|| { + ReferenceIndexError::CorruptMetadata( + "indexed canonical range contains a block-hash gap", + ) + })?; + if self.chain.canonical_hash(number)? == Some(indexed_hash) { + ancestor = Some(number); + break; + } + if number == jade_first_block { + break; + } + number -= 1; + } + + let ancestor = + ancestor.ok_or(ReferenceIndexError::ManualRebuildRequired { jade_first_block })?; + rollback_suffix(db, Some(ancestor), DEFAULT_BACKFILL_BATCH_BLOCKS) + } + + /// Perform one synchronous reconciliation/backfill turn. + pub fn synchronize_once(&mut self, defer: bool) -> Result<(), ReferenceIndexError> { + let started_at = Instant::now(); + let result = self.synchronize_once_inner(defer); + self.handle + .metrics() + .batch_duration_seconds + .record(started_at.elapsed()); + if result.is_err() { + self.handle.metrics().failures_total.increment(1); + self.handle.set_phase(ReferenceIndexPhase::Unavailable); + } + result + } + + fn synchronize_once_inner(&mut self, defer: bool) -> Result<(), ReferenceIndexError> { + let db = self.open_db()?; + let head = self.chain.head()?; + self.handle.metrics().target_block.set(head.number as f64); + if head.timestamp < self.config.jade_timestamp { + self.handle.metrics().indexed_block.set(0.0); + self.handle.metrics().lag_blocks.set(0.0); + self.handle.set_phase(ReferenceIndexPhase::PreJade); + return Ok(()); + } + if defer { + let indexed_to = db.indexed_to()?.unwrap_or_default(); + self.handle.metrics().indexed_block.set(indexed_to as f64); + self.handle + .metrics() + .lag_blocks + .set(head.number.saturating_sub(indexed_to) as f64); + self.handle.set_phase(ReferenceIndexPhase::Deferred); + return Ok(()); + } + + let jade_first_block = match db.jade_first_block_number()? { + Some(block) => block, + None => resolve_jade_first_block(&self.chain, head, self.config.jade_timestamp)? + .ok_or_else(|| { + ReferenceIndexError::Other(eyre::eyre!( + "Jade is active at the canonical head but its first block was not found" + )) + })?, + }; + self.reconcile_cursor(&db, jade_first_block, head)?; + let start = db + .indexed_to()? + .map_or(jade_first_block, |cursor| cursor.saturating_add(1)); + if start > head.number { + self.handle.metrics().indexed_block.set(head.number as f64); + self.handle.metrics().lag_blocks.set(0.0); + self.handle.set_phase(ReferenceIndexPhase::Live); + return Ok(()); + } + + self.handle.set_phase(ReferenceIndexPhase::Backfill); + let end = start + .saturating_add(DEFAULT_BACKFILL_BATCH_BLOCKS - 1) + .min(head.number); + let blocks = self.chain.canonical_blocks(start..=end)?; + let expected_len = end.saturating_sub(start).saturating_add(1) as usize; + if blocks.len() != expected_len { + return Err(ReferenceIndexError::Other(eyre::eyre!( + "canonical block range {start}..={end} returned {} of {expected_len} blocks", + blocks.len() + ))); + } + for (offset, block) in blocks.iter().enumerate() { + let expected_number = start + offset as u64; + let canonical_hash = self.chain.canonical_hash(expected_number)?; + if block.number != expected_number || canonical_hash != Some(block.hash) { + return Err(ReferenceIndexError::Other(eyre::eyre!( + "canonical block changed while reading backfill batch at {expected_number}" + ))); + } + } + commit_canonical_batch(&db, jade_first_block, start, &blocks)?; + self.handle + .metrics() + .indexed_blocks_total + .increment(blocks.len() as u64); + self.handle.metrics().committed_batches_total.increment(1); + + let current_head = self.chain.head()?; + self.handle + .metrics() + .target_block + .set(current_head.number as f64); + self.handle.metrics().indexed_block.set(end as f64); + let lag = current_head.number.saturating_sub(end); + self.handle.metrics().lag_blocks.set(lag as f64); + if lag > LIVE_LAG_SLO { + debug!(target: TARGET, lag, slo = LIVE_LAG_SLO, "reference index remains behind its live-lag SLO"); + } + if end == current_head.number + && blocks + .last() + .is_some_and(|block| block.hash == current_head.hash) + { + self.handle.set_phase(ReferenceIndexPhase::Live); + } + Ok(()) + } + + /// Run the non-critical background reconciler until node shutdown. + /// + /// Notifications are wake-ups and rate signals only. Canonical data is + /// always re-read through [`CanonicalChain`] before a durable commit. + pub async fn run( + mut self, + task_executor: TaskExecutor, + mut notifications: CanonStateNotifications, + ) { + let handle = self.handle.clone(); + let mut poll = tokio::time::interval(RECONCILE_POLL_INTERVAL); + poll.set_missed_tick_behavior(MissedTickBehavior::Skip); + // Consume tokio's immediate first tick so a queued canonical + // notification can classify rapid sync before the first DB batch. + poll.tick().await; + let mut detector = RapidSyncDetector::new(); + let mut notifications_open = true; + let mut sync_requested = false; + let mut retry_backoff = Duration::from_secs(1); + let mut logged_phase = handle.phase(); + + loop { + if notifications_open { + notifications_open = drain_ready_broadcast(&mut notifications, |event| { + sync_requested = true; + match event { + PendingBroadcastEvent::Item(notification) => { + observe_canonical_notification(notification, &mut detector); + } + PendingBroadcastEvent::Lagged(skipped) => { + observe_lagged_notification(skipped, &mut detector, &handle); + } + } + }); + if !notifications_open { + warn!(target: TARGET, "canonical notification channel closed; reference index falling back to polling"); + } + } + if sync_requested + && detector.should_defer(Instant::now()) + && handle.phase() == ReferenceIndexPhase::Deferred + { + // Stay completely off the index DB while rapid historical + // sync is still active. Notifications continue to extend the + // quiet-period deadline; polling resumes catch-up afterwards. + sync_requested = false; + } + if sync_requested { + let defer = detector.should_defer(Instant::now()); + let job = task_executor.spawn_blocking(move || { + let result = self.synchronize_once(defer); + (self, result) + }); + let (returned_runtime, result) = match job.await { + Ok(output) => output, + Err(error) if error.is_cancelled() => { + debug!(target: TARGET, "reference index blocking task cancelled during shutdown"); + return; + } + Err(error) => { + handle.set_phase(ReferenceIndexPhase::Unavailable); + warn!(target: TARGET, %error, "reference index blocking task stopped unexpectedly"); + return; + } + }; + self = returned_runtime; + + match result { + Ok(()) => { + retry_backoff = Duration::from_secs(1); + let phase = handle.phase(); + if phase != logged_phase { + info!(target: TARGET, ?phase, "reference index phase changed"); + logged_phase = phase; + } + sync_requested = phase == ReferenceIndexPhase::Backfill; + if sync_requested { + tokio::task::yield_now().await; + // Loop back through the non-blocking receiver drain before + // starting another bounded backfill batch. + continue; + } + } + Err(error) => { + if error.requires_manual_rebuild() { + warn!(target: TARGET, %error, "reference index requires manual rebuild; execution continues"); + return; + } + warn!(target: TARGET, %error, ?retry_backoff, "reference index reconciliation failed; execution continues"); + tokio::time::sleep(retry_backoff).await; + retry_backoff = retry_backoff.saturating_mul(2).min(MAX_RETRY_BACKOFF); + sync_requested = true; + continue; + } + } + } + + tokio::select! { + biased; + notification = notifications.recv(), if notifications_open => { + sync_requested = true; + match notification { + Ok(notification) => { + observe_canonical_notification(notification, &mut detector); + } + Err(RecvError::Lagged(skipped)) => { + observe_lagged_notification(skipped, &mut detector, &handle); + } + Err(RecvError::Closed) => { + notifications_open = false; + warn!(target: TARGET, "canonical notification channel closed; reference index falling back to polling"); + } + } + } + _ = poll.tick() => { + sync_requested = true; + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{CanonicalBlock, CanonicalChain, CanonicalTip, ReferenceQuery}; + use alloy_primitives::B256; + use parking_lot::RwLock; + use std::{ + ops::RangeInclusive, + sync::Arc, + time::{Duration, Instant}, + }; + + #[test] + fn rapid_sync_detector_defers_after_more_than_64_blocks_and_recovers_after_quiet_period() { + let start = Instant::now(); + let mut detector = RapidSyncDetector::new(); + + detector.record_blocks(start + Duration::from_secs(1), 64); + assert!(!detector.should_defer(start + Duration::from_secs(1))); + + detector.record_blocks(start + Duration::from_millis(1500), 1); + assert!(detector.should_defer(start + Duration::from_secs(4))); + assert!(!detector.should_defer(start + Duration::from_millis(6501))); + } + + #[test] + fn rapid_sync_detector_uses_a_sliding_two_second_window() { + let start = Instant::now(); + let mut detector = RapidSyncDetector::new(); + + detector.record_blocks(start + Duration::from_millis(1900), 32); + detector.record_blocks(start + Duration::from_millis(2100), 33); + + assert!(detector.should_defer(start + Duration::from_secs(3))); + } + + #[test] + fn deferred_sync_waits_five_seconds_after_low_rate_tail_activity() { + let start = Instant::now(); + let mut detector = RapidSyncDetector::new(); + + detector.record_blocks(start, 65); + detector.record_blocks(start + Duration::from_secs(3), 1); + + assert!(detector.should_defer(start + Duration::from_millis(7999))); + assert!(!detector.should_defer(start + Duration::from_millis(8001))); + } + + #[test] + fn lagged_notifications_enter_the_same_quiet_period() { + let start = Instant::now(); + let mut detector = RapidSyncDetector::new(); + + detector.record_lagged(start); + + assert!(detector.should_defer(start + Duration::from_secs(4))); + assert!(!detector.should_defer(start + Duration::from_millis(5001))); + } + + #[test] + fn queued_notifications_are_drained_before_the_next_batch() { + let (sender, mut receiver) = tokio::sync::broadcast::channel(128); + for _ in 0..65 { + sender.send(1usize).unwrap(); + } + let now = Instant::now(); + let mut detector = RapidSyncDetector::new(); + let mut received = 0usize; + + let remains_open = drain_ready_broadcast(&mut receiver, |event| match event { + PendingBroadcastEvent::Item(blocks) => { + received += 1; + detector.record_blocks(now, blocks); + } + PendingBroadcastEvent::Lagged(_) => panic!("receiver should not lag"), + }); + + assert!(remains_open); + assert_eq!(received, 65); + assert!(detector.should_defer(now)); + } + + #[derive(Clone, Debug)] + struct TestChain { + blocks: Arc>>, + } + + impl TestChain { + fn linear(count: u64, first_timestamp: u64) -> Self { + let blocks = (0..count) + .map(|number| CanonicalBlock { + number, + hash: B256::with_last_byte(number as u8), + timestamp: first_timestamp + number, + transactions: Vec::new(), + }) + .collect(); + Self { + blocks: Arc::new(RwLock::new(blocks)), + } + } + + fn replace_suffix(&self, from: u64) { + for block in self.blocks.write().iter_mut().skip(from as usize) { + block.hash = B256::repeat_byte(0x80 | block.number as u8); + } + } + + fn truncate(&self, len: u64) { + self.blocks.write().truncate(len as usize); + } + } + + impl CanonicalChain for TestChain { + fn head(&self) -> Result { + let blocks = self.blocks.read(); + let block = blocks.last().unwrap(); + Ok(CanonicalTip { + number: block.number, + hash: block.hash, + timestamp: block.timestamp, + }) + } + + fn canonical_hash(&self, number: u64) -> Result, crate::ReferenceIndexError> { + Ok(self + .blocks + .read() + .get(number as usize) + .map(|block| block.hash)) + } + + fn block_timestamp(&self, number: u64) -> Result, crate::ReferenceIndexError> { + Ok(self + .blocks + .read() + .get(number as usize) + .map(|block| block.timestamp)) + } + + fn canonical_blocks( + &self, + range: RangeInclusive, + ) -> Result, crate::ReferenceIndexError> { + let blocks = self.blocks.read(); + Ok(range + .filter_map(|number| blocks.get(number as usize).cloned()) + .collect()) + } + } + + #[test] + fn pre_jade_sync_serves_empty_results_without_a_cursor() { + let dir = tempfile::tempdir().unwrap(); + let chain = TestChain::linear(10, 100); + let config = ReferenceIndexConfig::new(dir.path(), 2818, B256::ZERO, 200); + let (mut runtime, handle) = ReferenceIndexRuntime::new(config, chain.clone()); + + runtime.synchronize_once(false).unwrap(); + + assert_eq!(handle.phase(), crate::ReferenceIndexPhase::PreJade); + let query = ReferenceQuery::new(B256::with_last_byte(0xaa), None, None).unwrap(); + assert!( + handle + .query_at(query, chain.head().unwrap()) + .unwrap() + .is_empty() + ); + assert_eq!(handle.db().unwrap().indexed_to().unwrap(), None); + } + + #[test] + fn post_jade_backfill_commits_at_most_512_blocks_per_turn() { + let dir = tempfile::tempdir().unwrap(); + let chain = TestChain::linear(600, 200); + let config = ReferenceIndexConfig::new(dir.path(), 2818, B256::ZERO, 200); + let (mut runtime, handle) = ReferenceIndexRuntime::new(config, chain.clone()); + + runtime.synchronize_once(false).unwrap(); + assert_eq!(handle.phase(), crate::ReferenceIndexPhase::Backfill); + assert_eq!(handle.db().unwrap().indexed_to().unwrap(), Some(511)); + + runtime.synchronize_once(false).unwrap(); + assert_eq!(handle.phase(), crate::ReferenceIndexPhase::Live); + assert_eq!(handle.db().unwrap().indexed_to().unwrap(), Some(599)); + + let query = ReferenceQuery::new(B256::with_last_byte(0xaa), None, None).unwrap(); + assert!( + handle + .query_at(query, chain.head().unwrap()) + .unwrap() + .is_empty() + ); + } + + #[test] + fn an_idle_reconciliation_keeps_the_live_snapshot_stable() { + let dir = tempfile::tempdir().unwrap(); + let chain = TestChain::linear(10, 200); + let config = ReferenceIndexConfig::new(dir.path(), 2818, B256::ZERO, 200); + let (mut runtime, handle) = ReferenceIndexRuntime::new(config, chain); + runtime.synchronize_once(false).unwrap(); + let generation = handle.generation(); + + runtime.synchronize_once(false).unwrap(); + + assert_eq!(handle.phase(), ReferenceIndexPhase::Live); + assert_eq!(handle.generation(), generation); + } + + #[test] + fn reorged_suffix_is_rolled_back_and_replayed() { + let dir = tempfile::tempdir().unwrap(); + let chain = TestChain::linear(10, 200); + let config = ReferenceIndexConfig::new(dir.path(), 2818, B256::ZERO, 200); + let (mut runtime, handle) = ReferenceIndexRuntime::new(config, chain.clone()); + runtime.synchronize_once(false).unwrap(); + assert_eq!(handle.phase(), crate::ReferenceIndexPhase::Live); + + chain.replace_suffix(7); + runtime.synchronize_once(false).unwrap(); + + assert_eq!(handle.phase(), crate::ReferenceIndexPhase::Live); + let query = ReferenceQuery::new(B256::with_last_byte(0xaa), None, None).unwrap(); + assert!( + handle + .query_at(query, chain.head().unwrap()) + .unwrap() + .is_empty() + ); + assert_eq!( + handle.db().unwrap().indexed_block_hash(7).unwrap(), + chain.canonical_hash(7).unwrap() + ); + } + + #[test] + fn pure_revert_rolls_the_cursor_back_to_the_new_head() { + let dir = tempfile::tempdir().unwrap(); + let chain = TestChain::linear(10, 200); + let config = ReferenceIndexConfig::new(dir.path(), 2818, B256::ZERO, 200); + let (mut runtime, handle) = ReferenceIndexRuntime::new(config, chain.clone()); + runtime.synchronize_once(false).unwrap(); + + chain.truncate(7); + runtime.synchronize_once(false).unwrap(); + + assert_eq!(handle.phase(), ReferenceIndexPhase::Live); + assert_eq!(handle.db().unwrap().indexed_to().unwrap(), Some(6)); + assert_eq!( + handle.db().unwrap().indexed_block_hash(6).unwrap(), + chain.canonical_hash(6).unwrap() + ); + } + + #[test] + fn reorg_without_a_post_jade_ancestor_requires_a_manual_rebuild() { + let dir = tempfile::tempdir().unwrap(); + let chain = TestChain::linear(10, 200); + let config = ReferenceIndexConfig::new(dir.path(), 2818, B256::ZERO, 200); + let (mut runtime, handle) = ReferenceIndexRuntime::new(config, chain.clone()); + runtime.synchronize_once(false).unwrap(); + + chain.replace_suffix(0); + let error = runtime.synchronize_once(false).unwrap_err(); + + assert!(matches!( + error, + ReferenceIndexError::ManualRebuildRequired { + jade_first_block: 0 + } + )); + assert_eq!(handle.phase(), ReferenceIndexPhase::Unavailable); + assert!( + dir.path().exists(), + "runtime must never delete the index DB" + ); + assert_eq!(handle.db().unwrap().indexed_to().unwrap(), Some(9)); + } + + #[derive(Clone, Debug)] + struct ShortRangeChain(TestChain); + + impl CanonicalChain for ShortRangeChain { + fn head(&self) -> Result { + self.0.head() + } + + fn canonical_hash(&self, number: u64) -> Result, ReferenceIndexError> { + self.0.canonical_hash(number) + } + + fn block_timestamp(&self, number: u64) -> Result, ReferenceIndexError> { + self.0.block_timestamp(number) + } + + fn canonical_blocks( + &self, + range: RangeInclusive, + ) -> Result, ReferenceIndexError> { + let mut blocks = self.0.canonical_blocks(range)?; + blocks.pop(); + Ok(blocks) + } + } + + #[test] + fn incomplete_provider_batch_does_not_advance_the_cursor() { + let dir = tempfile::tempdir().unwrap(); + let chain = ShortRangeChain(TestChain::linear(10, 200)); + let config = ReferenceIndexConfig::new(dir.path(), 2818, B256::ZERO, 200); + let (mut runtime, handle) = ReferenceIndexRuntime::new(config, chain); + + let error = runtime.synchronize_once(false).unwrap_err(); + + assert!(error.to_string().contains("returned 9 of 10 blocks")); + assert_eq!(handle.phase(), ReferenceIndexPhase::Unavailable); + assert_eq!(handle.db().unwrap().indexed_to().unwrap(), None); + } + + #[test] + fn deferred_sync_performs_no_per_block_index_writes() { + let dir = tempfile::tempdir().unwrap(); + let chain = TestChain::linear(10, 200); + let config = ReferenceIndexConfig::new(dir.path(), 2818, B256::ZERO, 200); + let (mut runtime, handle) = ReferenceIndexRuntime::new(config, chain); + + runtime.synchronize_once(true).unwrap(); + + assert_eq!(handle.phase(), ReferenceIndexPhase::Deferred); + assert_eq!(handle.db().unwrap().indexed_to().unwrap(), None); + assert_eq!(handle.db().unwrap().indexed_block_hash(0).unwrap(), None); + } +} diff --git a/crates/reference-index/src/source.rs b/crates/reference-index/src/source.rs new file mode 100644 index 00000000..1040b756 --- /dev/null +++ b/crates/reference-index/src/source.rs @@ -0,0 +1,86 @@ +//! Canonical-chain read seam used by the reference-index runtime. + +use crate::{CanonicalTip, ReferenceIndexError}; +use alloy_consensus::{BlockHeader, Sealable}; +use alloy_primitives::B256; +use morph_primitives::MorphTxEnvelope; +use reth_provider::{BlockHashReader, BlockReader, BlockReaderIdExt, HeaderProvider}; +use std::ops::RangeInclusive; + +/// Canonical block fields required to build the reference index. +#[derive(Debug, Clone)] +pub struct CanonicalBlock { + /// Canonical block number. + pub number: u64, + /// Header hash observed for this canonical block. + pub hash: B256, + /// Header timestamp. + pub timestamp: u64, + /// Transactions in their canonical block order. + pub transactions: Vec, +} + +/// Minimal canonical-chain API consumed by the runtime. +pub trait CanonicalChain: Clone + Send + Sync + 'static { + /// Return one internally consistent canonical head snapshot. + fn head(&self) -> Result; + + /// Return the canonical hash at `number`, if the block is retained. + fn canonical_hash(&self, number: u64) -> Result, ReferenceIndexError>; + + /// Return the canonical timestamp at `number`, if the header is retained. + fn block_timestamp(&self, number: u64) -> Result, ReferenceIndexError>; + + /// Load an inclusive range of canonical blocks with their transaction bodies. + fn canonical_blocks( + &self, + range: RangeInclusive, + ) -> Result, ReferenceIndexError>; +} + +impl

CanonicalChain for P +where + P: BlockReaderIdExt + + Clone + + Send + + Sync + + 'static, +{ + fn head(&self) -> Result { + let header = BlockReaderIdExt::latest_header(self)? + .ok_or_else(|| ReferenceIndexError::Other(eyre::eyre!("missing canonical head")))?; + Ok(CanonicalTip { + number: header.number(), + hash: header.hash(), + timestamp: header.timestamp(), + }) + } + + fn canonical_hash(&self, number: u64) -> Result, ReferenceIndexError> { + Ok(BlockHashReader::block_hash(self, number)?) + } + + fn block_timestamp(&self, number: u64) -> Result, ReferenceIndexError> { + Ok(HeaderProvider::header_by_number(self, number)?.map(|header| header.timestamp())) + } + + fn canonical_blocks( + &self, + range: RangeInclusive, + ) -> Result, ReferenceIndexError> { + BlockReader::block_range(self, range)? + .into_iter() + .map(|block| { + let number = block.header.number(); + let timestamp = block.header.timestamp(); + let hash = block.header.hash_slow(); + Ok(CanonicalBlock { + number, + hash, + timestamp, + transactions: block.body.transactions, + }) + }) + .collect() + } +} diff --git a/crates/reference-index/src/tables.rs b/crates/reference-index/src/tables.rs index 43b6e586..4cd8bf88 100644 --- a/crates/reference-index/src/tables.rs +++ b/crates/reference-index/src/tables.rs @@ -1,5 +1,7 @@ //! Reference index table declarations. +#![allow(unreachable_pub)] // `tables!` generates public marker types inside this private module. + use alloy_primitives::B256; use reth_codecs::DecompressError; use reth_db_api::{ diff --git a/crates/reference-index/src/types.rs b/crates/reference-index/src/types.rs index e8a060f7..3bfa3a50 100644 --- a/crates/reference-index/src/types.rs +++ b/crates/reference-index/src/types.rs @@ -1,33 +1,22 @@ use alloy_primitives::{B256, U64}; use serde::{Deserialize, Serialize}; -/// Current reference index database schema version. -pub const SCHEMA_VERSION: u32 = 1; - -/// Stored Jade activation sentinel for chains where Jade has not activated. -pub const JADE_NOT_ACTIVE_SENTINEL: u64 = u64::MAX; - -/// Persistent backfill progress state. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[repr(u8)] -pub enum BackfillState { - NotStarted = 0, - InProgress = 1, - Complete = 2, +/// Canonical chain snapshot used to linearize reference-index queries. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CanonicalTip { + /// Canonical head block number. + pub number: u64, + /// Canonical head block hash. + pub hash: B256, + /// Canonical head timestamp. + pub timestamp: u64, } -impl TryFrom for BackfillState { - type Error = ReferenceIndexError; - - fn try_from(value: u8) -> Result { - match value { - 0 => Ok(Self::NotStarted), - 1 => Ok(Self::InProgress), - 2 => Ok(Self::Complete), - other => Err(ReferenceIndexError::InvalidBackfillState(other)), - } - } -} +/// Current reference index database schema version. +/// +/// Version 1 was used only by the pre-release ExEx prototype. There is no +/// migration path: operators remove that derived database manually. +pub const SCHEMA_VERSION: u32 = 2; /// Validated query parameters for reference lookups. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -38,10 +27,14 @@ pub struct ReferenceQuery { } impl ReferenceQuery { + /// Default page size. pub const DEFAULT_LIMIT: u64 = 100; + /// Maximum accepted page size. pub const MAX_LIMIT: u64 = 100; + /// Maximum accepted result offset. pub const MAX_OFFSET: u64 = 10_000; + /// Validate and normalize public RPC query arguments. pub fn new( reference: B256, offset: Option, @@ -69,9 +62,13 @@ impl ReferenceQuery { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct ReferenceTransactionResult { + /// Hash of the matching Morph transaction. pub transaction_hash: B256, + /// Canonical block number containing the transaction. pub block_number: U64, + /// Timestamp of the containing canonical block. pub block_timestamp: U64, + /// Zero-based transaction position within the block. pub transaction_index: U64, } @@ -82,18 +79,24 @@ pub enum ReferenceIndexError { Initializing, #[error("reference index is behind")] IndexBehind, + #[error("reference index is unavailable")] + IndexUnavailable, #[error("reference query limit too large: {limit}")] LimitTooLarge { limit: u64 }, #[error("reference query offset too large: {offset}")] OffsetTooLarge { offset: u64 }, - #[error("invalid backfill state: {0}")] - InvalidBackfillState(u8), #[error("reference index backfill batch size must be greater than zero")] InvalidBackfillBatchSize, + #[error("reference index metadata is incomplete: {0}")] + CorruptMetadata(&'static str), #[error("reference index chain identity mismatch: {0}")] ChainIdentityMismatch(&'static str), #[error("reference index schema mismatch: expected {expected}, got {actual}")] SchemaMismatch { expected: u32, actual: u32 }, + #[error( + "reference index has no canonical ancestor at or after Jade block {jade_first_block}; manual rebuild required" + )] + ManualRebuildRequired { jade_first_block: u64 }, #[error(transparent)] Database(#[from] reth_db_api::DatabaseError), #[error(transparent)] @@ -101,3 +104,16 @@ pub enum ReferenceIndexError { #[error(transparent)] Other(#[from] eyre::Report), } + +impl ReferenceIndexError { + /// Errors that cannot be repaired while the existing derived DB remains. + pub(crate) fn requires_manual_rebuild(&self) -> bool { + matches!( + self, + Self::CorruptMetadata(_) + | Self::ChainIdentityMismatch(_) + | Self::SchemaMismatch { .. } + | Self::ManualRebuildRequired { .. } + ) + } +} diff --git a/crates/reference-index/src/writer.rs b/crates/reference-index/src/writer.rs index 9ce37108..936a59fc 100644 --- a/crates/reference-index/src/writer.rs +++ b/crates/reference-index/src/writer.rs @@ -8,10 +8,9 @@ use crate::{ db::{IndexMetaKey, encode_u64}, tables::{ BlockHashValue, BlockReferenceIndex, BlockReferenceKey, BlockTimestampValue, IndexMeta, - IndexedBlockKey, IndexedBlocks, MetaValue, ReferenceIndex, ReferenceIndexKey, - ReferenceValue, + IndexedBlockKey, IndexedBlocks, ReferenceIndex, ReferenceIndexKey, ReferenceValue, }, - types::{BackfillState, ReferenceIndexError}, + types::ReferenceIndexError, }; use alloy_consensus::transaction::TxHashRef; use alloy_primitives::B256; @@ -26,7 +25,7 @@ use reth_db_api::{cursor::DbCursorRO, transaction::DbTxMut}; /// the same block number before re-writing to avoid leaving stale entries /// (keys contain `tx_hash`, so a re-write with a different tx set would leave /// old-tx ghost rows). Reconcile and reorg paths follow this contract. -pub fn write_block( +pub(crate) fn write_block( tx: &Tx, block_number: u64, block_hash: B256, @@ -69,7 +68,10 @@ pub fn write_block( /// Implements the reverse of [`write_block`]: reads every /// `BlockReferenceIndex` row for `block_number`, reconstructs each /// `ReferenceIndex` key, and removes entries from all three tables. -pub fn delete_block(tx: &Tx, block_number: u64) -> Result<(), ReferenceIndexError> { +pub(crate) fn delete_block( + tx: &Tx, + block_number: u64, +) -> Result<(), ReferenceIndexError> { // 1. Collect all BlockReferenceIndex rows for this block. let mut entries = Vec::new(); { @@ -110,7 +112,7 @@ pub fn delete_block(tx: &Tx, block_number: u64) -> Result<(), Refer // ── metadata writes ────────────────────────────────────────────────────────── -pub fn update_indexed_to( +pub(crate) fn update_indexed_to( tx: &Tx, block_number: u64, ) -> Result<(), ReferenceIndexError> { @@ -118,37 +120,12 @@ pub fn update_indexed_to( Ok(()) } -pub fn update_indexed_from( - tx: &Tx, - block_number: u64, -) -> Result<(), ReferenceIndexError> { - tx.put::(IndexMetaKey::IndexedFrom.into(), encode_u64(block_number))?; +pub(crate) fn clear_indexed_to(tx: &Tx) -> Result<(), ReferenceIndexError> { + tx.delete::(IndexMetaKey::IndexedTo.into(), None)?; Ok(()) } -pub fn update_backfill_current( - tx: &Tx, - block_number: u64, -) -> Result<(), ReferenceIndexError> { - tx.put::( - IndexMetaKey::BackfillCurrent.into(), - encode_u64(block_number), - )?; - Ok(()) -} - -pub fn set_backfill_state( - tx: &Tx, - state: BackfillState, -) -> Result<(), ReferenceIndexError> { - tx.put::( - IndexMetaKey::BackfillState.into(), - MetaValue(vec![state as u8]), - )?; - Ok(()) -} - -pub fn set_jade_first_block_number( +pub(crate) fn set_jade_first_block_number( tx: &Tx, block_number: u64, ) -> Result<(), ReferenceIndexError> { @@ -162,7 +139,7 @@ pub fn set_jade_first_block_number( #[cfg(test)] mod tests { use super::*; - use crate::ReferenceIndexDb; + use crate::db::ReferenceIndexDb; use alloy_consensus::transaction::TxEip1559; use alloy_primitives::{Address, Signature, TxKind, U256}; use morph_primitives::MorphTxEnvelope; @@ -209,7 +186,7 @@ mod tests { tx.commit().unwrap(); assert_eq!(written, 0); - assert_eq!(db.indexed_to().unwrap(), 42); + assert_eq!(db.indexed_to().unwrap(), Some(42)); assert_eq!( db.indexed_block_hash(42).unwrap(), Some(B256::repeat_byte(0xaa)) @@ -237,32 +214,32 @@ mod tests { } #[test] - fn metadata_updates_are_visible_after_commit() { + fn cursor_and_jade_activation_are_visible_after_commit() { let dir = TempDir::new().unwrap(); let db = ReferenceIndexDb::open(dir.path(), 2818, B256::ZERO).unwrap(); let tx = db.tx_mut().unwrap(); - update_indexed_from(&tx, 100).unwrap(); update_indexed_to(&tx, 200).unwrap(); - update_backfill_current(&tx, 150).unwrap(); - set_backfill_state(&tx, BackfillState::Complete).unwrap(); set_jade_first_block_number(&tx, 100).unwrap(); tx.commit().unwrap(); - assert_eq!(db.indexed_from().unwrap(), Some(100)); - assert_eq!(db.indexed_to().unwrap(), 200); - assert_eq!(db.backfill_state().unwrap(), BackfillState::Complete); + assert_eq!(db.indexed_to().unwrap(), Some(200)); assert_eq!(db.jade_first_block_number().unwrap(), Some(100)); + } - // Sanity: a fresh read transaction also sees the backfill_current we set. - let tx = db.tx().unwrap(); - let raw = tx - .get::(IndexMetaKey::BackfillCurrent.into()) - .unwrap() - .unwrap(); - assert_eq!( - u64::from_be_bytes(raw.0.as_slice().try_into().unwrap()), - 150 - ); + #[test] + fn clearing_cursor_restores_no_progress_state() { + let dir = TempDir::new().unwrap(); + let db = ReferenceIndexDb::open(dir.path(), 2818, B256::ZERO).unwrap(); + + let tx = db.tx_mut().unwrap(); + update_indexed_to(&tx, 7).unwrap(); + tx.commit().unwrap(); + assert_eq!(db.indexed_to().unwrap(), Some(7)); + + let tx = db.tx_mut().unwrap(); + clear_indexed_to(&tx).unwrap(); + tx.commit().unwrap(); + assert_eq!(db.indexed_to().unwrap(), None); } } diff --git a/crates/rpc/Cargo.toml b/crates/rpc/Cargo.toml index 07885dba..356a7cb7 100644 --- a/crates/rpc/Cargo.toml +++ b/crates/rpc/Cargo.toml @@ -61,6 +61,7 @@ tracing.workspace = true [dev-dependencies] serde_json.workspace = true +tempfile.workspace = true [features] default = [] diff --git a/crates/rpc/src/morph/handler.rs b/crates/rpc/src/morph/handler.rs index 6616ca0f..928ee90a 100644 --- a/crates/rpc/src/morph/handler.rs +++ b/crates/rpc/src/morph/handler.rs @@ -1,14 +1,16 @@ //! `MorphRpc` handler implementation. use crate::morph::rpc::{MorphRpcServer, ReferenceQueryArgs}; +use alloy_consensus::BlockHeader as _; use jsonrpsee::{ core::RpcResult, types::{ErrorCode, ErrorObjectOwned}, }; use morph_reference_index::{ - ReferenceIndexError, ReferenceIndexReader, ReferenceQuery, ReferenceTransactionResult, + CanonicalTip, ReferenceIndexError, ReferenceIndexHandle, ReferenceQuery, + ReferenceTransactionResult, }; -use reth_storage_api::BlockNumReader; +use reth_storage_api::{BlockNumReader, HeaderProvider}; use tracing; const TARGET: &str = "morph::reference_index_rpc"; @@ -17,16 +19,16 @@ const TARGET: &str = "morph::reference_index_rpc"; /// `morph_` namespace context. All dependencies are required; no `Option<>`. /// -/// `Provider` must implement [`BlockNumReader`] so the handler can compare the -/// current canonical tip against the index's `indexed_to` for lag detection. +/// `Provider` supplies an exact canonical tip snapshot (number, hash, and +/// timestamp) for strict index-read consistency. #[derive(Debug, Clone)] pub struct MorphRpc { - pub reference_index: ReferenceIndexReader, + pub reference_index: ReferenceIndexHandle, pub provider: Provider, } -impl MorphRpc { - pub const fn new(reference_index: ReferenceIndexReader, provider: Provider) -> Self { +impl MorphRpc { + pub const fn new(reference_index: ReferenceIndexHandle, provider: Provider) -> Self { Self { reference_index, provider, @@ -42,14 +44,15 @@ pub struct MorphRpcHandler { ctx: MorphRpc, } -impl MorphRpcHandler { +impl MorphRpcHandler { pub const fn new(ctx: MorphRpc) -> Self { Self { ctx } } } -impl MorphRpcServer - for MorphRpcHandler +impl MorphRpcServer for MorphRpcHandler +where + Provider: BlockNumReader + HeaderProvider + Clone + Send + Sync + 'static, { fn get_transaction_hashes_by_reference( &self, @@ -58,17 +61,46 @@ impl MorphRpcServer let query = ReferenceQuery::new(args.reference, args.offset, args.limit).map_err(to_rpc_error)?; - let canonical_tip = self + let before = self .ctx .provider - .best_block_number() + .chain_info() .map_err(ReferenceIndexError::from) .map_err(to_rpc_error)?; + let header = self + .ctx + .provider + .header(before.best_hash) + .map_err(ReferenceIndexError::from) + .map_err(to_rpc_error)? + .ok_or(ReferenceIndexError::IndexBehind) + .map_err(to_rpc_error)?; + if header.number() != before.best_number { + return Err(to_rpc_error(ReferenceIndexError::IndexBehind)); + } + let canonical_tip = CanonicalTip { + number: before.best_number, + hash: before.best_hash, + timestamp: header.timestamp(), + }; - self.ctx + let result = self + .ctx .reference_index - .query(query, canonical_tip) - .map_err(to_rpc_error) + .query_at(query, canonical_tip) + .map_err(to_rpc_error)?; + + let after = self + .ctx + .provider + .chain_info() + .map_err(ReferenceIndexError::from) + .map_err(to_rpc_error)?; + if after != before { + return Err(to_rpc_error(ReferenceIndexError::IndexBehind)); + } + + Ok(result) } } @@ -82,6 +114,9 @@ fn to_rpc_error(error: ReferenceIndexError) -> ErrorObjectOwned { ReferenceIndexError::IndexBehind => { ErrorObjectOwned::owned(-32000, "reference index is behind", None::<()>) } + ReferenceIndexError::IndexUnavailable => { + ErrorObjectOwned::owned(-32000, "reference index is unavailable", None::<()>) + } ReferenceIndexError::LimitTooLarge { .. } | ReferenceIndexError::OffsetTooLarge { .. } => { ErrorObjectOwned::owned( ErrorCode::InvalidParams.code(), @@ -105,3 +140,255 @@ fn to_rpc_error(error: ReferenceIndexError) -> ErrorObjectOwned { } } } + +#[cfg(test)] +mod tests { + use super::*; + use alloy_primitives::B256; + use morph_primitives::MorphHeader; + use morph_reference_index::{ + CanonicalBlock, CanonicalChain, CanonicalTip, ReferenceIndexConfig, ReferenceIndexHandle, + ReferenceIndexRuntime, + }; + use reth_chainspec::ChainInfo; + use reth_errors::ProviderResult; + use reth_primitives_traits::SealedHeader; + use reth_storage_api::{BlockHashReader, HeaderProvider}; + use std::{ + collections::{BTreeMap, VecDeque}, + ops::{RangeBounds, RangeInclusive}, + sync::{Arc, Mutex}, + }; + + #[derive(Clone, Debug)] + struct TestChain { + block: CanonicalBlock, + } + + impl CanonicalChain for TestChain { + fn head(&self) -> Result { + Ok(CanonicalTip { + number: self.block.number, + hash: self.block.hash, + timestamp: self.block.timestamp, + }) + } + + fn canonical_hash(&self, number: u64) -> Result, ReferenceIndexError> { + Ok((number == self.block.number).then_some(self.block.hash)) + } + + fn block_timestamp(&self, number: u64) -> Result, ReferenceIndexError> { + Ok((number == self.block.number).then_some(self.block.timestamp)) + } + + fn canonical_blocks( + &self, + range: RangeInclusive, + ) -> Result, ReferenceIndexError> { + Ok(range + .contains(&self.block.number) + .then(|| self.block.clone()) + .into_iter() + .collect()) + } + } + + #[derive(Clone, Debug)] + struct TestProvider { + chain_info: Arc>>, + headers: Arc>, + } + + impl TestProvider { + fn new( + chain_info: impl IntoIterator, + headers: BTreeMap, + ) -> Self { + Self { + chain_info: Arc::new(Mutex::new(chain_info.into_iter().collect())), + headers: Arc::new(headers), + } + } + + fn info(&self) -> ChainInfo { + let mut infos = self.chain_info.lock().unwrap(); + if infos.len() > 1 { + infos.pop_front().unwrap() + } else { + *infos.front().unwrap() + } + } + } + + impl BlockHashReader for TestProvider { + fn block_hash(&self, number: u64) -> ProviderResult> { + Ok(self + .headers + .iter() + .find_map(|(hash, header)| (header.number() == number).then_some(*hash))) + } + + fn canonical_hashes_range(&self, start: u64, end: u64) -> ProviderResult> { + Ok(self + .headers + .iter() + .filter_map(|(hash, header)| { + (start..end).contains(&header.number()).then_some(*hash) + }) + .collect()) + } + } + + impl BlockNumReader for TestProvider { + fn chain_info(&self) -> ProviderResult { + Ok(self.info()) + } + + fn best_block_number(&self) -> ProviderResult { + Ok(self.info().best_number) + } + + fn last_block_number(&self) -> ProviderResult { + self.best_block_number() + } + + fn block_number(&self, hash: B256) -> ProviderResult> { + Ok(self.headers.get(&hash).map(|header| header.number())) + } + } + + impl HeaderProvider for TestProvider { + type Header = MorphHeader; + + fn header(&self, block_hash: B256) -> ProviderResult> { + Ok(self.headers.get(&block_hash).cloned()) + } + + fn header_by_number(&self, number: u64) -> ProviderResult> { + Ok(self + .headers + .values() + .find(|header| header.number() == number) + .cloned()) + } + + fn headers_range( + &self, + _range: impl RangeBounds, + ) -> ProviderResult> { + Ok(self.headers.values().cloned().collect()) + } + + fn sealed_header(&self, number: u64) -> ProviderResult>> { + Ok(self.headers.iter().find_map(|(hash, header)| { + (header.number() == number).then(|| SealedHeader::new(header.clone(), *hash)) + })) + } + + fn sealed_headers_while( + &self, + _range: impl RangeBounds, + mut predicate: impl FnMut(&SealedHeader) -> bool, + ) -> ProviderResult>> { + Ok(self + .headers + .iter() + .map(|(hash, header)| SealedHeader::new(header.clone(), *hash)) + .take_while(|header| predicate(header)) + .collect()) + } + } + + fn header(number: u64, timestamp: u64) -> MorphHeader { + let mut header = MorphHeader::default(); + header.inner.number = number; + header.inner.timestamp = timestamp; + header + } + + fn synced_handle( + jade_timestamp: u64, + block_timestamp: u64, + hash: B256, + ) -> (tempfile::TempDir, ReferenceIndexHandle) { + let dir = tempfile::tempdir().unwrap(); + let chain = TestChain { + block: CanonicalBlock { + number: 0, + hash, + timestamp: block_timestamp, + transactions: Vec::new(), + }, + }; + let config = ReferenceIndexConfig::new(dir.path(), 2818, B256::ZERO, jade_timestamp); + let (mut runtime, handle) = ReferenceIndexRuntime::new(config, chain); + runtime.synchronize_once(false).unwrap(); + (dir, handle) + } + + fn args() -> ReferenceQueryArgs { + ReferenceQueryArgs { + reference: B256::repeat_byte(0x42), + offset: None, + limit: None, + } + } + + #[test] + fn rejects_results_if_canonical_head_changes_during_query() { + let indexed_hash = B256::repeat_byte(0x11); + let changed_hash = B256::repeat_byte(0x22); + let (_dir, handle) = synced_handle(0, 100, indexed_hash); + let provider = TestProvider::new( + [ + ChainInfo { + best_hash: indexed_hash, + best_number: 0, + }, + ChainInfo { + best_hash: changed_hash, + best_number: 0, + }, + ], + BTreeMap::from([(indexed_hash, header(0, 100))]), + ); + let handler = MorphRpcHandler::new(MorphRpc::new(handle, provider)); + + let error = handler + .get_transaction_hashes_by_reference(args()) + .unwrap_err(); + + assert_eq!(error.code(), -32000); + assert_eq!(error.message(), "reference index is behind"); + } + + #[test] + fn passes_the_exact_head_timestamp_to_the_index() { + let hash = B256::repeat_byte(0x11); + let (_dir, handle) = synced_handle(200, 199, hash); + let provider = TestProvider::new( + [ChainInfo { + best_hash: hash, + best_number: 0, + }], + BTreeMap::from([(hash, header(0, 200))]), + ); + let handler = MorphRpcHandler::new(MorphRpc::new(handle, provider)); + + let error = handler + .get_transaction_hashes_by_reference(args()) + .unwrap_err(); + + assert_eq!(error.code(), -32000); + assert_eq!(error.message(), "reference index is behind"); + } + + #[test] + fn maps_unavailable_to_an_explicit_server_error() { + let error = to_rpc_error(ReferenceIndexError::IndexUnavailable); + + assert_eq!(error.code(), -32000); + assert_eq!(error.message(), "reference index is unavailable"); + } +} diff --git a/crates/rpc/src/morph/rpc.rs b/crates/rpc/src/morph/rpc.rs index 932fe227..2dd4109d 100644 --- a/crates/rpc/src/morph/rpc.rs +++ b/crates/rpc/src/morph/rpc.rs @@ -29,11 +29,14 @@ pub trait MorphRpc { /// ordered by (block_number, tx_index) ascending, with offset-based /// pagination. /// - /// Returns `-32000 "reference index initializing"` while the startup - /// backfill/reconcile has not yet completed. + /// Returns `-32000 "reference index initializing"` while the runtime is + /// opening the index. /// - /// Returns `-32000 "reference index is behind"` when the index lags the - /// current chain tip by more than the configured threshold (default 16). + /// Returns `-32000 "reference index is behind"` unless the durable index + /// cursor exactly matches the canonical head number and hash. + /// + /// Returns `-32000 "reference index is unavailable"` while the runtime is + /// repairing the index or after a non-recoverable index error. /// /// Marked `blocking` because the handler runs synchronous MDBX reads. #[method(name = "getTransactionHashesByReference", blocking)] From 372fd4686e99840d9caad5571469fb7a70e7649e Mon Sep 17 00:00:00 2001 From: panos-xyz Date: Tue, 14 Jul 2026 19:56:28 +0800 Subject: [PATCH 05/12] feat(proofs): add bounded historical proof history --- Cargo.lock | 215 +- Cargo.toml | 13 + README.md | 38 +- bin/morph-reth/Cargo.toml | 9 + bin/morph-reth/src/main.rs | 110 +- bin/morph-reth/src/proofs.rs | 254 ++ crates/node/Cargo.toml | 4 + crates/node/src/add_ons.rs | 44 +- crates/node/src/args.rs | 63 + crates/node/src/node.rs | 1 + crates/node/tests/it/main.rs | 1 + crates/node/tests/it/proof_history.rs | 178 + crates/proofs-exex/Cargo.toml | 37 + crates/proofs-exex/NOTICE.md | 31 + crates/proofs-exex/src/lib.rs | 1192 ++++++ crates/proofs-exex/src/sync_target.rs | 637 +++ crates/proofs/Cargo.toml | 50 + crates/proofs/NOTICE.md | 32 + crates/proofs/src/api.rs | 435 ++ crates/proofs/src/batch_provider.rs | 313 ++ crates/proofs/src/cursor.rs | 129 + crates/proofs/src/cursor_factory.rs | 260 ++ crates/proofs/src/db/batch.rs | 143 + crates/proofs/src/db/cursor.rs | 1545 +++++++ crates/proofs/src/db/mod.rs | 15 + crates/proofs/src/db/models/block.rs | 82 + crates/proofs/src/db/models/change_set.rs | 131 + crates/proofs/src/db/models/kv.rs | 85 + crates/proofs/src/db/models/metadata.rs | 56 + crates/proofs/src/db/models/mod.rs | 93 + crates/proofs/src/db/models/storage.rs | 272 ++ crates/proofs/src/db/models/version.rs | 205 + crates/proofs/src/db/store.rs | 4707 +++++++++++++++++++++ crates/proofs/src/error.rs | 233 + crates/proofs/src/in_memory.rs | 1138 +++++ crates/proofs/src/initialize.rs | 1357 ++++++ crates/proofs/src/lib.rs | 61 + crates/proofs/src/live.rs | 432 ++ crates/proofs/src/metrics.rs | 97 + crates/proofs/src/proof.rs | 667 +++ crates/proofs/src/provider.rs | 340 ++ crates/proofs/src/prune/error.rs | 102 + crates/proofs/src/prune/metrics.rs | 54 + crates/proofs/src/prune/mod.rs | 12 + crates/proofs/src/prune/pruner.rs | 735 ++++ crates/proofs/src/prune/task.rs | 80 + crates/proofs/tests/identity.rs | 24 + crates/rpc/Cargo.toml | 6 + crates/rpc/src/eth/mod.rs | 1 + crates/rpc/src/eth/proofs.rs | 146 + crates/rpc/src/lib.rs | 3 + crates/rpc/src/proof_status.rs | 130 + crates/rpc/src/state.rs | 185 + local-test/README.md | 2 +- local-test/reset.sh | 2 + local-test/reth-start.sh | 1 - 56 files changed, 17177 insertions(+), 11 deletions(-) create mode 100644 bin/morph-reth/src/proofs.rs create mode 100644 crates/node/tests/it/proof_history.rs create mode 100644 crates/proofs-exex/Cargo.toml create mode 100644 crates/proofs-exex/NOTICE.md create mode 100644 crates/proofs-exex/src/lib.rs create mode 100644 crates/proofs-exex/src/sync_target.rs create mode 100644 crates/proofs/Cargo.toml create mode 100644 crates/proofs/NOTICE.md create mode 100644 crates/proofs/src/api.rs create mode 100644 crates/proofs/src/batch_provider.rs create mode 100644 crates/proofs/src/cursor.rs create mode 100644 crates/proofs/src/cursor_factory.rs create mode 100644 crates/proofs/src/db/batch.rs create mode 100644 crates/proofs/src/db/cursor.rs create mode 100644 crates/proofs/src/db/mod.rs create mode 100644 crates/proofs/src/db/models/block.rs create mode 100644 crates/proofs/src/db/models/change_set.rs create mode 100644 crates/proofs/src/db/models/kv.rs create mode 100644 crates/proofs/src/db/models/metadata.rs create mode 100644 crates/proofs/src/db/models/mod.rs create mode 100644 crates/proofs/src/db/models/storage.rs create mode 100644 crates/proofs/src/db/models/version.rs create mode 100644 crates/proofs/src/db/store.rs create mode 100644 crates/proofs/src/error.rs create mode 100644 crates/proofs/src/in_memory.rs create mode 100644 crates/proofs/src/initialize.rs create mode 100644 crates/proofs/src/lib.rs create mode 100644 crates/proofs/src/live.rs create mode 100644 crates/proofs/src/metrics.rs create mode 100644 crates/proofs/src/proof.rs create mode 100644 crates/proofs/src/provider.rs create mode 100644 crates/proofs/src/prune/error.rs create mode 100644 crates/proofs/src/prune/metrics.rs create mode 100644 crates/proofs/src/prune/mod.rs create mode 100644 crates/proofs/src/prune/pruner.rs create mode 100644 crates/proofs/src/prune/task.rs create mode 100644 crates/proofs/tests/identity.rs create mode 100644 crates/rpc/src/eth/proofs.rs create mode 100644 crates/rpc/src/proof_status.rs create mode 100644 crates/rpc/src/state.rs diff --git a/Cargo.lock b/Cargo.lock index 16307f55..0b9cd49c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1488,6 +1488,26 @@ dependencies = [ "serde", ] +[[package]] +name = "bincode" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740" +dependencies = [ + "bincode_derive", + "serde", + "unty", +] + +[[package]] +name = "bincode_derive" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09" +dependencies = [ + "virtue", +] + [[package]] name = "bindgen" version = "0.72.1" @@ -2583,6 +2603,12 @@ dependencies = [ "litrs", ] +[[package]] +name = "downcast" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" + [[package]] name = "dunce" version = "1.0.5" @@ -2956,6 +2982,15 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fragile" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8878864ba14bb86e818a412bfd6f18f9eabd4ec0f008a28e8f7eb61db532fcf9" +dependencies = [ + "futures-core", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -4793,6 +4828,32 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "mockall" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f58d964098a5f9c6b63d0798e5372fd04708193510a7af313c22e9f29b7b620b" +dependencies = [ + "cfg-if", + "downcast", + "fragile", + "mockall_derive", + "predicates", + "predicates-tree", +] + +[[package]] +name = "mockall_derive" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca41ce716dda6a9be188b385aa78ee5260fc25cd3802cb2a8afdc6afbe6b6dbf" +dependencies = [ + "cfg-if", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "modular-bitfield" version = "0.13.1" @@ -4959,6 +5020,8 @@ dependencies = [ "morph-payload-builder", "morph-payload-types", "morph-primitives", + "morph-proofs", + "morph-proofs-exex", "morph-reference-index", "morph-rpc", "morph-txpool", @@ -4982,6 +5045,7 @@ dependencies = [ "reth-provider", "reth-rpc-builder", "reth-rpc-eth-api", + "reth-rpc-server-types", "reth-tasks", "reth-tracing", "reth-transaction-pool", @@ -4989,6 +5053,7 @@ dependencies = [ "reth-trie-db", "serde", "serde_json", + "tempfile", "tokio", "tokio-stream", "tracing", @@ -5068,6 +5133,63 @@ dependencies = [ "serde_json", ] +[[package]] +name = "morph-proofs" +version = "1.0.0" +dependencies = [ + "alloy-eips", + "alloy-primitives", + "auto_impl", + "bincode 2.0.1", + "bytes", + "derive_more", + "metrics", + "mockall", + "parking_lot", + "reth-codecs", + "reth-db", + "reth-db-api", + "reth-evm", + "reth-execution-errors", + "reth-metrics", + "reth-primitives-traits", + "reth-provider", + "reth-revm", + "reth-storage-errors", + "reth-tasks", + "reth-trie", + "reth-trie-common", + "serde", + "strum", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tracing", +] + +[[package]] +name = "morph-proofs-exex" +version = "1.0.0" +dependencies = [ + "alloy-consensus", + "alloy-eips", + "alloy-primitives", + "eyre", + "futures", + "morph-proofs", + "reth-db", + "reth-ethereum-primitives", + "reth-execution-types", + "reth-exex", + "reth-exex-test-utils", + "reth-node-api", + "reth-primitives-traits", + "reth-provider", + "reth-trie", + "tokio", + "tracing", +] + [[package]] name = "morph-reference-index" version = "1.0.0" @@ -5100,17 +5222,26 @@ dependencies = [ name = "morph-reth" version = "1.0.0" dependencies = [ + "alloy-consensus", "alloy-primitives", "clap", + "eyre", "morph-chainspec", "morph-consensus", "morph-evm", "morph-node", + "morph-proofs", + "morph-proofs-exex", + "reth-chainspec", "reth-cli", + "reth-cli-commands", + "reth-cli-runner", "reth-cli-util", "reth-ethereum-cli", "reth-node-builder", + "reth-provider", "reth-rpc-server-types", + "reth-tasks", "tracing", ] @@ -5148,26 +5279,32 @@ dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", "alloy-serde", + "async-trait", "derive_more", "eyre", "jsonrpsee", + "metrics", "morph-chainspec", "morph-evm", "morph-primitives", + "morph-proofs", "morph-reference-index", "morph-revm", "reth-chainspec", "reth-errors", "reth-evm", + "reth-metrics", "reth-node-api", "reth-node-builder", "reth-primitives-traits", "reth-provider", "reth-revm", "reth-rpc", + "reth-rpc-api", "reth-rpc-convert", "reth-rpc-eth-api", "reth-rpc-eth-types", + "reth-rpc-server-types", "reth-storage-api", "reth-tasks", "reth-transaction-pool", @@ -5887,6 +6024,32 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "predicates" +version = "3.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ada8f2932f28a27ee7b70dd6c1c39ea0675c55a36879ab92f3a715eaa1e63cfe" +dependencies = [ + "anstyle", + "predicates-core", +] + +[[package]] +name = "predicates-core" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cad38746f3166b4031b1a0d39ad9f954dd291e7854fcc0eed52ee41a0b50d144" + +[[package]] +name = "predicates-tree" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0de1b847b39c8131db0467e9df1ff60e6d0562ab8e9a16e568ad0fdb372e2f2" +dependencies = [ + "predicates-core", + "termtree", +] + [[package]] name = "prefix-trie" version = "0.8.4" @@ -7633,6 +7796,38 @@ dependencies = [ "tracing", ] +[[package]] +name = "reth-exex-test-utils" +version = "2.3.0" +source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" +dependencies = [ + "alloy-eips", + "eyre", + "futures-util", + "reth-chainspec", + "reth-config", + "reth-consensus", + "reth-db", + "reth-db-common", + "reth-ethereum-primitives", + "reth-evm-ethereum", + "reth-execution-types", + "reth-exex", + "reth-network", + "reth-node-api", + "reth-node-builder", + "reth-node-core", + "reth-node-ethereum", + "reth-payload-builder", + "reth-primitives-traits", + "reth-provider", + "reth-tasks", + "reth-transaction-pool", + "tempfile", + "thiserror 2.0.18", + "tokio", +] + [[package]] name = "reth-exex-types" version = "2.3.0" @@ -7907,7 +8102,7 @@ version = "2.3.0" source = "git+https://github.com/paradigmxyz/reth?rev=f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105#f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" dependencies = [ "anyhow", - "bincode", + "bincode 1.3.3", "derive_more", "lz4_flex", "memmap2", @@ -10425,6 +10620,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "termtree" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" + [[package]] name = "thiserror" version = "1.0.69" @@ -11188,6 +11389,12 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "unty" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" + [[package]] name = "url" version = "2.5.8" @@ -11287,6 +11494,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "virtue" +version = "0.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" + [[package]] name = "visibility" version = "0.1.1" diff --git a/Cargo.toml b/Cargo.toml index cd966912..c5674299 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,8 @@ members = [ "crates/payload/builder", "crates/payload/types", "crates/primitives", + "crates/proofs", + "crates/proofs-exex", "crates/reference-index", "crates/revm", "crates/txpool", @@ -94,6 +96,8 @@ morph-node = { path = "crates/node"} morph-payload-builder = { path = "crates/payload/builder", default-features = false } morph-payload-types = { path = "crates/payload/types", default-features = false } morph-primitives = { path = "crates/primitives", default-features = false } +morph-proofs = { path = "crates/proofs", default-features = false } +morph-proofs-exex = { path = "crates/proofs-exex", default-features = false } morph-reference-index = { path = "crates/reference-index", default-features = false } morph-rpc = { path = "crates/rpc" } morph-revm = { path = "crates/revm", default-features = false } @@ -104,6 +108,7 @@ reth-chain-state = { git = "https://github.com/paradigmxyz/reth", rev = "f0a4179 reth-chainspec = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-cli = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-cli-commands = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-cli-runner = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-cli-util = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-codecs = { version = "0.5.0", default-features = false } reth-codecs-derive = "0.5.0" @@ -116,6 +121,9 @@ reth-engine-local = { git = "https://github.com/paradigmxyz/reth", rev = "f0a417 reth-engine-primitives = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-engine-tree = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-errors = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-exex = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-exex-test-utils = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-execution-errors = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-eth-wire-types = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-ethereum = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-ethereum-cli = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } @@ -145,9 +153,11 @@ reth-rpc-eth-api = { git = "https://github.com/paradigmxyz/reth", rev = "f0a4179 reth-rpc-eth-types = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-rpc-server-types = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-storage-api = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-storage-errors = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105", default-features = false } reth-tasks = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-tracing = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-trie = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } +reth-trie-common = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-transaction-pool = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } reth-zstd-compressors = { version = "0.5.0", default-features = false } reth-execution-cache = { git = "https://github.com/paradigmxyz/reth", rev = "f0a41795d72ae5bb0b6c3661ac42ffb7b78ef105" } @@ -196,6 +206,7 @@ axum = "0.8.4" base64 = "0.22" blake3 = "1.8" bytes = "1.8" +bincode = "2" clap = { version = "4.5.45", features = ["derive"] } const-hex = { version = "1.15.0" } dashmap = "6.0" @@ -209,6 +220,7 @@ itertools = "0.14.0" jiff = { version = "0.2.15", default-features = false } jsonrpsee = { version = "0.26.0", features = ["server", "client", "macros"] } metrics = "0.24.0" +mockall = "0.14" p256 = "0.13" parking_lot = "0.12.4" prometheus-client = "0.24.0" @@ -219,6 +231,7 @@ rand_core = "0.6.4" rayon = "1.10" serde = { version = "1.0.219", features = ["derive"] } serde_json = "1.0.142" +strum = { version = "0.27", default-features = false } sha2 = "0.10" tempfile = "3.20.0" test-fuzz = "7" diff --git a/README.md b/README.md index a2239452..697f15ff 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ Morph Reth is the next-generation execution client for [Morph](https://www.morph - **Morph Hardforks**: Implements Morph hardfork logic through Jade, with bundled Mainnet and Hoodi chainspecs scheduled through Jade - **Custom Engine API**: L2-specific Engine API for sequencer block building and validation - **L1 Fee Validation**: Transaction pool with L1 data fee affordability checks +- **Bounded Historical Proofs**: Optional forward-only EIP-1186 proof history in an independent MDBX database ### Supported Networks @@ -42,6 +43,9 @@ morph-reth/ │ ├── builder/ # Block building logic │ └── types/ # Engine API types (MorphExecutionData, etc.) ├── primitives/ # Core types (transactions, receipts) + ├── proofs/ # Versioned MPT history and proof state provider + ├── proofs-exex/ # Forward accumulation, reorg, unwind, and pruning + ├── reference-index/ # Independent transaction reference index ├── revm/ # L1 fee calculation, token fee logic ├── rpc/ # RPC implementation and type conversions └── txpool/ # Transaction pool with L1 fee validation @@ -60,6 +64,9 @@ morph-reth/ | `morph-payload-types` | Engine API payload types | | `morph-payload-builder` | Block building implementation | | `morph-primitives` | Transaction and receipt types | +| `morph-proofs` | Bounded, versioned MPT history in MDBX | +| `morph-proofs-exex` | Forward-only proof-history execution extension | +| `morph-reference-index` | Independent Morph transaction reference index | | `morph-revm` | L1 fee and token fee calculations | | `morph-rpc` | RPC implementation and type conversions | | `morph-txpool` | Transaction pool with L1 fee and MorphTx validation | @@ -114,7 +121,36 @@ openssl rand -hex 32 > jwt.hex |------|---------|-------------| | `--morph.max-tx-payload-bytes` | 122880 (120KB) | Maximum transaction payload bytes per block | | `--morph.max-tx-per-block` | None (unlimited) | Maximum number of transactions per block | -| `--rpc.eth-proof-window` | 0 (disabled) | Max historical blocks for `eth_getProof` (up to 1209600) | +| `--proofs-history` | false | Enable historical `eth_getProof` and proof-history accumulation | +| `--proofs-history.storage-path` | `/historical-proofs` | Override the proof MDBX directory | +| `--proofs-history.window` | 1296000 | Number of canonical blocks retained | +| `--proofs-history.verification-interval` | 0 | Re-execute every Nth indexed block; 0 disables verification | +| `--rpc.eth-proof-window` | 0 | Reth overlay limit used only while proof history is disabled | + +#### Initializing Historical Proofs + +Proof history starts at the current canonical tip and accumulates forward. Stop the node before +initialization so the source state remains fixed: + +```bash +./target/release/morph-reth proofs init --chain hoodi --datadir /path/to/reth-data + +./target/release/morph-reth node \ + --chain hoodi \ + --datadir /path/to/reth-data \ + --proofs-history \ + --http \ + --authrpc.jwtsecret jwt.hex +``` + +The standard `eth_getProof` method is then served only for the inclusive range reported by +`debug_proofsSyncStatus`; requests outside that range never fall back to Reth's historical overlay. +The same `eth_getProof` override is installed on the authenticated RPC server. For manual +maintenance use `morph-reth proofs prune` and `morph-reth proofs unwind --target `. + +For cold copies, stop the source node and copy the complete chain data directory, including +`historical-proofs`, as one consistent unit. Startup validates the proof database schema, chain ID, +genesis hash, and latest canonical block hash before resuming. ### Running Tests diff --git a/bin/morph-reth/Cargo.toml b/bin/morph-reth/Cargo.toml index 485617c2..ecd361c0 100644 --- a/bin/morph-reth/Cargo.toml +++ b/bin/morph-reth/Cargo.toml @@ -20,17 +20,26 @@ morph-chainspec = { workspace = true, features = ["cli"] } morph-consensus.workspace = true morph-evm.workspace = true morph-node.workspace = true +morph-proofs.workspace = true +morph-proofs-exex.workspace = true # Reth CLI +reth-chainspec.workspace = true reth-cli.workspace = true +reth-cli-commands.workspace = true +reth-cli-runner.workspace = true reth-cli-util.workspace = true reth-ethereum-cli.workspace = true reth-node-builder.workspace = true +reth-provider.workspace = true reth-rpc-server-types.workspace = true +reth-tasks.workspace = true # Other alloy-primitives.workspace = true +alloy-consensus.workspace = true clap.workspace = true +eyre.workspace = true tracing.workspace = true [features] diff --git a/bin/morph-reth/src/main.rs b/bin/morph-reth/src/main.rs index a39d8adf..207597fd 100644 --- a/bin/morph-reth/src/main.rs +++ b/bin/morph-reth/src/main.rs @@ -3,6 +3,8 @@ //! This is the main entry point for the Morph L2 execution layer client. //! It extends reth with Morph-specific functionality. +mod proofs; + #[global_allocator] static ALLOC: reth_cli_util::allocator::Allocator = reth_cli_util::allocator::new_allocator(); @@ -21,6 +23,10 @@ use morph_chainspec::{MORPH_DEFAULT_PRIORITY_FEE, MorphChainSpec, MorphChainSpec use morph_consensus::MorphConsensus; use morph_evm::{MorphEvmConfig, evm::MorphEvmFactory}; use morph_node::{MorphAddOns, MorphArgs, MorphNode}; +use morph_proofs::{MdbxProofsStorage, MorphProofsStorage, MorphProofsStore, ProofDbIdentity}; +use morph_proofs_exex::MorphProofsExEx; +use proofs::MorphSubcommand; +use reth_chainspec::EthChainSpec; use reth_cli_util::sigsegv_handler; use reth_ethereum_cli::{Cli, Commands}; use reth_node_builder::Node; @@ -33,7 +39,7 @@ fn morph_default_suggested_fee() -> U256 { } fn apply_morph_cli_defaults( - cli: &mut Cli, + cli: &mut Cli, ) { if let Commands::Node(command) = &mut cli.command { command @@ -67,7 +73,8 @@ fn main() { ) }; - let mut cli = Cli::::parse(); + let mut cli = + Cli::::parse(); apply_morph_cli_defaults(&mut cli); // Run the node @@ -75,14 +82,76 @@ fn main() { cli.run_with_components::(components, async move |builder, morph_args| { info!(target: "morph::cli", "Starting Morph-Reth node"); + let chain_spec = builder.config().chain.clone(); + let datadir = builder.config().datadir(); + let chain_id = chain_spec.chain().id(); + let genesis_hash = chain_spec.genesis_hash(); + + let proof_history = if morph_args.proofs_history { + let path = morph_args + .proofs_history_storage_path + .clone() + .unwrap_or_else(|| datadir.data_dir().join("historical-proofs")); + info!( + target: "morph::proofs", + path = %path.display(), + chain_id, + "opening Morph historical proof database" + ); + let storage: MorphProofsStorage> = Arc::new( + MdbxProofsStorage::open( + &path, + ProofDbIdentity::new(chain_id, genesis_hash), + ) + .map_err(|error| { + eyre::eyre!( + "failed to open historical proof database at {}: {error}", + path.display() + ) + })?, + ); + if storage.get_earliest_block_number()?.is_none() + || storage.get_latest_block_number()?.is_none() + { + return Err(eyre::eyre!( + "proof history is enabled but {} is not initialized; run `morph-reth proofs init` first", + path.display() + )); + } + Some(( + storage, + morph_args.proofs_history_window, + morph_args.proofs_history_verification_interval, + )) + } else { + None + }; let node = MorphNode::new(morph_args); - let handle = builder + let mut add_ons = MorphAddOns::new(); + if let Some((storage, _, _)) = &proof_history { + add_ons = add_ons.with_proof_history(storage.clone()); + } + + let mut node_builder = builder .with_types::() .with_components(node.components_builder()) - .with_add_ons(MorphAddOns::new()) - .launch_with_debug_capabilities() - .await?; + .with_add_ons(add_ons); + + if let Some((storage, window, verification_interval)) = proof_history { + node_builder = node_builder.install_exex( + "morph-proof-history", + async move |ctx| { + let exex = MorphProofsExEx::builder(ctx, storage) + .with_proofs_history_window(window) + .with_verification_interval(verification_interval) + .build(); + Ok(async move { exex.run().await }) + }, + ); + } + + let handle = node_builder.launch_with_debug_capabilities().await?; info!(target: "morph::cli", "Node started successfully"); @@ -94,3 +163,32 @@ fn main() { std::process::exit(1); } } + +#[cfg(test)] +mod tests { + use super::*; + + type MorphCli = + Cli; + + #[test] + fn parses_top_level_proofs_command() { + let cli = MorphCli::try_parse_from(["morph-reth", "proofs", "init", "--chain", "hoodi"]) + .expect("top-level proofs command must parse"); + assert!(matches!( + cli.command, + Commands::Ext(MorphSubcommand::Proofs(_)) + )); + } + + #[test] + fn disabled_mode_keeps_reth_historical_overlay_off() { + let cli = MorphCli::try_parse_from(["morph-reth", "node", "--chain", "hoodi"]) + .expect("node command must parse"); + let Commands::Node(command) = cli.command else { + panic!("expected node command") + }; + assert!(!command.ext.proofs_history); + assert_eq!(command.rpc.rpc_eth_proof_window, 0); + } +} diff --git a/bin/morph-reth/src/proofs.rs b/bin/morph-reth/src/proofs.rs new file mode 100644 index 00000000..e7865c07 --- /dev/null +++ b/bin/morph-reth/src/proofs.rs @@ -0,0 +1,254 @@ +//! Management commands for the independent historical-proof database. + +use std::{path::PathBuf, sync::Arc}; + +use alloy_consensus::BlockHeader; +use clap::{Parser, Subcommand}; +use morph_chainspec::{MorphChainSpec, MorphChainSpecParser}; +use morph_node::{MorphNode, args::DEFAULT_PROOFS_HISTORY_WINDOW}; +use morph_proofs::{ + InitializationJob, MdbxProofsStorage, MorphProofStoragePruner, MorphProofsStorage, + MorphProofsStore, ProofDbIdentity, +}; +use reth_chainspec::EthChainSpec; +use reth_cli_commands::common::{AccessRights, Environment, EnvironmentArgs}; +use reth_cli_runner::CliRunner; +use reth_ethereum_cli::ExtendedCommand; +use reth_provider::{ + BlockNumReader, BlockReader, DBProvider, DatabaseProviderFactory, TransactionVariant, +}; +use tracing::{info, warn}; + +/// Morph-specific top-level CLI commands. +#[derive(Debug, Subcommand)] +pub(crate) enum MorphSubcommand { + /// Manage the historical proof database. + #[command(name = "proofs")] + Proofs(ProofsCommand), +} + +impl ExtendedCommand for MorphSubcommand { + fn execute(self, runner: CliRunner) -> eyre::Result<()> { + match self { + Self::Proofs(command) => { + let runtime = runner.runtime(); + runner.run_blocking_until_ctrl_c(command.execute(runtime)) + } + } + } +} + +/// Manage Morph historical proofs. +#[derive(Debug, Parser)] +pub(crate) struct ProofsCommand { + #[command(subcommand)] + command: ProofsSubcommand, +} + +impl ProofsCommand { + async fn execute(self, runtime: reth_tasks::Runtime) -> eyre::Result<()> { + match self.command { + ProofsSubcommand::Init(command) => command.execute(runtime), + ProofsSubcommand::Prune(command) => command.execute(runtime), + ProofsSubcommand::Unwind(command) => command.execute(runtime), + } + } +} + +#[derive(Debug, Subcommand)] +enum ProofsSubcommand { + /// Initialize proof history from the current canonical tip. + Init(InitCommand), + /// Remove proof history outside the configured window. + Prune(PruneCommand), + /// Remove proof history at and after a canonical block. + Unwind(UnwindCommand), +} + +#[derive(Debug, Parser)] +struct InitCommand { + #[command(flatten)] + env: EnvironmentArgs, + + /// Proof-history MDBX directory (defaults to `/historical-proofs`). + #[arg(long = "proofs-history.storage-path", value_name = "PATH")] + storage_path: Option, +} + +impl InitCommand { + fn execute(self, runtime: reth_tasks::Runtime) -> eyre::Result<()> { + let Environment { + provider_factory, + data_dir, + .. + } = self.env.init::(AccessRights::RO, runtime)?; + let path = self + .storage_path + .unwrap_or_else(|| data_dir.data_dir().join("historical-proofs")); + let storage = open_storage(&path, &self.env.chain)?; + + if let Some((number, hash)) = storage.get_earliest_block_number()? { + info!(target: "morph::proofs", block_number = number, ?hash, "Proof history is already initialized"); + return Ok(()); + } + + let chain_info = provider_factory.chain_info()?; + info!( + target: "morph::proofs", + path = %path.display(), + block_number = chain_info.best_number, + block_hash = ?chain_info.best_hash, + "Initializing proof history from canonical state" + ); + + let provider = provider_factory + .database_provider_ro()? + .disable_long_read_transaction_safety(); + InitializationJob::new(storage, provider.into_tx()) + .run(chain_info.best_number, chain_info.best_hash)?; + + info!(target: "morph::proofs", "Proof history initialized"); + Ok(()) + } +} + +#[derive(Debug, Parser)] +struct PruneCommand { + #[command(flatten)] + env: EnvironmentArgs, + + /// Proof-history MDBX directory (defaults to `/historical-proofs`). + #[arg(long = "proofs-history.storage-path", value_name = "PATH")] + storage_path: Option, + + /// Number of canonical blocks to retain. + #[arg( + long = "proofs-history.window", + value_name = "BLOCKS", + default_value_t = DEFAULT_PROOFS_HISTORY_WINDOW, + value_parser = clap::value_parser!(u64).range(1..) + )] + window: u64, + + /// Maximum number of blocks pruned per transaction. + #[arg( + long = "proofs-history.prune-batch-size", + value_name = "BLOCKS", + default_value_t = 1_000, + value_parser = clap::value_parser!(u64).range(1..) + )] + batch_size: u64, +} + +impl PruneCommand { + fn execute(self, runtime: reth_tasks::Runtime) -> eyre::Result<()> { + let Environment { + provider_factory, + data_dir, + .. + } = self.env.init::(AccessRights::RO, runtime)?; + let path = self + .storage_path + .unwrap_or_else(|| data_dir.data_dir().join("historical-proofs")); + let storage = open_storage(&path, &self.env.chain)?; + + let output = + MorphProofStoragePruner::new(storage, provider_factory, self.window, self.batch_size) + .try_run()?; + info!(target: "morph::proofs", ?output, "Proof-history pruning completed"); + Ok(()) + } +} + +#[derive(Debug, Parser)] +struct UnwindCommand { + #[command(flatten)] + env: EnvironmentArgs, + + /// Proof-history MDBX directory (defaults to `/historical-proofs`). + #[arg(long = "proofs-history.storage-path", value_name = "PATH")] + storage_path: Option, + + /// First block to remove, inclusive. + #[arg(long, value_name = "BLOCK")] + target: u64, +} + +impl UnwindCommand { + fn execute(self, runtime: reth_tasks::Runtime) -> eyre::Result<()> { + let Environment { + provider_factory, + data_dir, + .. + } = self.env.init::(AccessRights::RO, runtime)?; + let path = self + .storage_path + .unwrap_or_else(|| data_dir.data_dir().join("historical-proofs")); + let storage = open_storage(&path, &self.env.chain)?; + + let (Some((earliest, _)), Some((latest, _))) = ( + storage.get_earliest_block_number()?, + storage.get_latest_block_number()?, + ) else { + warn!(target: "morph::proofs", "Proof history is empty; nothing to unwind"); + return Ok(()); + }; + + if self.target <= earliest || self.target > latest { + return Err(eyre::eyre!( + "unwind target {} must be in ({earliest}, {latest}]", + self.target + )); + } + + let block = provider_factory + .recovered_block(self.target.into(), TransactionVariant::NoHash)? + .ok_or_else(|| eyre::eyre!("canonical block {} not found", self.target))?; + info!( + target: "morph::proofs", + block_number = block.number(), + block_hash = ?block.hash(), + "Unwinding proof history" + ); + storage.unwind_history(block.block_with_parent())?; + Ok(()) + } +} + +fn open_storage( + path: &std::path::Path, + chain: &MorphChainSpec, +) -> eyre::Result>> { + let identity = ProofDbIdentity::new(chain.chain().id(), chain.genesis_hash()); + let backend = MdbxProofsStorage::open(path, identity).map_err(|error| { + eyre::eyre!( + "failed to open proof history at {}: {error}", + path.display() + ) + })?; + Ok(Arc::new(backend)) +} + +#[cfg(test)] +mod tests { + use clap::Parser; + + use super::ProofsCommand; + + #[test] + fn parses_all_proof_commands() { + ProofsCommand::try_parse_from(["proofs", "init", "--chain", "hoodi"]) + .expect("init command must parse"); + ProofsCommand::try_parse_from([ + "proofs", + "prune", + "--chain", + "hoodi", + "--proofs-history.window", + "64", + ]) + .expect("prune command must parse"); + ProofsCommand::try_parse_from(["proofs", "unwind", "--chain", "hoodi", "--target", "42"]) + .expect("unwind command must parse"); + } +} diff --git a/crates/node/Cargo.toml b/crates/node/Cargo.toml index bb6427b6..df9b5617 100644 --- a/crates/node/Cargo.toml +++ b/crates/node/Cargo.toml @@ -18,6 +18,7 @@ morph-evm.workspace = true morph-payload-builder.workspace = true morph-payload-types.workspace = true morph-primitives = { workspace = true, features = ["reth-codec"] } +morph-proofs.workspace = true morph-reference-index.workspace = true morph-rpc.workspace = true morph-txpool.workspace = true @@ -84,6 +85,7 @@ reth-db = { workspace = true, features = ["test-utils"] } reth-e2e-test-utils.workspace = true reth-node-core.workspace = true reth-payload-primitives.workspace = true +reth-rpc-server-types.workspace = true reth-tasks.workspace = true reth-tracing.workspace = true alloy-consensus.workspace = true @@ -94,7 +96,9 @@ jsonrpsee.workspace = true morph-chainspec.workspace = true morph-payload-types.workspace = true morph-primitives.workspace = true +morph-proofs-exex.workspace = true serde_json.workspace = true +tempfile.workspace = true [[test]] name = "it" diff --git a/crates/node/src/add_ons.rs b/crates/node/src/add_ons.rs index 65249d57..b3bdd127 100644 --- a/crates/node/src/add_ons.rs +++ b/crates/node/src/add_ons.rs @@ -8,9 +8,12 @@ use alloy_hardforks::ForkCondition; use morph_chainspec::{MorphHardfork, MorphHardforks}; use morph_evm::MorphEvmConfig; use morph_primitives::{Block, MorphHeader, MorphReceipt}; +use morph_proofs::{MdbxProofsStorage, MorphProofsStorage}; use morph_reference_index::{ReferenceIndexConfig, ReferenceIndexRuntime}; use morph_rpc::{ - MorphEthApiBuilder, MorphEthConfigApiServer, MorphEthConfigHandler, + MorphEthApiBuilder, MorphEthConfigApiServer, MorphEthConfigHandler, ProofStatusApiExt, + ProofStatusApiOverrideServer, + eth::proofs::{EthProofApiExt, EthProofApiOverrideServer}, morph::{MorphRpc, MorphRpcHandler, MorphRpcServer}, }; use reth_chain_state::CanonStateSubscriptions; @@ -29,6 +32,7 @@ use reth_provider::{ use reth_rpc_builder::Identity; use reth_rpc_eth_api::RpcNodeCore; use reth_tracing::tracing; +use std::sync::Arc; /// Morph node add-ons for RPC and Engine API. /// @@ -47,6 +51,8 @@ pub struct MorphAddOns< > { /// Inner RPC add-ons from reth. inner: RpcAddOns, + /// Optional proof-history storage used to replace `eth_getProof` on normal and auth RPC. + proof_history: Option>>, } impl MorphAddOns, MorphEthApiBuilder> @@ -68,8 +74,18 @@ where Identity::default(), Identity::default(), ), + proof_history: None, } } + + /// Attach initialized proof-history storage to the RPC add-ons. + pub fn with_proof_history( + mut self, + storage: MorphProofsStorage>, + ) -> Self { + self.proof_history = Some(storage); + self + } } fn ensure_reference_index_pruning_compatible( @@ -171,6 +187,7 @@ where let morph_rpc_ctx = MorphRpc::new(reference_index_handle, provider.clone()); let reference_rpc_handler = MorphRpcHandler::new(morph_rpc_ctx); + let proof_history = self.proof_history; // Use launch_add_ons_with to register custom Engine API and eth_config self.inner @@ -178,9 +195,34 @@ where let reth_node_builder::rpc::RpcModuleContainer { modules, auth_module, + registry, .. } = container; + if let Some(storage) = proof_history { + let eth_api = registry.eth_api().clone(); + modules + .replace_configured( + EthProofApiExt::new(eth_api.clone(), storage.clone()).into_rpc(), + ) + .map_err(|error| { + eyre::eyre!("Failed to replace normal RPC eth_getProof: {error}") + })?; + auth_module + .replace_auth_methods( + EthProofApiExt::new(eth_api, storage.clone()).into_rpc(), + ) + .map_err(|error| { + eyre::eyre!("Failed to replace auth RPC eth_getProof: {error}") + })?; + modules + .replace_configured(ProofStatusApiExt::new(storage).into_rpc()) + .map_err(|error| { + eyre::eyre!("Failed to register debug_proofsSyncStatus: {error}") + })?; + tracing::info!(target: "morph::node", "Historical proof RPCs registered"); + } + // Register Morph eth_config handler (EIP-7910 + morph extension) // This provides eth_config on HTTP/WS/IPC for morphnode compatibility. tracing::debug!(target: "morph::node", "Registering Morph eth_config handler"); diff --git a/crates/node/src/args.rs b/crates/node/src/args.rs index 7e5e9233..16846daa 100644 --- a/crates/node/src/args.rs +++ b/crates/node/src/args.rs @@ -1,5 +1,7 @@ //! Morph node CLI arguments. +use std::path::PathBuf; + use clap::Args; /// Default maximum transaction payload bytes per block (120KB). @@ -7,6 +9,9 @@ use clap::Args; /// This matches Morph's go-ethereum configuration. pub const MORPH_DEFAULT_MAX_TX_PAYLOAD_BYTES: u64 = 122_880; +/// Default proof-history retention window: 30 days at a two-second block time. +pub const DEFAULT_PROOFS_HISTORY_WINDOW: u64 = 1_296_000; + /// Morph-specific CLI arguments. /// /// These arguments extend the standard reth CLI with Morph-specific options @@ -33,6 +38,33 @@ pub struct MorphArgs { /// Morph Holesky testnet uses 1000 as the default limit. #[arg(long = "morph.max-tx-per-block", value_name = "COUNT")] pub max_tx_per_block: Option, + + /// Enable the forward-only historical proof index and its RPC overrides. + #[arg(long = "proofs-history", default_value_t = false)] + pub proofs_history: bool, + + /// Proof-history MDBX directory. + /// + /// Defaults to `/historical-proofs` when proof history is enabled. + #[arg(long = "proofs-history.storage-path", value_name = "PATH")] + pub proofs_history_storage_path: Option, + + /// Number of canonical blocks retained in proof history. + #[arg( + long = "proofs-history.window", + value_name = "BLOCKS", + default_value_t = DEFAULT_PROOFS_HISTORY_WINDOW, + value_parser = clap::value_parser!(u64).range(1..) + )] + pub proofs_history_window: u64, + + /// Re-execute every Nth block to verify notification trie data; zero disables verification. + #[arg( + long = "proofs-history.verification-interval", + value_name = "BLOCKS", + default_value_t = 0 + )] + pub proofs_history_verification_interval: u64, } impl Default for MorphArgs { @@ -40,6 +72,10 @@ impl Default for MorphArgs { Self { max_tx_payload_bytes: MORPH_DEFAULT_MAX_TX_PAYLOAD_BYTES, max_tx_per_block: None, + proofs_history: false, + proofs_history_storage_path: None, + proofs_history_window: DEFAULT_PROOFS_HISTORY_WINDOW, + proofs_history_verification_interval: 0, } } } @@ -63,6 +99,10 @@ mod tests { MORPH_DEFAULT_MAX_TX_PAYLOAD_BYTES ); assert_eq!(args.max_tx_per_block, None); + assert!(!args.proofs_history); + assert_eq!(args.proofs_history_storage_path, None); + assert_eq!(args.proofs_history_window, 1_296_000); + assert_eq!(args.proofs_history_verification_interval, 0); } #[test] @@ -105,6 +145,29 @@ mod tests { ); } + #[test] + fn test_proofs_history_args() { + let args = CommandParser::::parse_from([ + "test", + "--proofs-history", + "--proofs-history.storage-path", + "/tmp/morph-proofs", + "--proofs-history.window", + "2592000", + "--proofs-history.verification-interval", + "64", + ]) + .args; + + assert!(args.proofs_history); + assert_eq!( + args.proofs_history_storage_path.as_deref(), + Some(std::path::Path::new("/tmp/morph-proofs")) + ); + assert_eq!(args.proofs_history_window, 2_592_000); + assert_eq!(args.proofs_history_verification_interval, 64); + } + #[test] fn test_default_trait_impl() { let args = MorphArgs::default(); diff --git a/crates/node/src/node.rs b/crates/node/src/node.rs index 347cef93..1f6d7887 100644 --- a/crates/node/src/node.rs +++ b/crates/node/src/node.rs @@ -252,6 +252,7 @@ mod tests { let args = super::super::args::MorphArgs { max_tx_payload_bytes: 200_000, max_tx_per_block: Some(500), + ..Default::default() }; let node = MorphNode::new(args); assert_eq!(node.args.max_tx_payload_bytes, 200_000); diff --git a/crates/node/tests/it/main.rs b/crates/node/tests/it/main.rs index fd66dea6..66388bc5 100644 --- a/crates/node/tests/it/main.rs +++ b/crates/node/tests/it/main.rs @@ -13,6 +13,7 @@ mod evm; mod hardfork; mod l1_messages; mod morph_tx; +mod proof_history; mod reference_index; mod rpc; mod sync; diff --git a/crates/node/tests/it/proof_history.rs b/crates/node/tests/it/proof_history.rs new file mode 100644 index 00000000..2e8a95f3 --- /dev/null +++ b/crates/node/tests/it/proof_history.rs @@ -0,0 +1,178 @@ +//! End-to-end coexistence coverage for proof history and the reference index. + +use std::{sync::Arc, time::Duration}; + +use alloy_genesis::Genesis; +use alloy_primitives::{Address, B256}; +use alloy_rpc_types_eth::EIP1186AccountProofResponse; +use jsonrpsee::{core::client::ClientT, http_client::HttpClient, rpc_params}; +use morph_chainspec::MorphChainSpec; +use morph_node::{ + MorphAddOns, MorphNode, + test_utils::{MorphTestNode, advance_empty_block, morph_payload_attributes}, +}; +use morph_proofs::{InitializationJob, MdbxProofsStorage, MorphProofsStorage, ProofDbIdentity}; +use morph_proofs_exex::MorphProofsExEx; +use morph_reference_index::ReferenceTransactionResult; +use morph_rpc::{ProofsSyncStatus, ReferenceQueryArgs}; +use reth_chainspec::EthChainSpec; +use reth_e2e_test_utils::node::NodeTestContext; +use reth_node_builder::{EngineNodeLauncher, Node, NodeBuilder, NodeConfig, NodeHandle}; +use reth_node_core::args::{DiscoveryArgs, NetworkArgs, RpcServerArgs}; +use reth_provider::{DBProvider, DatabaseProviderFactory, providers::BlockchainProvider}; +use reth_rpc_server_types::RpcModuleSelection; +use reth_tasks::Runtime; + +async fn launch_node_with_proof_history( + chain_spec: Arc, + storage: MorphProofsStorage>, +) -> eyre::Result { + let runtime = Runtime::test(); + let network = NetworkArgs { + discovery: DiscoveryArgs { + disable_discovery: true, + ..DiscoveryArgs::default() + }, + ..NetworkArgs::default() + }; + let node_config = NodeConfig::new(chain_spec.clone()) + .with_network(network) + .with_unused_ports() + .with_rpc( + RpcServerArgs::default() + .with_unused_ports() + .with_http() + .with_http_api(RpcModuleSelection::All), + ); + let node = MorphNode::default(); + let exex_storage = storage.clone(); + + let NodeHandle { + node, + node_exit_future: _, + } = NodeBuilder::new(node_config) + .testing_node(runtime.clone()) + .with_types_and_provider::>() + .with_components(node.components_builder()) + .with_add_ons(MorphAddOns::new().with_proof_history(storage)) + .install_exex("morph-proof-history", async move |ctx| { + let head = ctx.head; + let provider = ctx + .provider() + .database_provider_ro()? + .disable_long_read_transaction_safety(); + InitializationJob::new(exex_storage.clone(), provider.into_tx()) + .run(head.number, head.hash)?; + + let exex = MorphProofsExEx::builder(ctx, exex_storage).build(); + Ok(async move { exex.run().await }) + }) + .launch_with_fn(|builder| { + let launcher = EngineNodeLauncher::new( + builder.task_executor().clone(), + builder.config().datadir(), + reth_node_api::TreeConfig::default().with_cross_block_cache_size(1024 * 1024), + ); + builder.launch_with(launcher) + }) + .await?; + + let node = NodeTestContext::new(node, morph_payload_attributes).await?; + let genesis_hash = chain_spec.genesis_hash(); + node.update_forkchoice(genesis_hash, genesis_hash).await?; + Ok(node) +} + +async fn wait_for_proof_tip(client: &HttpClient, expected: u64) -> eyre::Result { + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + let mut last_status = None; + while tokio::time::Instant::now() < deadline { + if let Ok(status) = client + .request::("debug_proofsSyncStatus", rpc_params![]) + .await + { + if status.latest == Some(expected) { + return Ok(status); + } + last_status = Some(status); + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + Err(eyre::eyre!( + "proof history did not reach block {expected}: {last_status:?}" + )) +} + +async fn wait_for_reference_index(client: &HttpClient) -> eyre::Result<()> { + let args = ReferenceQueryArgs { + reference: B256::repeat_byte(0x99), + offset: None, + limit: None, + }; + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + let mut last_error = None; + while tokio::time::Instant::now() < deadline { + match client + .request::, _>( + "morph_getTransactionHashesByReference", + (args.clone(),), + ) + .await + { + Ok(results) if results.is_empty() => return Ok(()), + Ok(results) => { + return Err(eyre::eyre!( + "unexpected reference-index results: {results:?}" + )); + } + Err(error) => last_error = Some(error.to_string()), + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + Err(eyre::eyre!( + "reference index did not become ready: {}", + last_error.as_deref().unwrap_or("no RPC response") + )) +} + +#[tokio::test(flavor = "multi_thread")] +async fn proof_history_and_reference_index_advance_together() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let genesis: Genesis = serde_json::from_str(include_str!("../assets/test-genesis.json"))?; + let chain_spec = Arc::new(MorphChainSpec::from_genesis(genesis)); + let proof_directory = tempfile::tempdir()?; + let proof_storage = Arc::new(MdbxProofsStorage::open( + proof_directory.path(), + ProofDbIdentity::new(chain_spec.chain().id(), chain_spec.genesis_hash()), + )?); + let mut node = launch_node_with_proof_history(chain_spec, proof_storage).await?; + + advance_empty_block(&mut node).await?; + let client = node + .rpc_client() + .ok_or_else(|| eyre::eyre!("HTTP RPC client not available"))?; + + let status = wait_for_proof_tip(&client, 1).await?; + assert_eq!(status.earliest, Some(0)); + wait_for_reference_index(&client).await?; + + let normal_proof: EIP1186AccountProofResponse = client + .request( + "eth_getProof", + rpc_params![Address::ZERO, Vec::::new(), "0x0"], + ) + .await?; + let auth_proof: EIP1186AccountProofResponse = node + .auth_server_handle() + .http_client() + .request( + "eth_getProof", + rpc_params![Address::ZERO, Vec::::new(), "0x0"], + ) + .await?; + assert_eq!(normal_proof.address, Address::ZERO); + assert_eq!(normal_proof, auth_proof); + + Ok(()) +} diff --git a/crates/proofs-exex/Cargo.toml b/crates/proofs-exex/Cargo.toml new file mode 100644 index 00000000..6c1a6728 --- /dev/null +++ b/crates/proofs-exex/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "morph-proofs-exex" +description = "Forward-only proof-history execution extension for Morph" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[lints] +workspace = true + +[dependencies] +morph-proofs.workspace = true + +alloy-consensus.workspace = true +alloy-eips.workspace = true +eyre.workspace = true +futures.workspace = true +reth-execution-types.workspace = true +reth-exex.workspace = true +reth-node-api.workspace = true +reth-provider.workspace = true +reth-trie.workspace = true +tokio.workspace = true +tracing.workspace = true + +[dev-dependencies] +alloy-primitives.workspace = true +morph-proofs = { workspace = true, features = ["test-utils"] } +reth-db = { workspace = true, features = ["test-utils"] } +reth-ethereum-primitives = { workspace = true, features = ["test-utils"] } +reth-exex-test-utils.workspace = true +reth-primitives-traits = { workspace = true, features = ["test-utils"] } + +[features] +default = [] diff --git a/crates/proofs-exex/NOTICE.md b/crates/proofs-exex/NOTICE.md new file mode 100644 index 00000000..db03f0a4 --- /dev/null +++ b/crates/proofs-exex/NOTICE.md @@ -0,0 +1,31 @@ +# Attribution + +This execution extension is adapted from Base at commit +`b2673bbd927cb34d7cfad4d448bfbd5bd30eae88` under the MIT license: + + + +Morph-specific changes include MDBX-only storage, canonical-tip validation, Morph CLI +messages, and Morph metrics/logging names. + +## Base MIT License + +Copyright (c) 2025 Base + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/proofs-exex/src/lib.rs b/crates/proofs-exex/src/lib.rs new file mode 100644 index 00000000..e1ad5d98 --- /dev/null +++ b/crates/proofs-exex/src/lib.rs @@ -0,0 +1,1192 @@ +//! Forward-only proof-history execution extension for Morph. +//! +//! Adapted from Base commit `b2673bbd927cb34d7cfad4d448bfbd5bd30eae88` +//! under the MIT license. + +#![cfg_attr(not(test), warn(unused_crate_dependencies))] + +mod sync_target; +use std::{sync::Arc, time::Duration}; + +use alloy_consensus::BlockHeader; +use alloy_eips::eip1898::BlockWithParent; +use futures::TryStreamExt; +use morph_proofs::{ + MorphProofStoragePrunerTask, MorphProofsBatchStore, MorphProofsStorage, + live::{BatchBlock, LiveTrieCollector}, + metrics::ProofHistoryMetrics, +}; +use reth_execution_types::Chain; +use reth_exex::{ExExContext, ExExEvent, ExExNotification, ExExNotificationsStream}; +use reth_node_api::{FullNodeComponents, NodePrimitives, NodeTypes}; +use reth_provider::{BlockHashReader, BlockNumReader, BlockReader, TransactionVariant}; +pub use sync_target::{CachedBlockTrieData, SyncTarget, SyncTargetState}; +use tokio::task; +use tracing::{debug, error, info}; + +/// Default safety threshold for the gap between stored earliest block and the configured +/// window target. When exceeded on startup, the node refuses to auto-prune and asks the +/// operator to run `proofs prune` manually. The threshold is a backstop against accidental +/// misconfiguration (e.g. shrinking the proofs window by orders of magnitude) — it is not a +/// measure of pruner throughput. Override via +/// [`MorphProofsExExBuilder::with_max_prune_blocks_startup`]. +const DEFAULT_MAX_PRUNE_BLOCKS_STARTUP: u64 = 1_000; + +/// How many blocks to process in a single sync turn before yielding. +const SYNC_BLOCKS_PER_TURN: usize = 50; + +/// Default proofs history window: 1 month of blocks at 2s block time +const DEFAULT_PROOFS_HISTORY_WINDOW: u64 = 1_296_000; + +/// Default interval between proof-storage prune runs. Default is 15 seconds. +const DEFAULT_PRUNE_INTERVAL: Duration = Duration::from_secs(15); + +/// Default verification interval: disabled +const DEFAULT_VERIFICATION_INTERVAL: u64 = 0; // disabled + +/// Builder for [`MorphProofsExEx`]. +#[derive(Debug)] +pub struct MorphProofsExExBuilder +where + Node: FullNodeComponents, +{ + ctx: ExExContext, + storage: MorphProofsStorage, + proofs_history_window: u64, + proofs_history_prune_interval: Duration, + verification_interval: u64, + max_prune_blocks_startup: u64, +} + +impl MorphProofsExExBuilder +where + Node: FullNodeComponents, +{ + /// Create a new builder with required parameters and defaults. + pub const fn new(ctx: ExExContext, storage: MorphProofsStorage) -> Self { + Self { + ctx, + storage, + proofs_history_window: DEFAULT_PROOFS_HISTORY_WINDOW, + proofs_history_prune_interval: DEFAULT_PRUNE_INTERVAL, + verification_interval: DEFAULT_VERIFICATION_INTERVAL, + max_prune_blocks_startup: DEFAULT_MAX_PRUNE_BLOCKS_STARTUP, + } + } + + /// Sets the window to span blocks for proofs history. + pub const fn with_proofs_history_window(mut self, window: u64) -> Self { + self.proofs_history_window = window; + self + } + + /// Sets the interval between proof-storage prune runs. + pub const fn with_proofs_history_prune_interval(mut self, interval: Duration) -> Self { + self.proofs_history_prune_interval = interval; + self + } + + /// Sets the verification interval. + pub const fn with_verification_interval(mut self, interval: u64) -> Self { + self.verification_interval = interval; + self + } + + /// Sets the safety threshold for blocks that may be auto-pruned at startup. + /// See `DEFAULT_MAX_PRUNE_BLOCKS_STARTUP`. + pub const fn with_max_prune_blocks_startup(mut self, max_blocks: u64) -> Self { + self.max_prune_blocks_startup = max_blocks; + self + } + + /// Builds the [`MorphProofsExEx`]. + pub fn build(self) -> MorphProofsExEx { + MorphProofsExEx { + ctx: self.ctx, + storage: self.storage, + proofs_history_window: self.proofs_history_window, + proofs_history_prune_interval: self.proofs_history_prune_interval, + verification_interval: self.verification_interval, + max_prune_blocks_startup: self.max_prune_blocks_startup, + } + } +} + +/// Proofs `ExEx` - processes blocks and tracks state changes within fault proof window. +/// +/// Saves and serves trie nodes to make proofs faster. This handles the process of +/// saving the current state, new blocks as they're added, and serving proof RPCs +/// based on the saved data. +/// +#[derive(Debug)] +pub struct MorphProofsExEx +where + Node: FullNodeComponents, +{ + /// The `ExEx` context containing the node related utilities e.g. provider, notifications, + /// events. + ctx: ExExContext, + /// The type of storage DB. + storage: MorphProofsStorage, + /// The window to span blocks for proofs history. Value is the number of blocks, received as + /// cli arg. + proofs_history_window: u64, + /// Interval between proof-storage prune runs + proofs_history_prune_interval: Duration, + /// Verification interval: perform full block execution every N blocks for data integrity. + /// If 0, verification is disabled (always use fast path when available). + /// If 1, verification is always enabled (always execute blocks). + verification_interval: u64, + /// Maximum blocks the startup check is willing to schedule for auto-prune. See + /// [`DEFAULT_MAX_PRUNE_BLOCKS_STARTUP`]. + max_prune_blocks_startup: u64, +} + +impl MorphProofsExEx +where + Node: FullNodeComponents, +{ + /// Create a new `MorphProofsExEx` instance. + pub fn new(ctx: ExExContext, storage: MorphProofsStorage) -> Self { + MorphProofsExExBuilder::new(ctx, storage).build() + } + + /// Create a new builder for `MorphProofsExEx`. + pub const fn builder( + ctx: ExExContext, + storage: MorphProofsStorage, + ) -> MorphProofsExExBuilder { + MorphProofsExExBuilder::new(ctx, storage) + } +} + +impl MorphProofsExEx +where + Node: FullNodeComponents>, + Primitives: NodePrimitives, + Storage: MorphProofsBatchStore + Clone + 'static, +{ + /// Main execution loop for the `ExEx` + pub async fn run(mut self) -> eyre::Result<()> { + self.ensure_initialized()?; + let sync_target = self.spawn_sync_task(); + + // If storage is behind tip, start syncing immediately rather than waiting + // for the first notification. + let best_block = self.ctx.provider().best_block_number()?; + let latest_stored = self + .storage + .get_latest_block_number()? + .map(|(n, _)| n) + .unwrap_or(0); + if latest_stored < best_block { + info!( + target: "morph::proofs_exex", + latest_stored, + best_block, + "Storage behind tip, starting sync immediately" + ); + sync_target.update_state(SyncTargetState::SyncUpTo { to: best_block }); + } + + let prune_task = MorphProofStoragePrunerTask::new( + self.storage.clone(), + self.ctx.provider().clone(), + self.proofs_history_window, + self.proofs_history_prune_interval, + ); + self.ctx + .task_executor() + .spawn_with_graceful_shutdown_signal(|signal| Box::pin(prune_task.run(signal))); + + self.ctx.notifications.set_without_head(); + + while let Some(notification) = self.ctx.notifications.try_next().await? { + self.handle_notification(notification, &sync_target)?; + } + + Ok(()) + } + + /// Ensure proofs storage is initialized + fn ensure_initialized(&self) -> eyre::Result<()> { + // Check if proofs storage is initialized + let earliest_block_number = match self.storage.get_earliest_block_number()? { + Some((n, _)) => n, + None => { + return Err(eyre::eyre!( + "Proofs storage not initialized. Please run 'morph-reth proofs init --proofs-history.storage-path ' first." + )); + } + }; + + let (latest_block_number, latest_block_hash) = match self + .storage + .get_latest_block_number()? + { + Some(latest) => latest, + None => { + return Err(eyre::eyre!( + "Proofs storage not initialized. Please run 'morph-reth proofs init --proofs-history.storage-path ' first." + )); + } + }; + + // Check if we have accumulated too much history for the configured window. + // If the gap between what we have and what we want to keep is too large, the auto-pruner + // will stall the node. + let target_earliest = latest_block_number.saturating_sub(self.proofs_history_window); + if target_earliest > earliest_block_number { + let blocks_to_prune = target_earliest - earliest_block_number; + if blocks_to_prune > self.max_prune_blocks_startup { + return Err(eyre::eyre!( + "Configuration requires pruning {} blocks, which exceeds the safety threshold of {}. \ + Huge prune operations can stall the node. \ + Please run 'morph-reth proofs prune' manually before starting the node.", + blocks_to_prune, + self.max_prune_blocks_startup + )); + } + } + + let canonical_hash = self + .ctx + .provider() + .block_hash(latest_block_number)? + .ok_or_else(|| eyre::eyre!( + "Proofs storage latest block {latest_block_number} is missing from the canonical chain" + ))?; + if canonical_hash != latest_block_hash { + return Err(eyre::eyre!( + "Proofs storage latest block hash mismatch at {latest_block_number}: stored {latest_block_hash}, canonical {canonical_hash}" + )); + } + + ProofHistoryMetrics::set_window(earliest_block_number, latest_block_number); + + Ok(()) + } + + /// Spawn the background sync task and return the shared [`SyncTarget`]. + /// + /// The sync target buffers trie data from notifications so the sync loop + /// can use pre-computed data for blocks even when it is many blocks behind + /// the chain tip. Blocks whose trie data was evicted from the cache fall + /// back to full execution. + fn spawn_sync_task(&self) -> Arc { + let sync_target = Arc::new(SyncTarget::new()); + let task_sync_target = Arc::clone(&sync_target); + + let task_storage = self.storage.clone(); + let task_provider = self.ctx.provider().clone(); + let task_evm_config = self.ctx.evm_config().clone(); + let verification_interval = self.verification_interval; + + self.ctx.task_executor().spawn_critical_task( + "morph::proofs_exex::proofs_storage_sync_loop", + async move { + let storage = task_storage.clone(); + let task_collector = + LiveTrieCollector::new(task_evm_config, task_provider.clone(), &storage); + Self::sync_loop( + task_sync_target, + task_storage, + task_provider, + &task_collector, + verification_interval, + ) + .await + .unwrap_or_else(|error| panic!("proof storage sync loop failed: {error:?}")); + }, + ); + + sync_target + } + + async fn sync_loop( + sync_target: Arc, + storage: MorphProofsStorage, + provider: Node::Provider, + collector: &LiveTrieCollector<'_, Node::Evm, Node::Provider, Storage>, + verification_interval: u64, + ) -> eyre::Result<()> { + info!(target: "morph::proofs_exex", "Starting proofs storage sync loop"); + + loop { + let Some(state) = sync_target.take_state() else { + sync_target.notified().await; + continue; + }; + + match state { + SyncTargetState::Revert { revert_to } => { + Self::handle_revert(&storage, collector, revert_to)?; + sync_target.mark_revert_complete(&revert_to); + } + SyncTargetState::RevertThenSync { revert_to, sync_to } => { + Self::handle_revert(&storage, collector, revert_to)?; + sync_target.mark_revert_complete(&revert_to); + Self::sync_forward( + &sync_target, + &storage, + &provider, + collector, + verification_interval, + sync_to, + ) + .await; + } + SyncTargetState::SyncUpTo { to } => { + Self::sync_forward( + &sync_target, + &storage, + &provider, + collector, + verification_interval, + to, + ) + .await; + } + } + } + } + + fn handle_revert( + storage: &MorphProofsStorage, + collector: &LiveTrieCollector<'_, Node::Evm, Node::Provider, Storage>, + revert_to: BlockWithParent, + ) -> eyre::Result<()> { + let latest = match storage.get_latest_block_number()? { + Some((n, _)) => n, + None => return Ok(()), + }; + + if latest >= revert_to.block.number { + info!( + target: "morph::proofs_exex", + revert_to = revert_to.block.number, + latest, + "Reverting proofs storage" + ); + collector.unwind_history(revert_to).map_err(|error| { + error!(target: "morph::proofs_exex", ?error, "Failed to revert proofs storage"); + error + })?; + } else { + debug!( + target: "morph::proofs_exex", + revert_to = revert_to.block.number, + latest, + "Revert target beyond stored blocks, skipping" + ); + } + Ok(()) + } + + #[cfg(test)] + fn handle_revert_for_test(&self, revert_to: BlockWithParent) -> eyre::Result<()> { + let collector = LiveTrieCollector::new( + self.ctx.evm_config().clone(), + self.ctx.provider().clone(), + &self.storage, + ); + Self::handle_revert(&self.storage, &collector, revert_to) + } + + async fn sync_forward( + sync_target: &SyncTarget, + storage: &MorphProofsStorage, + provider: &Node::Provider, + collector: &LiveTrieCollector<'_, Node::Evm, Node::Provider, Storage>, + verification_interval: u64, + target: u64, + ) { + loop { + // Check for higher-priority state (e.g. revert) before processing. + if sync_target.has_pending_state() { + return; + } + + let latest = match storage.get_latest_block_number() { + Ok(Some((n, _))) => n, + Ok(None) => { + error!(target: "morph::proofs_exex", "No blocks stored in proofs storage during sync"); + return; + } + Err(e) => { + error!(target: "morph::proofs_exex", error = ?e, "Failed to get latest block"); + return; + } + }; + + if latest >= target { + return; + } + + let end = latest + .saturating_add(SYNC_BLOCKS_PER_TURN as u64) + .min(target); + info!( + target: "morph::proofs_exex", + start = latest + 1, + end, + target, + blocks = end - latest, + "Processing proofs storage sync turn" + ); + + let mut batch: Vec> = + Vec::with_capacity((end - latest) as usize); + for block_num in (latest + 1)..=end { + let cached = sync_target.take(block_num); + match Self::build_batch_entry(block_num, cached, provider, verification_interval) { + Ok(entry) => batch.push(entry), + Err(e) => { + error!(target: "morph::proofs_exex", block_number = block_num, error = ?e, "Preparing block for batch failed"); + return; + } + } + } + + if let Err(e) = collector.execute_and_store_batch(batch) { + error!(target: "morph::proofs_exex", start = latest + 1, end, error = ?e, "Batch processing failed"); + return; + } + + info!(target: "morph::proofs_exex", latest_stored = latest, target, "Batch processed, yielding"); + task::yield_now().await; + } + } + + fn build_batch_entry( + block_number: u64, + cached: Option, + provider: &Node::Provider, + verification_interval: u64, + ) -> eyre::Result> { + let should_verify = + verification_interval > 0 && block_number.is_multiple_of(verification_interval); + let has_cached = cached.is_some(); + + if let Some(cached) = cached + && !should_verify + { + debug!( + target: "morph::proofs_exex", + block_number, + "Using pre-computed state from notification" + ); + return Ok(BatchBlock::Cached { + block_with_parent: cached.block_with_parent, + sorted_trie_updates: cached.trie_data.trie_updates(), + sorted_post_state: cached.trie_data.hashed_state(), + }); + } + + if has_cached { + info!( + target: "morph::proofs_exex", + block_number, + verification_interval, + "Periodic verification: performing full block execution despite cached data" + ); + } else { + debug!( + target: "morph::proofs_exex", + block_number, + "No cached trie data, falling back to full execution" + ); + } + + debug!( + target: "morph::proofs_exex", + block_number, + "Fetching block from provider for execution", + ); + + let block = provider + .recovered_block(block_number.into(), TransactionVariant::NoHash)? + .ok_or_else(|| eyre::eyre!("Missing block {} in provider", block_number))?; + + Ok(BatchBlock::Execute(Box::new(block))) + } + + fn handle_notification( + &self, + notification: ExExNotification, + sync_target: &SyncTarget, + ) -> eyre::Result<()> { + match ¬ification { + ExExNotification::ChainCommitted { new } => { + self.handle_chain_committed(Arc::clone(new), sync_target)? + } + ExExNotification::ChainReorged { old, new } => { + self.handle_chain_reorged(Arc::clone(old), Arc::clone(new), sync_target)? + } + ExExNotification::ChainReverted { old } => { + self.handle_chain_reverted(Arc::clone(old), sync_target)? + } + } + + if let Some(committed_chain) = notification.committed_chain() { + let tip = committed_chain.tip().num_hash(); + debug!( + target: "morph::proofs_exex", + block_number = tip.number, + block_hash = ?tip.hash, + "Sending FinishedHeight event" + ); + self.ctx.events.send(ExExEvent::FinishedHeight(tip))?; + } + + Ok(()) + } + + fn handle_chain_committed( + &self, + new: Arc>, + sync_target: &SyncTarget, + ) -> eyre::Result<()> { + debug!( + target: "morph::proofs_exex", + block_number = new.tip().number(), + block_hash = ?new.tip().hash(), + "ChainCommitted notification received", + ); + + // Cache trie data for all blocks in the chain so the sync loop can + // use pre-computed data even when it is many blocks behind. + let total_blocks = new.blocks().len(); + let mut cached_count = 0usize; + for (&block_number, block) in new.blocks() { + if let Some(trie_data) = new.trie_data_at(block_number) { + sync_target.insert( + block_number, + CachedBlockTrieData { + block_with_parent: block.block_with_parent(), + trie_data: trie_data.clone(), + }, + ); + cached_count += 1; + } else { + debug!( + target: "morph::proofs_exex", + block_number, + "Notification block missing trie data" + ); + } + } + + debug!( + target: "morph::proofs_exex", + tip = new.tip().number(), + total_blocks, + cached_count, + missing = total_blocks - cached_count, + "Cached notification trie data" + ); + + sync_target.update_state(SyncTargetState::SyncUpTo { + to: new.tip().number(), + }); + Ok(()) + } + + fn handle_chain_reorged( + &self, + old: Arc>, + new: Arc>, + sync_target: &SyncTarget, + ) -> eyre::Result<()> { + info!( + target: "morph::proofs_exex", + old_block_number = old.tip().number(), + old_block_hash = ?old.tip().hash(), + new_block_number = new.tip().number(), + new_block_hash = ?new.tip().hash(), + "ChainReorged notification received", + ); + + if old.fork_block() != new.fork_block() { + return Err(eyre::eyre!( + "Fork blocks do not match: old fork block {:?}, new fork block {:?}", + old.fork_block(), + new.fork_block() + )); + } + + let first_old = old.first().block_with_parent(); + + // Invalidate any cached blocks from the old chain. + sync_target.clear_from(first_old.block.number); + + // Cache trie data for all blocks in the new chain. + for (&block_number, block) in new.blocks() { + if let Some(trie_data) = new.trie_data_at(block_number) { + sync_target.insert( + block_number, + CachedBlockTrieData { + block_with_parent: block.block_with_parent(), + trie_data: trie_data.clone(), + }, + ); + } else { + debug!( + target: "morph::proofs_exex", + block_number, + "Reorged block missing trie data" + ); + } + } + + sync_target.update_state(SyncTargetState::RevertThenSync { + revert_to: first_old, + sync_to: new.tip().number(), + }); + + Ok(()) + } + + fn handle_chain_reverted( + &self, + old: Arc>, + sync_target: &SyncTarget, + ) -> eyre::Result<()> { + info!( + target: "morph::proofs_exex", + old_block_number = old.tip().number(), + old_block_hash = ?old.tip().hash(), + "ChainReverted notification received", + ); + + let first_old = old.first().block_with_parent(); + + // Invalidate any cached blocks that are being reverted. + sync_target.clear_from(first_old.block.number); + + sync_target.update_state(SyncTargetState::Revert { + revert_to: first_old, + }); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::{collections::BTreeMap, default::Default, sync::Arc, time::Duration}; + + use alloy_consensus::private::alloy_primitives::B256; + use alloy_eips::{BlockNumHash, NumHash, eip1898::BlockWithParent}; + use morph_proofs::{BlockStateDiff, MdbxProofsStorage, MorphProofsStorage, MorphProofsStore}; + use reth_db::test_utils::tempdir_path; + use reth_ethereum_primitives::{Block, Receipt}; + use reth_execution_types::{Chain, ExecutionOutcome}; + use reth_primitives_traits::RecoveredBlock; + use reth_trie::{ + ComputedTrieData, HashedPostStateSorted, LazyTrieData, updates::TrieUpdatesSorted, + }; + + use super::*; + + // ------------------------------------------------------------------------- + // Helpers: deterministic blocks and deterministic Chain with precomputed updates + // ------------------------------------------------------------------------- + fn b256(byte: u8) -> B256 { + B256::new([byte; 32]) + } + + // deterministic hash from block number: 0 -> 0x00.., 1 -> 0x01.., etc. + fn hash_for_num(num: u64) -> B256 { + // if you only care about small test numbers, this is enough: + b256(num as u8) + + // If you want to avoid wrapping when num > 255, use something like: + // let mut out = [0u8; 32]; + // out[0..8].copy_from_slice(&num.to_be_bytes()); + // B256::new(out) + } + + fn mk_block(num: u64) -> RecoveredBlock { + let mut b: RecoveredBlock = Default::default(); + b.set_block_number(num); + b.set_hash(hash_for_num(num)); + b.set_parent_hash(hash_for_num(num - 1)); + b + } + + fn mk_chain_with_updates( + from: u64, + to: u64, + hash_override: Option, + ) -> Chain { + let mut blocks: Vec> = Vec::new(); + let mut trie_data = BTreeMap::new(); + + for n in from..=to { + let mut b = mk_block(n); + if let Some(hash) = hash_override { + b.set_hash(hash); + } + blocks.push(b); + + let data = LazyTrieData::ready(ComputedTrieData::new( + Arc::new(HashedPostStateSorted::default()), + Arc::new(TrieUpdatesSorted::default()), + )); + trie_data.insert(n, data); + } + + let execution_outcome: ExecutionOutcome = ExecutionOutcome { + bundle: Default::default(), + receipts: Vec::new(), + requests: Vec::new(), + first_block: from, + }; + + Chain::new(blocks, execution_outcome, trie_data) + } + + /// Store blocks directly into proofs storage (bypasses the sync loop). + fn store_blocks(from: u64, to: u64, storage: &MorphProofsStorage) { + for n in from..=to { + let chain = mk_chain_with_updates(n, n, None); + let block = chain.blocks().get(&n).unwrap(); + storage + .store_trie_updates(block.block_with_parent(), BlockStateDiff::default()) + .expect("store trie update"); + } + } + + fn init_storage_at( + storage: MorphProofsStorage, + genesis_block: NumHash, + ) { + storage + .set_earliest_block_number(genesis_block.number, genesis_block.hash) + .expect("set earliest"); + storage + .store_trie_updates( + BlockWithParent::new(genesis_block.hash, genesis_block), + BlockStateDiff::default(), + ) + .expect("store trie update"); + } + + fn init_storage(storage: MorphProofsStorage) { + init_storage_at(storage, NumHash::new(0, b256(0x00))); + } + + // Initialize exex with config + fn build_test_exex( + ctx: ExExContext, + storage: MorphProofsStorage, + ) -> MorphProofsExEx + where + NodeT: FullNodeComponents, + Store: MorphProofsStore + Clone + 'static, + { + MorphProofsExEx::builder(ctx, storage) + .with_proofs_history_window(20) + .with_proofs_history_prune_interval(Duration::from_secs(3600)) + .with_verification_interval(1000) + .with_max_prune_blocks_startup(1000) + .build() + } + + #[tokio::test] + async fn handle_notification_chain_committed() { + // MDBX proofs storage + let dir = tempdir_path(); + let store = Arc::new(MdbxProofsStorage::new(dir.as_path()).expect("env")); + let proofs: MorphProofsStorage> = Arc::clone(&store); + + init_storage(proofs.clone()); + + let (ctx, _handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + + let exex = build_test_exex(ctx, proofs.clone()); + + let new_chain = Arc::new(mk_chain_with_updates(1, 1, None)); + let notif = ExExNotification::ChainCommitted { new: new_chain }; + + let sync_target = SyncTarget::new(); + + exex.handle_notification(notif, &sync_target) + .expect("handle chain commit"); + + // Committed blocks are cached in sync target, not stored directly + assert!(sync_target.take(1).is_some(), "block 1 should be cached"); + let state = sync_target.take_state().expect("should have pending state"); + assert!(matches!(state, SyncTargetState::SyncUpTo { to: 1 })); + } + + #[tokio::test] + async fn handle_notification_chain_committed_caches_already_stored_blocks() { + // MDBX proofs storage + let dir = tempdir_path(); + let store = Arc::new(MdbxProofsStorage::new(dir.as_path()).expect("env")); + let proofs: MorphProofsStorage> = Arc::clone(&store); + + init_storage(proofs.clone()); + + // Pre-store blocks 1..5 so storage is at block 5 + store_blocks(1, 5, &proofs); + + let (ctx, _handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + + let exex = build_test_exex(ctx, proofs.clone()); + + let sync_target = SyncTarget::new(); + + // Handle notification for block 5 which is already stored - still caches + let new_chain = Arc::new(mk_chain_with_updates(5, 5, Some(hash_for_num(10)))); + let notif = ExExNotification::ChainCommitted { new: new_chain }; + exex.handle_notification(notif, &sync_target) + .expect("handle chain commit"); + + // State is set (sync loop will see latest >= target and skip) + let state = sync_target.take_state().expect("should have pending state"); + assert!(matches!(state, SyncTargetState::SyncUpTo { to: 5 })); + + // Storage is unchanged (notification handler doesn't write to storage) + let latest = proofs + .get_latest_block_number() + .expect("get latest block") + .expect("ok"); + assert_eq!(latest.0, 5); + assert_eq!(latest.1, hash_for_num(5)); + } + + #[tokio::test] + async fn handle_notification_chain_reorged() { + // MDBX proofs storage + let dir = tempdir_path(); + let store = Arc::new(MdbxProofsStorage::new(dir.as_path()).expect("env")); + let proofs: MorphProofsStorage> = Arc::clone(&store); + + init_storage(proofs.clone()); + store_blocks(1, 10, &proofs); + + let (ctx, _handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + + let exex = build_test_exex(ctx, proofs.clone()); + + let sync_target = SyncTarget::new(); + + // Now the tip is 10, and we want to reorg from block 6..12 + let old_chain = Arc::new(mk_chain_with_updates(6, 10, None)); + let new_chain = Arc::new(mk_chain_with_updates(6, 12, None)); + + // Notification: chain reorged 6..12 + let notif = ExExNotification::ChainReorged { + new: new_chain, + old: old_chain, + }; + + exex.handle_notification(notif, &sync_target) + .expect("handle chain re-orged"); + + // Should have RevertThenSync state + let state = sync_target.take_state().expect("should have pending state"); + assert!(matches!( + state, + SyncTargetState::RevertThenSync { revert_to, sync_to: 12 } + if revert_to.block.number == 6 + )); + + // New chain blocks should be cached + for n in 6..=12 { + assert!(sync_target.take(n).is_some(), "block {n} should be cached"); + } + + // Storage unchanged (sync loop handles the actual revert) + let latest = proofs + .get_latest_block_number() + .expect("get latest block") + .expect("ok") + .0; + assert_eq!(latest, 10); + } + + #[tokio::test] + async fn handle_notification_chain_reorged_beyond_stored_blocks() { + // MDBX proofs storage + let dir = tempdir_path(); + let store = Arc::new(MdbxProofsStorage::new(dir.as_path()).expect("env")); + let proofs: MorphProofsStorage> = Arc::clone(&store); + + init_storage(proofs.clone()); + store_blocks(1, 10, &proofs); + + let (ctx, _handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + + let exex = build_test_exex(ctx, proofs.clone()); + + let sync_target = SyncTarget::new(); + + // Now the tip is 10, and we want to reorg from block 12..15 + // Both chains share the same fork block (block 11) + let old_chain = Arc::new(mk_chain_with_updates(12, 15, None)); + let new_chain = Arc::new(mk_chain_with_updates(12, 20, None)); + + // Notification: chain reorged 12..20 + let notif = ExExNotification::ChainReorged { + new: new_chain, + old: old_chain, + }; + + exex.handle_notification(notif, &sync_target) + .expect("handle chain re-orged"); + + // State is set; sync loop will detect revert is beyond stored blocks + let state = sync_target.take_state().expect("should have pending state"); + assert!(matches!( + state, + SyncTargetState::RevertThenSync { revert_to, sync_to: 20 } + if revert_to.block.number == 12 + )); + + // Storage unchanged + let latest = proofs + .get_latest_block_number() + .expect("get latest block") + .expect("ok") + .0; + assert_eq!(latest, 10); + } + + #[tokio::test] + async fn handle_notification_chain_reverted() { + // MDBX proofs storage + let dir = tempdir_path(); + let store = Arc::new(MdbxProofsStorage::new(dir.as_path()).expect("env")); + let proofs: MorphProofsStorage> = Arc::clone(&store); + + init_storage(proofs.clone()); + store_blocks(1, 10, &proofs); + + let (ctx, _handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + + let exex = build_test_exex(ctx, proofs.clone()); + + let sync_target = SyncTarget::new(); + + // Now the tip is 10, and we want to revert from block 9..10 + let old_chain = Arc::new(mk_chain_with_updates(9, 10, None)); + + // Notification: chain reverted 9..10 + let notif = ExExNotification::ChainReverted { old: old_chain }; + + exex.handle_notification(notif, &sync_target) + .expect("handle chain reverted"); + + // Should have Revert state + let state = sync_target.take_state().expect("should have pending state"); + assert!(matches!( + state, + SyncTargetState::Revert { revert_to } + if revert_to.block.number == 9 + )); + + // Storage unchanged (sync loop handles the actual revert) + let latest = proofs + .get_latest_block_number() + .expect("get latest block") + .expect("ok") + .0; + assert_eq!(latest, 10); + } + + #[tokio::test] + async fn handle_notification_chain_reverted_beyond_stored_blocks() { + // MDBX proofs storage + let dir = tempdir_path(); + let store = Arc::new(MdbxProofsStorage::new(dir.as_path()).expect("env")); + let proofs: MorphProofsStorage> = Arc::clone(&store); + + init_storage(proofs.clone()); + store_blocks(1, 5, &proofs); + + let (ctx, _handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + + let exex = build_test_exex(ctx, proofs.clone()); + + let sync_target = SyncTarget::new(); + + // Now the tip is 5, and we want to revert from block 9..10 + let old_chain = Arc::new(mk_chain_with_updates(9, 10, None)); + + // Notification: chain reverted 9..10 + let notif = ExExNotification::ChainReverted { old: old_chain }; + + exex.handle_notification(notif, &sync_target) + .expect("handle chain reverted"); + + // State is set; sync loop will detect revert is beyond stored blocks + let state = sync_target.take_state().expect("should have pending state"); + assert!(matches!( + state, + SyncTargetState::Revert { revert_to } + if revert_to.block.number == 9 + )); + + // Storage unchanged + let latest = proofs + .get_latest_block_number() + .expect("get latest block") + .expect("ok") + .0; + assert_eq!(latest, 5); + } + + #[tokio::test] + async fn handle_revert_propagates_unwind_failure() { + let dir = tempdir_path(); + let store = Arc::new(MdbxProofsStorage::new(dir.as_path()).expect("env")); + let proofs: MorphProofsStorage> = Arc::clone(&store); + init_storage(proofs.clone()); + + let (ctx, _handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + let exex = build_test_exex(ctx, proofs.clone()); + + let initial_anchor = BlockWithParent::new(b256(0x00), NumHash::new(0, b256(0x00))); + let error = exex + .handle_revert_for_test(initial_anchor) + .expect_err("unwinding the initial anchor must fail"); + + assert!( + error.to_string().contains("earliest"), + "unexpected unwind error: {error:?}" + ); + assert_eq!( + proofs + .get_latest_block_number() + .expect("latest") + .expect("initialized") + .0, + 0, + "failed unwind must leave proof history unchanged" + ); + } + + #[tokio::test] + async fn ensure_initialized_errors_on_storage_not_initialized() { + // MDBX proofs storage + let dir = tempdir_path(); + let store = Arc::new(MdbxProofsStorage::new(dir.as_path()).expect("env")); + let proofs: MorphProofsStorage> = Arc::clone(&store); + + let (ctx, _handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + + let exex = build_test_exex(ctx, proofs.clone()); + let _ = exex.ensure_initialized().expect_err("should return error"); + } + + #[tokio::test] + async fn ensure_initialized_errors_when_prune_exceeds_threshold() { + // MDBX proofs storage + let dir = tempdir_path(); + let store = Arc::new(MdbxProofsStorage::new(dir.as_path()).expect("env")); + let proofs: MorphProofsStorage> = Arc::clone(&store); + + init_storage(proofs.clone()); + + for i in 1..1100 { + proofs + .store_trie_updates( + BlockWithParent::new( + hash_for_num(i - 1), + BlockNumHash::new(i, hash_for_num(i)), + ), + BlockStateDiff::default(), + ) + .expect("store trie update"); + } + + let (ctx, _handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + + let exex = build_test_exex(ctx, proofs.clone()); + let _ = exex.ensure_initialized().expect_err("should return error"); + } + + #[tokio::test] + async fn ensure_initialized_succeeds() { + // MDBX proofs storage + let dir = tempdir_path(); + let store = Arc::new(MdbxProofsStorage::new(dir.as_path()).expect("env")); + let proofs: MorphProofsStorage> = Arc::clone(&store); + + let (ctx, handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + init_storage_at(proofs.clone(), handle.genesis.num_hash()); + + let exex = build_test_exex(ctx, proofs.clone()); + exex.ensure_initialized().expect("should not return error"); + } + + #[tokio::test] + async fn handle_notification_schedules_async_on_gap() { + // MDBX proofs storage + let dir = tempdir_path(); + let store = Arc::new(MdbxProofsStorage::new(dir.as_path()).expect("env")); + let proofs: MorphProofsStorage> = Arc::clone(&store); + + init_storage(proofs.clone()); + + let (ctx, _handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + + let exex = build_test_exex(ctx, proofs.clone()); + + // Notification: chain committed 5..10 (Blocks 1,2,3,4 are missing from storage) + let new_chain = Arc::new(mk_chain_with_updates(5, 10, None)); + let notif = ExExNotification::ChainCommitted { new: new_chain }; + + let sync_target = SyncTarget::new(); + + // Process notification + exex.handle_notification(notif, &sync_target) + .expect("handle chain commit should return ok immediately"); + + // Verify the sync target state was set + let state = sync_target.take_state().expect("should have pending state"); + assert!( + matches!(state, SyncTargetState::SyncUpTo { to: 10 }), + "Should have scheduled sync to block 10" + ); + + // Verify Main Thread did NOT process it + // Because we didn't spawn the actual worker thread in this test, storage should still be at + // 0. This proves the 'handle_notification' returned instantly without doing the + // heavy lifting. + let latest = proofs + .get_latest_block_number() + .expect("get") + .expect("ok") + .0; + assert_eq!( + latest, 0, + "Main thread should not have processed the blocks synchronously" + ); + } +} diff --git a/crates/proofs-exex/src/sync_target.rs b/crates/proofs-exex/src/sync_target.rs new file mode 100644 index 00000000..5a26b6fa --- /dev/null +++ b/crates/proofs-exex/src/sync_target.rs @@ -0,0 +1,637 @@ +//! Sync target with trie data cache for the background sync loop. +//! +//! Buffers trie data from exex notifications so the sync loop can use +//! pre-computed data even when it is many blocks behind the chain tip. +//! Routes committed, reverted, and reorged notifications through a +//! [`SyncTargetState`] state machine so the sync loop is the single +//! writer to proofs storage. + +use std::{collections::BTreeMap, sync::Mutex}; + +use alloy_eips::eip1898::BlockWithParent; +use reth_trie::LazyTrieData; +use tokio::sync::Notify; +use tracing::debug; + +/// Maximum number of blocks to cache trie data for. +const CACHE_CAPACITY: usize = 1024; + +/// Cached trie data for a single block. +#[derive(Debug)] +pub struct CachedBlockTrieData { + /// The block identifier with its parent hash. + pub block_with_parent: BlockWithParent, + /// The lazy trie data (hashed state + trie updates). + pub trie_data: LazyTrieData, +} + +/// The state of the sync target, describing what the sync loop should do next. +#[derive(Debug)] +pub enum SyncTargetState { + /// Sync forward to a specific block number. + SyncUpTo { + /// The target block number to sync up to. + to: u64, + }, + /// Revert to a specific block, then sync forward to a target. + RevertThenSync { + /// The first block to remove (inclusive) during the revert. + revert_to: BlockWithParent, + /// The target block number to sync up to after reverting. + sync_to: u64, + }, + /// Revert to a specific block without syncing forward. + Revert { + /// The first block to remove (inclusive) during the revert. + revert_to: BlockWithParent, + }, +} + +impl SyncTargetState { + const fn apply_next(&mut self, new: Self) { + *self = match (&*self, new) { + // If we are just syncing to tip already, replace with the new state. + (Self::SyncUpTo { .. }, new) => new, + + // If the new state is a revert, replace with the new state. + (_, Self::Revert { revert_to }) => Self::Revert { revert_to }, + (_, Self::RevertThenSync { revert_to, sync_to }) => { + Self::RevertThenSync { revert_to, sync_to } + } + + // If we're currently reverting, replace the sync to value with the new + // state. + ( + Self::RevertThenSync { revert_to, .. } | Self::Revert { revert_to }, + Self::SyncUpTo { to }, + ) => Self::RevertThenSync { + revert_to: *revert_to, + sync_to: to, + }, + }; + } +} + +/// Sync target that buffers trie data from recent exex notifications. +/// +/// Routes all notification types (committed, reverted, reorged) through a +/// [`SyncTargetState`] state machine so the sync loop is the single writer +/// to proofs storage. Uses a [`Notify`] to wake the sync loop when new +/// state is available. +/// +/// Trie data is accumulated in a bounded [`BTreeMap`] so the sync loop +/// can still use pre-computed trie data for blocks from earlier notifications. +pub struct SyncTarget { + cache: Mutex>, + state: Mutex>, + notify: Notify, +} + +impl Default for SyncTarget { + fn default() -> Self { + Self::new() + } +} + +impl SyncTarget { + /// Create a new `SyncTarget` with no cached data and no pending state. + pub fn new() -> Self { + Self { + cache: Mutex::new(BTreeMap::new()), + state: Mutex::new(None), + notify: Notify::new(), + } + } + + /// Update the sync target state and wake the sync loop. + /// + /// If there is already a pending state, the new state is merged using + /// `SyncTargetState::apply_next`. + pub fn update_state(&self, new: SyncTargetState) { + let mut state = self.state.lock().expect("SyncTarget lock poisoned"); + match state.as_mut() { + Some(current) => current.apply_next(new), + None => *state = Some(new), + } + drop(state); + self.notify.notify_one(); + } + + /// Take the current pending state, leaving `None` in its place. + /// + /// Used by the sync loop to consume the next action to perform. + pub fn take_state(&self) -> Option { + self.state.lock().expect("SyncTarget lock poisoned").take() + } + + /// Notify the sync target that a revert has been processed up to the given block. + /// + /// Only strips the revert portion if the pending revert target is at or above + /// `reverted_to` (i.e. already covered by the completed revert). If a deeper + /// revert arrived while processing, the pending state is left unchanged. + /// + /// - Covered `RevertThenSync` → `SyncUpTo` (keeps the sync target) + /// - Covered `Revert` → `None` + /// - Deeper revert or other states → unchanged + pub fn mark_revert_complete(&self, reverted_to: &BlockWithParent) { + let mut state = self.state.lock().expect("SyncTarget lock poisoned"); + match &*state { + Some(SyncTargetState::RevertThenSync { revert_to, sync_to }) + if revert_to.block.number >= reverted_to.block.number => + { + *state = Some(SyncTargetState::SyncUpTo { to: *sync_to }); + } + Some(SyncTargetState::Revert { revert_to }) + if revert_to.block.number >= reverted_to.block.number => + { + *state = None; + } + _ => {} + } + } + + /// Check if there is a pending state without consuming it. + /// + /// Used by the sync loop to interrupt forward sync when a higher-priority + /// state (e.g. revert) arrives. + pub fn has_pending_state(&self) -> bool { + self.state + .lock() + .expect("SyncTarget lock poisoned") + .is_some() + } + + /// Wait for a state change notification. + /// + /// Returns immediately if a notification arrived since the last call. + pub async fn notified(&self) { + self.notify.notified().await; + } + + /// Insert cached trie data for a block. + /// + /// Evicts the oldest entries when the cache exceeds capacity. + pub fn insert(&self, block_number: u64, data: CachedBlockTrieData) { + let mut cache = self.cache.lock().expect("SyncTarget lock poisoned"); + cache.insert(block_number, data); + let mut evicted = 0u64; + while cache.len() > CACHE_CAPACITY { + cache.pop_first(); + evicted += 1; + } + if evicted > 0 { + debug!( + target: "morph::proofs_exex::sync_target", + block_number, + evicted, + "Cache full, evicted oldest entries" + ); + } + debug!( + target: "morph::proofs_exex::sync_target", + block_number, + cached_blocks = cache.len(), + "Cached trie data for block" + ); + } + + /// Take cached trie data for a specific block, removing it from the cache. + pub fn take(&self, block_number: u64) -> Option { + let result = self + .cache + .lock() + .expect("SyncTarget lock poisoned") + .remove(&block_number); + if result.is_some() { + debug!( + target: "morph::proofs_exex::sync_target", + block_number, + "Cache hit: trie data found for block" + ); + } else { + debug!( + target: "morph::proofs_exex::sync_target", + block_number, + "Cache miss: no trie data for block, will re-execute" + ); + } + result + } + + /// Remove all cached entries at or above the given block number. + /// + /// Used when a revert or reorg invalidates cached blocks. + pub fn clear_from(&self, block_number: u64) { + let mut cache = self.cache.lock().expect("SyncTarget lock poisoned"); + let removed = cache.split_off(&block_number); + if !removed.is_empty() { + debug!( + target: "morph::proofs_exex::sync_target", + block_number, + cleared = removed.len(), + "Cleared cached entries from block onward" + ); + } + } +} + +impl std::fmt::Debug for SyncTarget { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SyncTarget") + .field( + "has_pending_state", + &self + .state + .lock() + .expect("SyncTarget lock poisoned") + .is_some(), + ) + .field( + "cached_blocks", + &self.cache.lock().expect("SyncTarget lock poisoned").len(), + ) + .finish() + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use alloy_consensus::private::alloy_primitives::B256; + use alloy_eips::{NumHash, eip1898::BlockWithParent}; + use reth_trie::{ + ComputedTrieData, HashedPostStateSorted, LazyTrieData, updates::TrieUpdatesSorted, + }; + + use super::*; + + fn b256(byte: u8) -> B256 { + B256::new([byte; 32]) + } + + fn block_with_parent(num: u64) -> BlockWithParent { + BlockWithParent::new( + b256(num.wrapping_sub(1) as u8), + NumHash::new(num, b256(num as u8)), + ) + } + + fn dummy_cached_data(num: u64) -> CachedBlockTrieData { + CachedBlockTrieData { + block_with_parent: block_with_parent(num), + trie_data: LazyTrieData::ready(ComputedTrieData::new( + Arc::new(HashedPostStateSorted::default()), + Arc::new(TrieUpdatesSorted::default()), + )), + } + } + + // ---- SyncTargetState::apply_next tests ---- + + #[test] + fn apply_next_sync_replaced_by_sync() { + let mut state = SyncTargetState::SyncUpTo { to: 10 }; + state.apply_next(SyncTargetState::SyncUpTo { to: 20 }); + assert!(matches!(state, SyncTargetState::SyncUpTo { to: 20 })); + } + + #[test] + fn apply_next_sync_replaced_by_revert() { + let mut state = SyncTargetState::SyncUpTo { to: 10 }; + let revert_to = block_with_parent(5); + state.apply_next(SyncTargetState::Revert { revert_to }); + assert!( + matches!(state, SyncTargetState::Revert { revert_to } if revert_to.block.number == 5) + ); + } + + #[test] + fn apply_next_sync_replaced_by_revert_then_sync() { + let mut state = SyncTargetState::SyncUpTo { to: 10 }; + let revert_to = block_with_parent(5); + state.apply_next(SyncTargetState::RevertThenSync { + revert_to, + sync_to: 20, + }); + assert!(matches!( + state, + SyncTargetState::RevertThenSync { revert_to, sync_to: 20 } + if revert_to.block.number == 5 + )); + } + + #[test] + fn apply_next_revert_then_sync_keeps_revert_on_new_sync() { + let revert_to = block_with_parent(5); + let mut state = SyncTargetState::RevertThenSync { + revert_to, + sync_to: 10, + }; + state.apply_next(SyncTargetState::SyncUpTo { to: 30 }); + assert!(matches!( + state, + SyncTargetState::RevertThenSync { revert_to, sync_to: 30 } + if revert_to.block.number == 5 + )); + } + + #[test] + fn apply_next_revert_then_sync_replaced_by_new_revert() { + let revert_to = block_with_parent(5); + let mut state = SyncTargetState::RevertThenSync { + revert_to, + sync_to: 10, + }; + let new_revert = block_with_parent(3); + state.apply_next(SyncTargetState::Revert { + revert_to: new_revert, + }); + assert!(matches!( + state, + SyncTargetState::Revert { revert_to } if revert_to.block.number == 3 + )); + } + + #[test] + fn apply_next_revert_gains_sync_target() { + let revert_to = block_with_parent(5); + let mut state = SyncTargetState::Revert { revert_to }; + state.apply_next(SyncTargetState::SyncUpTo { to: 15 }); + assert!(matches!( + state, + SyncTargetState::RevertThenSync { revert_to, sync_to: 15 } + if revert_to.block.number == 5 + )); + } + + #[test] + fn apply_next_revert_replaced_by_revert_then_sync() { + let revert_to = block_with_parent(5); + let mut state = SyncTargetState::Revert { revert_to }; + let new_revert = block_with_parent(3); + state.apply_next(SyncTargetState::RevertThenSync { + revert_to: new_revert, + sync_to: 20, + }); + assert!(matches!( + state, + SyncTargetState::RevertThenSync { revert_to, sync_to: 20 } + if revert_to.block.number == 3 + )); + } + + // ---- SyncTarget state management tests ---- + + #[test] + fn new_sync_target_has_no_state() { + let target = SyncTarget::new(); + assert!(!target.has_pending_state()); + assert!(target.take_state().is_none()); + } + + #[test] + fn update_state_sets_pending() { + let target = SyncTarget::new(); + target.update_state(SyncTargetState::SyncUpTo { to: 10 }); + assert!(target.has_pending_state()); + } + + #[test] + fn take_state_clears_pending() { + let target = SyncTarget::new(); + target.update_state(SyncTargetState::SyncUpTo { to: 10 }); + let state = target.take_state(); + assert!(matches!(state, Some(SyncTargetState::SyncUpTo { to: 10 }))); + assert!(!target.has_pending_state()); + } + + #[test] + fn update_state_merges_with_existing() { + let target = SyncTarget::new(); + let revert_to = block_with_parent(5); + target.update_state(SyncTargetState::Revert { revert_to }); + target.update_state(SyncTargetState::SyncUpTo { to: 20 }); + + let state = target.take_state().expect("should have state"); + assert!(matches!( + state, + SyncTargetState::RevertThenSync { revert_to, sync_to: 20 } + if revert_to.block.number == 5 + )); + } + + // ---- mark_revert_complete tests ---- + + #[test] + fn mark_revert_complete_clears_covered_revert() { + let target = SyncTarget::new(); + let revert_to = block_with_parent(5); + target.update_state(SyncTargetState::Revert { revert_to }); + + target.mark_revert_complete(&block_with_parent(5)); + assert!(!target.has_pending_state()); + } + + #[test] + fn mark_revert_complete_strips_covered_revert_then_sync() { + let target = SyncTarget::new(); + let revert_to = block_with_parent(5); + target.update_state(SyncTargetState::RevertThenSync { + revert_to, + sync_to: 20, + }); + + target.mark_revert_complete(&block_with_parent(5)); + let state = target.take_state().expect("should have state"); + assert!(matches!(state, SyncTargetState::SyncUpTo { to: 20 })); + } + + #[test] + fn mark_revert_complete_keeps_deeper_revert() { + let target = SyncTarget::new(); + // A deeper revert (block 3) arrived while we were reverting to block 5 + let revert_to = block_with_parent(3); + target.update_state(SyncTargetState::Revert { revert_to }); + + target.mark_revert_complete(&block_with_parent(5)); + let state = target.take_state().expect("should still have state"); + assert!(matches!( + state, + SyncTargetState::Revert { revert_to } if revert_to.block.number == 3 + )); + } + + #[test] + fn mark_revert_complete_keeps_deeper_revert_then_sync() { + let target = SyncTarget::new(); + let revert_to = block_with_parent(3); + target.update_state(SyncTargetState::RevertThenSync { + revert_to, + sync_to: 20, + }); + + target.mark_revert_complete(&block_with_parent(5)); + let state = target.take_state().expect("should still have state"); + assert!(matches!( + state, + SyncTargetState::RevertThenSync { revert_to, sync_to: 20 } + if revert_to.block.number == 3 + )); + } + + #[test] + fn mark_revert_complete_clears_shallower_revert() { + let target = SyncTarget::new(); + // Pending revert to block 8 is shallower than what we reverted to (block 5) + let revert_to = block_with_parent(8); + target.update_state(SyncTargetState::Revert { revert_to }); + + target.mark_revert_complete(&block_with_parent(5)); + assert!(!target.has_pending_state()); + } + + #[test] + fn mark_revert_complete_noop_on_sync_up_to() { + let target = SyncTarget::new(); + target.update_state(SyncTargetState::SyncUpTo { to: 20 }); + + target.mark_revert_complete(&block_with_parent(5)); + let state = target.take_state().expect("should still have state"); + assert!(matches!(state, SyncTargetState::SyncUpTo { to: 20 })); + } + + #[test] + fn mark_revert_complete_noop_on_empty() { + let target = SyncTarget::new(); + target.mark_revert_complete(&block_with_parent(5)); + assert!(!target.has_pending_state()); + } + + // ---- Cache tests ---- + + #[test] + fn insert_and_take_cached_data() { + let target = SyncTarget::new(); + target.insert(42, dummy_cached_data(42)); + + let taken = target.take(42); + assert!(taken.is_some()); + assert_eq!(taken.unwrap().block_with_parent.block.number, 42); + + // Second take returns None (was removed) + assert!(target.take(42).is_none()); + } + + #[test] + fn take_nonexistent_block_returns_none() { + let target = SyncTarget::new(); + assert!(target.take(999).is_none()); + } + + #[test] + fn clear_from_removes_blocks_at_and_above() { + let target = SyncTarget::new(); + for n in 1..=10 { + target.insert(n, dummy_cached_data(n)); + } + + target.clear_from(6); + + // Blocks 1..=5 remain + for n in 1..=5 { + assert!(target.take(n).is_some(), "block {n} should still be cached"); + } + // Blocks 6..=10 removed + for n in 6..=10 { + assert!( + target.take(n).is_none(), + "block {n} should have been cleared" + ); + } + } + + #[test] + fn clear_from_at_start_clears_everything() { + let target = SyncTarget::new(); + for n in 5..=10 { + target.insert(n, dummy_cached_data(n)); + } + target.clear_from(5); + for n in 5..=10 { + assert!(target.take(n).is_none()); + } + } + + #[test] + fn clear_from_beyond_cache_is_noop() { + let target = SyncTarget::new(); + for n in 1..=5 { + target.insert(n, dummy_cached_data(n)); + } + target.clear_from(100); + for n in 1..=5 { + assert!(target.take(n).is_some()); + } + } + + #[test] + fn cache_evicts_oldest_when_full() { + let target = SyncTarget::new(); + + // Fill to capacity + 10 + for n in 1..=(CACHE_CAPACITY as u64 + 10) { + target.insert(n, dummy_cached_data(n)); + } + + // Oldest 10 blocks should have been evicted + for n in 1..=10 { + assert!( + target.take(n).is_none(), + "block {n} should have been evicted" + ); + } + + // Block 11 onward should still be present + assert!(target.take(11).is_some()); + assert!(target.take(CACHE_CAPACITY as u64 + 10).is_some()); + } + + #[test] + fn default_creates_empty_sync_target() { + let target = SyncTarget::default(); + assert!(!target.has_pending_state()); + assert!(target.take(0).is_none()); + } + + #[test] + fn debug_impl_does_not_panic() { + let target = SyncTarget::new(); + target.insert(1, dummy_cached_data(1)); + target.update_state(SyncTargetState::SyncUpTo { to: 5 }); + let debug_str = format!("{target:?}"); + assert!(debug_str.contains("SyncTarget")); + assert!(debug_str.contains("has_pending_state")); + assert!(debug_str.contains("cached_blocks")); + } + + #[tokio::test] + async fn notified_wakes_on_update_state() { + let target = Arc::new(SyncTarget::new()); + let target_clone = Arc::clone(&target); + + // Spawn a task that waits for notification + let handle = tokio::spawn(async move { + target_clone.notified().await; + target_clone.take_state() + }); + + // Give the task time to start waiting + tokio::task::yield_now().await; + + target.update_state(SyncTargetState::SyncUpTo { to: 42 }); + + let result = handle.await.expect("task should complete"); + assert!(matches!(result, Some(SyncTargetState::SyncUpTo { to: 42 }))); + } +} diff --git a/crates/proofs/Cargo.toml b/crates/proofs/Cargo.toml new file mode 100644 index 00000000..db4b1603 --- /dev/null +++ b/crates/proofs/Cargo.toml @@ -0,0 +1,50 @@ +[package] +name = "morph-proofs" +description = "Bounded historical MPT storage for Morph eth_getProof" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[lints] +workspace = true + +[dependencies] +alloy-eips.workspace = true +alloy-primitives.workspace = true +auto_impl.workspace = true +bincode = { workspace = true, features = ["serde"] } +bytes.workspace = true +derive_more.workspace = true +metrics.workspace = true +parking_lot.workspace = true +serde.workspace = true +strum = { workspace = true, features = ["derive"] } +thiserror.workspace = true +tokio = { workspace = true, features = ["sync"] } +tracing.workspace = true + +reth-codecs.workspace = true +reth-db = { workspace = true, features = ["mdbx"] } +reth-evm.workspace = true +reth-execution-errors.workspace = true +reth-metrics.workspace = true +reth-primitives-traits.workspace = true +reth-provider.workspace = true +reth-revm = { workspace = true, features = ["witness"] } +reth-tasks.workspace = true +reth-trie = { workspace = true, features = ["serde"] } +reth-trie-common = { workspace = true, features = ["serde"] } + +[dev-dependencies] +alloy-primitives = { workspace = true, features = ["rand"] } +mockall.workspace = true +reth-db = { workspace = true, features = ["test-utils"] } +reth-db-api = { workspace = true, features = ["test-utils"] } +reth-storage-errors.workspace = true +tempfile.workspace = true + +[features] +default = [] +test-utils = [] diff --git a/crates/proofs/NOTICE.md b/crates/proofs/NOTICE.md new file mode 100644 index 00000000..f60916ef --- /dev/null +++ b/crates/proofs/NOTICE.md @@ -0,0 +1,32 @@ +# Attribution + +The versioned trie storage and proof-history algorithms in this crate are adapted from +the Base repository at commit `b2673bbd927cb34d7cfad4d448bfbd5bd30eae88`: + + + +The adapted code is used under Base's MIT license. Morph-specific changes include an +MDBX-only surface, durable chain-identity metadata, and integration with Morph's node, +RPC, and lifecycle rules. + +## Base MIT License + +Copyright (c) 2025 Base + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/proofs/src/api.rs b/crates/proofs/src/api.rs new file mode 100644 index 00000000..0d014093 --- /dev/null +++ b/crates/proofs/src/api.rs @@ -0,0 +1,435 @@ +//! Storage API for external storage of intermediary trie nodes. + +use std::{fmt::Debug, time::Duration}; + +use alloy_eips::{BlockNumHash, eip1898::BlockWithParent}; +use alloy_primitives::{B256, U256}; +use auto_impl::auto_impl; +use derive_more::{AddAssign, Constructor}; +use reth_primitives_traits::Account; +use reth_trie::{ + hashed_cursor::{HashedCursor, HashedStorageCursor}, + trie_cursor::{TrieCursor, TrieStorageCursor}, +}; +use reth_trie_common::{ + BranchNodeCompact, HashedPostStateSorted, Nibbles, StoredNibbles, updates::TrieUpdatesSorted, +}; + +use crate::{ + MorphProofsStorageResult, + db::{HashedStorageKey, StorageTrieKey}, +}; + +/// Type alias for bulk storage branch entries: a list of (hashed address, storage nodes) pairs. +pub type StorageBranchEntries = Vec<(B256, Vec<(Nibbles, Option)>)>; + +/// Diff of trie updates and post state for a block. +#[derive(Debug, Clone, Default)] +pub struct BlockStateDiff { + /// Trie updates for branch nodes + pub sorted_trie_updates: TrieUpdatesSorted, + /// Post state for leaf nodes (accounts and storage) + pub sorted_post_state: HashedPostStateSorted, +} + +impl BlockStateDiff { + /// Extend the [` BlockStateDiff`] from other latest [`BlockStateDiff`] + pub fn extend_ref(&mut self, other: &Self) { + self.sorted_trie_updates + .extend_ref_and_sort(&other.sorted_trie_updates); + self.sorted_post_state + .extend_ref_and_sort(&other.sorted_post_state); + } +} + +/// Counts of trie updates written to storage. +#[derive(Debug, Clone, Default, AddAssign, Constructor, Eq, PartialEq)] +pub struct WriteCounts { + /// Number of account trie updates written + pub account_trie_updates_written_total: u64, + /// Number of storage trie updates written + pub storage_trie_updates_written_total: u64, + /// Number of hashed accounts written + pub hashed_accounts_written_total: u64, + /// Number of hashed storages written + pub hashed_storages_written_total: u64, +} + +/// Duration metrics for block processing. +#[derive(Debug, Default, Clone)] +pub struct OperationDurations { + /// Total time to process a block (end-to-end) in seconds + pub total_duration_seconds: Duration, + /// Time spent executing the block (EVM) in seconds + pub execution_duration_seconds: Duration, + /// Time spent calculating state root in seconds + pub state_root_duration_seconds: Duration, + /// Time spent writing trie updates to storage in seconds + pub write_duration_seconds: Duration, +} + +/// Trait for reading trie nodes from the database. +/// +/// Only leaf nodes and some branch nodes are stored. The bottom layer of branch nodes +/// are not stored to reduce write amplification. This matches Reth's non-historical trie storage. +#[auto_impl(Arc)] +pub trait MorphProofsStore: Send + Sync + Debug { + /// Cursor for iterating over trie branches. + type StorageTrieCursor<'tx>: TrieStorageCursor + 'tx + where + Self: 'tx; + + /// Cursor for iterating over account trie branches. + type AccountTrieCursor<'tx>: TrieCursor + 'tx + where + Self: 'tx; + + /// Cursor for iterating over storage leaves. + type StorageCursor<'tx>: HashedStorageCursor + Send + Sync + 'tx + where + Self: 'tx; + + /// Cursor for iterating over account leaves. + type AccountHashedCursor<'tx>: HashedCursor + Send + Sync + 'tx + where + Self: 'tx; + + /// Transaction type used by request-scoped cursor factories. + type Tx<'tx>: Debug + Send + Sync + where + Self: 'tx; + + /// Get the earliest block number and hash that has been stored + /// + /// This is used to determine the block number of trie nodes with block number 0. + /// All earliest block numbers are stored in 0 to reduce updates required to prune trie nodes. + fn get_earliest_block_number(&self) -> MorphProofsStorageResult>; + + /// Get the latest block number and hash that has been stored + fn get_latest_block_number(&self) -> MorphProofsStorageResult>; + + /// Get the earliest block number and hash visible to an existing read transaction. + fn get_earliest_block_number_with_tx<'tx, 'db>( + &self, + tx: &'tx Self::Tx<'db>, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx; + + /// Get the latest block number and hash visible to an existing read transaction. + fn get_latest_block_number_with_tx<'tx, 'db>( + &self, + tx: &'tx Self::Tx<'db>, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx; + + /// Get a trie cursor for the storage backend + fn storage_trie_cursor<'tx>( + &'tx self, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult>; + + /// Get a trie cursor for the account backend + fn account_trie_cursor<'tx>( + &'tx self, + max_block_number: u64, + ) -> MorphProofsStorageResult>; + + /// Get a storage cursor for the storage backend + fn storage_hashed_cursor<'tx>( + &'tx self, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult>; + + /// Get an account hashed cursor for the storage backend + fn account_hashed_cursor<'tx>( + &'tx self, + max_block_number: u64, + ) -> MorphProofsStorageResult>; + + /// Open a transaction for a request-scoped cursor factory. + fn ro_tx<'tx>(&'tx self) -> MorphProofsStorageResult>; + + /// Storage trie cursor reusing `tx`. + fn storage_trie_cursor_with_tx<'tx, 'db>( + &self, + tx: &'tx Self::Tx<'db>, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx; + + /// Account trie cursor reusing `tx`. + fn account_trie_cursor_with_tx<'tx, 'db>( + &self, + tx: &'tx Self::Tx<'db>, + max_block_number: u64, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx; + + /// Storage hashed cursor reusing `tx`. + fn storage_hashed_cursor_with_tx<'tx, 'db>( + &self, + tx: &'tx Self::Tx<'db>, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx; + + /// Account hashed cursor reusing `tx`. + fn account_hashed_cursor_with_tx<'tx, 'db>( + &self, + tx: &'tx Self::Tx<'db>, + max_block_number: u64, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx; + + /// Store a batch of trie updates. + /// + /// If wiped is true, the entire storage trie is wiped, but this is unsupported going forward, + /// so should only happen for legacy reasons. + fn store_trie_updates( + &self, + block_ref: BlockWithParent, + block_state_diff: BlockStateDiff, + ) -> MorphProofsStorageResult; + + /// Fetch all updates for a given block number. + fn fetch_trie_updates(&self, block_number: u64) -> MorphProofsStorageResult; + + /// Applies [`BlockStateDiff`] to the earliest state (updating/deleting nodes) and updates the + /// earliest block number. + fn prune_earliest_state( + &self, + new_earliest_block_ref: BlockWithParent, + ) -> MorphProofsStorageResult; + + /// Remove account, storage and trie updates from historical storage for all blocks till + /// the specified block (inclusive). + fn unwind_history(&self, to: BlockWithParent) -> MorphProofsStorageResult<()>; + + /// Deletes all updates > `latest_common_block` and replaces them with the new updates. + fn replace_updates( + &self, + latest_common_block: BlockNumHash, + blocks_to_add: Vec<(BlockWithParent, BlockStateDiff)>, + ) -> MorphProofsStorageResult<()>; + + /// Set the earliest block number and hash that has been stored + fn set_earliest_block_number( + &self, + block_number: u64, + hash: B256, + ) -> MorphProofsStorageResult<()>; +} + +/// Session-scoped store of trie updates that amortizes write overhead across multiple +/// block writes within a single batch. +/// +/// Reads through the session observe writes from earlier [`MorphProofsBatchSession::store_trie_updates`] +/// calls in the same session, enabling cold catch-up where `block N+1` is computed against +/// `block N` written earlier in the same session. +/// +/// All cursor methods mirror [`MorphProofsStore`] but read through the session's active view. +#[auto_impl(&mut)] +pub trait MorphProofsBatchSession: Send + Sync + Debug { + /// Cursor for iterating over storage trie branches in the active session. + type StorageTrieCursor<'a>: TrieStorageCursor + 'a + where + Self: 'a; + + /// Cursor for iterating over account trie branches in the active session. + type AccountTrieCursor<'a>: TrieCursor + 'a + where + Self: 'a; + + /// Cursor for iterating over storage leaves in the active session. + type StorageCursor<'a>: HashedStorageCursor + Send + Sync + 'a + where + Self: 'a; + + /// Cursor for iterating over account leaves in the active session. + type AccountHashedCursor<'a>: HashedCursor + Send + Sync + 'a + where + Self: 'a; + + /// Earliest stored block number/hash visible to the active transaction. + fn get_earliest_block_number(&self) -> MorphProofsStorageResult>; + + /// Latest stored block number/hash visible to this session (including writes from + /// earlier [`Self::store_trie_updates`] calls in this session). + fn get_latest_block_number(&self) -> MorphProofsStorageResult>; + + /// Storage trie cursor reading through the active transaction. + fn storage_trie_cursor( + &self, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult>; + + /// Account trie cursor reading through the active transaction. + fn account_trie_cursor( + &self, + max_block_number: u64, + ) -> MorphProofsStorageResult>; + + /// Storage hashed cursor reading through the active transaction. + fn storage_hashed_cursor( + &self, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult>; + + /// Account hashed cursor reading through the active transaction. + fn account_hashed_cursor( + &self, + max_block_number: u64, + ) -> MorphProofsStorageResult>; + + /// Append-only write of `block_state_diff` for `block_ref` to the active session. + /// Subsequent reads through this session observe the new state immediately. + /// Durability semantics are backend-dependent; see [`MorphProofsBatchStore`]. + fn store_trie_updates( + &mut self, + block_ref: BlockWithParent, + block_state_diff: BlockStateDiff, + ) -> MorphProofsStorageResult; +} + +/// Storage that can open a [`MorphProofsBatchSession`] for amortized multi-block writes. +/// +/// MDBX commits the session atomically on `Ok` and rolls it back on `Err`. +/// The in-memory implementation is a test double and does not provide rollback. +pub trait MorphProofsBatchStore: MorphProofsStore { + /// Session type for this backend. + type BatchSession<'a>: MorphProofsBatchSession + 'a + where + Self: 'a; + + /// Run `f` inside one batch session. Atomicity and error-recovery semantics are + /// backend-dependent; see the trait-level documentation. + fn with_batch_session(&self, f: F) -> MorphProofsStorageResult + where + F: FnOnce(&mut Self::BatchSession<'_>) -> MorphProofsStorageResult; +} + +impl MorphProofsBatchStore for std::sync::Arc { + type BatchSession<'a> + = T::BatchSession<'a> + where + Self: 'a; + + fn with_batch_session(&self, f: F) -> MorphProofsStorageResult + where + F: FnOnce(&mut Self::BatchSession<'_>) -> MorphProofsStorageResult, + { + (**self).with_batch_session(f) + } +} + +/// Status of the initial state anchor. +#[derive(Debug, Clone, Copy, Default)] +pub enum InitialStateStatus { + /// Init isn't yet started + #[default] + NotStarted, + /// Init is in progress (some tables may already be populated) + InProgress, + /// Init completed successfully (all tables done + earliest block set) + Completed, +} + +/// Anchor for the initial state. +#[derive(Debug, Clone, Default)] +pub struct InitialStateAnchor { + /// The block for which the initial state is being initialized. None if initialization is not + /// yet started. + pub block: Option, + /// Whether initialization is still running or completed. + pub status: InitialStateStatus, + /// The latest key stored for `AccountTrieHistory`. + pub latest_account_trie_key: Option, + /// The latest key stored for `StorageTrieHistory`. + pub latest_storage_trie_key: Option, + /// The latest key stored for `HashedAccountHistory`. + pub latest_hashed_account_key: Option, + /// The latest key stored for `HashedStorageHistory`. + pub latest_hashed_storage_key: Option, +} + +/// Trait for storing and retrieving the initial state anchor. +#[auto_impl(Arc)] +pub trait MorphProofsInitialStateStore: Send + Sync + Debug { + /// Read the current anchor. + fn initial_state_anchor(&self) -> MorphProofsStorageResult; + + /// Create the anchor if it doesn't exist. + /// Returns `Err` if an anchor already exists (prevents accidental overwrite). + fn set_initial_state_anchor(&self, anchor: BlockNumHash) -> MorphProofsStorageResult<()>; + + /// Store a batch of account trie branches. Used for saving existing state. For live state + /// capture, use [store_trie_updates](MorphProofsStore::store_trie_updates). + fn store_account_branches( + &self, + account_nodes: Vec<(Nibbles, Option)>, + ) -> MorphProofsStorageResult<()>; + + /// Store a batch of storage trie branches. Used for saving existing state. + fn store_storage_branches( + &self, + hashed_address: B256, + storage_nodes: Vec<(Nibbles, Option)>, + ) -> MorphProofsStorageResult<()>; + + /// Store storage trie branches for multiple addresses in one initialization batch. + fn store_storage_branches_bulk( + &self, + entries: StorageBranchEntries, + ) -> MorphProofsStorageResult<()> { + for (hashed_address, nodes) in entries { + self.store_storage_branches(hashed_address, nodes)?; + } + Ok(()) + } + + /// Store a batch of account trie leaf nodes. Used for saving existing state. + fn store_hashed_accounts( + &self, + accounts: Vec<(B256, Option)>, + ) -> MorphProofsStorageResult<()>; + + /// Store a batch of storage trie leaf nodes. Used for saving existing state. + fn store_hashed_storages( + &self, + hashed_address: B256, + storages: Vec<(B256, U256)>, + ) -> MorphProofsStorageResult<()>; + + /// Store hashed storage slots for multiple addresses in one initialization batch. + fn store_hashed_storages_bulk( + &self, + entries: Vec<(B256, Vec<(B256, U256)>)>, + ) -> MorphProofsStorageResult<()> { + for (hashed_address, storages) in entries { + self.store_hashed_storages(hashed_address, storages)?; + } + Ok(()) + } + + /// Commit the initial state - mark the anchor as completed and also set the earliest block + /// number to anchor. + fn commit_initial_state(&self) -> MorphProofsStorageResult; +} diff --git a/crates/proofs/src/batch_provider.rs b/crates/proofs/src/batch_provider.rs new file mode 100644 index 00000000..11b67e09 --- /dev/null +++ b/crates/proofs/src/batch_provider.rs @@ -0,0 +1,313 @@ +//! State provider for an active [`MorphProofsBatchSession`] enabling reads to observe +//! uncommitted writes performed earlier in the same session. + +use std::fmt::Debug; + +use alloy_primitives::{ + keccak256, + map::{B256Map, HashMap}, +}; +use derive_more::Constructor; +use reth_primitives_traits::{Account, Bytecode}; +use reth_provider::{ + AccountReader, BlockHashReader, BytecodeReader, HashedPostStateProvider, ProviderError, + ProviderResult, StateProofProvider, StateProvider, StateRootProvider, StorageRootProvider, +}; +use reth_revm::{ + db::BundleState, + primitives::{Address, B256, Bytes, StorageValue, alloy_primitives::BlockNumber}, +}; +use reth_trie::{ + StateRoot, StorageRoot, TrieType, + hashed_cursor::{HashedCursor, HashedPostStateCursorFactory}, + metrics::TrieRootMetrics, + proof, + trie_cursor::InMemoryTrieCursorFactory, + witness::TrieWitness, +}; +use reth_trie_common::{ + AccountProof, ExecutionWitnessMode, HashedPostState, HashedPostStateSorted, HashedStorage, + KeccakKeyHasher, MultiProof, MultiProofTargets, StorageMultiProof, StorageProof, TrieInput, + updates::TrieUpdates, +}; + +use crate::{ + MorphProofsBatchHashedAccountCursorFactory, MorphProofsBatchTrieCursorFactory, + api::MorphProofsBatchSession, +}; + +/// State provider that reads through an active [`MorphProofsBatchSession`]'s transaction. +#[derive(Constructor)] +pub struct MorphProofsBatchStateProviderRef<'a, S: MorphProofsBatchSession> { + latest: Box, + session: &'a S, + block_number: BlockNumber, +} + +impl Debug for MorphProofsBatchStateProviderRef<'_, S> +where + S: MorphProofsBatchSession, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MorphProofsBatchStateProviderRef") + .field("session", &self.session) + .field("block_number", &self.block_number) + .finish() + } +} + +impl<'a, S: MorphProofsBatchSession> MorphProofsBatchStateProviderRef<'a, S> { + const fn factories( + &self, + ) -> ( + MorphProofsBatchTrieCursorFactory<'a, S>, + MorphProofsBatchHashedAccountCursorFactory<'a, S>, + ) { + ( + MorphProofsBatchTrieCursorFactory::new(self.session, self.block_number), + MorphProofsBatchHashedAccountCursorFactory::new(self.session, self.block_number), + ) + } +} + +impl BlockHashReader for MorphProofsBatchStateProviderRef<'_, S> { + fn block_hash(&self, number: BlockNumber) -> ProviderResult> { + self.latest.block_hash(number) + } + + fn canonical_hashes_range( + &self, + start: BlockNumber, + end: BlockNumber, + ) -> ProviderResult> { + self.latest.canonical_hashes_range(start, end) + } +} + +impl StateRootProvider for MorphProofsBatchStateProviderRef<'_, S> { + fn state_root(&self, state: HashedPostState) -> ProviderResult { + let prefix_sets = state.construct_prefix_sets().freeze(); + let state_sorted = state.into_sorted(); + let (trie_factory, hashed_factory) = self.factories(); + StateRoot::new( + trie_factory, + HashedPostStateCursorFactory::new(hashed_factory, &state_sorted), + ) + .with_prefix_sets(prefix_sets) + .root() + .map_err(ProviderError::from) + } + + fn state_root_from_nodes(&self, input: TrieInput) -> ProviderResult { + let state_sorted = input.state.into_sorted(); + let nodes_sorted = input.nodes.into_sorted(); + let (trie_factory, hashed_factory) = self.factories(); + StateRoot::new( + InMemoryTrieCursorFactory::new(trie_factory, &nodes_sorted), + HashedPostStateCursorFactory::new(hashed_factory, &state_sorted), + ) + .with_prefix_sets(input.prefix_sets.freeze()) + .root() + .map_err(ProviderError::from) + } + + fn state_root_with_updates( + &self, + state: HashedPostState, + ) -> ProviderResult<(B256, TrieUpdates)> { + let prefix_sets = state.construct_prefix_sets().freeze(); + let state_sorted = state.into_sorted(); + let (trie_factory, hashed_factory) = self.factories(); + StateRoot::new( + trie_factory, + HashedPostStateCursorFactory::new(hashed_factory, &state_sorted), + ) + .with_prefix_sets(prefix_sets) + .root_with_updates() + .map_err(ProviderError::from) + } + + fn state_root_from_nodes_with_updates( + &self, + input: TrieInput, + ) -> ProviderResult<(B256, TrieUpdates)> { + let state_sorted = input.state.into_sorted(); + let nodes_sorted = input.nodes.into_sorted(); + let (trie_factory, hashed_factory) = self.factories(); + StateRoot::new( + InMemoryTrieCursorFactory::new(trie_factory, &nodes_sorted), + HashedPostStateCursorFactory::new(hashed_factory, &state_sorted), + ) + .with_prefix_sets(input.prefix_sets.freeze()) + .root_with_updates() + .map_err(ProviderError::from) + } +} + +impl StorageRootProvider for MorphProofsBatchStateProviderRef<'_, S> { + fn storage_root(&self, address: Address, storage: HashedStorage) -> ProviderResult { + let prefix_set = storage.construct_prefix_set().freeze(); + let state_sorted = + HashedPostState::from_hashed_storage(keccak256(address), storage).into_sorted(); + let (trie_factory, hashed_factory) = self.factories(); + StorageRoot::new( + trie_factory, + HashedPostStateCursorFactory::new(hashed_factory, &state_sorted), + address, + prefix_set, + TrieRootMetrics::new(TrieType::Custom("morph_historical_proofs_storage_batch")), + ) + .root() + .map_err(|err| ProviderError::Database(err.into())) + } + + fn storage_proof( + &self, + address: Address, + slot: B256, + hashed_storage: HashedStorage, + ) -> ProviderResult { + let hashed_address = keccak256(address); + let prefix_set = hashed_storage.construct_prefix_set(); + let state_sorted = HashedPostStateSorted::new( + Default::default(), + HashMap::from_iter([(hashed_address, hashed_storage.into_sorted())]), + ); + let (trie_factory, hashed_factory) = self.factories(); + proof::StorageProof::new(trie_factory, hashed_factory.clone(), address) + .with_hashed_cursor_factory(HashedPostStateCursorFactory::new( + hashed_factory, + &state_sorted, + )) + .with_prefix_set_mut(prefix_set) + .storage_proof(slot) + .map_err(ProviderError::from) + } + + fn storage_multiproof( + &self, + address: Address, + slots: &[B256], + hashed_storage: HashedStorage, + ) -> ProviderResult { + let hashed_address = keccak256(address); + let targets = slots.iter().map(keccak256).collect(); + let prefix_set = hashed_storage.construct_prefix_set(); + let state_sorted = HashedPostStateSorted::new( + Default::default(), + HashMap::from_iter([(hashed_address, hashed_storage.into_sorted())]), + ); + let (trie_factory, hashed_factory) = self.factories(); + proof::StorageProof::new(trie_factory, hashed_factory.clone(), address) + .with_hashed_cursor_factory(HashedPostStateCursorFactory::new( + hashed_factory, + &state_sorted, + )) + .with_prefix_set_mut(prefix_set) + .storage_multiproof(targets) + .map_err(ProviderError::from) + } +} + +impl StateProofProvider for MorphProofsBatchStateProviderRef<'_, S> { + fn proof( + &self, + input: TrieInput, + address: Address, + slots: &[B256], + ) -> ProviderResult { + let nodes_sorted = input.nodes.into_sorted(); + let state_sorted = input.state.into_sorted(); + let (trie_factory, hashed_factory) = self.factories(); + proof::Proof::new(trie_factory.clone(), hashed_factory.clone()) + .with_trie_cursor_factory(InMemoryTrieCursorFactory::new(trie_factory, &nodes_sorted)) + .with_hashed_cursor_factory(HashedPostStateCursorFactory::new( + hashed_factory, + &state_sorted, + )) + .with_prefix_sets_mut(input.prefix_sets) + .account_proof(address, slots) + .map_err(ProviderError::from) + } + + fn multiproof( + &self, + input: TrieInput, + targets: MultiProofTargets, + ) -> ProviderResult { + let nodes_sorted = input.nodes.into_sorted(); + let state_sorted = input.state.into_sorted(); + let (trie_factory, hashed_factory) = self.factories(); + proof::Proof::new(trie_factory.clone(), hashed_factory.clone()) + .with_trie_cursor_factory(InMemoryTrieCursorFactory::new(trie_factory, &nodes_sorted)) + .with_hashed_cursor_factory(HashedPostStateCursorFactory::new( + hashed_factory, + &state_sorted, + )) + .with_prefix_sets_mut(input.prefix_sets) + .multiproof(targets) + .map_err(ProviderError::from) + } + + fn witness( + &self, + input: TrieInput, + target: HashedPostState, + _mode: ExecutionWitnessMode, + ) -> ProviderResult> { + let nodes_sorted = input.nodes.into_sorted(); + let state_sorted = input.state.into_sorted(); + let (trie_factory, hashed_factory) = self.factories(); + let result: B256Map = TrieWitness::new(trie_factory.clone(), hashed_factory.clone()) + .with_trie_cursor_factory(InMemoryTrieCursorFactory::new(trie_factory, &nodes_sorted)) + .with_hashed_cursor_factory(HashedPostStateCursorFactory::new( + hashed_factory, + &state_sorted, + )) + .with_prefix_sets_mut(input.prefix_sets) + .always_include_root_node() + .compute(target) + .map_err(ProviderError::from)?; + Ok(result.into_values().collect()) + } +} + +impl HashedPostStateProvider + for MorphProofsBatchStateProviderRef<'_, S> +{ + fn hashed_post_state(&self, bundle_state: &BundleState) -> HashedPostState { + HashedPostState::from_bundle_state::(bundle_state.state()) + } +} + +impl AccountReader for MorphProofsBatchStateProviderRef<'_, S> { + fn basic_account(&self, address: &Address) -> ProviderResult> { + let hashed_key = keccak256(address.0); + Ok(self + .session + .account_hashed_cursor(self.block_number) + .map_err(Into::::into)? + .seek(hashed_key) + .map_err(Into::::into)? + .and_then(|(key, account)| (key == hashed_key).then_some(account))) + } +} + +impl StateProvider for MorphProofsBatchStateProviderRef<'_, S> { + fn storage(&self, address: Address, storage_key: B256) -> ProviderResult> { + let hashed_key = keccak256(storage_key); + Ok(self + .session + .storage_hashed_cursor(keccak256(address.0), self.block_number) + .map_err(Into::::into)? + .seek(hashed_key) + .map_err(Into::::into)? + .and_then(|(key, storage)| (key == hashed_key).then_some(storage))) + } +} + +impl BytecodeReader for MorphProofsBatchStateProviderRef<'_, S> { + fn bytecode_by_hash(&self, code_hash: &B256) -> ProviderResult> { + self.latest.bytecode_by_hash(code_hash) + } +} diff --git a/crates/proofs/src/cursor.rs b/crates/proofs/src/cursor.rs new file mode 100644 index 00000000..ec04141e --- /dev/null +++ b/crates/proofs/src/cursor.rs @@ -0,0 +1,129 @@ +//! Implementation of [`HashedCursor`] and [`TrieCursor`] for +//! [`MorphProofsStorage`](crate::MorphProofsStorage). + +use alloy_primitives::{B256, U256}; +use derive_more::Constructor; +use reth_db::DatabaseError; +use reth_primitives_traits::Account; +use reth_trie::{ + hashed_cursor::{HashedCursor, HashedStorageCursor}, + trie_cursor::{TrieCursor, TrieStorageCursor}, +}; +use reth_trie_common::{BranchNodeCompact, Nibbles}; + +/// Manages reading storage or account trie nodes from [`TrieCursor`]. +#[derive(Debug, Clone, Constructor)] +pub struct MorphProofsTrieCursor(pub C); + +impl TrieCursor for MorphProofsTrieCursor +where + C: TrieCursor, +{ + #[inline] + fn seek_exact( + &mut self, + key: Nibbles, + ) -> Result, DatabaseError> { + self.0.seek_exact(key) + } + + #[inline] + fn seek( + &mut self, + key: Nibbles, + ) -> Result, DatabaseError> { + self.0.seek(key) + } + + #[inline] + fn next(&mut self) -> Result, DatabaseError> { + self.0.next() + } + + #[inline] + fn current(&mut self) -> Result, DatabaseError> { + self.0.current() + } + + #[inline] + fn reset(&mut self) { + self.0.reset() + } +} + +impl TrieStorageCursor for MorphProofsTrieCursor +where + C: TrieStorageCursor, +{ + #[inline] + fn set_hashed_address(&mut self, hashed_address: B256) { + self.0.set_hashed_address(hashed_address) + } +} + +/// Manages reading hashed account nodes from external storage. +#[derive(Debug, Clone, Constructor)] +pub struct MorphProofsHashedAccountCursor(pub C); + +impl HashedCursor for MorphProofsHashedAccountCursor +where + C: HashedCursor + Send + Sync, +{ + type Value = Account; + + #[inline] + fn seek(&mut self, key: B256) -> Result, DatabaseError> { + self.0.seek(key) + } + + #[inline] + fn next(&mut self) -> Result, DatabaseError> { + self.0.next() + } + + #[inline] + fn reset(&mut self) { + self.0.reset() + } +} + +/// Manages reading hashed storage nodes from external storage. +#[derive(Debug, Clone, Constructor)] +pub struct MorphProofsHashedStorageCursor(pub C); + +impl HashedCursor for MorphProofsHashedStorageCursor +where + C: HashedCursor + Send + Sync, +{ + type Value = U256; + + #[inline] + fn seek(&mut self, key: B256) -> Result, DatabaseError> { + self.0.seek(key) + } + + #[inline] + fn next(&mut self) -> Result, DatabaseError> { + self.0.next() + } + + #[inline] + fn reset(&mut self) { + self.0.reset() + } +} + +impl HashedStorageCursor for MorphProofsHashedStorageCursor +where + C: HashedStorageCursor + Send + Sync, +{ + #[inline] + fn is_storage_empty(&mut self) -> Result { + self.0.is_storage_empty() + } + + #[inline] + fn set_hashed_address(&mut self, hashed_address: B256) { + self.0.set_hashed_address(hashed_address) + } +} diff --git a/crates/proofs/src/cursor_factory.rs b/crates/proofs/src/cursor_factory.rs new file mode 100644 index 00000000..fa2dfcbb --- /dev/null +++ b/crates/proofs/src/cursor_factory.rs @@ -0,0 +1,260 @@ +//! Implements [`TrieCursorFactory`] and [`HashedCursorFactory`] for [`MorphProofsStore`] types. +//! +//! Both factories borrow one read-only [`MorphProofsStore::Tx`] for their entire lifetime +//! and route every cursor allocation through the `*_with_tx` fast path. This mirrors +//! reth's own `DatabaseTrieCursorFactory` / `DatabaseHashedCursorFactory` pattern and is +//! what lets proof, state-root, and witness requests acquire exactly one MDBX +//! transaction. + +use alloy_primitives::B256; +use reth_db::DatabaseError; +use reth_trie::{hashed_cursor::HashedCursorFactory, trie_cursor::TrieCursorFactory}; + +use crate::{ + MorphProofsHashedAccountCursor, MorphProofsHashedStorageCursor, MorphProofsStorage, + MorphProofsStore, MorphProofsTrieCursor, + api::MorphProofsBatchSession, + cursor::{ + MorphProofsHashedAccountCursor as RawHashedAccountCursor, + MorphProofsHashedStorageCursor as RawHashedStorageCursor, + MorphProofsTrieCursor as RawTrieCursor, + }, +}; + +/// Request-scoped factory that opens trie cursors against a shared read-only transaction. +/// +/// Holds a borrow of the transaction so every cursor allocation reuses the same MDBX +/// reader slot. See [`MorphProofsStore::Tx`] for the underlying contention story. +#[derive(Debug, Clone)] +pub struct MorphProofsTrieCursorFactory<'tx, 'db, S: MorphProofsStore> { + storage: &'db MorphProofsStorage, + tx: &'tx as MorphProofsStore>::Tx<'db>, + block_number: u64, +} + +impl<'tx, 'db, S: MorphProofsStore> MorphProofsTrieCursorFactory<'tx, 'db, S> { + /// Initializes a request-scoped trie cursor factory bound to `tx`. + pub const fn new( + storage: &'db MorphProofsStorage, + tx: &'tx as MorphProofsStore>::Tx<'db>, + block_number: u64, + ) -> Self { + Self { + storage, + tx, + block_number, + } + } +} + +impl<'tx, 'db, S> TrieCursorFactory for MorphProofsTrieCursorFactory<'tx, 'db, S> +where + for<'a> S: MorphProofsStore + 'db, + 'db: 'tx, +{ + type AccountTrieCursor<'a> + = MorphProofsTrieCursor> + where + Self: 'a; + type StorageTrieCursor<'a> + = MorphProofsTrieCursor> + where + Self: 'a; + + fn account_trie_cursor(&self) -> Result, DatabaseError> { + Ok(MorphProofsTrieCursor::new( + self.storage + .account_trie_cursor_with_tx(self.tx, self.block_number) + .map_err(Into::::into)?, + )) + } + + fn storage_trie_cursor( + &self, + hashed_address: B256, + ) -> Result, DatabaseError> { + Ok(MorphProofsTrieCursor::new( + self.storage + .storage_trie_cursor_with_tx(self.tx, hashed_address, self.block_number) + .map_err(Into::::into)?, + )) + } +} + +/// Request-scoped factory that opens hashed cursors against a shared read-only transaction. +/// +/// Mirror of [`MorphProofsTrieCursorFactory`] for the hashed account/storage tries. +#[derive(Debug, Clone)] +pub struct MorphProofsHashedAccountCursorFactory<'tx, 'db, S: MorphProofsStore> { + storage: &'db MorphProofsStorage, + tx: &'tx as MorphProofsStore>::Tx<'db>, + block_number: u64, +} + +impl<'tx, 'db, S: MorphProofsStore> MorphProofsHashedAccountCursorFactory<'tx, 'db, S> { + /// Initializes a request-scoped hashed cursor factory bound to `tx`. + pub const fn new( + storage: &'db MorphProofsStorage, + tx: &'tx as MorphProofsStore>::Tx<'db>, + block_number: u64, + ) -> Self { + Self { + storage, + tx, + block_number, + } + } +} + +impl<'tx, 'db, S> HashedCursorFactory for MorphProofsHashedAccountCursorFactory<'tx, 'db, S> +where + for<'a> S: MorphProofsStore + 'db, + 'db: 'tx, +{ + type AccountCursor<'a> + = MorphProofsHashedAccountCursor> + where + Self: 'a; + type StorageCursor<'a> + = MorphProofsHashedStorageCursor> + where + Self: 'a; + + fn hashed_account_cursor(&self) -> Result, DatabaseError> { + Ok(MorphProofsHashedAccountCursor::new( + self.storage + .account_hashed_cursor_with_tx(self.tx, self.block_number)?, + )) + } + + fn hashed_storage_cursor( + &self, + hashed_address: B256, + ) -> Result, DatabaseError> { + Ok(MorphProofsHashedStorageCursor::new( + self.storage.storage_hashed_cursor_with_tx( + self.tx, + hashed_address, + self.block_number, + )?, + )) + } +} + +/// Session-scoped trie cursor factory backed by a [`MorphProofsBatchSession`]. +/// +/// Cursors read from the session's active transaction and therefore observe writes +/// from earlier `store_trie_updates` calls in the same session. +#[derive(Debug)] +pub struct MorphProofsBatchTrieCursorFactory<'a, S: MorphProofsBatchSession> { + session: &'a S, + block_number: u64, +} + +impl Clone for MorphProofsBatchTrieCursorFactory<'_, S> { + fn clone(&self) -> Self { + Self { + session: self.session, + block_number: self.block_number, + } + } +} + +impl<'a, S: MorphProofsBatchSession> MorphProofsBatchTrieCursorFactory<'a, S> { + /// Initializes a session-scoped trie cursor factory. + pub const fn new(session: &'a S, block_number: u64) -> Self { + Self { + session, + block_number, + } + } +} + +impl TrieCursorFactory for MorphProofsBatchTrieCursorFactory<'_, S> +where + S: MorphProofsBatchSession, +{ + type AccountTrieCursor<'a> + = RawTrieCursor> + where + Self: 'a; + type StorageTrieCursor<'a> + = RawTrieCursor> + where + Self: 'a; + + fn account_trie_cursor(&self) -> Result, DatabaseError> { + Ok(RawTrieCursor::new( + self.session + .account_trie_cursor(self.block_number) + .map_err(Into::::into)?, + )) + } + + fn storage_trie_cursor( + &self, + hashed_address: B256, + ) -> Result, DatabaseError> { + Ok(RawTrieCursor::new( + self.session + .storage_trie_cursor(hashed_address, self.block_number) + .map_err(Into::::into)?, + )) + } +} + +/// Session-scoped hashed cursor factory backed by a [`MorphProofsBatchSession`]. +#[derive(Debug)] +pub struct MorphProofsBatchHashedAccountCursorFactory<'a, S: MorphProofsBatchSession> { + session: &'a S, + block_number: u64, +} + +impl Clone for MorphProofsBatchHashedAccountCursorFactory<'_, S> { + fn clone(&self) -> Self { + Self { + session: self.session, + block_number: self.block_number, + } + } +} + +impl<'a, S: MorphProofsBatchSession> MorphProofsBatchHashedAccountCursorFactory<'a, S> { + /// Initializes a session-scoped hashed cursor factory. + pub const fn new(session: &'a S, block_number: u64) -> Self { + Self { + session, + block_number, + } + } +} + +impl HashedCursorFactory for MorphProofsBatchHashedAccountCursorFactory<'_, S> +where + S: MorphProofsBatchSession, +{ + type AccountCursor<'a> + = RawHashedAccountCursor> + where + Self: 'a; + type StorageCursor<'a> + = RawHashedStorageCursor> + where + Self: 'a; + + fn hashed_account_cursor(&self) -> Result, DatabaseError> { + Ok(RawHashedAccountCursor::new( + self.session.account_hashed_cursor(self.block_number)?, + )) + } + + fn hashed_storage_cursor( + &self, + hashed_address: B256, + ) -> Result, DatabaseError> { + Ok(RawHashedStorageCursor::new( + self.session + .storage_hashed_cursor(hashed_address, self.block_number)?, + )) + } +} diff --git a/crates/proofs/src/db/batch.rs b/crates/proofs/src/db/batch.rs new file mode 100644 index 00000000..1b8ba868 --- /dev/null +++ b/crates/proofs/src/db/batch.rs @@ -0,0 +1,143 @@ +//! Batch write session for [`MdbxProofsStorage`] enabling multiple block writes inside one MDBX +//! RW transaction. Reads through the session observe uncommitted writes from earlier blocks in +//! the same session, which is required for cold catch-up where block `N+1` must execute against +//! block `N` written but not yet committed. + +use alloy_eips::eip1898::BlockWithParent; +use alloy_primitives::B256; +use reth_db::{ + Database, DatabaseEnv, + table::{DupSort, Table}, + transaction::DbTx, +}; + +use crate::{ + BlockStateDiff, MorphProofsStorageError, MorphProofsStorageResult, + api::{MorphProofsBatchSession, WriteCounts}, + db::{ + AccountTrieHistory, HashedAccountHistory, HashedStorageHistory, MdbxAccountCursor, + MdbxProofsStorage, MdbxStorageCursor, MdbxTrieCursor, StorageTrieHistory, + }, +}; + +/// Alias for the dup-sorted cursor type produced by an MDBX RW transaction. +pub type DupRw<'tx, T> = <::TXMut as DbTx>::DupCursor; + +/// Active write batch holding one MDBX RW transaction across multiple block writes. +#[derive(Debug)] +pub struct MdbxBatchSession<'tx> { + storage: &'tx MdbxProofsStorage, + tx: Option<::TXMut>, +} + +impl<'tx> MdbxBatchSession<'tx> { + pub(crate) const fn new( + storage: &'tx MdbxProofsStorage, + tx: ::TXMut, + ) -> Self { + Self { + storage, + tx: Some(tx), + } + } + + pub(crate) fn commit(mut self) -> MorphProofsStorageResult<()> { + if let Some(tx) = self.tx.take() { + tx.commit()?; + } + Ok(()) + } + + fn tx_ref(&self) -> MorphProofsStorageResult<&::TXMut> { + self.tx + .as_ref() + .ok_or(MorphProofsStorageError::BatchSessionClosed) + } + + fn dup_cursor(&self) -> MorphProofsStorageResult> { + Ok(self.tx_ref()?.cursor_dup_read::()?) + } +} + +impl MorphProofsBatchSession for MdbxBatchSession<'_> { + type StorageTrieCursor<'a> + = MdbxTrieCursor> + where + Self: 'a; + type AccountTrieCursor<'a> + = MdbxTrieCursor> + where + Self: 'a; + type StorageCursor<'a> + = MdbxStorageCursor> + where + Self: 'a; + type AccountHashedCursor<'a> + = MdbxAccountCursor> + where + Self: 'a; + + fn get_earliest_block_number(&self) -> MorphProofsStorageResult> { + self.storage + .inner_get_earliest_block_number_hash(self.tx_ref()?) + } + + fn get_latest_block_number(&self) -> MorphProofsStorageResult> { + self.storage + .inner_get_latest_block_number_hash(self.tx_ref()?) + } + + fn storage_trie_cursor( + &self, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + Ok(MdbxTrieCursor::new( + self.dup_cursor::()?, + max_block_number, + Some(hashed_address), + )) + } + + fn account_trie_cursor( + &self, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + Ok(MdbxTrieCursor::new( + self.dup_cursor::()?, + max_block_number, + None, + )) + } + + fn storage_hashed_cursor( + &self, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + Ok(MdbxStorageCursor::new( + self.dup_cursor::()?, + max_block_number, + hashed_address, + )) + } + + fn account_hashed_cursor( + &self, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + Ok(MdbxAccountCursor::new( + self.dup_cursor::()?, + max_block_number, + )) + } + + fn store_trie_updates( + &mut self, + block_ref: BlockWithParent, + block_state_diff: BlockStateDiff, + ) -> MorphProofsStorageResult { + self.storage + .store_trie_updates_append_only(self.tx_ref()?, block_ref, block_state_diff) + } +} diff --git a/crates/proofs/src/db/cursor.rs b/crates/proofs/src/db/cursor.rs new file mode 100644 index 00000000..057a1a23 --- /dev/null +++ b/crates/proofs/src/db/cursor.rs @@ -0,0 +1,1545 @@ +use std::marker::PhantomData; + +use alloy_primitives::{B256, U256}; +use reth_db::{ + Database, DatabaseEnv, DatabaseError, + cursor::{DbCursorRO, DbDupCursorRO}, + table::{DupSort, Table}, + transaction::DbTx, +}; +use reth_primitives_traits::Account; +use reth_trie::{ + hashed_cursor::{HashedCursor, HashedStorageCursor}, + trie_cursor::{TrieCursor, TrieStorageCursor}, +}; +use reth_trie_common::{BranchNodeCompact, Nibbles, StoredNibbles}; + +use crate::{ + MorphProofsStorageResult, + db::{ + AccountTrieHistory, HashedAccountHistory, HashedStorageHistory, HashedStorageKey, + MaybeDeleted, StorageTrieHistory, StorageTrieKey, VersionedValue, + }, +}; + +/// Generic alias for dup cursor for T +pub type Dup<'tx, T> = <::TX as DbTx>::DupCursor; + +/// Iterates versioned dup-sorted rows and returns the latest value (<= `max_block_number`), +/// skipping tombstones. +#[derive(Debug, Clone)] +pub struct BlockNumberVersionedCursor { + _table: PhantomData, + cursor: Cursor, + max_block_number: u64, +} + +impl BlockNumberVersionedCursor +where + T: Table> + DupSort, + Cursor: DbCursorRO + DbDupCursorRO, +{ + /// Initializes new [`BlockNumberVersionedCursor`]. + pub const fn new(cursor: Cursor, max_block_number: u64) -> Self { + Self { + _table: PhantomData, + cursor, + max_block_number, + } + } + + /// Check if the cursor is currently positioned at a valid row. + fn is_positioned(&mut self) -> MorphProofsStorageResult { + Ok(self.cursor.current()?.is_some()) + } + + /// Resolve the latest version for `key` with `block_number` <= `max_block_number`. + /// Strategy: + /// - `seek_by_key_subkey(key, max)` gives first dup >= max. + /// - if exactly == max → it's our latest + /// - if > max → `prev_dup()` is latest < max (or None) + /// - if no dup >= max: + /// - if key exists → `last_dup()` is latest < max + /// - else → None + fn latest_version_for_key( + &mut self, + key: T::Key, + ) -> MorphProofsStorageResult> { + // First dup with subkey >= max_block_number + let seek_res = self + .cursor + .seek_by_key_subkey(key.clone(), self.max_block_number)?; + + if let Some(vv) = seek_res { + if vv.block_number > self.max_block_number { + // step back to the last dup < max + return Ok(self.cursor.prev_dup()?); + } + // already at the dup = max + return Ok(Some((key, vv))); + } + + // No dup >= max ⇒ either key absent or all dups < max. Check if key exists: + if self.cursor.seek_exact(key.clone())?.is_none() { + return Ok(None); + } + + // Key exists ⇒ take last dup (< max). + if let Some(vv) = self.cursor.last_dup()? { + return Ok(Some((key, vv))); + } + Ok(None) + } + + /// Returns a non-deleted latest version for exactly `key`, if any. + fn seek_exact(&mut self, key: T::Key) -> MorphProofsStorageResult> { + if let Some((latest_key, latest_value)) = self.latest_version_for_key(key)? + && let MaybeDeleted(Some(v)) = latest_value.value + { + return Ok(Some((latest_key, v))); + } + Ok(None) + } + + /// Walk forward from `first_key` (inclusive) until we find a *live* latest-≤-max value. + /// `first_key` must already be a *real key* in the table. + fn next_live_from( + &mut self, + mut first_key: T::Key, + ) -> MorphProofsStorageResult> { + loop { + // Compute latest version ≤ max for this key + if let Some((k, v)) = self.seek_exact(first_key.clone())? { + return Ok(Some((k, v))); + } + + // Move to next distinct key, or EOF + let Some((next_key, _)) = self.cursor.next_no_dup()? else { + return Ok(None); + }; + + first_key = next_key; + } + } + + /// Seek to the first non-deleted latest version at or after `start_key`. + /// Logic: + /// - Try exact key first (above). If alive, return it. + /// - Otherwise hop to next distinct key and repeat until we find a live version or hit EOF. + fn seek(&mut self, start_key: T::Key) -> MorphProofsStorageResult> { + // Position MDBX at first key >= start_key + if let Some((first_key, _)) = self.cursor.seek(start_key)? { + return self.next_live_from(first_key); + } + Ok(None) + } + + /// Advance to the next distinct key from the current MDBX position + /// and return its non-deleted latest version, if any. + /// Next distinct key; if not positioned, start from `T::Key::default()`. + fn next(&mut self) -> MorphProofsStorageResult> + where + T::Key: Default, + { + // If not positioned, start from the beginning (default key). + if self.cursor.current()?.is_none() { + let Some((first_key, _)) = self.cursor.seek(T::Key::default())? else { + return Ok(None); + }; + return self.next_live_from(first_key); + } + + // Otherwise advance to next distinct key and resume the walk. + let Some((next_key, _)) = self.cursor.next_no_dup()? else { + return Ok(None); + }; + self.next_live_from(next_key) + } +} + +/// MDBX implementation of [`TrieCursor`]. +#[derive(Debug)] +pub struct MdbxTrieCursor { + inner: BlockNumberVersionedCursor, + hashed_address: Option, +} + +impl< + V, + T: Table> + DupSort, + Cursor: DbCursorRO + DbDupCursorRO, +> MdbxTrieCursor +{ + /// Initializes new [`MdbxTrieCursor`]. + pub const fn new(cursor: Cursor, max_block_number: u64, hashed_address: Option) -> Self { + Self { + inner: BlockNumberVersionedCursor::new(cursor, max_block_number), + hashed_address, + } + } +} + +impl TrieCursor for MdbxTrieCursor +where + Cursor: DbCursorRO + DbDupCursorRO + Send + Sync, +{ + fn seek_exact( + &mut self, + path: Nibbles, + ) -> Result, DatabaseError> { + Ok(self + .inner + .seek_exact(StoredNibbles(path)) + .map(|opt| opt.map(|(StoredNibbles(n), node)| (n, node)))?) + } + + fn seek( + &mut self, + path: Nibbles, + ) -> Result, DatabaseError> { + Ok(self + .inner + .seek(StoredNibbles(path)) + .map(|opt| opt.map(|(StoredNibbles(n), node)| (n, node)))?) + } + + fn next(&mut self) -> Result, DatabaseError> { + Ok(self + .inner + .next() + .map(|opt| opt.map(|(StoredNibbles(n), node)| (n, node)))?) + } + + fn current(&mut self) -> Result, DatabaseError> { + self.inner + .cursor + .current() + .map(|opt| opt.map(|(StoredNibbles(n), _)| n)) + } + + fn reset(&mut self) { + // Database cursors are stateless, no reset needed + } +} + +impl TrieCursor for MdbxTrieCursor +where + Cursor: DbCursorRO + DbDupCursorRO + Send + Sync, +{ + fn seek_exact( + &mut self, + path: Nibbles, + ) -> Result, DatabaseError> { + if let Some(address) = self.hashed_address { + let key = StorageTrieKey::new(address, StoredNibbles(path)); + return Ok(self.inner.seek_exact(key).map(|opt| { + opt.and_then(|(k, node)| (k.hashed_address == address).then_some((k.path.0, node))) + })?); + } + Ok(None) + } + + fn seek( + &mut self, + path: Nibbles, + ) -> Result, DatabaseError> { + if let Some(address) = self.hashed_address { + let key = StorageTrieKey::new(address, StoredNibbles(path)); + return Ok(self.inner.seek(key).map(|opt| { + opt.and_then(|(k, node)| (k.hashed_address == address).then_some((k.path.0, node))) + })?); + } + Ok(None) + } + + fn next(&mut self) -> Result, DatabaseError> { + if let Some(address) = self.hashed_address { + // If the cursor is not positioned, we need to seek to the first key for our bound + // address to ensure we start iterating from the correct position in the + // table. This is necessary because BlockNumberVersionedCursor::next() would + // otherwise start from T::Key::default() (the beginning of the entire + // table), which would cause us to miss entries for non-first addresses. + if !self.inner.is_positioned()? { + return self.seek(Nibbles::default()); + } + + return Ok(self.inner.next().map(|opt| { + opt.and_then(|(k, node)| (k.hashed_address == address).then_some((k.path.0, node))) + })?); + } + Ok(None) + } + + fn current(&mut self) -> Result, DatabaseError> { + if let Some(address) = self.hashed_address { + return self.inner.cursor.current().map(|opt| { + opt.and_then(|(k, _)| (k.hashed_address == address).then_some(k.path.0)) + }); + } + Ok(None) + } + + fn reset(&mut self) { + // Database cursors are stateless, no reset needed + } +} + +impl TrieStorageCursor for MdbxTrieCursor +where + Cursor: DbCursorRO + DbDupCursorRO + Send + Sync, +{ + fn set_hashed_address(&mut self, hashed_address: B256) { + self.hashed_address = Some(hashed_address); + } +} + +/// MDBX implementation of [`HashedCursor`] for storage state. +#[derive(Debug)] +pub struct MdbxStorageCursor { + inner: BlockNumberVersionedCursor, + hashed_address: B256, +} + +impl MdbxStorageCursor +where + Cursor: DbCursorRO + DbDupCursorRO + Send + Sync, +{ + /// Initializes new [`MdbxStorageCursor`] + pub const fn new(cursor: Cursor, block_number: u64, hashed_address: B256) -> Self { + Self { + inner: BlockNumberVersionedCursor::new(cursor, block_number), + hashed_address, + } + } +} + +impl HashedCursor for MdbxStorageCursor +where + Cursor: DbCursorRO + DbDupCursorRO + Send + Sync, +{ + type Value = U256; + + fn seek(&mut self, key: B256) -> Result, DatabaseError> { + let storage_key = HashedStorageKey::new(self.hashed_address, key); + + // hashed storage values can be zero, which means the storage slot is deleted, so we should + // skip those + let result = self.inner.seek(storage_key).map(|opt| { + opt.and_then(|(k, v)| { + // Only return entries that belong to the bound address + (k.hashed_address == self.hashed_address).then_some((k.hashed_storage_key, v.0)) + }) + })?; + + if let Some((_, v)) = result + && v.is_zero() + { + return self.next(); + } + + Ok(result) + } + + fn next(&mut self) -> Result, DatabaseError> { + // If the cursor is not positioned, we need to seek to the first key for our bound address + // to ensure we start iterating from the correct position in the table. + // This is necessary because BlockNumberVersionedCursor::next() would otherwise start + // from T::Key::default() (the beginning of the entire table), which would cause us + // to miss entries for non-first addresses. + if !self.inner.is_positioned()? { + return self.seek(B256::ZERO); + } + + loop { + let result = self.inner.next().map(|opt| { + opt.and_then(|(k, v)| { + // Only return entries that belong to the bound address + (k.hashed_address == self.hashed_address).then_some((k.hashed_storage_key, v.0)) + }) + })?; + + // hashed storage values can be zero, which means the storage slot is deleted, so we + // should skip those + if let Some((_, v)) = result + && v.is_zero() + { + continue; + } + + return Ok(result); + } + } + + fn reset(&mut self) { + // Database cursors are stateless, no reset needed + } +} + +impl HashedStorageCursor for MdbxStorageCursor +where + Cursor: DbCursorRO + DbDupCursorRO + Send + Sync, +{ + fn is_storage_empty(&mut self) -> Result { + Ok(self.seek(B256::ZERO)?.is_none()) + } + + fn set_hashed_address(&mut self, hashed_address: B256) { + self.hashed_address = hashed_address + } +} + +/// MDBX implementation of [`HashedCursor`] for account state. +#[derive(Debug)] +pub struct MdbxAccountCursor { + inner: BlockNumberVersionedCursor, +} + +impl MdbxAccountCursor +where + Cursor: DbCursorRO + DbDupCursorRO + Send + Sync, +{ + /// Initializes new `MdbxAccountCursor` + pub const fn new(cursor: Cursor, block_number: u64) -> Self { + Self { + inner: BlockNumberVersionedCursor::new(cursor, block_number), + } + } +} + +impl HashedCursor for MdbxAccountCursor +where + Cursor: DbCursorRO + DbDupCursorRO + Send + Sync, +{ + type Value = Account; + + fn seek(&mut self, key: B256) -> Result, DatabaseError> { + Ok(self.inner.seek(key)?) + } + + fn next(&mut self) -> Result, DatabaseError> { + Ok(self.inner.next()?) + } + + fn reset(&mut self) { + // Database cursors are stateless, no reset needed + } +} + +#[cfg(test)] +mod tests { + use reth_db::{ + DatabaseEnv, + mdbx::{DatabaseArguments, init_db_for}, + }; + use reth_db_api::{ + Database, + cursor::DbDupCursorRW, + transaction::{DbTx, DbTxMut}, + }; + use reth_trie::{BranchNodeCompact, Nibbles, StoredNibbles}; + use tempfile::TempDir; + + use super::*; + use crate::db::{StorageValue, models}; + + fn setup_db() -> DatabaseEnv { + let tmp = TempDir::new().expect("create tmpdir"); + init_db_for::<_, models::Tables>(tmp, DatabaseArguments::default()).expect("init db") + } + + fn stored(path: Nibbles) -> StoredNibbles { + StoredNibbles(path) + } + + fn node() -> BranchNodeCompact { + BranchNodeCompact::default() + } + + fn append_account_trie( + wtx: &::TXMut, + key: StoredNibbles, + block: u64, + val: Option, + ) { + let mut c = wtx + .cursor_dup_write::() + .expect("dup write cursor"); + let vv = VersionedValue { + block_number: block, + value: MaybeDeleted(val), + }; + c.append_dup(key, vv).expect("append dup"); + } + + fn append_storage_trie( + wtx: &::TXMut, + address: B256, + path: Nibbles, + block: u64, + val: Option, + ) { + let mut c = wtx + .cursor_dup_write::() + .expect("dup write cursor"); + let key = StorageTrieKey::new(address, StoredNibbles(path)); + let vv = VersionedValue { + block_number: block, + value: MaybeDeleted(val), + }; + c.append_dup(key, vv).expect("append dup"); + } + + fn append_hashed_storage( + wtx: &::TXMut, + addr: B256, + slot: B256, + block: u64, + val: Option, + ) { + let mut c = wtx + .cursor_dup_write::() + .expect("dup write"); + let key = HashedStorageKey::new(addr, slot); + let vv = VersionedValue { + block_number: block, + value: MaybeDeleted(val.map(StorageValue)), + }; + c.append_dup(key, vv).expect("append dup"); + } + + fn append_hashed_account( + wtx: &::TXMut, + key: B256, + block: u64, + val: Option, + ) { + let mut c = wtx + .cursor_dup_write::() + .expect("dup write"); + let vv = VersionedValue { + block_number: block, + value: MaybeDeleted(val), + }; + c.append_dup(key, vv).expect("append dup"); + } + + // Open a dup-RO cursor and wrap it in a BlockNumberVersionedCursor with a given bound. + fn version_cursor( + tx: &::TX, + max_block: u64, + ) -> BlockNumberVersionedCursor> { + let cur = tx + .cursor_dup_read::() + .expect("dup ro cursor"); + BlockNumberVersionedCursor::new(cur, max_block) + } + + fn account_trie_cursor( + tx: &'_ ::TX, + max_block: u64, + ) -> MdbxTrieCursor> { + let c = tx + .cursor_dup_read::() + .expect("dup ro cursor"); + // For account trie the address is not used; pass None. + MdbxTrieCursor::new(c, max_block, None) + } + + // Helper: build a Storage trie cursor bound to an address + fn storage_trie_cursor( + tx: &'_ ::TX, + max_block: u64, + address: B256, + ) -> MdbxTrieCursor> { + let c = tx + .cursor_dup_read::() + .expect("dup ro cursor"); + MdbxTrieCursor::new(c, max_block, Some(address)) + } + + fn storage_cursor( + tx: &'_ ::TX, + max_block: u64, + address: B256, + ) -> MdbxStorageCursor> { + let c = tx + .cursor_dup_read::() + .expect("dup ro cursor"); + MdbxStorageCursor::new(c, max_block, address) + } + + fn account_cursor( + tx: &'_ ::TX, + max_block: u64, + ) -> MdbxAccountCursor> { + let c = tx + .cursor_dup_read::() + .expect("dup ro cursor"); + MdbxAccountCursor::new(c, max_block) + } + + // Assert helper: ensure the chosen VersionedValue has the expected block and deletion flag. + fn assert_block( + got: Option<(StoredNibbles, VersionedValue)>, + expected_block: u64, + expect_deleted: bool, + ) { + let (_, vv) = got.expect("expected Some(..)"); + assert_eq!(vv.block_number, expected_block, "wrong block chosen"); + let is_deleted = matches!(vv.value, MaybeDeleted(None)); + assert_eq!(is_deleted, expect_deleted, "tombstone mismatch"); + } + + /// No entry for key → None. + #[test] + fn latest_version_for_key_none_when_key_absent() { + let db = setup_db(); + let tx = db.tx().expect("ro tx"); + let mut cursor = version_cursor(&tx, 100); + + let out = cursor + .latest_version_for_key(stored(Nibbles::default())) + .expect("should not return error"); + assert!(out.is_none(), "absent key must return None"); + } + + /// Exact match at max (live) → pick it. + #[test] + fn latest_version_for_key_picks_value_at_max_if_present() { + let db = setup_db(); + let k = stored(Nibbles::from_nibbles([0x0A])); + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k.clone(), 10, Some(node())); + append_account_trie(&wtx, k.clone(), 50, Some(node())); // == max + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut core = version_cursor(&tx, 50); + + let out = core.latest_version_for_key(k).expect("ok"); + assert_block(out, 50, false); + } + + /// When `seek_by_key_subkey` points to the subkey > max - fallback to the prev. + #[test] + fn latest_version_for_key_picks_latest_below_max_when_next_is_above() { + let db = setup_db(); + let k = stored(Nibbles::from_nibbles([0x0A])); + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k.clone(), 10, Some(node())); + append_account_trie(&wtx, k.clone(), 30, Some(node())); // expected + append_account_trie(&wtx, k.clone(), 70, Some(node())); // > max + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut core = version_cursor(&tx, 50); + + let out = core.latest_version_for_key(k).expect("ok"); + assert_block(out, 30, false); + } + + /// No ≥ max but key exists → use last < max. + #[test] + fn latest_version_for_key_picks_last_below_max_when_none_at_or_above() { + let db = setup_db(); + let k = stored(Nibbles::from_nibbles([0x0A])); + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k.clone(), 10, Some(node())); + append_account_trie(&wtx, k.clone(), 40, Some(node())); // expected (max=100) + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut core = version_cursor(&tx, 100); + + let out = core.latest_version_for_key(k).expect("ok"); + assert_block(out, 40, false); + } + + /// All entries are > max → None. + #[test] + fn latest_version_for_key_none_when_everything_is_above_max() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0B])); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k1.clone(), 60, Some(node())); + append_account_trie(&wtx, k1.clone(), 70, Some(node())); + append_account_trie(&wtx, k2, 40, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut core = version_cursor(&tx, 50); + + let out = core.latest_version_for_key(k1).expect("ok"); + assert!(out.is_none(), "no dup ≤ max ⇒ None"); + } + + /// Single dup < max → pick it. + #[test] + fn latest_version_for_key_picks_single_below_max() { + let db = setup_db(); + let k = stored(Nibbles::from_nibbles([0x0A])); + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k.clone(), 25, Some(node())); // < max + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut core = version_cursor(&tx, 50); + + let out = core.latest_version_for_key(k).expect("ok"); + assert_block(out, 25, false); + } + + /// Single dup == max → pick it. + #[test] + fn latest_version_for_key_picks_single_at_max() { + let db = setup_db(); + let k = stored(Nibbles::from_nibbles([0x0A])); + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k.clone(), 50, Some(node())); // == max + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut core = version_cursor(&tx, 50); + + let out = core.latest_version_for_key(k).expect("ok"); + assert_block(out, 50, false); + } + + /// Latest ≤ max is a tombstone → return it (this API doesn't filter). + #[test] + fn latest_version_for_key_returns_tombstone_if_latest_is_deleted() { + let db = setup_db(); + let k = stored(Nibbles::from_nibbles([0x0A])); + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k.clone(), 10, Some(node())); + append_account_trie(&wtx, k.clone(), 90, None); // latest ≤ max, but deleted + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut core = version_cursor(&tx, 100); + + let out = core.latest_version_for_key(k).expect("ok"); + assert_block(out, 90, true); + } + + /// Should skip tombstones and return None when the latest ≤ max is deleted. + #[test] + fn seek_exact_skips_tombstone_returns_none() { + let db = setup_db(); + let k = stored(Nibbles::from_nibbles([0x0A])); + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k.clone(), 10, Some(node())); + append_account_trie(&wtx, k.clone(), 90, None); // latest ≤ max is tombstoned + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut core = version_cursor(&tx, 100); + + let out = core.seek_exact(k).expect("ok"); + assert!( + out.is_none(), + "seek_exact must filter out deleted latest value" + ); + } + + /// Empty table → None. + #[test] + fn seek_empty_returns_none() { + let db = setup_db(); + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 100); + + let out = cur.seek(stored(Nibbles::from_nibbles([0x0A]))).expect("ok"); + assert!(out.is_none()); + } + + /// Start at an existing key whose latest ≤ max is live → returns that key. + #[test] + fn seek_at_live_key_returns_it() { + let db = setup_db(); + let k = stored(Nibbles::from_nibbles([0x0A])); + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k.clone(), 10, Some(node())); + append_account_trie(&wtx, k.clone(), 20, Some(node())); // latest ≤ max + wtx.commit().expect("commit"); + } + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 50); + + let out = cur.seek(k.clone()).expect("ok").expect("some"); + assert_eq!(out.0, k); + } + + /// Start at an existing key whose latest ≤ max is tombstoned → skip to next key with live + /// value. + #[test] + fn seek_skips_tombstoned_key_to_next_live_key() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0B])); + + { + let wtx = db.tx_mut().expect("rw tx"); + // Key 0x10 latest ≤ max is deleted + append_account_trie(&wtx, k1.clone(), 10, Some(node())); + append_account_trie(&wtx, k1.clone(), 20, None); // tombstone at latest ≤ max + // Next key has live + append_account_trie(&wtx, k2.clone(), 5, Some(node())); + wtx.commit().expect("commit"); + } + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 50); + + let out = cur.seek(k1).expect("ok").expect("some"); + assert_eq!(out.0, k2); + } + + /// Start between keys → returns the next key’s live latest ≤ max. + #[test] + fn seek_between_keys_returns_next_key() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0C])); + let k3 = stored(Nibbles::from_nibbles([0x0B])); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k1, 10, Some(node())); + append_account_trie(&wtx, k2.clone(), 10, Some(node())); + wtx.commit().expect("commit"); + } + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 100); + + // Start at 0x15 (between 0x10 and 0x20) + + let out = cur.seek(k3).expect("ok").expect("some"); + assert_eq!(out.0, k2); + } + + /// Start after the last key → None. + #[test] + fn seek_after_last_returns_none() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0B])); + let k3 = stored(Nibbles::from_nibbles([0x0C])); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k1, 10, Some(node())); + append_account_trie(&wtx, k2, 10, Some(node())); + wtx.commit().expect("commit"); + } + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 100); + + let out = cur.seek(k3).expect("ok"); + assert!(out.is_none()); + } + + /// If the first key at-or-after has only versions > max, it is effectively not visible → skip + /// to next. + #[test] + fn seek_skips_keys_with_only_versions_above_max() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0B])); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k1.clone(), 60, Some(node())); + append_account_trie(&wtx, k2.clone(), 40, Some(node())); + wtx.commit().expect("commit"); + } + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 50); + + let out = cur.seek(k1).expect("ok").expect("some"); + assert_eq!(out.0, k2); + } + + /// Start at a key with mixed versions; latest ≤ max is tombstone → skip to next key with live. + #[test] + fn seek_mixed_versions_tombstone_latest_skips_to_next_key() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0B])); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k1.clone(), 10, Some(node())); + append_account_trie(&wtx, k1.clone(), 30, None); + append_account_trie(&wtx, k2.clone(), 5, Some(node())); + wtx.commit().expect("commit"); + } + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 30); + + let out = cur.seek(k1).expect("ok").expect("some"); + assert_eq!(out.0, k2); + } + + /// When not positioned should start from default key and return the first live key. + #[test] + fn next_unpositioned_starts_from_default_returns_first_live() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0B])); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k1.clone(), 10, Some(node())); // first live + append_account_trie(&wtx, k2, 10, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + // Unpositioned cursor + let mut cur = version_cursor(&tx, 100); + + let out = cur.next().expect("ok").expect("some"); + assert_eq!(out.0, k1); + } + + /// After positioning on a live key via `seek()`, `next()` should advance to the next live key. + #[test] + fn next_advances_from_current_live_to_next_live() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0B])); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k1.clone(), 10, Some(node())); // live + append_account_trie(&wtx, k2.clone(), 10, Some(node())); // next live + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 100); + + // Position at k1 + let _ = cur.seek(k1).expect("ok").expect("some"); + // Next should yield k2 + let out = cur.next().expect("ok").expect("some"); + assert_eq!(out.0, k2); + } + + /// If the next key's latest ≤ max is tombstone, `next()` should skip to the next live key. + #[test] + fn next_skips_tombstoned_key_to_next_live() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0B])); // will be tombstoned at latest ≤ max + let k3 = stored(Nibbles::from_nibbles([0x0C])); // next live + + { + let wtx = db.tx_mut().expect("rw tx"); + // k1 live + append_account_trie(&wtx, k1.clone(), 10, Some(node())); + // k2: latest ≤ max is tombstone + append_account_trie(&wtx, k2.clone(), 10, Some(node())); + append_account_trie(&wtx, k2, 20, None); + // k3 live + append_account_trie(&wtx, k3.clone(), 10, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 50); + + // Position at k1 + let _ = cur.seek(k1).expect("ok").expect("some"); + // next should skip k2 (tombstoned latest) and return k3 + let out = cur.next().expect("ok").expect("some"); + assert_eq!(out.0, k3); + } + + /// If positioned on the last live key, `next()` should return None (EOF). + #[test] + fn next_returns_none_at_eof() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); + let k2 = stored(Nibbles::from_nibbles([0x0B])); // last key + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, k1, 10, Some(node())); + append_account_trie(&wtx, k2.clone(), 10, Some(node())); // last live + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 100); + + // Position at the last key k2 + let _ = cur.seek(k2).expect("ok").expect("some"); + // `next()` should hit EOF + let out = cur.next().expect("ok"); + assert!(out.is_none()); + } + + /// If the first key has only versions > max, `next()` should skip it and return the next live + /// key. + #[test] + fn next_skips_keys_with_only_versions_above_max() { + let db = setup_db(); + let k1 = stored(Nibbles::from_nibbles([0x0A])); // only > max + let k2 = stored(Nibbles::from_nibbles([0x0B])); // ≤ max live + + { + let wtx = db.tx_mut().expect("rw tx"); + // k1 only above max (max=50) + append_account_trie(&wtx, k1, 60, Some(node())); + // k2 within max + append_account_trie(&wtx, k2.clone(), 40, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + // Unpositioned; `next()` will start from default and walk + let mut cur = version_cursor(&tx, 50); + + let out = cur.next().expect("ok").expect("some"); + assert_eq!(out.0, k2); + } + + /// Empty table: `next()` should return None. + #[test] + fn next_on_empty_returns_none() { + let db = setup_db(); + let tx = db.tx().expect("ro tx"); + let mut cur = version_cursor(&tx, 100); + + let out = cur.next().expect("ok"); + assert!(out.is_none()); + } + + // ----------------- Account trie cursor thin-wrapper checks ----------------- + + #[test] + fn account_seek_exact_live_maps_key_and_value() { + let db = setup_db(); + let k = Nibbles::from_nibbles([0x0A]); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, StoredNibbles(k), 10, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + + // Build wrapper + let mut cur = account_trie_cursor(&tx, 100); + + // Wrapper should return (Nibbles, BranchNodeCompact) + let out = TrieCursor::seek_exact(&mut cur, k) + .expect("ok") + .expect("some"); + assert_eq!(out.0, k); + } + + #[test] + fn account_seek_exact_filters_tombstone() { + let db = setup_db(); + let k = Nibbles::from_nibbles([0x0B]); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, StoredNibbles(k), 5, Some(node())); + append_account_trie(&wtx, StoredNibbles(k), 9, None); // latest ≤ max tombstone + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut cur = account_trie_cursor(&tx, 10); + + let out = TrieCursor::seek_exact(&mut cur, k).expect("ok"); + assert!(out.is_none(), "account seek_exact must filter tombstone"); + } + + #[test] + fn account_seek_and_next_and_current_roundtrip() { + let db = setup_db(); + let k1 = Nibbles::from_nibbles([0x01]); + let k2 = Nibbles::from_nibbles([0x02]); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_account_trie(&wtx, StoredNibbles(k1), 10, Some(node())); + append_account_trie(&wtx, StoredNibbles(k2), 10, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut cur = account_trie_cursor(&tx, 100); + + // seek at k1 + let out1 = TrieCursor::seek(&mut cur, k1).expect("ok").expect("some"); + assert_eq!(out1.0, k1); + + // current should be k1 + let cur_k = TrieCursor::current(&mut cur).expect("ok").expect("some"); + assert_eq!(cur_k, k1); + + // next should move to k2 + let out2 = TrieCursor::next(&mut cur).expect("ok").expect("some"); + assert_eq!(out2.0, k2); + } + + // ----------------- Storage trie cursor thin-wrapper checks ----------------- + + #[test] + fn storage_seek_exact_respects_address_filter() { + let db = setup_db(); + + let addr_a = B256::from([0xAA; 32]); + let addr_b = B256::from([0xBB; 32]); + + let path = Nibbles::from_nibbles([0x0D]); + + { + let wtx = db.tx_mut().expect("rw tx"); + // insert only under B + append_storage_trie(&wtx, addr_b, path, 10, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + + // Cursor bound to A must not see B’s data + let mut cur_a = storage_trie_cursor(&tx, 100, addr_a); + let out_a = TrieCursor::seek_exact(&mut cur_a, path).expect("ok"); + assert!(out_a.is_none(), "no data for addr A"); + + // Cursor bound to B should see it + let mut cur_b = storage_trie_cursor(&tx, 100, addr_b); + let out_b = TrieCursor::seek_exact(&mut cur_b, path) + .expect("ok") + .expect("some"); + assert_eq!(out_b.0, path); + } + + #[test] + fn storage_seek_returns_first_key_for_bound_address() { + let db = setup_db(); + + let addr_a = B256::from([0x11; 32]); + let addr_b = B256::from([0x22; 32]); + + let p1 = Nibbles::from_nibbles([0x01]); + let p2 = Nibbles::from_nibbles([0x02]); + let p3 = Nibbles::from_nibbles([0x03]); + + { + let wtx = db.tx_mut().expect("rw tx"); + // For A: only p2 + append_storage_trie(&wtx, addr_a, p2, 10, Some(node())); + // For B: p1 + append_storage_trie(&wtx, addr_b, p1, 10, Some(node())); + wtx.commit().expect("commit"); + } + + // test seek behaviour + { + let tx = db.tx().expect("ro tx"); + let mut cur_a = storage_trie_cursor(&tx, 100, addr_a); + + // seek at p1: for A there is no p1; the next key >= p1 under A is p2 + let out = TrieCursor::seek(&mut cur_a, p1).expect("ok").expect("some"); + assert_eq!(out.0, p2); + + // seek at p2: exact match + let out = TrieCursor::seek(&mut cur_a, p2).expect("ok").expect("some"); + assert_eq!(out.0, p2); + + // seek at p3: no p3 under A; no next key ≥ p3 under A → None + let out = TrieCursor::seek(&mut cur_a, p3).expect("ok"); + assert!(out.is_none(), "no key ≥ p3 under A"); + } + + // test next behaviour + { + let tx = db.tx().expect("ro tx"); + let mut cur_a = storage_trie_cursor(&tx, 100, addr_a); + + let out = TrieCursor::next(&mut cur_a).expect("ok").expect("some"); + assert_eq!(out.0, p2); + + // next should yield None as there is no further key under A + let out = TrieCursor::next(&mut cur_a).expect("ok"); + assert!(out.is_none(), "no more keys under A"); + + // current should return None + let out = TrieCursor::current(&mut cur_a).expect("ok"); + assert!(out.is_none(), "no current key after EOF"); + } + + // test seek_exact behaviour + { + let tx = db.tx().expect("ro tx"); + let mut cur_a = storage_trie_cursor(&tx, 100, addr_a); + + // seek_exact at p1: no exact match + let out = TrieCursor::seek_exact(&mut cur_a, p1).expect("ok"); + assert!(out.is_none(), "no exact p1 under A"); + + // seek_exact at p2: exact match + let out = TrieCursor::seek_exact(&mut cur_a, p2) + .expect("ok") + .expect("some"); + assert_eq!(out.0, p2); + + // seek_exact at p3: no exact match + let out = TrieCursor::seek_exact(&mut cur_a, p3).expect("ok"); + assert!(out.is_none(), "no exact p3 under A"); + } + } + + #[test] + fn storage_next_stops_at_address_boundary() { + let db = setup_db(); + + let addr_a = B256::from([0x33; 32]); + let addr_b = B256::from([0x44; 32]); + + let p1 = Nibbles::from_nibbles([0x05]); // under A + let p2 = Nibbles::from_nibbles([0x06]); // under B (next key overall) + + { + let wtx = db.tx_mut().expect("rw tx"); + append_storage_trie(&wtx, addr_a, p1, 10, Some(node())); + append_storage_trie(&wtx, addr_b, p2, 10, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut cur_a = storage_trie_cursor(&tx, 100, addr_a); + + // position at p1 (A) + let _ = TrieCursor::seek_exact(&mut cur_a, p1) + .expect("ok") + .expect("some"); + + // next should reach boundary; impl filters different address and returns None + let out = TrieCursor::next(&mut cur_a).expect("ok"); + assert!( + out.is_none(), + "next() should stop when next key is a different address" + ); + } + + #[test] + fn storage_current_maps_key() { + let db = setup_db(); + + let addr = B256::from([0x55; 32]); + let p = Nibbles::from_nibbles([0x09]); + + { + let wtx = db.tx_mut().expect("rw tx"); + append_storage_trie(&wtx, addr, p, 10, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro tx"); + let mut cur = storage_trie_cursor(&tx, 100, addr); + + let _ = TrieCursor::seek_exact(&mut cur, p) + .expect("ok") + .expect("some"); + + let now = TrieCursor::current(&mut cur).expect("ok").expect("some"); + assert_eq!(now, p); + } + + #[test] + fn hashed_storage_seek_maps_slot_and_value() { + let db = setup_db(); + let addr = B256::from([0xAA; 32]); + let slot = B256::from([0x10; 32]); + + { + let wtx = db.tx_mut().expect("rw"); + append_hashed_storage(&wtx, addr, slot, 10, Some(U256::from(7))); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro"); + let mut cur = storage_cursor(&tx, 100, addr); + + let (got_slot, got_val) = cur.seek(slot).expect("ok").expect("some"); + assert_eq!(got_slot, slot); + assert_eq!(got_val, U256::from(7)); + } + + #[test] + fn hashed_storage_seek_filters_tombstone() { + let db = setup_db(); + let addr = B256::from([0xAB; 32]); + let slot = B256::from([0x11; 32]); + + { + let wtx = db.tx_mut().expect("rw"); + append_hashed_storage(&wtx, addr, slot, 5, Some(U256::from(1))); + append_hashed_storage(&wtx, addr, slot, 9, None); // latest ≤ max is tombstone + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro"); + let mut cur = storage_cursor(&tx, 10, addr); + + let out = cur.seek(slot).expect("ok"); + assert!(out.is_none(), "wrapper must filter tombstoned latest"); + } + + #[test] + fn hashed_storage_seek_and_next_roundtrip() { + let db = setup_db(); + let addr = B256::from([0xAC; 32]); + let s1 = B256::from([0x01; 32]); + let s2 = B256::from([0x02; 32]); + + { + let wtx = db.tx_mut().expect("rw"); + append_hashed_storage(&wtx, addr, s1, 10, Some(U256::from(11))); + append_hashed_storage(&wtx, addr, s2, 10, Some(U256::from(22))); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro"); + let mut cur = storage_cursor(&tx, 100, addr); + + let (k1, v1) = cur.seek(s1).expect("ok").expect("some"); + assert_eq!((k1, v1), (s1, U256::from(11))); + + let (k2, v2) = cur.next().expect("ok").expect("some"); + assert_eq!((k2, v2), (s2, U256::from(22))); + } + + #[test] + fn hashed_storage_address_boundary() { + let db = setup_db(); + let addr1 = B256::from([0xAC; 32]); + let addr2 = B256::from([0xAD; 32]); + let s1 = B256::from([0x01; 32]); + let s2 = B256::from([0x02; 32]); + let s3 = B256::from([0x03; 32]); + + { + let wtx = db.tx_mut().expect("rw"); + append_hashed_storage(&wtx, addr1, s1, 10, Some(U256::from(11))); + append_hashed_storage(&wtx, addr1, s2, 10, Some(U256::from(22))); + wtx.commit().expect("commit"); + } + + { + let wtx = db.tx_mut().expect("rw"); + append_hashed_storage(&wtx, addr2, s1, 10, Some(U256::from(33))); + append_hashed_storage(&wtx, addr2, s2, 10, Some(U256::from(44))); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro"); + let mut cur = storage_cursor(&tx, 100, addr1); + + let (k1, v1) = cur.next().expect("ok").expect("some"); + assert_eq!((k1, v1), (s1, U256::from(11))); + + let (k2, v2) = cur.next().expect("ok").expect("some"); + assert_eq!((k2, v2), (s2, U256::from(22))); + + let out = cur.next().expect("ok"); + assert!(out.is_none(), "should stop at address boundary"); + + let (k1, v1) = cur.seek(s1).expect("ok").expect("some"); + assert_eq!((k1, v1), (s1, U256::from(11))); + + let (k2, v2) = cur.seek(s2).expect("ok").expect("some"); + assert_eq!((k2, v2), (s2, U256::from(22))); + + let out = cur.seek(s3).expect("ok"); + assert!(out.is_none(), "should not see keys from other address"); + } + + #[test] + fn hashed_account_seek_maps_key_and_value() { + let db = setup_db(); + let key = B256::from([0x20; 32]); + + { + let wtx = db.tx_mut().expect("rw"); + append_hashed_account(&wtx, key, 10, Some(Account::default())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro"); + let mut cur = account_cursor(&tx, 100); + + let (got_key, _acc) = cur.seek(key).expect("ok").expect("some"); + assert_eq!(got_key, key); + } + + #[test] + fn hashed_account_seek_filters_tombstone() { + let db = setup_db(); + let key = B256::from([0x21; 32]); + + { + let wtx = db.tx_mut().expect("rw"); + append_hashed_account(&wtx, key, 5, Some(Account::default())); + append_hashed_account(&wtx, key, 9, None); // latest ≤ max is tombstone + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro"); + let mut cur = account_cursor(&tx, 10); + + let out = cur.seek(key).expect("ok"); + assert!(out.is_none(), "wrapper must filter tombstoned latest"); + } + + #[test] + fn hashed_account_seek_and_next_roundtrip() { + let db = setup_db(); + let k1 = B256::from([0x01; 32]); + let k2 = B256::from([0x02; 32]); + + { + let wtx = db.tx_mut().expect("rw"); + append_hashed_account(&wtx, k1, 10, Some(Account::default())); + append_hashed_account(&wtx, k2, 10, Some(Account::default())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro"); + let mut cur = account_cursor(&tx, 100); + + let (got1, _) = cur.seek(k1).expect("ok").expect("some"); + assert_eq!(got1, k1); + + let (got2, _) = cur.next().expect("ok").expect("some"); + assert_eq!(got2, k2); + } + + /// Regression test: `MdbxStorageCursor` `next()` should work without explicit `seek()` + /// when cursor is constructed for a non-first key. + /// + /// Bug: When a storage cursor is created for a specific address (e.g., 0x02), + /// calling `next()` without first calling `seek()` returns None instead of the first + /// slot for that address. This only manifests when the address is not the first + /// in the table. + #[test] + fn storage_cursor_next_without_seek_for_non_first_address() { + let db = setup_db(); + let addr1 = B256::from([0x01; 32]); // First address + let addr2 = B256::from([0x02; 32]); // Second address (non-first) + let slot1 = B256::from([0x11; 32]); + let slot2 = B256::from([0x12; 32]); + + { + let wtx = db.tx_mut().expect("rw"); + // Add storage for first address + append_hashed_storage(&wtx, addr1, slot1, 10, Some(U256::from(100))); + + // Add storage for second address + append_hashed_storage(&wtx, addr2, slot2, 10, Some(U256::from(200))); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro"); + + // Test with addr1 (first address) - this typically works + let mut cur1 = storage_cursor(&tx, 100, addr1); + let result1 = cur1.next().expect("ok"); + assert!( + result1.is_some(), + "next() should return data for first address without seek()" + ); + if let Some((key, val)) = result1 { + assert_eq!(key, slot1); + assert_eq!(val, U256::from(100)); + } + + // Test with addr2 (non-first address) - this demonstrates the bug fix + let mut cur2 = storage_cursor(&tx, 100, addr2); + let result2_without_seek = cur2.next().expect("ok"); + + assert!( + result2_without_seek.is_some(), + "next() should return data for non-first address without seek()" + ); + if let Some((key, val)) = result2_without_seek { + assert_eq!(key, slot2); + assert_eq!(val, U256::from(200)); + } + + // Verify that seek() works correctly + let mut cur3 = storage_cursor(&tx, 100, addr2); + let result3_with_seek = cur3.seek(slot2).expect("ok"); + assert!( + result3_with_seek.is_some(), + "seek() should find the slot for addr2" + ); + if let Some((key, val)) = result3_with_seek { + assert_eq!(key, slot2); + assert_eq!(val, U256::from(200)); + } + } + + /// Regression test: `MdbxTrieCursor` `next()` should work without `seek()` + /// for non-first addresses. + #[test] + fn storage_trie_cursor_next_without_seek_for_non_first_address() { + let db = setup_db(); + let addr1 = B256::from([0x01; 32]); + let addr2 = B256::from([0x02; 32]); + let path1 = Nibbles::from_nibbles([0x0A]); + let path2 = Nibbles::from_nibbles([0x0B]); + + { + let wtx = db.tx_mut().expect("rw"); + append_storage_trie(&wtx, addr1, path1, 10, Some(node())); + append_storage_trie(&wtx, addr2, path2, 10, Some(node())); + wtx.commit().expect("commit"); + } + + let tx = db.tx().expect("ro"); + + // Test addr1 (first) - works + let mut cur1 = storage_trie_cursor(&tx, 100, addr1); + let result1 = TrieCursor::next(&mut cur1).expect("ok"); + assert!(result1.is_some()); + assert_eq!(result1.unwrap().0, path1); + + // Test addr2 (non-first) - should also work now + let mut cur2 = storage_trie_cursor(&tx, 100, addr2); + let result2 = TrieCursor::next(&mut cur2).expect("ok"); + assert!( + result2.is_some(), + "next() should work for non-first address without seek()" + ); + assert_eq!(result2.unwrap().0, path2); + } +} diff --git a/crates/proofs/src/db/mod.rs b/crates/proofs/src/db/mod.rs new file mode 100644 index 00000000..7ebcfc34 --- /dev/null +++ b/crates/proofs/src/db/mod.rs @@ -0,0 +1,15 @@ +//! MDBX-backed implementation of [`MorphProofsStore`](crate::MorphProofsStore). + +mod models; +pub use models::*; + +mod store; +pub use store::{MdbxProofsStorage, MdbxProofsStorageOptions, ProofDbIdentity, ProofWindowBounds}; + +mod cursor; +pub use cursor::{ + BlockNumberVersionedCursor, Dup, MdbxAccountCursor, MdbxStorageCursor, MdbxTrieCursor, +}; + +mod batch; +pub use batch::{DupRw, MdbxBatchSession}; diff --git a/crates/proofs/src/db/models/block.rs b/crates/proofs/src/db/models/block.rs new file mode 100644 index 00000000..7475c364 --- /dev/null +++ b/crates/proofs/src/db/models/block.rs @@ -0,0 +1,82 @@ +use alloy_eips::BlockNumHash; +use alloy_primitives::B256; +use bytes::BufMut; +use derive_more::{From, Into}; +use reth_codecs::DecompressError; +use reth_db::{ + DatabaseError, + table::{Compress, Decompress}, +}; +use serde::{Deserialize, Serialize}; + +/// Wrapper for block number and block hash tuple to implement [`Compress`]/[`Decompress`]. +/// +/// Used for storing block metadata (number + hash). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, From, Into)] +pub struct BlockNumberHash(BlockNumHash); + +impl Compress for BlockNumberHash { + type Compressed = Vec; + + fn compress_to_buf>(&self, buf: &mut B) { + // Encode block number (8 bytes, big-endian) + hash (32 bytes) = 40 bytes total + buf.put_u64(self.0.number); + buf.put_slice(self.0.hash.as_slice()); + } +} + +impl Decompress for BlockNumberHash { + fn decompress(value: &[u8]) -> Result { + if value.len() != 40 { + return Err(DecompressError::new(DatabaseError::Decode)); + } + + let number = u64::from_be_bytes( + value[..8] + .try_into() + .map_err(|_| DecompressError::new(DatabaseError::Decode))?, + ); + let hash = B256::from_slice(&value[8..40]); + + Ok(Self(BlockNumHash { number, hash })) + } +} + +impl BlockNumberHash { + /// Create new instance. + pub const fn new(number: u64, hash: B256) -> Self { + Self(BlockNumHash { number, hash }) + } + + /// Get the block number. + pub const fn number(&self) -> u64 { + self.0.number + } + + /// Get the block hash. + pub const fn hash(&self) -> &B256 { + &self.0.hash + } +} + +#[cfg(test)] +mod tests { + use alloy_primitives::B256; + + use super::*; + + #[test] + fn test_block_number_hash_roundtrip() { + let test_cases = vec![ + BlockNumberHash::new(0, B256::ZERO), + BlockNumberHash::new(42, B256::repeat_byte(0xaa)), + BlockNumberHash::new(u64::MAX, B256::repeat_byte(0xff)), + ]; + + for original in test_cases { + let compressed = original.compress(); + let decompressed = BlockNumberHash::decompress(&compressed).unwrap(); + assert_eq!(original, decompressed); + } + } +} diff --git a/crates/proofs/src/db/models/change_set.rs b/crates/proofs/src/db/models/change_set.rs new file mode 100644 index 00000000..d9f7d806 --- /dev/null +++ b/crates/proofs/src/db/models/change_set.rs @@ -0,0 +1,131 @@ +use alloy_primitives::B256; +use reth_codecs::DecompressError; +use reth_db::{ + DatabaseError, + table::{self, Decode, Encode}, +}; +use reth_trie_common::StoredNibbles; +use serde::{Deserialize, Serialize}; + +use crate::db::{HashedStorageKey, StorageTrieKey}; + +/// The keys of the entries in the history tables. +#[derive(Debug, Default, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct ChangeSet { + /// Keys changed in [`AccountTrieHistory`](super::AccountTrieHistory) table. + pub account_trie_keys: Vec, + /// Keys changed in [`StorageTrieHistory`](super::StorageTrieHistory) table. + pub storage_trie_keys: Vec, + /// Keys changed in [`HashedAccountHistory`](super::HashedAccountHistory) table. + pub hashed_account_keys: Vec, + /// Keys changed in [`HashedStorageHistory`](super::HashedStorageHistory) table. + pub hashed_storage_keys: Vec, +} + +impl table::Encode for ChangeSet { + type Encoded = Vec; + + fn encode(self) -> Self::Encoded { + bincode::serde::encode_to_vec(&self, bincode::config::standard()) + .expect("ChangeSet serialization should not fail") + } +} + +impl table::Decode for ChangeSet { + fn decode(value: &[u8]) -> Result { + bincode::serde::decode_from_slice(value, bincode::config::standard()) + .map(|(v, _)| v) + .map_err(|_| DatabaseError::Decode) + } +} + +impl table::Compress for ChangeSet { + type Compressed = Vec; + + fn compress_to_buf>(&self, buf: &mut B) { + let encoded = self.clone().encode(); + buf.put_slice(&encoded); + } +} + +impl table::Decompress for ChangeSet { + fn decompress(value: &[u8]) -> Result { + Self::decode(value).map_err(DecompressError::new) + } +} + +#[cfg(test)] +mod tests { + use alloy_primitives::B256; + use reth_db::table::{Compress, Decompress}; + + use super::*; + + #[test] + fn test_encode_decode_empty_change_set() { + let change_set = ChangeSet { + account_trie_keys: vec![], + storage_trie_keys: vec![], + hashed_account_keys: vec![], + hashed_storage_keys: vec![], + }; + + let encoded = change_set.clone().encode(); + let decoded = ChangeSet::decode(&encoded).expect("Failed to decode"); + assert_eq!(change_set, decoded); + } + + #[test] + fn test_encode_decode_populated_change_set() { + let account_key = StoredNibbles::from(vec![1, 2, 3, 4]); + let storage_key = StorageTrieKey { + hashed_address: B256::repeat_byte(0x11), + path: StoredNibbles::from(vec![5, 6, 7, 8]), + }; + let hashed_storage_key = HashedStorageKey { + hashed_address: B256::repeat_byte(0x22), + hashed_storage_key: B256::repeat_byte(0x33), + }; + + let change_set = ChangeSet { + account_trie_keys: vec![account_key], + storage_trie_keys: vec![storage_key], + hashed_account_keys: vec![B256::repeat_byte(0x44)], + hashed_storage_keys: vec![hashed_storage_key], + }; + + let encoded = change_set.clone().encode(); + let decoded = ChangeSet::decode(&encoded).expect("Failed to decode"); + assert_eq!(change_set, decoded); + } + + #[test] + fn test_decode_invalid_data() { + let invalid_data = vec![0xFF; 32]; + let result = ChangeSet::decode(&invalid_data); + assert!(result.is_err()); + assert!(matches!(result.unwrap_err(), DatabaseError::Decode)); + } + + #[test] + fn test_compress_decompress() { + let change_set = ChangeSet { + account_trie_keys: vec![StoredNibbles::from(vec![1, 2, 3])], + storage_trie_keys: vec![StorageTrieKey { + hashed_address: B256::ZERO, + path: StoredNibbles::from(vec![4, 5, 6]), + }], + hashed_account_keys: vec![B256::ZERO], + hashed_storage_keys: vec![HashedStorageKey { + hashed_address: B256::ZERO, + hashed_storage_key: B256::repeat_byte(0x42), + }], + }; + + let mut buf = Vec::new(); + change_set.compress_to_buf(&mut buf); + + let decompressed = ChangeSet::decompress(&buf).expect("Failed to decompress"); + assert_eq!(change_set, decompressed); + } +} diff --git a/crates/proofs/src/db/models/kv.rs b/crates/proofs/src/db/models/kv.rs new file mode 100644 index 00000000..05f85807 --- /dev/null +++ b/crates/proofs/src/db/models/kv.rs @@ -0,0 +1,85 @@ +use alloy_primitives::B256; +use reth_db::table::{DupSort, Table}; +use reth_primitives_traits::Account; +use reth_trie_common::{BranchNodeCompact, Nibbles, StoredNibbles}; + +use crate::db::{ + AccountTrieHistory, HashedAccountHistory, HashedStorageHistory, HashedStorageKey, MaybeDeleted, + StorageTrieHistory, StorageTrieKey, StorageValue, VersionedValue, +}; + +/// Helper to convert inputs into a table key or kv pair. +pub trait IntoKV { + /// Convert `self` into the table key. + fn into_key(self) -> Tab::Key; + + /// Convert `self` into kv for the given `block_number`. + fn into_kv(self, block_number: u64) -> (Tab::Key, Tab::Value); +} + +impl IntoKV for (Nibbles, Option) { + fn into_key(self) -> StoredNibbles { + StoredNibbles::from(self.0) + } + + fn into_kv(self, block_number: u64) -> (StoredNibbles, VersionedValue) { + let (path, node) = self; + ( + StoredNibbles::from(path), + VersionedValue { + block_number, + value: MaybeDeleted(node), + }, + ) + } +} + +impl IntoKV for (B256, Nibbles, Option) { + fn into_key(self) -> StorageTrieKey { + let (hashed_address, path, _) = self; + StorageTrieKey::new(hashed_address, StoredNibbles::from(path)) + } + fn into_kv(self, block_number: u64) -> (StorageTrieKey, VersionedValue) { + let (hashed_address, path, node) = self; + ( + StorageTrieKey::new(hashed_address, StoredNibbles::from(path)), + VersionedValue { + block_number, + value: MaybeDeleted(node), + }, + ) + } +} + +impl IntoKV for (B256, Option) { + fn into_key(self) -> B256 { + self.0 + } + fn into_kv(self, block_number: u64) -> (B256, VersionedValue) { + let (hashed_address, account) = self; + ( + hashed_address, + VersionedValue { + block_number, + value: MaybeDeleted(account), + }, + ) + } +} + +impl IntoKV for (B256, B256, Option) { + fn into_key(self) -> HashedStorageKey { + let (hashed_address, hashed_storage_key, _) = self; + HashedStorageKey::new(hashed_address, hashed_storage_key) + } + fn into_kv(self, block_number: u64) -> (HashedStorageKey, VersionedValue) { + let (hashed_address, hashed_storage_key, value) = self; + ( + HashedStorageKey::new(hashed_address, hashed_storage_key), + VersionedValue { + block_number, + value: MaybeDeleted(value), + }, + ) + } +} diff --git a/crates/proofs/src/db/models/metadata.rs b/crates/proofs/src/db/models/metadata.rs new file mode 100644 index 00000000..999795e7 --- /dev/null +++ b/crates/proofs/src/db/models/metadata.rs @@ -0,0 +1,56 @@ +use bytes::BufMut; +use reth_codecs::DecompressError; +use reth_db::{ + DatabaseError, + table::{Compress, Decode, Decompress, Encode}, +}; +use serde::{Deserialize, Serialize}; + +/// Durable proof-database metadata keys. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[repr(u8)] +pub enum ProofMetadataKey { + /// EIP-155 chain identifier. + ChainId = 0, + /// Canonical genesis block hash. + GenesisHash = 1, + /// Proof-database schema version. + SchemaVersion = 2, +} + +impl Encode for ProofMetadataKey { + type Encoded = [u8; 1]; + + fn encode(self) -> Self::Encoded { + [self as u8] + } +} + +impl Decode for ProofMetadataKey { + fn decode(value: &[u8]) -> Result { + match value.first() { + Some(0) => Ok(Self::ChainId), + Some(1) => Ok(Self::GenesisHash), + Some(2) => Ok(Self::SchemaVersion), + _ => Err(DatabaseError::Decode), + } + } +} + +/// Opaque metadata bytes whose shape is validated by the store. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ProofMetadataValue(pub Vec); + +impl Compress for ProofMetadataValue { + type Compressed = Vec; + + fn compress_to_buf>(&self, buf: &mut B) { + buf.put_slice(&self.0); + } +} + +impl Decompress for ProofMetadataValue { + fn decompress(value: &[u8]) -> Result { + Ok(Self(value.to_vec())) + } +} diff --git a/crates/proofs/src/db/models/mod.rs b/crates/proofs/src/db/models/mod.rs new file mode 100644 index 00000000..de5b9786 --- /dev/null +++ b/crates/proofs/src/db/models/mod.rs @@ -0,0 +1,93 @@ +//! MDBX implementation of [`MorphProofsStore`](crate::MorphProofsStore). +//! +//! This module provides a complete MDBX implementation of the +//! [`MorphProofsStore`](crate::MorphProofsStore) trait. It uses the [`reth_db`] crate for +//! database interactions and defines the necessary tables and models for storing trie branches, +//! accounts, and storage leaves. + +mod block; +pub use block::*; +mod version; +pub use version::*; +mod storage; +pub use storage::*; +mod change_set; +mod kv; +mod metadata; +use std::fmt; + +use alloy_primitives::B256; +pub use change_set::*; +pub use kv::*; +pub use metadata::*; +use reth_db::{ + TableSet, TableType, TableViewer, + table::{DupSort, TableInfo}, + tables, +}; +use reth_primitives_traits::Account; +use reth_trie_common::{BranchNodeCompact, StoredNibbles}; + +tables! { + /// Stores historical branch nodes for the account state trie. + /// + /// Each entry maps a compact-encoded trie path (`StoredNibbles`) to its versioned branch node. + /// Multiple versions of the same node are stored using the block number as a subkey. + table AccountTrieHistory { + type Key = StoredNibbles; + type Value = VersionedValue; + type SubKey = u64; // block number + } + + /// Stores historical branch nodes for the storage trie of each account. + /// + /// Each entry is identified by a composite key combining the account’s hashed address and the + /// compact-encoded trie path. Versions are tracked using block numbers as subkeys. + table StorageTrieHistory { + type Key = StorageTrieKey; + type Value = VersionedValue; + type SubKey = u64; // block number + } + + /// Stores versioned account state across block history. + /// + /// Each entry maps a hashed account address to its serialized account data (balance, nonce, + /// code hash, storage root). + table HashedAccountHistory { + type Key = B256; + type Value = VersionedValue; + type SubKey = u64; // block number + } + + /// Stores versioned storage state across block history. + /// + /// Each entry maps a composite key of (hashed address, storage key) to its stored value. + /// Used for reconstructing contract storage at any historical block height. + table HashedStorageHistory { + type Key = HashedStorageKey; + type Value = VersionedValue; + type SubKey = u64; // block number + } + + /// Tracks the active proof window in the external historical storage. + /// + /// Stores the earliest and latest block numbers (and corresponding hashes) + /// for which historical trie data is retained. + table ProofWindow { + type Key = ProofWindowKey; + type Value = BlockNumberHash; + } + + /// Stores schema and canonical-chain identity for fail-fast cold-copy validation. + table ProofMetadata { + type Key = ProofMetadataKey; + type Value = ProofMetadataValue; + } + + /// A reverse mapping of block numbers to a keys of the tables. + /// This is used for efficiently locating data by block number. + table BlockChangeSet { + type Key = u64; // Block number + type Value = ChangeSet; + } +} diff --git a/crates/proofs/src/db/models/storage.rs b/crates/proofs/src/db/models/storage.rs new file mode 100644 index 00000000..bcf30dfb --- /dev/null +++ b/crates/proofs/src/db/models/storage.rs @@ -0,0 +1,272 @@ +use alloy_primitives::{B256, U256}; +use derive_more::{Constructor, From, Into}; +use reth_codecs::DecompressError; +use reth_db::{ + DatabaseError, + table::{Compress, Decode, Decompress, Encode}, +}; +use reth_trie_common::StoredNibbles; +use serde::{Deserialize, Serialize}; + +/// Composite key: `(hashed-address, path)` for storage trie branches +/// +/// Used to efficiently index storage branches by both account address and trie path. +/// The encoding ensures lexicographic ordering: first by address, then by path. +#[derive(Default, Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct StorageTrieKey { + /// Hashed account address + pub hashed_address: B256, + /// Trie path as nibbles + pub path: StoredNibbles, +} + +impl StorageTrieKey { + /// Create a new storage branch key + pub const fn new(hashed_address: B256, path: StoredNibbles) -> Self { + Self { + hashed_address, + path, + } + } +} + +impl Encode for StorageTrieKey { + type Encoded = Vec; + + fn encode(self) -> Self::Encoded { + let mut buf = Vec::with_capacity(32 + self.path.0.len()); + // First encode the address (32 bytes) + buf.extend_from_slice(self.hashed_address.as_slice()); + // Then encode the path + buf.extend_from_slice(&self.path.encode()); + buf + } +} + +impl Decode for StorageTrieKey { + fn decode(value: &[u8]) -> Result { + if value.len() < 32 { + return Err(DatabaseError::Decode); + } + + // First 32 bytes are the address + let hashed_address = B256::from_slice(&value[..32]); + + // Remaining bytes are the path + let path = StoredNibbles::decode(&value[32..])?; + + Ok(Self { + hashed_address, + path, + }) + } +} + +/// Composite key: (`hashed_address`, `hashed_storage_key`) for hashed storage values +/// +/// Used to efficiently index storage values by both account address and storage key. +/// The encoding ensures lexicographic ordering: first by address, then by storage key. +#[derive(Debug, Default, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct HashedStorageKey { + /// Hashed account address + pub hashed_address: B256, + /// Hashed storage key + pub hashed_storage_key: B256, +} + +impl HashedStorageKey { + /// Create a new hashed storage key + pub const fn new(hashed_address: B256, hashed_storage_key: B256) -> Self { + Self { + hashed_address, + hashed_storage_key, + } + } +} + +impl Encode for HashedStorageKey { + type Encoded = [u8; 64]; + + fn encode(self) -> Self::Encoded { + let mut buf = [0u8; 64]; + // First 32 bytes: address + buf[..32].copy_from_slice(self.hashed_address.as_slice()); + // Next 32 bytes: storage key + buf[32..].copy_from_slice(self.hashed_storage_key.as_slice()); + buf + } +} + +impl Decode for HashedStorageKey { + fn decode(value: &[u8]) -> Result { + if value.len() != 64 { + return Err(DatabaseError::Decode); + } + + let hashed_address = B256::from_slice(&value[..32]); + let hashed_storage_key = B256::from_slice(&value[32..64]); + + Ok(Self { + hashed_address, + hashed_storage_key, + }) + } +} + +/// Storage value wrapper for U256 values +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, From, Into, Constructor)] +pub struct StorageValue(pub U256); + +impl Compress for StorageValue { + type Compressed = Vec; + + fn compress_to_buf>(&self, buf: &mut B) { + let be: [u8; 32] = self.0.to_be_bytes::<32>(); + buf.put_slice(&be); + } +} + +impl Decompress for StorageValue { + fn decompress(value: &[u8]) -> Result { + if value.len() != 32 { + return Err(DecompressError::new(DatabaseError::Decode)); + } + let bytes: [u8; 32] = value + .try_into() + .map_err(|_| DecompressError::new(DatabaseError::Decode))?; + Ok(Self(U256::from_be_bytes(bytes))) + } +} + +/// Proof Window key for tracking active proof window bounds +/// +/// Used to store earliest and latest block numbers in the external storage. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[repr(u8)] +pub enum ProofWindowKey { + /// Earliest block number stored in external storage + EarliestBlock = 0, + /// Latest block number stored in external storage + LatestBlock = 1, + /// Anchor block from where the initial state initialization started + InitialStateAnchor = 2, +} + +impl Encode for ProofWindowKey { + type Encoded = [u8; 1]; + + fn encode(self) -> Self::Encoded { + [self as u8] + } +} + +impl Decode for ProofWindowKey { + fn decode(value: &[u8]) -> Result { + match value.first() { + Some(&0) => Ok(Self::EarliestBlock), + Some(&1) => Ok(Self::LatestBlock), + Some(&2) => Ok(Self::InitialStateAnchor), + _ => Err(DatabaseError::Decode), + } + } +} + +#[cfg(test)] +mod tests { + use reth_trie::Nibbles; + + use super::*; + + #[test] + fn test_storage_branch_subkey_encode_decode() { + let addr = B256::from([1u8; 32]); + let path = StoredNibbles(Nibbles::from_nibbles_unchecked([1, 2, 3, 4])); + let key = StorageTrieKey::new(addr, path.clone()); + + let encoded = key.clone().encode(); + let decoded = StorageTrieKey::decode(&encoded).unwrap(); + + assert_eq!(key, decoded); + assert_eq!(decoded.hashed_address, addr); + assert_eq!(decoded.path, path); + } + + #[test] + fn test_storage_branch_subkey_ordering() { + let addr1 = B256::from([1u8; 32]); + let addr2 = B256::from([2u8; 32]); + let path1 = StoredNibbles(Nibbles::from_nibbles_unchecked([1, 2])); + let path2 = StoredNibbles(Nibbles::from_nibbles_unchecked([1, 3])); + + let key1 = StorageTrieKey::new(addr1, path1.clone()); + let key2 = StorageTrieKey::new(addr1, path2); + let key3 = StorageTrieKey::new(addr2, path1); + + // Encoded bytes should be sortable: first by address, then by path + let enc1 = key1.encode(); + let enc2 = key2.encode(); + let enc3 = key3.encode(); + + assert!(enc1 < enc2, "Same address, path1 < path2"); + assert!(enc1 < enc3, "addr1 < addr2"); + assert!(enc2 < enc3, "addr1 < addr2 (even with larger path)"); + } + + #[test] + fn test_hashed_storage_subkey_encode_decode() { + let addr = B256::from([1u8; 32]); + let storage_key = B256::from([2u8; 32]); + let key = HashedStorageKey::new(addr, storage_key); + + let encoded = key.clone().encode(); + let decoded = HashedStorageKey::decode(&encoded).unwrap(); + + assert_eq!(key, decoded); + assert_eq!(decoded.hashed_address, addr); + assert_eq!(decoded.hashed_storage_key, storage_key); + } + + #[test] + fn test_hashed_storage_subkey_ordering() { + let addr1 = B256::from([1u8; 32]); + let addr2 = B256::from([2u8; 32]); + let storage1 = B256::from([10u8; 32]); + let storage2 = B256::from([20u8; 32]); + + let key1 = HashedStorageKey::new(addr1, storage1); + let key2 = HashedStorageKey::new(addr1, storage2); + let key3 = HashedStorageKey::new(addr2, storage1); + + // Encoded bytes should be sortable: first by address, then by storage key + let enc1 = key1.encode(); + let enc2 = key2.encode(); + let enc3 = key3.encode(); + + assert!(enc1 < enc2, "Same address, storage1 < storage2"); + assert!(enc1 < enc3, "addr1 < addr2"); + assert!(enc2 < enc3, "addr1 < addr2 (even with larger storage key)"); + } + + #[test] + fn test_hashed_storage_subkey_size() { + let addr = B256::from([1u8; 32]); + let storage_key = B256::from([2u8; 32]); + let key = HashedStorageKey::new(addr, storage_key); + + let encoded = key.encode(); + assert_eq!(encoded.len(), 64, "Encoded size should be exactly 64 bytes"); + } + + #[test] + fn test_metadata_key_encode_decode() { + let key = ProofWindowKey::EarliestBlock; + let encoded = key.encode(); + let decoded = ProofWindowKey::decode(&encoded).unwrap(); + assert_eq!(key, decoded); + + let key = ProofWindowKey::LatestBlock; + let encoded = key.encode(); + let decoded = ProofWindowKey::decode(&encoded).unwrap(); + assert_eq!(key, decoded); + } +} diff --git a/crates/proofs/src/db/models/version.rs b/crates/proofs/src/db/models/version.rs new file mode 100644 index 00000000..5e1ea7d2 --- /dev/null +++ b/crates/proofs/src/db/models/version.rs @@ -0,0 +1,205 @@ +use bytes::{Buf, BufMut}; +use reth_codecs::DecompressError; +use reth_db::{ + DatabaseError, + table::{Compress, Decompress}, +}; +use reth_primitives_traits::ValueWithSubKey; +use serde::{Deserialize, Serialize}; + +/// Wrapper type for `Option` that implements [`Compress`] and [`Decompress`] +/// +/// Encoding: +/// - `None` => empty byte array (length 0) +/// - `Some(value)` => compressed bytes of value (length > 0) +/// +/// This assumes the inner type `T` always compresses to non-empty bytes when it exists. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct MaybeDeleted(pub Option); + +impl From> for MaybeDeleted { + fn from(opt: Option) -> Self { + Self(opt) + } +} + +impl From> for Option { + fn from(maybe: MaybeDeleted) -> Self { + maybe.0 + } +} + +impl Compress for MaybeDeleted { + type Compressed = Vec; + + fn compress_to_buf>(&self, buf: &mut B) { + match &self.0 { + None => { + // Empty = deleted, write nothing + } + Some(value) => { + // Compress the inner value to the buffer + value.compress_to_buf(buf); + } + } + } +} + +impl Decompress for MaybeDeleted { + fn decompress(value: &[u8]) -> Result { + if value.is_empty() { + // Empty = deleted + Ok(Self(None)) + } else { + // Non-empty = present + let inner = T::decompress(value)?; + Ok(Self(Some(inner))) + } + } +} + +/// Versioned value wrapper for [`DupSort`] tables +/// +/// For [`DupSort`] tables in MDBX, the Value type must contain the [`DupSort::SubKey`] as a field. +/// This wrapper combines a [`block_number`] (the [`DupSort::SubKey`]) with +/// the actual value. +/// +/// [`DupSort`]: reth_db::table::DupSort +/// [`DupSort::SubKey`]: reth_db::table::DupSort::SubKey +/// [`block_number`]: Self::block_number +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct VersionedValue { + /// Block number ([`DupSort::SubKey`] for [`DupSort`]) + /// + /// [`DupSort`]: reth_db::table::DupSort + /// [`DupSort::SubKey`]: reth_db::table::DupSort::SubKey + pub block_number: u64, + /// The actual value (may be deleted) + pub value: MaybeDeleted, +} + +impl VersionedValue { + /// Create a new versioned value + pub const fn new(block_number: u64, value: MaybeDeleted) -> Self { + Self { + block_number, + value, + } + } +} + +impl Compress for VersionedValue { + type Compressed = Vec; + + fn compress_to_buf>(&self, buf: &mut B) { + // Encode block number first (8 bytes, big-endian) + buf.put_u64(self.block_number); + // Then encode the value + self.value.compress_to_buf(buf); + } +} + +impl Decompress for VersionedValue { + fn decompress(value: &[u8]) -> Result { + if value.len() < 8 { + return Err(DecompressError::new(DatabaseError::Decode)); + } + + let mut buf: &[u8] = value; + let block_number = buf.get_u64(); + let value = MaybeDeleted::::decompress(&value[8..])?; + + Ok(Self { + block_number, + value, + }) + } +} + +impl ValueWithSubKey for VersionedValue { + type SubKey = u64; + + fn get_subkey(&self) -> Self::SubKey { + self.block_number + } +} + +#[cfg(test)] +mod tests { + use reth_primitives_traits::Account; + use reth_trie::BranchNodeCompact; + + use super::*; + + #[test] + fn test_maybe_deleted_none() { + let none: MaybeDeleted = MaybeDeleted(None); + let compressed = none.compress(); + assert!(compressed.is_empty(), "None should compress to empty bytes"); + + let decompressed = MaybeDeleted::::decompress(&compressed).unwrap(); + assert_eq!(decompressed.0, None); + } + + #[test] + fn test_maybe_deleted_some_account() { + let account = Account { + nonce: 42, + balance: alloy_primitives::U256::from(1000u64), + bytecode_hash: None, + }; + let some = MaybeDeleted(Some(account)); + let compressed = some.compress(); + assert!( + !compressed.is_empty(), + "Some should compress to non-empty bytes" + ); + + let decompressed = MaybeDeleted::::decompress(&compressed).unwrap(); + assert_eq!(decompressed.0, Some(account)); + } + + #[test] + fn test_maybe_deleted_some_branch() { + // Create a simple valid BranchNodeCompact (empty is valid) + let branch = BranchNodeCompact::new( + 0, // state_mask + 0, // tree_mask + 0, // hash_mask + vec![], // hashes + None, // root_hash + ); + let some = MaybeDeleted(Some(branch.clone())); + let compressed = some.compress(); + assert!( + !compressed.is_empty(), + "Some should compress to non-empty bytes" + ); + + let decompressed = MaybeDeleted::::decompress(&compressed).unwrap(); + assert_eq!(decompressed.0, Some(branch)); + } + + #[test] + fn test_maybe_deleted_roundtrip() { + let test_cases = vec![ + MaybeDeleted(None), + MaybeDeleted(Some(Account { + nonce: 0, + balance: alloy_primitives::U256::ZERO, + bytecode_hash: None, + })), + MaybeDeleted(Some(Account { + nonce: 999, + balance: alloy_primitives::U256::MAX, + bytecode_hash: Some([0xff; 32].into()), + })), + ]; + + for original in test_cases { + let compressed = original.clone().compress(); + let decompressed = MaybeDeleted::::decompress(&compressed).unwrap(); + assert_eq!(original, decompressed); + } + } +} diff --git a/crates/proofs/src/db/store.rs b/crates/proofs/src/db/store.rs new file mode 100644 index 00000000..0f7064d5 --- /dev/null +++ b/crates/proofs/src/db/store.rs @@ -0,0 +1,4707 @@ +use std::{collections::BTreeMap, ops::RangeBounds, path::Path, time::Duration}; + +use alloy_eips::{BlockNumHash, NumHash, eip1898::BlockWithParent}; +use alloy_primitives::{B256, U256, map::HashMap}; +use reth_db::{ + Database, DatabaseEnv, DatabaseError, + cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO, DbDupCursorRW}, + mdbx::{DatabaseArguments, MaxReadTransactionDuration, init_db_for}, + table::{DupSort, Table}, + transaction::{DbTx, DbTxMut}, +}; +use reth_primitives_traits::Account; +use reth_trie::{hashed_cursor::HashedCursor, trie_cursor::TrieCursor}; +use reth_trie_common::{ + BranchNodeCompact, HashedPostState, Nibbles, StoredNibbles, + updates::{StorageTrieUpdates, TrieUpdates}, +}; + +use super::{ + BlockNumberHash, ProofMetadata, ProofMetadataKey, ProofMetadataValue, ProofWindow, + ProofWindowKey, Tables, +}; +use crate::{ + BlockStateDiff, MorphProofsStorageError, + MorphProofsStorageError::NoBlocksFound, + MorphProofsStorageResult, MorphProofsStore, + api::{ + InitialStateAnchor, InitialStateStatus, MorphProofsBatchStore, + MorphProofsInitialStateStore, WriteCounts, + }, + db::{ + MdbxAccountCursor, MdbxBatchSession, MdbxStorageCursor, MdbxTrieCursor, + cursor::Dup, + models::{ + AccountTrieHistory, BlockChangeSet, ChangeSet, HashedAccountHistory, + HashedStorageHistory, HashedStorageKey, IntoKV, MaybeDeleted, StorageTrieHistory, + StorageTrieKey, StorageValue, VersionedValue, + }, + }, +}; + +/// MDBX implementation of [`MorphProofsStore`]. +#[derive(Debug)] +pub struct MdbxProofsStorage { + env: DatabaseEnv, +} + +/// Options for opening [`MdbxProofsStorage`]. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct MdbxProofsStorageOptions { + /// Maximum duration of a read transaction. + pub max_read_transaction_duration: Option, +} + +/// Canonical-chain identity persisted in the proof database. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ProofDbIdentity { + /// EIP-155 chain identifier. + pub chain_id: u64, + /// Canonical genesis block hash. + pub genesis_hash: B256, +} + +impl ProofDbIdentity { + /// Creates a proof-database identity. + pub const fn new(chain_id: u64, genesis_hash: B256) -> Self { + Self { + chain_id, + genesis_hash, + } + } +} + +impl Default for ProofDbIdentity { + fn default() -> Self { + Self::new(0, B256::ZERO) + } +} + +/// Inclusive range currently served by the proof database. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ProofWindowBounds { + /// Oldest block whose proof data is retained. + pub earliest: (u64, B256), + /// Newest block durably indexed by proof history. + pub latest: (u64, B256), +} + +/// Current on-disk schema. There is intentionally no migration path before launch. +const SCHEMA_VERSION: u32 = 1; + +fn decode_u64_metadata( + value: ProofMetadataValue, + field: &'static str, +) -> MorphProofsStorageResult { + let bytes: [u8; 8] = value + .0 + .try_into() + .map_err(|_| MorphProofsStorageError::CorruptMetadata(field))?; + Ok(u64::from_be_bytes(bytes)) +} + +fn decode_u32_metadata( + value: ProofMetadataValue, + field: &'static str, +) -> MorphProofsStorageResult { + let bytes: [u8; 4] = value + .0 + .try_into() + .map_err(|_| MorphProofsStorageError::CorruptMetadata(field))?; + Ok(u32::from_be_bytes(bytes)) +} + +fn decode_hash_metadata( + value: ProofMetadataValue, + field: &'static str, +) -> MorphProofsStorageResult { + let bytes: [u8; 32] = value + .0 + .try_into() + .map_err(|_| MorphProofsStorageError::CorruptMetadata(field))?; + Ok(B256::new(bytes)) +} + +struct ProofWindowValue { + earliest: NumHash, + latest: NumHash, +} + +/// Preprocessed prune plan for a target block number +#[derive(Debug, Clone)] +struct PrunePlan { + earliest_block: u64, + acc_survivors: Vec<(StoredNibbles, u64)>, + storage_survivors: Vec<(StorageTrieKey, u64)>, + hashed_acc_survivors: Vec<(B256, u64)>, + hashed_storage_survivors: Vec<(HashedStorageKey, u64)>, +} + +/// Preprocessed delete work for a prune range +#[derive(Debug, Default, Clone)] +struct HistoryDeleteBatch { + account_trie: Vec<(::Key, u64)>, + storage_trie: Vec<(::Key, u64)>, + hashed_account: Vec<(::Key, u64)>, + hashed_storage: Vec<(::Key, u64)>, +} + +impl MdbxProofsStorage { + /// Creates a new [`MdbxProofsStorage`] instance with the given path. + pub fn new(path: &Path) -> Result { + Self::open(path, ProofDbIdentity::default()) + } + + /// Opens an MDBX proof database and validates its canonical-chain identity. + pub fn open(path: &Path, identity: ProofDbIdentity) -> Result { + Self::open_with_options(path, identity, MdbxProofsStorageOptions::default()) + } + + /// Creates a new [`MdbxProofsStorage`] instance with the given path and options. + pub fn new_with_options( + path: &Path, + options: MdbxProofsStorageOptions, + ) -> Result { + Self::open_with_options(path, ProofDbIdentity::default(), options) + } + + /// Opens an MDBX proof database with runtime options and validates chain identity. + pub fn open_with_options( + path: &Path, + identity: ProofDbIdentity, + options: MdbxProofsStorageOptions, + ) -> Result { + let mut args = DatabaseArguments::default(); + if let Some(duration) = options.max_read_transaction_duration { + args = args.with_max_read_transaction_duration(Some(MaxReadTransactionDuration::Set( + duration, + ))); + } + let env = init_db_for::<_, Tables>(path, args) + .map_err(|e| DatabaseError::Other(format!("Failed to open database: {e}")))?; + let storage = Self { env }; + storage.validate_or_initialize_identity(identity)?; + Ok(storage) + } + + fn validate_or_initialize_identity( + &self, + identity: ProofDbIdentity, + ) -> MorphProofsStorageResult<()> { + // Use one explicit write transaction so the empty-database check and identity + // initialization are atomic. Dropping the transaction on any error aborts it instead of + // committing a partially-written identity. + let tx = self.env.tx_mut()?; + let stored_chain_id = tx.get::(ProofMetadataKey::ChainId)?; + let stored_genesis_hash = tx.get::(ProofMetadataKey::GenesisHash)?; + let stored_schema_version = tx.get::(ProofMetadataKey::SchemaVersion)?; + let metadata_entries = tx.entries::()?; + + match (stored_chain_id, stored_genesis_hash, stored_schema_version) { + (None, None, None) if metadata_entries == 0 => { + if !Self::proof_tables_are_empty(&tx)? { + return Err(MorphProofsStorageError::ProofDataWithoutIdentityMetadata); + } + tx.put::( + ProofMetadataKey::ChainId, + ProofMetadataValue(identity.chain_id.to_be_bytes().to_vec()), + )?; + tx.put::( + ProofMetadataKey::GenesisHash, + ProofMetadataValue(identity.genesis_hash.as_slice().to_vec()), + )?; + tx.put::( + ProofMetadataKey::SchemaVersion, + ProofMetadataValue(SCHEMA_VERSION.to_be_bytes().to_vec()), + )?; + } + (Some(chain_id), Some(genesis_hash), Some(schema_version)) if metadata_entries == 3 => { + let chain_id = decode_u64_metadata(chain_id, "chain_id")?; + if chain_id != identity.chain_id { + return Err(MorphProofsStorageError::ChainIdentityMismatch("chain_id")); + } + + let genesis_hash = decode_hash_metadata(genesis_hash, "genesis_hash")?; + if genesis_hash != identity.genesis_hash { + return Err(MorphProofsStorageError::ChainIdentityMismatch( + "genesis_hash", + )); + } + + let schema_version = decode_u32_metadata(schema_version, "schema_version")?; + if schema_version != SCHEMA_VERSION { + return Err(MorphProofsStorageError::SchemaMismatch { + expected: SCHEMA_VERSION, + actual: schema_version, + }); + } + } + _ => return Err(MorphProofsStorageError::IncompleteIdentityMetadata), + } + + tx.commit()?; + Ok(()) + } + + fn proof_tables_are_empty(tx: &impl DbTx) -> Result { + Ok(tx.entries::()? == 0 + && tx.entries::()? == 0 + && tx.entries::()? == 0 + && tx.entries::()? == 0 + && tx.entries::()? == 0 + && tx.entries::()? == 0) + } + + /// Returns the inclusive range currently served by proof history. + pub fn proof_window(&self) -> MorphProofsStorageResult> { + self.env.view(|tx| { + Ok(self + .inner_get_proof_window(tx)? + .map(|window| ProofWindowBounds { + earliest: (window.earliest.number, window.earliest.hash), + latest: (window.latest.number, window.latest.hash), + })) + })? + } + + pub(crate) fn inner_get_latest_block_number_hash( + &self, + tx: &impl DbTx, + ) -> MorphProofsStorageResult> { + let block = self.inner_get_block_number_hash(tx, ProofWindowKey::LatestBlock)?; + if block.is_some() { + return Ok(block); + } + + self.inner_get_block_number_hash(tx, ProofWindowKey::EarliestBlock) + } + + pub(crate) fn inner_get_earliest_block_number_hash( + &self, + tx: &impl DbTx, + ) -> MorphProofsStorageResult> { + self.inner_get_block_number_hash(tx, ProofWindowKey::EarliestBlock) + } + + fn inner_get_block_number_hash( + &self, + tx: &impl DbTx, + key: ProofWindowKey, + ) -> MorphProofsStorageResult> { + let mut cursor = tx.cursor_read::()?; + let value = cursor.seek_exact(key)?; + Ok(value.map(|(_, val)| (val.number(), *val.hash()))) + } + + fn inner_get_proof_window( + &self, + tx: &impl DbTx, + ) -> MorphProofsStorageResult> { + let mut cursor = tx.cursor_read::()?; + + let earliest = match cursor.seek_exact(ProofWindowKey::EarliestBlock)? { + Some((_, val)) => NumHash::new(val.number(), *val.hash()), + None => return Ok(None), + }; + + let latest = match cursor.seek_exact(ProofWindowKey::LatestBlock)? { + Some((_, val)) => NumHash::new(val.number(), *val.hash()), + None => earliest, + }; + + Ok(Some(ProofWindowValue { earliest, latest })) + } + + fn set_earliest_block_number_hash( + &self, + block_number: u64, + hash: B256, + ) -> MorphProofsStorageResult<()> { + let _ = self.env.update(|tx| { + Self::inner_set_earliest_block_number(tx, block_number, hash)?; + Ok::<(), DatabaseError>(()) + })?; + Ok(()) + } + + /// Internal helper to set earliest block number hash within an existing transaction + fn inner_set_earliest_block_number( + tx: &(impl DbTxMut + DbTx), + block_number: u64, + hash: B256, + ) -> MorphProofsStorageResult<()> { + let mut cursor = tx.cursor_write::()?; + cursor.upsert( + ProofWindowKey::EarliestBlock, + &BlockNumberHash::new(block_number, hash), + )?; + Ok(()) + } + + /// Internal helper to set latest block number hash within an existing transaction + fn inner_set_latest_block_number( + tx: &(impl DbTxMut + DbTx), + block_number: u64, + hash: B256, + ) -> MorphProofsStorageResult<()> { + let mut cursor = tx.cursor_write::()?; + cursor.upsert( + ProofWindowKey::LatestBlock, + &BlockNumberHash::new(block_number, hash), + )?; + Ok(()) + } + + /// Persist a batch of versioned history entries to a dup-sorted table. + /// + /// # Parameters + /// - `block_number`: Target block number for versioning entries + /// - `items`: **Must be sorted** - iterator of entries to persist + /// - `append_mode`: Mode selector for write strategy: + /// - `true` (Append): Appends all entries including tombstones for forward progress + /// - `false` (Prune): Removes tombstones, writes non-tombstones to block 0 + /// + /// The cost of pruning is the cost of (append + deleting tombstones + deleting old block 0). + /// The tombstones deletion is expensive as it requires a seek for each (key + subkey). + /// + /// Uses [`reth_db::mdbx::cursor::Cursor::upsert`] for upsert operation. + fn persist_history_batch( + &self, + tx: &(impl DbTxMut + DbTx), + block_number: T::SubKey, + items: I, + append_mode: bool, + ) -> MorphProofsStorageResult> + where + T: Table> + DupSort, + T::Key: Clone, + I: IntoIterator, + I::Item: IntoKV, + { + let mut cur = tx.cursor_dup_write::()?; + let mut keys = Vec::::new(); + + // Materialize iterator to enable partitioning and collect keys + let mut pairs: Vec<(T::Key, T::Value)> = Vec::new(); + for it in items { + let (k, vv) = it.into_kv(block_number); + pairs.push((k.clone(), vv)); + keys.push(k) + } + + if append_mode { + // Append all entries (including tombstones) to preserve full history + for (k, vv) in pairs { + cur.append_dup(k.clone(), vv)?; + } + return Ok(keys); + } + + // Drop current cursor to start clean for Phase 1 + drop(cur); + + // Phase 1: Batch Delete (Sequential) + // Remove all existing state at Block 0 for these keys. + { + let mut del_cur = tx.cursor_dup_write::()?; + for (k, _) in &pairs { + // Seek to (Key, Block 0) + if let Some(vv) = del_cur.seek_by_key_subkey(k.clone(), 0)? + && vv.block_number == 0 + { + del_cur.delete_current()?; + } + } + } + + // Phase 2: Batch Write (Sequential) + // Write new values (skipping tombstones). + { + let mut write_cur = tx.cursor_dup_write::()?; + for (k, vv) in pairs { + if vv.value.0.is_some() { + write_cur.upsert(k, &vv)?; + } + } + } + + Ok(keys) + } + + /// Delete entries for `items` at exactly `block_number` in a dup-sorted table. + /// Seeks (key, block) and deletes current if the subkey matches. + fn delete_dup_sorted( + &self, + tx: &(impl DbTxMut + DbTx), + items: I, + ) -> MorphProofsStorageResult<()> + where + T: Table> + DupSort, + T::Key: Clone, + T::SubKey: PartialEq + Clone, + I: IntoIterator, + { + let mut cur = tx.cursor_dup_write::()?; + for (key, subkey) in items { + if let Some(vv) = cur.seek_by_key_subkey(key, subkey)? { + // ensure we didn't land on a >subkey + if vv.block_number == subkey { + cur.delete_current()?; + } + } + } + Ok(()) + } + + /// Phase 1 of pruning: Calculate survivors. + /// Scans change sets to find the LATEST update for every key in the range. + fn calculate_prune_plan( + &self, + target_block: u64, + ) -> MorphProofsStorageResult> { + self.env.view(|tx| { + let Some((earliest, _)) = + self.inner_get_block_number_hash(tx, ProofWindowKey::EarliestBlock)? + else { + return Ok(None); + }; + + if earliest >= target_block { + return Ok(None); + } + + // Walk the change set in REVERSE so the first time we see a key, its block_number + // is by definition the latest version in [earliest+1, target_block] — the only + // one that must survive. This lets us use `or_insert` instead of an + // `and_modify(max).or_insert` per key, halving the per-insert work. + // + // HashMaps are pre-sized to bound rehash overhead. The chosen capacity is a rough + // average of unique keys per block on Base mainnet; the maps grow if exceeded. + const PER_BLOCK_CAPACITY_HINT: usize = 4096; + let blocks_in_range = target_block + .saturating_sub(earliest) + .try_into() + .unwrap_or(usize::MAX); + let cap = blocks_in_range + .saturating_mul(PER_BLOCK_CAPACITY_HINT) + .min(1 << 20); + + let mut acc_candidates: HashMap = + HashMap::with_capacity_and_hasher(cap, Default::default()); + let mut storage_candidates: HashMap = + HashMap::with_capacity_and_hasher(cap, Default::default()); + let mut hashed_acc_candidates: HashMap = + HashMap::with_capacity_and_hasher(cap, Default::default()); + let mut hashed_storage_candidates: HashMap = + HashMap::with_capacity_and_hasher(cap, Default::default()); + + let mut cs_cursor = tx.cursor_read::()?; + let mut walker = cs_cursor.walk_back(Some(target_block))?; + while let Some(Ok((block_number, cs))) = walker.next() { + if block_number <= earliest { + break; + } + for k in cs.account_trie_keys { + acc_candidates.entry(k).or_insert(block_number); + } + for k in cs.storage_trie_keys { + storage_candidates.entry(k).or_insert(block_number); + } + for k in cs.hashed_account_keys { + hashed_acc_candidates.entry(k).or_insert(block_number); + } + for k in cs.hashed_storage_keys { + hashed_storage_candidates.entry(k).or_insert(block_number); + } + } + + Ok(Some(PrunePlan { + earliest_block: earliest, + acc_survivors: Self::flatten_and_sort(acc_candidates), + storage_survivors: Self::flatten_and_sort(storage_candidates), + hashed_acc_survivors: Self::flatten_and_sort(hashed_acc_candidates), + hashed_storage_survivors: Self::flatten_and_sort(hashed_storage_candidates), + })) + })? + } + + /// Helper to flatten `HashMap` into a sorted Vector of survivors. + /// Sorting is required to ensure optimal sequential seek performance in MDBX. + fn flatten_and_sort(map: HashMap) -> Vec<(K, u64)> { + let mut v: Vec<_> = map.into_iter().collect(); + v.sort_unstable_by(|a, b| a.0.cmp(&b.0)); + v + } + + /// Delete history versions for `items` that are strictly older than the provided block number. + /// `items` is a list of (Key, `SurvivorBlock`). Everything strictly older than `SurvivorBlock` + /// is deleted. Returns the number of entries deleted. + fn prune_history_preceding( + &self, + tx: &(impl DbTxMut + DbTx), + cutoff_items: Vec<(T::Key, u64)>, + ) -> MorphProofsStorageResult + where + T: Table> + DupSort, + T::Key: Clone + Ord, + { + if cutoff_items.is_empty() { + return Ok(0); + } + + let mut deleted_count = 0; + let mut cur = tx.cursor_dup_write::()?; + for (key, survivor_block) in cutoff_items { + let Some(mut entry) = cur.seek_by_key_subkey(key, 0)? else { + continue; + }; + loop { + if entry.block_number >= survivor_block { + // Reached the survivor version (or newer). Stop deleting for this key. + // If the survivor is a tombstone (None), delete it too — with all older + // history already gone, a tombstone at the new earliest is redundant + // (it implies "does not exist"). + if entry.block_number == survivor_block && entry.value.0.is_none() { + cur.delete_current()?; + deleted_count += 1; + } + break; + } + + // Entry is strictly older than survivor. Delete it. + cur.delete_current()?; + deleted_count += 1; + + // libmdbx semantics: after `delete_current()` the cursor sits directly + // before the just-deleted slot, so `next_dup_val` (MDBX_NEXT_DUP) yields the + // following dup of the same key, or `None` once this key's dup-group is + // exhausted. That bound removes the need for an explicit key comparison. + match cur.next_dup_val()? { + Some(next_entry) => entry = next_entry, + None => break, + } + } + } + Ok(deleted_count) + } + + /// Tombstone every key returned by `next`, then overlay `new_entries` so collisions + /// resolve in favor of `new_entries`, and `append_dup` the merged set in sorted-key order at + /// `block_number`. Used by the wipe branches of `store_trie_updates_for_block` to keep new + /// slots / nodes that arrive in the same block as a `wiped`/`is_deleted` entry — + /// `HashedStorage::from_plain_storage` sets `wiped = was_destroyed()` (true for + /// `DestroyedChanged`), so a SELFDESTRUCT + same-block recreate produces both at once. + fn wipe_and_overlay( + &self, + tx: &(impl DbTxMut + DbTx), + block_number: u64, + hashed_address: B256, + mut next: Next, + new_entries: I, + ) -> MorphProofsStorageResult> + where + T: Table> + DupSort, + Next: FnMut() -> MorphProofsStorageResult>, + I: IntoIterator)>, + (B256, K, Option): IntoKV, + T::Key: Clone, + K: Ord, + { + let mut merged: BTreeMap> = BTreeMap::new(); + while let Some((k, _vv)) = next()? { + merged.insert(k, None); + } + for (k, v) in new_entries { + merged.insert(k, v); + } + + let mut cur = tx.cursor_dup_write::()?; + let mut keys: Vec = Vec::with_capacity(merged.len()); + for (k, value) in merged { + let key: T::Key = (hashed_address, k, Option::::None).into_key(); + let vv: T::Value = VersionedValue { + block_number, + value: MaybeDeleted(value), + }; + cur.append_dup(key.clone(), vv)?; + keys.push(key); + } + Ok(keys) + } + + /// Collect versioned history over `block_range` using `BlockChangeSet`. + fn collect_history_ranged( + &self, + tx: &impl DbTx, + block_range: impl RangeBounds, + ) -> MorphProofsStorageResult { + let mut history = HistoryDeleteBatch::default(); + let mut change_set_cursor = tx.cursor_read::()?; + let mut walker = change_set_cursor.walk_range(block_range)?; + + while let Some(Ok((block_number, change_set))) = walker.next() { + // Push (key, subkey=block_number) pairs + history.account_trie.extend( + change_set + .account_trie_keys + .into_iter() + .map(|k| (k, block_number)), + ); + history.storage_trie.extend( + change_set + .storage_trie_keys + .into_iter() + .map(|k| (k, block_number)), + ); + history.hashed_account.extend( + change_set + .hashed_account_keys + .into_iter() + .map(|k| (k, block_number)), + ); + history.hashed_storage.extend( + change_set + .hashed_storage_keys + .into_iter() + .map(|k| (k, block_number)), + ); + } + + // Sorting by tuple sorts by key first, then by block_number. + history + .account_trie + .sort_by(|(k1, b1), (k2, b2)| k1.cmp(k2).then_with(|| b1.cmp(b2))); + history + .storage_trie + .sort_by(|(k1, b1), (k2, b2)| k1.cmp(k2).then_with(|| b1.cmp(b2))); + history + .hashed_account + .sort_by(|(k1, b1), (k2, b2)| k1.cmp(k2).then_with(|| b1.cmp(b2))); + history + .hashed_storage + .sort_by(|(k1, b1), (k2, b2)| k1.cmp(k2).then_with(|| b1.cmp(b2))); + + Ok(history) + } + + /// Delete versioned history over `block_range` using history batch. + fn delete_history_ranged( + &self, + tx: &(impl DbTxMut + DbTx), + block_range: impl RangeBounds, + history: HistoryDeleteBatch, + ) -> MorphProofsStorageResult { + let mut change_set_cursor = tx.cursor_write::()?; + let mut walker = change_set_cursor.walk_range(block_range)?; + + while let Some(Ok((_, _))) = walker.next() { + walker.delete_current()?; + } + + // Delete using the simplified API: iterator of (key, subkey) + self.delete_dup_sorted::(tx, history.clone().account_trie)?; + self.delete_dup_sorted::(tx, history.clone().storage_trie)?; + self.delete_dup_sorted::(tx, history.clone().hashed_account)?; + self.delete_dup_sorted::(tx, history.clone().hashed_storage)?; + + Ok(WriteCounts { + account_trie_updates_written_total: history.account_trie.len() as u64, + storage_trie_updates_written_total: history.storage_trie.len() as u64, + hashed_accounts_written_total: history.hashed_account.len() as u64, + hashed_storages_written_total: history.hashed_storage.len() as u64, + }) + } + + /// Write trie/state history for `block_number` from `block_state_diff`. + fn store_trie_updates_for_block( + &self, + tx: &::TXMut, + block_number: u64, + block_state_diff: BlockStateDiff, + append_mode: bool, + ) -> MorphProofsStorageResult { + let BlockStateDiff { + sorted_trie_updates, + sorted_post_state, + } = block_state_diff; + + let storage_trie_len = sorted_trie_updates.storage_tries_ref().len(); + let hashed_storage_len = sorted_post_state.storages.len(); + + let account_trie_keys = self.persist_history_batch( + tx, + block_number, + sorted_trie_updates.account_nodes_ref().iter().cloned(), + append_mode, + )?; + let hashed_account_keys = self.persist_history_batch( + tx, + block_number, + sorted_post_state.accounts.iter().copied(), + append_mode, + )?; + + let mut storage_trie_keys = Vec::::with_capacity(storage_trie_len); + for (hashed_address, nodes) in sorted_trie_updates.storage_tries_ref() { + if nodes.is_deleted && append_mode { + // Lookback must use the active tx so it sees uncommitted writes from earlier + // blocks in the same batch session. Opening a fresh RO tx here would read the + // pre-batch committed snapshot and emit wrong tombstones for wipe blocks whose + // parent is staged in the same batch. + let mut ro = MdbxTrieCursor::::new( + tx.cursor_dup_read::()?, + block_number - 1, + Some(*hashed_address), + ); + let keys = self.wipe_and_overlay( + tx, + block_number, + *hashed_address, + || Ok(ro.next()?), + nodes.storage_nodes_ref().iter().cloned(), + )?; + storage_trie_keys.extend(keys); + continue; + } + + let keys = self.persist_history_batch( + tx, + block_number, + nodes + .storage_nodes_ref() + .iter() + .cloned() + .map(|(path, node)| (*hashed_address, path, node)), + append_mode, + )?; + storage_trie_keys.extend(keys); + } + + let mut hashed_storage_keys = Vec::::with_capacity(hashed_storage_len); + for (hashed_address, storage) in sorted_post_state.storages { + if append_mode && storage.is_wiped() { + // See lookback note above: must use the active tx, not a fresh RO tx. + let mut ro = MdbxStorageCursor::new( + tx.cursor_dup_read::()?, + block_number - 1, + hashed_address, + ); + let keys = self.wipe_and_overlay( + tx, + block_number, + hashed_address, + || Ok(ro.next()?), + storage + .storage_slots_ref() + .iter() + .map(|(slot, val)| (*slot, Some(StorageValue(*val)))), + )?; + hashed_storage_keys.extend(keys); + continue; + } + let keys = self.persist_history_batch( + tx, + block_number, + storage + .storage_slots_ref() + .iter() + .map(|(key, val)| (hashed_address, *key, Some(StorageValue(*val)))), + append_mode, + )?; + hashed_storage_keys.extend(keys); + } + + Ok(ChangeSet { + account_trie_keys, + storage_trie_keys, + hashed_account_keys, + hashed_storage_keys, + }) + } + + /// Append-only writer for a block: validates parent, persists diff (soft-delete=true), + /// records a `BlockChangeSet`, and advances `ProofWindow::LatestBlock`. + pub(crate) fn store_trie_updates_append_only( + &self, + tx: &::TXMut, + block_ref: BlockWithParent, + block_state_diff: BlockStateDiff, + ) -> MorphProofsStorageResult { + let block_number = block_ref.block.number; + + // Check the latest stored block is the parent of the incoming block + let latest_block_hash = self + .inner_get_latest_block_number_hash(tx)? + .map_or(B256::ZERO, |(_num, hash)| hash); + + if latest_block_hash != block_ref.parent { + return Err(MorphProofsStorageError::OutOfOrder { + block_number, + parent_block_hash: block_ref.parent, + latest_block_hash, + }); + } + + let change_set = + &self.store_trie_updates_for_block(tx, block_number, block_state_diff, true)?; + + // Cursor for recording all changes made in this block for all history tables + let mut change_set_cursor = tx.new_cursor::()?; + change_set_cursor.append(block_number, change_set)?; + + // Update proof window's latest block + Self::inner_set_latest_block_number(tx, block_number, block_ref.block.hash)?; + + Ok(WriteCounts { + account_trie_updates_written_total: change_set.account_trie_keys.len() as u64, + storage_trie_updates_written_total: change_set.storage_trie_keys.len() as u64, + hashed_accounts_written_total: change_set.hashed_account_keys.len() as u64, + hashed_storages_written_total: change_set.hashed_storage_keys.len() as u64, + }) + } + + /// Return `BlockNumHash` for the initial state anchor. + fn get_initial_state_anchor(&self) -> MorphProofsStorageResult> { + self.env.view(|tx| { + let mut cur = tx.cursor_read::()?; + Ok(cur + .seek_exact(ProofWindowKey::InitialStateAnchor)? + .map(|(_k, v)| v.into())) + })? + } + + /// Return latest key for a table + fn get_latest_key(&self) -> MorphProofsStorageResult> + where + T: Table, + { + self.env.view(|tx| { + let mut cursor = tx.cursor_read::()?; + Ok(cursor.last()?.map(|(k, _)| k)) + })? + } +} + +impl MorphProofsStore for MdbxProofsStorage { + type StorageTrieCursor<'tx> + = MdbxTrieCursor> + where + Self: 'tx; + type AccountTrieCursor<'tx> + = MdbxTrieCursor> + where + Self: 'tx; + type StorageCursor<'tx> + = MdbxStorageCursor> + where + Self: 'tx; + type AccountHashedCursor<'tx> + = MdbxAccountCursor> + where + Self: 'tx; + type Tx<'tx> + = ::TX + where + Self: 'tx; + + fn ro_tx<'tx>(&'tx self) -> MorphProofsStorageResult> { + Ok(self.env.tx()?) + } + + fn get_earliest_block_number(&self) -> MorphProofsStorageResult> { + self.env + .view(|tx| self.inner_get_earliest_block_number_hash(tx))? + } + + fn get_latest_block_number(&self) -> MorphProofsStorageResult> { + self.env + .view(|tx| self.inner_get_latest_block_number_hash(tx))? + } + + fn get_earliest_block_number_with_tx<'tx, 'db>( + &self, + tx: &'tx Self::Tx<'db>, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx, + { + self.inner_get_earliest_block_number_hash(tx) + } + + fn get_latest_block_number_with_tx<'tx, 'db>( + &self, + tx: &'tx Self::Tx<'db>, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx, + { + self.inner_get_latest_block_number_hash(tx) + } + + fn storage_trie_cursor<'tx>( + &'tx self, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + let tx = self.env.tx()?; + let cursor = tx.cursor_dup_read::()?; + + Ok(MdbxTrieCursor::new( + cursor, + max_block_number, + Some(hashed_address), + )) + } + + fn account_trie_cursor<'tx>( + &'tx self, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + let tx = self.env.tx()?; + let cursor = tx.cursor_dup_read::()?; + + Ok(MdbxTrieCursor::new(cursor, max_block_number, None)) + } + + fn storage_hashed_cursor<'tx>( + &'tx self, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + let tx = self.env.tx()?; + let cursor = tx.cursor_dup_read::()?; + + Ok(MdbxStorageCursor::new( + cursor, + max_block_number, + hashed_address, + )) + } + + fn account_hashed_cursor<'tx>( + &'tx self, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + let tx = self.env.tx()?; + let cursor = tx.cursor_dup_read::()?; + + Ok(MdbxAccountCursor::new(cursor, max_block_number)) + } + + fn storage_trie_cursor_with_tx<'tx, 'db>( + &self, + tx: &'tx Self::Tx<'db>, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx, + { + let cursor = tx.cursor_dup_read::()?; + + Ok(MdbxTrieCursor::new( + cursor, + max_block_number, + Some(hashed_address), + )) + } + + fn account_trie_cursor_with_tx<'tx, 'db>( + &self, + tx: &'tx Self::Tx<'db>, + max_block_number: u64, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx, + { + let cursor = tx.cursor_dup_read::()?; + + Ok(MdbxTrieCursor::new(cursor, max_block_number, None)) + } + + fn storage_hashed_cursor_with_tx<'tx, 'db>( + &self, + tx: &'tx Self::Tx<'db>, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx, + { + let cursor = tx.cursor_dup_read::()?; + + Ok(MdbxStorageCursor::new( + cursor, + max_block_number, + hashed_address, + )) + } + + fn account_hashed_cursor_with_tx<'tx, 'db>( + &self, + tx: &'tx Self::Tx<'db>, + max_block_number: u64, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx, + { + let cursor = tx.cursor_dup_read::()?; + + Ok(MdbxAccountCursor::new(cursor, max_block_number)) + } + + fn store_trie_updates( + &self, + block_ref: BlockWithParent, + block_state_diff: BlockStateDiff, + ) -> MorphProofsStorageResult { + self.env + .update(|tx| self.store_trie_updates_append_only(tx, block_ref, block_state_diff))? + } + + fn fetch_trie_updates(&self, block_number: u64) -> MorphProofsStorageResult { + self.env.view(|tx| { + let mut change_set_cursor = tx.cursor_read::()?; + let (_, change_set) = change_set_cursor + .seek_exact(block_number)? + .ok_or(MorphProofsStorageError::NoChangeSetForBlock(block_number))?; + + let mut account_trie_cursor = tx.new_cursor::()?; + let mut storage_trie_cursor = tx.new_cursor::()?; + let mut hashed_account_cursor = tx.new_cursor::()?; + let mut hashed_storage_cursor = tx.new_cursor::()?; + + let mut trie_updates = TrieUpdates::default(); + for key in change_set.account_trie_keys { + let entry = + match account_trie_cursor.seek_by_key_subkey(key.clone(), block_number)? { + Some(v) if v.block_number == block_number => v.value.0, + _ => { + return Err(MorphProofsStorageError::MissingAccountTrieHistory( + key.0, + block_number, + )); + } + }; + + if let Some(value) = entry { + trie_updates.account_nodes.insert(key.0, value); + } else { + trie_updates.removed_nodes.insert(key.0); + } + } + + for key in change_set.storage_trie_keys { + let entry = + match storage_trie_cursor.seek_by_key_subkey(key.clone(), block_number)? { + Some(v) if v.block_number == block_number => v.value.0, + _ => { + return Err(MorphProofsStorageError::MissingStorageTrieHistory( + key.hashed_address, + key.path.0, + block_number, + )); + } + }; + + let stu = trie_updates + .storage_tries + .entry(key.hashed_address) + .or_insert_with(StorageTrieUpdates::default); + + // handle is_deleted scenario + // Issue: https://github.com/op-rs/op-reth/issues/323 + if let Some(value) = entry { + stu.storage_nodes.insert(key.path.0, value); + } else { + stu.removed_nodes.insert(key.path.0); + } + } + + let mut post_state = + HashedPostState::with_capacity(change_set.hashed_account_keys.len()); + for key in change_set.hashed_account_keys { + let entry = match hashed_account_cursor.seek_by_key_subkey(key, block_number)? { + Some(v) if v.block_number == block_number => v.value.0, + _ => { + return Err(MorphProofsStorageError::MissingHashedAccountHistory( + key, + block_number, + )); + } + }; + + post_state.accounts.insert(key, entry); + } + + for key in change_set.hashed_storage_keys { + let entry = + match hashed_storage_cursor.seek_by_key_subkey(key.clone(), block_number)? { + Some(v) if v.block_number == block_number => v.value.0, + _ => { + return Err(MorphProofsStorageError::MissingHashedStorageHistory { + hashed_address: key.hashed_address, + hashed_storage_key: key.hashed_storage_key, + block_number, + }); + } + }; + + let hs = post_state.storages.entry(key.hashed_address).or_default(); + + // handle wiped storage scenario + // Issue: https://github.com/op-rs/op-reth/issues/323 + if let Some(value) = entry { + hs.storage.insert(key.hashed_storage_key, value.0); + } else { + hs.storage.insert(key.hashed_storage_key, U256::ZERO); + } + } + + Ok(BlockStateDiff { + sorted_trie_updates: trie_updates.into_sorted(), + sorted_post_state: post_state.into_sorted(), + }) + })? + } + + /// Update the initial state with the provided diff. + /// Prune all historical trie data till `new_earliest_block_number` (inclusive) using + /// the [`BlockChangeSet`] index. + /// + /// Arguments: + /// - `new_earliest_block_ref`: The new earliest block reference (with parent hash). + /// - `diff`: The state diff to apply to the initial state (block 0). This diff represents all + /// the changes from the old earliest block to the new earliest block (inclusive). + fn prune_earliest_state( + &self, + new_earliest_block_ref: BlockWithParent, + ) -> MorphProofsStorageResult { + let target_block = new_earliest_block_ref.block.number; + + // --- PHASE 1: READ (Calculate Deletions) --- + let plan = self.calculate_prune_plan(target_block)?; + let Some(plan) = plan else { + return Ok(WriteCounts::default()); + }; + + // --- PHASE 2: WRITE (Execute Deletions) --- + self.env.update(|tx| { + // 1. Execute Sparse Deletions and track actual deleted rows + let acc_deleted = + self.prune_history_preceding::(tx, plan.acc_survivors)?; + + let st_deleted = + self.prune_history_preceding::(tx, plan.storage_survivors)?; + + let ha_deleted = self.prune_history_preceding::( + tx, + plan.hashed_acc_survivors, + )?; + + let hs_deleted = self.prune_history_preceding::( + tx, + plan.hashed_storage_survivors, + )?; + + let counts = WriteCounts { + account_trie_updates_written_total: acc_deleted, + storage_trie_updates_written_total: st_deleted, + hashed_accounts_written_total: ha_deleted, + hashed_storages_written_total: hs_deleted, + }; + + // 2. Delete ChangeSets + let range = (plan.earliest_block + 1)..=target_block; + let mut cs_cursor = tx.cursor_write::()?; + let mut walker = cs_cursor.walk_range(range)?; + while walker.next().is_some() { + walker.delete_current()?; + } + + // 3. Update Earliest Pointer + Self::inner_set_earliest_block_number( + tx, + target_block, + new_earliest_block_ref.block.hash, + )?; + + Ok(counts) + })? + } + + /// Unwind the historical state to `unwind_upto_block` (inclusive), deleting all history + /// starting from provided block. Also updates the `ProofWindow::LatestBlock` to parent of + /// `unwind_upto_block`. + fn unwind_history(&self, to: BlockWithParent) -> MorphProofsStorageResult<()> { + let history_to_delete = self + .env + .view(|tx| self.collect_history_ranged(tx, to.block.number..))??; + + self.env.update(|tx| { + let proof_window = match self.inner_get_proof_window(tx)? { + Some(pw) => pw, + None => return Ok(()), // Nothing to unwind + }; + + if to.block.number > proof_window.latest.number { + return Ok(()); // Nothing to unwind + } + + if to.block.number <= proof_window.earliest.number { + return Err(MorphProofsStorageError::UnwindBeyondEarliest { + unwind_block_number: to.block.number, + earliest_block_number: proof_window.earliest.number, + }); + } + + self.delete_history_ranged(tx, to.block.number.., history_to_delete)?; + + let new_latest_block = + BlockNumberHash::new(to.block.number.saturating_sub(1), to.parent); + + // Update proof window's Latest block + Self::inner_set_latest_block_number( + tx, + new_latest_block.number(), + *new_latest_block.hash(), + )?; + + Ok(()) + })? + } + + fn replace_updates( + &self, + latest_common_block: BlockNumHash, + mut blocks_to_add: Vec<(BlockWithParent, BlockStateDiff)>, + ) -> MorphProofsStorageResult<()> { + // Sort the vec list by block number + blocks_to_add.sort_unstable_by_key(|(bwp, _)| bwp.block.number); + + let history_to_delete = self + .env + .view(|tx| self.collect_history_ranged(tx, latest_common_block.number + 1..))??; + + self.env.update(|tx| { + // Remove the old history + self.delete_history_ranged(tx, latest_common_block.number + 1.., history_to_delete)?; + + // Update the ProofWindow Latest Block to latest_common_block so we can perform + // `store_trie_updates_append_only`. + Self::inner_set_latest_block_number( + tx, + latest_common_block.number, + latest_common_block.hash, + )?; + + // Apply the new history + for (block_with_parent, diff) in blocks_to_add { + self.store_trie_updates_append_only(tx, block_with_parent, diff)?; + } + Ok(()) + })? + } + + fn set_earliest_block_number( + &self, + block_number: u64, + hash: B256, + ) -> MorphProofsStorageResult<()> { + self.set_earliest_block_number_hash(block_number, hash) + } +} + +impl MorphProofsBatchStore for MdbxProofsStorage { + type BatchSession<'a> + = MdbxBatchSession<'a> + where + Self: 'a; + + fn with_batch_session(&self, f: F) -> MorphProofsStorageResult + where + F: FnOnce(&mut Self::BatchSession<'_>) -> MorphProofsStorageResult, + { + let tx = self.env.tx_mut()?; + let mut session = MdbxBatchSession::new(self, tx); + match f(&mut session) { + Ok(result) => { + session.commit()?; + Ok(result) + } + Err(err) => Err(err), + } + } +} + +impl MorphProofsInitialStateStore for MdbxProofsStorage { + fn initial_state_anchor(&self) -> MorphProofsStorageResult { + // 1) NotStarted: no anchor row + let Some(block) = self.get_initial_state_anchor()? else { + return Ok(InitialStateAnchor::default()); + }; + + // 2) Completed: anchor exists + earliest is set + let completed = self.get_earliest_block_number()?.is_some(); + + // 3) InProgress / Completed: populate details + Ok(InitialStateAnchor { + block: Some(block), + status: if completed { + InitialStateStatus::Completed + } else { + InitialStateStatus::InProgress + }, + latest_account_trie_key: self.get_latest_key::()?, + latest_storage_trie_key: self.get_latest_key::()?, + latest_hashed_account_key: self.get_latest_key::()?, + latest_hashed_storage_key: self.get_latest_key::()?, + }) + } + + fn set_initial_state_anchor(&self, anchor: BlockNumHash) -> MorphProofsStorageResult<()> { + self.env.update(|tx| { + let mut cur = tx.cursor_write::()?; + cur.insert(ProofWindowKey::InitialStateAnchor, &anchor.into())?; + Ok(()) + })? + } + + fn store_account_branches( + &self, + account_nodes: Vec<(Nibbles, Option)>, + ) -> MorphProofsStorageResult<()> { + let mut account_nodes = account_nodes; + if account_nodes.is_empty() { + return Ok(()); + } + + account_nodes.sort_by_key(|(key, _)| *key); + + self.env.update(|tx| { + self.persist_history_batch(tx, 0, account_nodes, true)?; + Ok(()) + })? + } + + fn store_storage_branches( + &self, + hashed_address: B256, + storage_nodes: Vec<(Nibbles, Option)>, + ) -> MorphProofsStorageResult<()> { + let mut storage_nodes = storage_nodes; + if storage_nodes.is_empty() { + return Ok(()); + } + + storage_nodes.sort_by_key(|(key, _)| *key); + + self.env.update(|tx| { + self.persist_history_batch( + tx, + 0, + storage_nodes + .into_iter() + .map(|(path, node)| (hashed_address, path, node)), + true, + )?; + Ok(()) + })? + } + + fn store_hashed_accounts( + &self, + accounts: Vec<(B256, Option)>, + ) -> MorphProofsStorageResult<()> { + let mut accounts = accounts; + if accounts.is_empty() { + return Ok(()); + } + + // sort the accounts by key to ensure insertion is efficient + accounts.sort_by_key(|(key, _)| *key); + + self.env.update(|tx| { + self.persist_history_batch(tx, 0, accounts, true)?; + Ok(()) + })? + } + + fn store_hashed_storages( + &self, + hashed_address: B256, + storages: Vec<(B256, U256)>, + ) -> MorphProofsStorageResult<()> { + let mut storages = storages; + if storages.is_empty() { + return Ok(()); + } + + // sort the storages by key to ensure insertion is efficient + storages.sort_by_key(|(key, _)| *key); + + self.env.update(|tx| { + self.persist_history_batch( + tx, + 0, + storages + .into_iter() + .map(|(key, val)| (hashed_address, key, Some(StorageValue(val)))), + true, + )?; + Ok(()) + })? + } + + fn commit_initial_state(&self) -> MorphProofsStorageResult { + let anchor = self.get_initial_state_anchor()?.ok_or(NoBlocksFound)?; + self.set_earliest_block_number(anchor.number, anchor.hash)?; + Ok(anchor) + } +} + +#[cfg(test)] +mod tests { + use alloy_eips::NumHash; + use alloy_primitives::B256; + use reth_db::{ + DatabaseError, + cursor::DbDupCursorRO, + transaction::{DbTx, DbTxMut}, + }; + use reth_trie::{ + BranchNodeCompact, HashedPostStateSorted, HashedStorage, Nibbles, StoredNibbles, + updates::{StorageTrieUpdates, TrieUpdatesSorted}, + }; + use tempfile::TempDir; + + use super::*; + use crate::db::{ + StorageTrieKey, + models::{AccountTrieHistory, StorageTrieHistory}, + }; + + const B0: u64 = 0; + + #[test] + fn open_rejects_partial_identity_metadata_and_requires_manual_deletion() { + let dir = TempDir::new().unwrap(); + let env = init_db_for::<_, Tables>(dir.path(), DatabaseArguments::default()) + .expect("raw proof database"); + env.update(|tx| { + tx.put::( + ProofMetadataKey::ChainId, + ProofMetadataValue(2818u64.to_be_bytes().to_vec()), + ) + }) + .expect("write transaction") + .expect("write partial metadata"); + drop(env); + + let error = MdbxProofsStorage::open( + dir.path(), + ProofDbIdentity::new(2818, B256::repeat_byte(0x11)), + ) + .expect_err("partial identity metadata must fail closed"); + + assert!( + error.to_string().contains("manually delete"), + "operator must receive an explicit manual-deletion instruction: {error}" + ); + } + + #[test] + fn open_rejects_proof_data_without_identity_metadata() { + let identity = ProofDbIdentity::new(2818, B256::repeat_byte(0x11)); + + let proof_window_dir = TempDir::new().unwrap(); + let env = init_db_for::<_, Tables>(proof_window_dir.path(), DatabaseArguments::default()) + .expect("raw proof database"); + env.update(|tx| { + tx.put::( + ProofWindowKey::EarliestBlock, + BlockNumberHash::new(1, B256::repeat_byte(0x22)), + ) + }) + .expect("write transaction") + .expect("write proof window"); + drop(env); + + let error = MdbxProofsStorage::open(proof_window_dir.path(), identity) + .expect_err("proof-window data without identity metadata must fail closed"); + assert!( + error.to_string().contains("manually delete"), + "operator must receive an explicit manual-deletion instruction: {error}" + ); + + let history_dir = TempDir::new().unwrap(); + let env = init_db_for::<_, Tables>(history_dir.path(), DatabaseArguments::default()) + .expect("raw proof database"); + env.update(|tx| tx.put::(1, ChangeSet::default())) + .expect("write transaction") + .expect("write history change set"); + drop(env); + + let error = MdbxProofsStorage::open(history_dir.path(), identity) + .expect_err("history data without identity metadata must fail closed"); + assert!( + error.to_string().contains("manually delete"), + "operator must receive an explicit manual-deletion instruction: {error}" + ); + } + + #[test] + fn new_with_options_accepts_custom_max_read_transaction_duration() { + let dir = TempDir::new().unwrap(); + let options = MdbxProofsStorageOptions { + max_read_transaction_duration: Some(Duration::from_secs(30)), + }; + let store = MdbxProofsStorage::new_with_options(dir.path(), options).expect("env"); + + let _tx = store.env.tx().expect("ro tx"); + } + + #[test] + fn store_hashed_accounts_writes_versioned_values() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr = B256::from([0xAA; 32]); + let account = Account::default(); + store + .store_hashed_accounts(vec![(addr, Some(account))]) + .expect("write accounts"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + let vv = cur.seek_by_key_subkey(addr, B0).expect("seek"); + let vv = vv.expect("entry exists"); + + assert_eq!(vv.block_number, B0); + assert_eq!(vv.value.0, Some(account)); + } + + #[test] + fn store_hashed_accounts_multiple_items_unsorted() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // Unsorted input, mixed Some/None (deletion) + let a1 = B256::from([0x01; 32]); + let a2 = B256::from([0x02; 32]); + let a3 = B256::from([0x03; 32]); + let acc1 = Account { + nonce: 2, + balance: U256::from(1000u64), + ..Default::default() + }; + let acc3 = Account { + nonce: 1, + balance: U256::from(10000u64), + ..Default::default() + }; + + store + .store_hashed_accounts(vec![(a2, None), (a1, Some(acc1)), (a3, Some(acc3))]) + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + let v1 = cur + .seek_by_key_subkey(a1, B0) + .expect("seek a1") + .expect("exists a1"); + assert_eq!(v1.block_number, B0); + assert_eq!(v1.value.0, Some(acc1)); + + let v2 = cur + .seek_by_key_subkey(a2, B0) + .expect("seek a2") + .expect("exists a2"); + assert_eq!(v2.block_number, B0); + assert!(v2.value.0.is_none(), "a2 is none"); + + let v3 = cur + .seek_by_key_subkey(a3, B0) + .expect("seek a3") + .expect("exists a3"); + assert_eq!(v3.block_number, B0); + assert_eq!(v3.value.0, Some(acc3)); + } + + #[test] + fn store_hashed_accounts_multiple_calls() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // Unsorted input, mixed Some/None (deletion) + let a1 = B256::from([0x01; 32]); + let a2 = B256::from([0x02; 32]); + let a3 = B256::from([0x03; 32]); + let a4 = B256::from([0x04; 32]); + let a5 = B256::from([0x05; 32]); + let acc1 = Account { + nonce: 2, + balance: U256::from(1000u64), + ..Default::default() + }; + let acc3 = Account { + nonce: 1, + balance: U256::from(10000u64), + ..Default::default() + }; + let acc4 = Account { + nonce: 5, + balance: U256::from(5000u64), + ..Default::default() + }; + let acc5 = Account { + nonce: 10, + balance: U256::from(20000u64), + ..Default::default() + }; + + { + store + .store_hashed_accounts(vec![(a2, None), (a1, Some(acc1)), (a4, Some(acc4))]) + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + let v1 = cur + .seek_by_key_subkey(a1, B0) + .expect("seek a1") + .expect("exists a1"); + assert_eq!(v1.block_number, B0); + assert_eq!(v1.value.0, Some(acc1)); + + let v2 = cur + .seek_by_key_subkey(a2, B0) + .expect("seek a2") + .expect("exists a2"); + assert_eq!(v2.block_number, B0); + assert!(v2.value.0.is_none(), "a2 is none"); + + let v4 = cur + .seek_by_key_subkey(a4, B0) + .expect("seek a4") + .expect("exists a4"); + assert_eq!(v4.block_number, B0); + assert_eq!(v4.value.0, Some(acc4)); + } + + { + // Second call + store + .store_hashed_accounts(vec![(a5, Some(acc5)), (a3, Some(acc3))]) + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + let v3 = cur + .seek_by_key_subkey(a3, B0) + .expect("seek a3") + .expect("exists a3"); + assert_eq!(v3.block_number, B0); + assert_eq!(v3.value.0, Some(acc3)); + + let v5 = cur + .seek_by_key_subkey(a5, B0) + .expect("seek a5") + .expect("exists a5"); + assert_eq!(v5.block_number, B0); + assert_eq!(v5.value.0, Some(acc5)); + } + } + + #[test] + fn store_hashed_storages_writes_versioned_values() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr = B256::from([0x11; 32]); + let slot = B256::from([0x22; 32]); + let val = U256::from(0x1234u64); + + store + .store_hashed_storages(addr, vec![(slot, val)]) + .expect("write storage"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + let key = HashedStorageKey::new(addr, slot); + let vv = cur.seek_by_key_subkey(key, B0).expect("seek"); + let vv = vv.expect("entry exists"); + + assert_eq!(vv.block_number, B0); + let inner = vv.value.0.as_ref().expect("Some(StorageValue)"); + assert_eq!(inner.0, val); + } + + #[test] + fn store_hashed_storages_multiple_slots_unsorted() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr = B256::from([0x11; 32]); + let s1 = B256::from([0x01; 32]); + let v1 = U256::from(1u64); + let s2 = B256::from([0x02; 32]); + let v2 = U256::from(2u64); + let s3 = B256::from([0x03; 32]); + let v3 = U256::from(3u64); + + store + .store_hashed_storages(addr, vec![(s2, v2), (s1, v1), (s3, v3)]) + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + for (slot, expected) in [(s1, v1), (s2, v2), (s3, v3)] { + let key = HashedStorageKey::new(addr, slot); + let vv = cur + .seek_by_key_subkey(key, B0) + .expect("seek") + .expect("exists"); + assert_eq!(vv.block_number, B0); + let inner = vv.value.0.as_ref().expect("Some(StorageValue)"); + assert_eq!(inner.0, expected); + } + } + + #[test] + fn store_hashed_storages_multiple_calls() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr = B256::from([0x11; 32]); + let s1 = B256::from([0x01; 32]); + let v1 = U256::from(1u64); + let s2 = B256::from([0x02; 32]); + let v2 = U256::from(2u64); + let s3 = B256::from([0x03; 32]); + let v3 = U256::from(3u64); + let s4 = B256::from([0x04; 32]); + let v4 = U256::from(4u64); + let s5 = B256::from([0x05; 32]); + let v5 = U256::from(5u64); + + { + store + .store_hashed_storages(addr, vec![(s2, v2), (s1, v1), (s5, v5)]) + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + for (slot, expected) in [(s1, v1), (s2, v2), (s5, v5)] { + let key = HashedStorageKey::new(addr, slot); + let vv = cur + .seek_by_key_subkey(key, B0) + .expect("seek") + .expect("exists"); + assert_eq!(vv.block_number, B0); + let inner = vv.value.0.as_ref().expect("Some(StorageValue)"); + assert_eq!(inner.0, expected); + } + } + + { + // Second call + store + .store_hashed_storages(addr, vec![(s4, v4), (s3, v3)]) + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + for (slot, expected) in [(s4, v4), (s3, v3)] { + let key = HashedStorageKey::new(addr, slot); + let vv = cur + .seek_by_key_subkey(key, B0) + .expect("seek") + .expect("exists"); + assert_eq!(vv.block_number, B0); + let inner = vv.value.0.as_ref().expect("Some(StorageValue)"); + assert_eq!(inner.0, expected); + } + } + } + + #[test] + fn test_store_account_branches_writes_versioned_values() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let nibble = Nibbles::from_nibbles_unchecked([0x12, 0x34]); + let branch_node = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let updates = vec![(nibble, Some(branch_node.clone()))]; + + store.store_account_branches(updates).expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.cursor_dup_read::().expect("cursor"); + + let vv = cur + .seek_by_key_subkey(StoredNibbles::from(nibble), B0) + .expect("seek") + .expect("entry exists"); + + assert_eq!(vv.block_number, B0); + assert_eq!(vv.value.0, Some(branch_node)); + } + + #[test] + fn test_store_account_branches_multiple_items_unsorted() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let n1 = Nibbles::from_nibbles_unchecked([0x01]); + let b1 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let n2 = Nibbles::from_nibbles_unchecked([0x02]); + let n3 = Nibbles::from_nibbles_unchecked([0x03]); + let b3 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + + let updates = vec![(n2, None), (n1, Some(b1)), (n3, Some(b3))]; + store + .store_account_branches(updates.clone()) + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.cursor_dup_read::().expect("cursor"); + + for (nibble, branch) in updates { + let v = cur + .seek_by_key_subkey(StoredNibbles::from(nibble), B0) + .expect("seek") + .expect("exists"); + assert_eq!(v.block_number, B0); + assert_eq!(v.value.0, branch); + } + } + + #[test] + fn store_account_branches_multiple_calls() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let n1 = Nibbles::from_nibbles_unchecked([0x01]); + let b1 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let n2 = Nibbles::from_nibbles_unchecked([0x02]); + let n3 = Nibbles::from_nibbles_unchecked([0x03]); + let b3 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let n4 = Nibbles::from_nibbles_unchecked([0x04]); + let b4 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let n5 = Nibbles::from_nibbles_unchecked([0x05]); + let b5 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + + { + let updates1 = vec![(n2, None), (n1, Some(b1)), (n4, Some(b4))]; + store + .store_account_branches(updates1.clone()) + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.cursor_dup_read::().expect("cursor"); + + for (nibble, branch) in updates1 { + let v = cur + .seek_by_key_subkey(StoredNibbles::from(nibble), B0) + .expect("seek") + .expect("exists"); + assert_eq!(v.block_number, B0); + assert_eq!(v.value.0, branch); + } + } + + { + // Second call + let updates2 = vec![(n5, Some(b5)), (n3, Some(b3))]; + store + .store_account_branches(updates2.clone()) + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.cursor_dup_read::().expect("cursor"); + + for (nibble, branch) in updates2 { + let v = cur + .seek_by_key_subkey(StoredNibbles::from(nibble), B0) + .expect("seek") + .expect("exists"); + assert_eq!(v.block_number, B0); + assert_eq!(v.value.0, branch); + } + } + } + + #[test] + fn test_store_storage_branches_writes_versioned_values() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let hashed_address = B256::random(); + let nibble = Nibbles::from_nibbles_unchecked([0x12, 0x34]); + let branch_node = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let items = vec![(nibble, Some(branch_node.clone()))]; + + store + .store_storage_branches(hashed_address, items) + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.cursor_dup_read::().expect("cursor"); + + let key = StorageTrieKey::new(hashed_address, StoredNibbles::from(nibble)); + let vv = cur + .seek_by_key_subkey(key, B0) + .expect("seek") + .expect("entry exists"); + + assert_eq!(vv.block_number, B0); + assert_eq!(vv.value.0, Some(branch_node)); + } + + #[test] + fn store_storage_branches_multiple_items_unsorted() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let hashed_address = B256::random(); + let n1 = Nibbles::from_nibbles_unchecked([0x01]); + let b1 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let n2 = Nibbles::from_nibbles_unchecked([0x02]); + let n3 = Nibbles::from_nibbles_unchecked([0x03]); + let b3 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + + let items = vec![(n2, None), (n1, Some(b1)), (n3, Some(b3))]; + store + .store_storage_branches(hashed_address, items.clone()) + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.cursor_dup_read::().expect("cursor"); + + for (nibble, branch) in items { + let key = StorageTrieKey::new(hashed_address, StoredNibbles::from(nibble)); + let v = cur + .seek_by_key_subkey(key, B0) + .expect("seek") + .expect("exists"); + assert_eq!(v.block_number, B0); + assert_eq!(v.value.0, branch); + } + } + + #[test] + fn store_storage_branches_multiple_calls() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let hashed_address = B256::random(); + let n1 = Nibbles::from_nibbles_unchecked([0x01]); + let b1 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let n2 = Nibbles::from_nibbles_unchecked([0x02]); + let n3 = Nibbles::from_nibbles_unchecked([0x03]); + let b3 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let n4 = Nibbles::from_nibbles_unchecked([0x04]); + let b4 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let n5 = Nibbles::from_nibbles_unchecked([0x05]); + let b5 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + + { + let items1 = vec![(n2, None), (n1, Some(b1)), (n5, Some(b5))]; + store + .store_storage_branches(hashed_address, items1.clone()) + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.cursor_dup_read::().expect("cursor"); + + for (nibble, branch) in items1 { + let key = StorageTrieKey::new(hashed_address, StoredNibbles::from(nibble)); + let v = cur + .seek_by_key_subkey(key, B0) + .expect("seek") + .expect("exists"); + assert_eq!(v.block_number, B0); + assert_eq!(v.value.0, branch); + } + } + + { + // Second call + let items2 = vec![(n4, Some(b4)), (n3, Some(b3))]; + store + .store_storage_branches(hashed_address, items2.clone()) + .expect("write"); + + let tx = store.env.tx().expect("ro tx"); + let mut cur = tx.cursor_dup_read::().expect("cursor"); + + for (nibble, branch) in items2 { + let key = StorageTrieKey::new(hashed_address, StoredNibbles::from(nibble)); + let v = cur + .seek_by_key_subkey(key, B0) + .expect("seek") + .expect("exists"); + assert_eq!(v.block_number, B0); + assert_eq!(v.value.0, branch); + } + } + } + + #[test] + fn test_store_trie_updates_comprehensive() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // Sample block number + const BLOCK: BlockWithParent = + BlockWithParent::new(B256::ZERO, NumHash::new(42, B256::ZERO)); + + // Sample addresses and keys + let addr1 = B256::from([0x11; 32]); + let addr2 = B256::from([0x22; 32]); + let slot1 = B256::from([0xA1; 32]); + let slot2 = B256::from([0xA2; 32]); + + // Sample accounts + let acc1 = Account { + nonce: 1, + balance: U256::from(100), + ..Default::default() + }; + + // Sample storage values + let val1 = U256::from(1234u64); + let val2 = U256::from(5678u64); + + // Sample trie paths + let account_path1 = Nibbles::from_nibbles_unchecked(vec![0, 1, 2, 3]); + let account_path2 = Nibbles::from_nibbles_unchecked(vec![4, 5, 6, 7]); + let removed_account_path = Nibbles::from_nibbles_unchecked(vec![7, 8, 9]); + + let account_node1 = BranchNodeCompact::default(); + let account_node2 = BranchNodeCompact::default(); + + let storage_path1 = Nibbles::from_nibbles_unchecked(vec![1, 2, 3, 4]); + let storage_path2 = Nibbles::from_nibbles_unchecked(vec![8, 9, 0, 1]); + + let storage_node1 = BranchNodeCompact::default(); + let storage_node2 = BranchNodeCompact::default(); + + // Construct test BlockStateDiff + let mut block_state_diff_trie_updates = TrieUpdates::default(); + let mut block_state_diff_post_state = HashedPostState::default(); + + // Add account trie nodes + block_state_diff_trie_updates + .account_nodes + .insert(account_path1, account_node1); + block_state_diff_trie_updates + .account_nodes + .insert(account_path2, account_node2); + block_state_diff_trie_updates + .removed_nodes + .insert(removed_account_path); + + // Add storage trie nodes for two addresses + let mut storage_nodes1 = StorageTrieUpdates::default(); + storage_nodes1 + .storage_nodes + .insert(storage_path1, storage_node1); + block_state_diff_trie_updates + .storage_tries + .insert(addr1, storage_nodes1); + + let mut storage_nodes2 = StorageTrieUpdates::default(); + storage_nodes2 + .storage_nodes + .insert(storage_path2, storage_node2); + block_state_diff_trie_updates + .storage_tries + .insert(addr2, storage_nodes2); + + // Add hashed accounts (one Some, one None) + block_state_diff_post_state + .accounts + .insert(addr1, Some(acc1)); + block_state_diff_post_state.accounts.insert(addr2, None); // Deletion + + // Add storage slots for both addresses + let mut storage1 = HashedStorage::default(); + storage1.storage.insert(slot1, val1); + block_state_diff_post_state.storages.insert(addr1, storage1); + + let mut storage2 = HashedStorage::default(); + storage2.storage.insert(slot2, val2); + block_state_diff_post_state.storages.insert(addr2, storage2); + + // Store everything + let block_state_diff = BlockStateDiff { + sorted_trie_updates: block_state_diff_trie_updates.into_sorted(), + sorted_post_state: block_state_diff_post_state.into_sorted(), + }; + store + .store_trie_updates(BLOCK, block_state_diff) + .expect("store"); + + // Verify account trie nodes + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + // Check first node + let vv1 = cur + .seek_by_key_subkey(account_path1.into(), BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv1.block_number, BLOCK.block.number); + assert!(vv1.value.0.is_some()); + + // Check second node + let vv2 = cur + .seek_by_key_subkey(account_path2.into(), BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv2.block_number, BLOCK.block.number); + assert!(vv2.value.0.is_some()); + + // Check removed node + let vv3 = cur + .seek_by_key_subkey(removed_account_path.into(), BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv3.block_number, BLOCK.block.number); + assert!(vv3.value.0.is_none(), "Expected node deletion"); + } + + // Verify storage trie nodes + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + // Check node for addr1 + let key1 = StorageTrieKey::new(addr1, storage_path1.into()); + let vv1 = cur + .seek_by_key_subkey(key1, BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv1.block_number, BLOCK.block.number); + assert!(vv1.value.0.is_some()); + + // Check node for addr2 + let key2 = StorageTrieKey::new(addr2, storage_path2.into()); + let vv2 = cur + .seek_by_key_subkey(key2, BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv2.block_number, BLOCK.block.number); + assert!(vv2.value.0.is_some()); + } + + // Verify hashed accounts + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + // Check account1 (exists) + let vv1 = cur + .seek_by_key_subkey(addr1, BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv1.block_number, BLOCK.block.number); + assert_eq!(vv1.value.0, Some(acc1)); + + // Check account2 (deletion) + let vv2 = cur + .seek_by_key_subkey(addr2, BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv2.block_number, BLOCK.block.number); + assert!(vv2.value.0.is_none(), "Expected account deletion"); + } + + // Verify hashed storages + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + // Check storage for addr1 + let key1 = HashedStorageKey::new(addr1, slot1); + let vv1 = cur + .seek_by_key_subkey(key1, BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv1.block_number, BLOCK.block.number); + let inner1 = vv1.value.0.as_ref().expect("Some(StorageValue)"); + assert_eq!(inner1.0, val1); + + // Check storage for addr2 + let key2 = HashedStorageKey::new(addr2, slot2); + let vv2 = cur + .seek_by_key_subkey(key2, BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv2.block_number, BLOCK.block.number); + let inner2 = vv2.value.0.as_ref().expect("Some(StorageValue)"); + assert_eq!(inner2.0, val2); + } + + // Verify BlockChangeSet entries + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + let entries: Vec<_> = cur.walk(Some(BLOCK.block.number)).expect("walk").collect(); + assert_eq!(entries.len(), 1, "Expected 1 BlockChangeSet entry"); + } + + // check the latest block number in proof window + { + let tx = store.env.tx().expect("tx"); + let mut proof_window_cursor = tx.new_cursor::().expect("cursor"); + let latest_block = proof_window_cursor + .seek(ProofWindowKey::LatestBlock) + .expect("seek") + .expect("exists"); + assert_eq!(latest_block.1.number(), BLOCK.block.number); + assert_eq!(*latest_block.1.hash(), BLOCK.block.hash); + } + } + + #[test] + fn store_trie_updates_out_of_order_rejects() { + let dir = tempfile::TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // set latest to some hash H1 + let existing_block = BlockWithParent::new(B256::random(), NumHash::new(1, B256::random())); + store + .set_earliest_block_number(existing_block.block.number, existing_block.block.hash) + .expect("set"); + + // incoming block whose parent != existing latest + let bad_parent = B256::from([0xFF; 32]); + let bad_block = BlockWithParent::new(bad_parent, NumHash::new(2, B256::ZERO)); + let diff = BlockStateDiff::default(); + + let res = store.store_trie_updates(bad_block, diff); + assert!(matches!( + res, + Err(MorphProofsStorageError::OutOfOrder { .. }) + )); + // verify nothing written: proof window still unchanged + let latest = store.get_latest_block_number().expect("get latest"); + assert_eq!(latest.unwrap().1, existing_block.block.hash); + } + + #[test] + fn store_trie_updates_multiple_blocks_append_versions() { + let dir = tempfile::TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr = B256::from([0x21; 32]); + // block A (parent = ZERO) + let block_a = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + let mut diff_a_post_state = HashedPostState::default(); + diff_a_post_state + .accounts + .insert(addr, Some(Account::default())); + + let diff_a = BlockStateDiff { + sorted_trie_updates: TrieUpdatesSorted::default(), + sorted_post_state: diff_a_post_state.into_sorted(), + }; + store.store_trie_updates(block_a, diff_a).expect("store A"); + + // block B (parent = hash of A) + let block_b = BlockWithParent::new(block_a.block.hash, NumHash::new(2, B256::random())); + let mut diff_b_post_state = HashedPostState::default(); + diff_b_post_state.accounts.insert( + addr, + Some(Account { + nonce: 5, + ..Default::default() + }), + ); + + let diff_b = BlockStateDiff { + sorted_trie_updates: TrieUpdatesSorted::default(), + sorted_post_state: diff_b_post_state.into_sorted(), + }; + store.store_trie_updates(block_b, diff_b).expect("store B"); + + // verify we can retrieve entries for both block numbers + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + let v_a = cur + .seek_by_key_subkey(addr, block_a.block.number) + .expect("seek") + .expect("exists"); + let v_b = cur + .seek_by_key_subkey(addr, block_b.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(v_a.block_number, block_a.block.number); + assert_eq!(v_b.block_number, block_b.block.number); + } + + #[test] + fn test_store_trie_updates_empty_collections() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + const BLOCK: BlockWithParent = + BlockWithParent::new(B256::ZERO, NumHash::new(42, B256::ZERO)); + + // Create BlockStateDiff with empty collections + let block_state_diff = BlockStateDiff::default(); + + // This should work without errors + store + .store_trie_updates(BLOCK, block_state_diff) + .expect("store"); + + // Verify nothing was written (should be empty) + let tx = store.env.tx().expect("tx"); + + let mut cur1 = tx.new_cursor::().expect("cursor"); + assert!( + cur1.next_dup_val().expect("first").is_none(), + "Account trie should be empty" + ); + + let mut cur2 = tx.new_cursor::().expect("cursor"); + assert!( + cur2.next_dup_val().expect("first").is_none(), + "Storage trie should be empty" + ); + + let mut cur3 = tx.new_cursor::().expect("cursor"); + assert!( + cur3.next_dup_val().expect("first").is_none(), + "Hashed accounts should be empty" + ); + + let mut cur4 = tx.new_cursor::().expect("cursor"); + assert!( + cur4.next_dup_val().expect("first").is_none(), + "Hashed storage should be empty" + ); + + let mut cur5 = tx.new_cursor::().expect("cursor"); + assert!( + cur5.next().expect("first").is_some(), + "Pruning index SHOULD populate the change set even for empty diffs" + ); + } + + #[test] + fn fetch_trie_updates_missing_changeset_returns_error() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let res = store.fetch_trie_updates(99); + assert!(matches!( + res, + Err(MorphProofsStorageError::NoChangeSetForBlock(99)) + )); + } + + #[test] + fn fetch_trie_updates_empty_changeset() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let block = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + let diff = BlockStateDiff::default(); + + store.store_trie_updates(block, diff).expect("store"); + let got = store.fetch_trie_updates(1).expect("fetch"); + assert!(got.sorted_trie_updates.account_nodes_ref().is_empty()); + assert!(got.sorted_trie_updates.storage_tries_ref().is_empty()); + assert!(got.sorted_post_state.accounts.is_empty()); + assert!(got.sorted_post_state.storages.is_empty()); + } + + #[test] + fn fetch_trie_updates_missing_account_history_entry_returns_error() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // prepare ChangeSet that references StoredNibbles for account key + // (insert ChangeSet into BlockChangeSet directly using tx) + { + let tx = store.env.tx_mut().unwrap(); + let mut cur = tx.cursor_write::().unwrap(); + cur.insert( + 1, + &ChangeSet { + account_trie_keys: vec![StoredNibbles::default()], + ..Default::default() + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + let res = store.fetch_trie_updates(1); + assert!(matches!( + res, + Err(MorphProofsStorageError::MissingAccountTrieHistory(..)) + )); + } + + #[test] + fn fetch_trie_updates_account_history_seek_returns_later_block_treated_as_missing() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // manually insert account history and ChangeSet for block 1 referencing same key + { + let tx = store.env.tx_mut().unwrap(); + let mut acc_cur = tx.cursor_write::().unwrap(); + acc_cur + .insert( + StoredNibbles::from(Nibbles::from_nibbles_unchecked([0x1])), + &VersionedValue::new(2, MaybeDeleted(Some(BranchNodeCompact::default()))), + ) + .unwrap(); + + let mut cur = tx.cursor_write::().unwrap(); + cur.insert( + 1, + &ChangeSet { + account_trie_keys: vec![StoredNibbles::from(Nibbles::from_nibbles_unchecked( + [0x1], + ))], + ..Default::default() + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + // fetch block 1 -> seek will find block 2 but block_number != 1 so expect + // MissingAccountTrieHistory + let res = store.fetch_trie_updates(1); + assert!(matches!( + res, + Err(MorphProofsStorageError::MissingAccountTrieHistory(..)) + )); + } + + #[test] + fn fetch_trie_updates_missing_storage_history_entry_returns_error() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // prepare ChangeSet that references StorageTrieKey for storage trie + // (insert ChangeSet into BlockChangeSet directly using tx) + { + let tx = store.env.tx_mut().unwrap(); + let mut cur = tx.cursor_write::().unwrap(); + cur.insert( + 1, + &ChangeSet { + storage_trie_keys: vec![StorageTrieKey::new( + B256::from([0u8; 32]), + StoredNibbles::default(), + )], + ..Default::default() + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + let res = store.fetch_trie_updates(1); + assert!(matches!( + res, + Err(MorphProofsStorageError::MissingStorageTrieHistory(..)) + )); + } + + #[test] + fn fetch_trie_updates_storage_history_seek_returns_later_block_treated_as_missing() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // manually insert storage history and ChangeSet for block 1 referencing same key + { + let tx = store.env.tx_mut().unwrap(); + let mut stor_cur = tx.cursor_write::().unwrap(); + stor_cur + .insert( + StorageTrieKey::new( + B256::from([0u8; 32]), + StoredNibbles::from(Nibbles::from_nibbles_unchecked([0x1])), + ), + &VersionedValue::new(2, MaybeDeleted(Some(BranchNodeCompact::default()))), + ) + .unwrap(); + + let mut cur = tx.cursor_write::().unwrap(); + cur.insert( + 1, + &ChangeSet { + storage_trie_keys: vec![StorageTrieKey::new( + B256::from([0u8; 32]), + StoredNibbles::from(Nibbles::from_nibbles_unchecked([0x1])), + )], + ..Default::default() + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + // fetch block 1 -> seek will find block 2 but block_number != 1 so expect + // MissingStorageTrieHistory + let res = store.fetch_trie_updates(1); + assert!(matches!( + res, + Err(MorphProofsStorageError::MissingStorageTrieHistory(..)) + )); + } + + #[test] + fn fetch_trie_updates_missing_hashed_account_entry_returns_error() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // prepare ChangeSet that references hashed account address + // (insert ChangeSet into BlockChangeSet directly using tx) + { + let tx = store.env.tx_mut().unwrap(); + let mut cur = tx.cursor_write::().unwrap(); + cur.insert( + 1, + &ChangeSet { + hashed_account_keys: vec![B256::from([0u8; 32])], + ..Default::default() + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + let res = store.fetch_trie_updates(1); + assert!(matches!( + res, + Err(MorphProofsStorageError::MissingHashedAccountHistory(..)) + )); + } + + #[test] + fn fetch_trie_updates_hashed_account_seek_returns_later_block_treated_as_missing() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // manually insert hashed account history and ChangeSet for block 1 referencing same key + { + let tx = store.env.tx_mut().unwrap(); + let mut acc_cur = tx.cursor_write::().unwrap(); + acc_cur + .insert( + B256::from([0u8; 32]), + &VersionedValue::new(2, MaybeDeleted(Some(Account::default()))), + ) + .unwrap(); + + let mut cur = tx.cursor_write::().unwrap(); + cur.insert( + 1, + &ChangeSet { + hashed_account_keys: vec![B256::from([0u8; 32])], + ..Default::default() + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + // fetch block 1 -> seek will find block 2 but block_number != 1 so expect + // MissingHashedAccountHistory + let res = store.fetch_trie_updates(1); + assert!(matches!( + res, + Err(MorphProofsStorageError::MissingHashedAccountHistory(..)) + )); + } + + #[test] + fn fetch_trie_updates_missing_hashed_storage_entry_returns_error() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // prepare ChangeSet that references hashed storage key + // (insert ChangeSet into BlockChangeSet directly using tx) + { + let tx = store.env.tx_mut().unwrap(); + let mut cur = tx.cursor_write::().unwrap(); + cur.insert( + 1, + &ChangeSet { + hashed_storage_keys: vec![HashedStorageKey::new( + B256::from([0u8; 32]), + B256::from([0u8; 32]), + )], + ..Default::default() + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + let res = store.fetch_trie_updates(1); + assert!(matches!( + res, + Err(MorphProofsStorageError::MissingHashedStorageHistory { .. }) + )); + } + + #[test] + fn fetch_trie_updates_hashed_storage_seek_returns_later_block_treated_as_missing() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // manually insert hashed storage history and ChangeSet for block 1 referencing same key + { + let tx = store.env.tx_mut().unwrap(); + let mut stor_cur = tx.cursor_write::().unwrap(); + stor_cur + .insert( + HashedStorageKey::new(B256::from([0u8; 32]), B256::from([0u8; 32])), + &VersionedValue::new(2, MaybeDeleted(Some(StorageValue::new(U256::ZERO)))), + ) + .unwrap(); + + let mut cur = tx.cursor_write::().unwrap(); + cur.insert( + 1, + &ChangeSet { + hashed_storage_keys: vec![HashedStorageKey::new( + B256::from([0u8; 32]), + B256::from([0u8; 32]), + )], + ..Default::default() + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + // fetch block 1 -> seek will find block 2 but block_number != 1 so expect + // MissingHashedStorageHistory + let res = store.fetch_trie_updates(1); + assert!(matches!( + res, + Err(MorphProofsStorageError::MissingHashedStorageHistory { .. }) + )); + } + + #[test] + fn fetch_trie_updates_basic() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // Build a block with mixed changes (accounts, trie nodes, hashed storages) + let block = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + + // prepare data + let addr1 = B256::from([0x11; 32]); + let addr2 = B256::from([0x22; 32]); + let slot1 = B256::from([0xA1; 32]); + let slot2 = B256::from([0xA2; 32]); + + let acc1 = Account { + nonce: 1, + balance: U256::from(100), + ..Default::default() + }; + + let val1 = U256::from(1234u64); + let val2 = U256::from(5678u64); + + let account_path1 = Nibbles::from_nibbles_unchecked(vec![0, 1, 2, 3]); + let account_path2 = Nibbles::from_nibbles_unchecked(vec![4, 5, 6, 7]); + let account_node1 = BranchNodeCompact { + root_hash: Some(B256::random()), + ..Default::default() + }; + let account_node2 = BranchNodeCompact { + root_hash: Some(B256::random()), + ..Default::default() + }; + + let storage_path1 = Nibbles::from_nibbles_unchecked(vec![1, 2, 3, 4]); + let storage_node1 = BranchNodeCompact { + root_hash: Some(B256::random()), + ..Default::default() + }; + + // Construct BlockStateDiff + let mut block_state_diff_trie_updates = TrieUpdates::default(); + block_state_diff_trie_updates + .account_nodes + .insert(account_path1, account_node1); + block_state_diff_trie_updates + .account_nodes + .insert(account_path2, account_node2); + // storage trie for addr1 + let mut storage_nodes1 = StorageTrieUpdates::default(); + storage_nodes1 + .storage_nodes + .insert(storage_path1, storage_node1); + block_state_diff_trie_updates + .storage_tries + .insert(addr1, storage_nodes1); + + // hashed accounts: addr1 -> Some, addr2 -> None + let mut block_state_diff_post_state = HashedPostState::default(); + block_state_diff_post_state + .accounts + .insert(addr1, Some(acc1)); + block_state_diff_post_state.accounts.insert(addr2, None); + + // hashed storages + let mut storage1 = HashedStorage::default(); + storage1.storage.insert(slot1, val1); + block_state_diff_post_state.storages.insert(addr1, storage1); + + let mut storage2 = HashedStorage::default(); + storage2.storage.insert(slot2, val2); + block_state_diff_post_state.storages.insert(addr2, storage2); + + // store then fetch + let block_state_diff = BlockStateDiff { + sorted_trie_updates: block_state_diff_trie_updates.into_sorted(), + sorted_post_state: block_state_diff_post_state.into_sorted(), + }; + store + .store_trie_updates(block, block_state_diff.clone()) + .expect("store"); + let got = store.fetch_trie_updates(1).expect("fetch"); + + // verify trie updates + assert_eq!( + got.sorted_trie_updates.account_nodes_ref(), + block_state_diff.sorted_trie_updates.account_nodes_ref(), + ); + assert_eq!( + got.sorted_trie_updates.storage_tries_ref(), + block_state_diff.sorted_trie_updates.storage_tries_ref(), + ); + + // verify post state + assert_eq!( + got.sorted_post_state.accounts, + block_state_diff.sorted_post_state.accounts + ); + assert_eq!( + got.sorted_post_state.storages, + block_state_diff.sorted_post_state.storages + ); + } + + #[test] + fn test_prune_earliest_state_single_entry() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + let block = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + store.set_earliest_block_number(0, B256::ZERO).unwrap(); + + // Insert a single entry to be pruned + let addr = B256::random(); + let mut state_diff_post_state = HashedPostState::default(); + state_diff_post_state + .accounts + .insert(addr, Some(Account::default())); + let state_diff = BlockStateDiff { + sorted_post_state: state_diff_post_state.into_sorted(), + ..Default::default() + }; + store.store_trie_updates(block, state_diff).unwrap(); + + // Prune the entry - pass empty diff since we're just removing data + let next_block = BlockWithParent::new(block.block.hash, NumHash::new(2, B256::random())); + store.prune_earliest_state(next_block).unwrap(); + + // Verify the entry was pruned + let tx = store.env.tx().unwrap(); + let mut cur = tx.new_cursor::().unwrap(); + + // The survivor at block 1 should remain + let val = cur + .seek_by_key_subkey(addr, block.block.number) + .unwrap() + .expect("Survivor should exist"); + assert_eq!(val.block_number, 1); + + let mut pruning_cur = tx.new_cursor::().unwrap(); + assert!( + pruning_cur + .seek_exact(block.block.number) + .unwrap() + .is_none() + ); + + // Verify earliest block was updated + let earliest = store.get_earliest_block_number().unwrap(); + assert_eq!(earliest, Some((2, next_block.block.hash))); + } + + #[test] + fn test_prune_earliest_state_multiple_entries_same_block() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + let block = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + store.set_earliest_block_number(0, B256::ZERO).unwrap(); + + // Insert multiple entries for the same block + let addr1 = B256::random(); + let addr2 = B256::random(); + let mut state_diff_post_state = HashedPostState::default(); + state_diff_post_state + .accounts + .insert(addr1, Some(Account::default())); + state_diff_post_state + .accounts + .insert(addr2, Some(Account::default())); + let state_diff = BlockStateDiff { + sorted_post_state: state_diff_post_state.into_sorted(), + ..Default::default() + }; + store.store_trie_updates(block, state_diff).unwrap(); + + // Prune the entries + let next_block = BlockWithParent::new(block.block.hash, NumHash::new(2, B256::random())); + store.prune_earliest_state(next_block).unwrap(); + + // Verify the entries were pruned + let tx = store.env.tx().unwrap(); + let mut cur = tx.new_cursor::().unwrap(); + + // All entries should survive at block 1 + let v1 = cur + .seek_by_key_subkey(addr1, block.block.number) + .unwrap() + .expect("addr1 survivor"); + assert_eq!(v1.block_number, 1); + let v2 = cur + .seek_by_key_subkey(addr2, block.block.number) + .unwrap() + .expect("addr2 survivor"); + assert_eq!(v2.block_number, 1); + + let mut pruning_cur = tx.new_cursor::().unwrap(); + assert!( + pruning_cur + .seek_exact(block.block.number) + .unwrap() + .is_none() + ); + } + + #[test] + fn test_prune_earliest_state_multiple_blocks() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + let block_1 = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + let block_2 = BlockWithParent::new(block_1.block.hash, NumHash::new(2, B256::random())); + let block_3 = BlockWithParent::new(block_2.block.hash, NumHash::new(3, B256::random())); + store.set_earliest_block_number(0, B256::ZERO).unwrap(); + + // Insert entries for multiple blocks + let addr1 = B256::random(); + let addr2 = B256::random(); + let mut state_diff1_post_state = HashedPostState::default(); + state_diff1_post_state + .accounts + .insert(addr1, Some(Account::default())); + let state_diff1 = BlockStateDiff { + sorted_post_state: state_diff1_post_state.into_sorted(), + ..Default::default() + }; + store.store_trie_updates(block_1, state_diff1).unwrap(); + + let mut state_diff2_post_state = HashedPostState::default(); + state_diff2_post_state + .accounts + .insert(addr2, Some(Account::default())); + let state_diff2 = BlockStateDiff { + sorted_post_state: state_diff2_post_state.into_sorted(), + ..Default::default() + }; + store.store_trie_updates(block_2, state_diff2).unwrap(); + + // Prune up to block 3 (should remove blocks 1 and 2) + store.prune_earliest_state(block_3).unwrap(); + + // Verify the entries were pruned + let tx = store.env.tx().unwrap(); + let mut cur = tx.new_cursor::().unwrap(); + + // addr1 survivor at block 1 must exist + let v1 = cur + .seek_by_key_subkey(addr1, 1) + .unwrap() + .expect("addr1 survivor"); + assert_eq!(v1.block_number, 1); + + // addr2 survivor at block 2 must exist + let v2 = cur + .seek_by_key_subkey(addr2, 2) + .unwrap() + .expect("addr2 survivor"); + assert_eq!(v2.block_number, 2); + + let mut pruning_cur = tx.new_cursor::().unwrap(); + assert!(pruning_cur.seek_exact(1).unwrap().is_none()); + assert!(pruning_cur.seek_exact(2).unwrap().is_none()); + } + + #[test] + fn test_prune_earliest_state_no_op() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + store.set_earliest_block_number(1, B256::random()).unwrap(); + + // Attempt to prune with a new earliest block that is not newer + let block_1 = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + let block_0 = BlockWithParent::new(B256::ZERO, NumHash::new(0, B256::random())); + store.prune_earliest_state(block_1).unwrap(); + store.prune_earliest_state(block_0).unwrap(); + + // Nothing should have been pruned, this call should not panic or error + } + + #[test] + fn test_prune_earliest_state_no_entries_to_prune() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + store.set_earliest_block_number(1, B256::random()).unwrap(); + + // Prune a range where no entries exist + let block_10 = BlockWithParent::new(B256::ZERO, NumHash::new(10, B256::random())); + store.prune_earliest_state(block_10).unwrap(); + + // Nothing should have been pruned, this call should not panic or error + } + + #[test] + fn test_prune_earliest_state_with_diff_insertion() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + store.set_earliest_block_number(0, B256::ZERO).unwrap(); + + // Insert entries for blocks 1 and 2 + let addr1 = B256::random(); + let addr2 = B256::random(); + let acc1 = Account { + nonce: 1, + balance: U256::from(100), + ..Default::default() + }; + let acc2 = Account { + nonce: 2, + balance: U256::from(200), + ..Default::default() + }; + + let block_1 = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + let mut state_diff1_post_state = HashedPostState::default(); + state_diff1_post_state.accounts.insert(addr1, Some(acc1)); + let state_diff1 = BlockStateDiff { + sorted_post_state: state_diff1_post_state.into_sorted(), + ..Default::default() + }; + store.store_trie_updates(block_1, state_diff1).unwrap(); + + let block_2 = BlockWithParent::new(block_1.block.hash, NumHash::new(2, B256::random())); + let mut state_diff2_post_state = HashedPostState::default(); + state_diff2_post_state.accounts.insert(addr2, Some(acc2)); + let state_diff2 = BlockStateDiff { + sorted_post_state: state_diff2_post_state.into_sorted(), + ..Default::default() + }; + store.store_trie_updates(block_2, state_diff2).unwrap(); + + // Now prune to block 3 + let block_3 = BlockWithParent::new(block_2.block.hash, NumHash::new(3, B256::random())); + store.prune_earliest_state(block_3).unwrap(); + + let tx = store.env.tx().unwrap(); + let mut cur = tx.new_cursor::().unwrap(); + + // Verify that blocks 1 and 2 entries were NOT pruned (they are survivors for their + // respective keys) + let vv1 = cur + .seek_by_key_subkey(addr1, 0) + .unwrap() + .expect("addr1 should survive"); + assert_eq!(vv1.block_number, 1); + assert_eq!(vv1.value.0, Some(acc1)); + + let vv2 = cur + .seek_by_key_subkey(addr2, 0) + .unwrap() + .expect("addr2 should survive"); + assert_eq!(vv2.block_number, 2); + assert_eq!(vv2.value.0, Some(acc2)); + + // Verify change sets for blocks 1 and 2 were removed + let mut pruning_cur = tx.new_cursor::().unwrap(); + assert!(pruning_cur.seek_exact(1).unwrap().is_none()); + assert!(pruning_cur.seek_exact(2).unwrap().is_none()); + + // Verify earliest block was updated + let earliest = store.get_earliest_block_number().unwrap(); + assert_eq!(earliest, Some((3, block_3.block.hash))); + } + + #[test] + fn test_prune_earliest_state_with_removed_nodes() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + store.set_earliest_block_number(0, B256::ZERO).unwrap(); + + // Create some trie nodes in blocks 1, 2, 3 + let path1 = Nibbles::from_nibbles_unchecked([0x01, 0x02]); + let path2 = Nibbles::from_nibbles_unchecked([0x03, 0x04]); + let node1 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let node2 = BranchNodeCompact::new(0b10, 0, 0, vec![], Some(B256::random())); + + let block_1 = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + let mut diff1_trie_updates = TrieUpdates::default(); + diff1_trie_updates.account_nodes.insert(path1, node1); + let diff1 = BlockStateDiff { + sorted_trie_updates: diff1_trie_updates.into_sorted(), + ..Default::default() + }; + store.store_trie_updates(block_1, diff1).unwrap(); + + let block_2 = BlockWithParent::new(block_1.block.hash, NumHash::new(2, B256::random())); + let mut diff2_trie_updates = TrieUpdates::default(); + diff2_trie_updates + .account_nodes + .insert(path2, node2.clone()); + let diff2 = BlockStateDiff { + sorted_trie_updates: diff2_trie_updates.into_sorted(), + ..Default::default() + }; + store.store_trie_updates(block_2, diff2).unwrap(); + + // In block 3, path1 is deleted (stored as None in the database) + // This happens when we store trie updates with path1 mapped to None + let block_3 = BlockWithParent::new(block_2.block.hash, NumHash::new(3, B256::random())); + // Simulate storing a deletion by directly writing to DB + store + .env + .update(|tx| { + let mut cursor = tx.new_cursor::()?; + let vv = VersionedValue { + block_number: 3, + value: MaybeDeleted(None), + }; + cursor.upsert(StoredNibbles::from(path1), &vv)?; + + // Record in change set + let mut change_set_cursor = tx.new_cursor::()?; + change_set_cursor.upsert( + 3, + &ChangeSet { + account_trie_keys: vec![StoredNibbles::from(path1)], + storage_trie_keys: vec![], + hashed_account_keys: vec![], + hashed_storage_keys: vec![], + }, + )?; + + // Update proof window + let mut proof_window_cursor = tx.new_cursor::()?; + proof_window_cursor.upsert( + ProofWindowKey::LatestBlock, + &BlockNumberHash::new(3, block_3.block.hash), + )?; + Ok::<(), DatabaseError>(()) + }) + .unwrap() + .unwrap(); + + // Now prune to block 5, with the new initial state: + // - path1 should be in removed_nodes (it was deleted in block 3) + // - path2 should be included with its value (it still exists from block 2) + let block_5 = BlockWithParent::new(B256::random(), NumHash::new(5, B256::random())); + store.prune_earliest_state(block_5).unwrap(); + + // Verify that all entries for path1 before block 5 were removed + let tx = store.env.tx().unwrap(); + let mut cur = tx.cursor_dup_read::().unwrap(); + + // path1 at block 1 should be gone; seeking 1 finds survivor (tombstone at 3) + if let Some(v) = cur + .seek_by_key_subkey(StoredNibbles::from(path1), 1) + .unwrap() + { + assert!(v.block_number >= 3, "path1 at block 1 should be pruned"); + } + + // path1 at block 1 should be gone. + // path1 survivor at block 3 (tombstone) should ALSO be gone now (optimization). + // So seeking for path1 should return None. + assert!( + cur.seek_by_key_subkey(StoredNibbles::from(path1), 0) + .unwrap() + .is_none(), + "path1 should be completely removed including tombstone" + ); + + // path2 entries should be pruned (blocks < 5) + // Survivor for path2 is at block 2. + let v2 = cur + .seek_by_key_subkey(StoredNibbles::from(path2), 0) + .unwrap() + .expect("path2 survivor"); + assert_eq!(v2.block_number, 2); + assert_eq!(v2.value.0, Some(node2)); + } + + #[test] + fn test_prune_earliest_state_overlapping_keys() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + store.set_earliest_block_number(0, B256::ZERO).unwrap(); + + // Use overlapping key (addr1 updated in blocks 1 and 2) + let addr1 = B256::random(); + let acc1 = Account { + nonce: 1, + balance: U256::from(100), + ..Default::default() + }; + let acc2 = Account { + nonce: 2, + balance: U256::from(200), + ..Default::default() + }; + + let block_1 = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + let mut diff1_post_state = HashedPostState::default(); + diff1_post_state.accounts.insert(addr1, Some(acc1)); + let diff1 = BlockStateDiff { + sorted_post_state: diff1_post_state.into_sorted(), + ..Default::default() + }; + store.store_trie_updates(block_1, diff1).unwrap(); + + let block_2 = BlockWithParent::new(block_1.block.hash, NumHash::new(2, B256::random())); + let mut diff2_post_state = HashedPostState::default(); + diff2_post_state.accounts.insert(addr1, Some(acc2)); + let diff2 = BlockStateDiff { + sorted_post_state: diff2_post_state.into_sorted(), + ..Default::default() + }; + store.store_trie_updates(block_2, diff2).unwrap(); + + // Prune to block 3 + let block_3 = BlockWithParent::new(block_2.block.hash, NumHash::new(3, B256::random())); + store.prune_earliest_state(block_3).unwrap(); + + let tx = store.env.tx().unwrap(); + let mut cur = tx.new_cursor::().unwrap(); + + // Verify survivor (acc2 at block 2) remains + let vv1 = cur + .seek_by_key_subkey(addr1, 0) + .unwrap() + .expect("addr1 should having surviving state"); + assert_eq!(vv1.block_number, 2); + assert_eq!(vv1.value.0, Some(acc2)); + } + + #[test] + fn test_prune_earliest_state_comprehensive() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + store.set_earliest_block_number(0, B256::ZERO).unwrap(); + + // Setup complex scenario with accounts, storage, and trie nodes + let addr1 = B256::random(); + let slot1 = B256::random(); + let path1 = Nibbles::from_nibbles_unchecked([0x01]); + let storage_path1 = Nibbles::from_nibbles_unchecked([0x03]); + + let acc1 = Account { + nonce: 1, + balance: U256::from(100), + ..Default::default() + }; + let node1 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let storage_node1 = BranchNodeCompact::new(0b10, 0, 0, vec![], Some(B256::random())); + + // Block 1: Insert account, trie node, and storage for addr1 + let block_1 = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + + let mut diff1_trie_updates = TrieUpdates::default(); + let mut diff1_post_state = HashedPostState::default(); + + diff1_post_state.accounts.insert(addr1, Some(acc1)); + diff1_trie_updates + .account_nodes + .insert(path1, node1.clone()); + let mut storage1 = HashedStorage::default(); + storage1.storage.insert(slot1, U256::from(1234)); + diff1_post_state.storages.insert(addr1, storage1.clone()); + let mut storage_updates1 = StorageTrieUpdates::default(); + storage_updates1 + .storage_nodes + .insert(storage_path1, storage_node1.clone()); + diff1_trie_updates + .storage_tries + .insert(addr1, storage_updates1.clone()); + + let diff_1 = BlockStateDiff { + sorted_post_state: diff1_post_state.into_sorted(), + sorted_trie_updates: diff1_trie_updates.into_sorted(), + }; + store.store_trie_updates(block_1, diff_1).unwrap(); + + // Block 2: Update account (overwriting addr1) + let acc2 = Account { + nonce: 2, + balance: U256::from(200), + ..Default::default() + }; + let block_2 = BlockWithParent::new(block_1.block.hash, NumHash::new(2, B256::random())); + + let mut diff2_post_state = HashedPostState::default(); + diff2_post_state.accounts.insert(addr1, Some(acc2)); + let diff2 = BlockStateDiff { + sorted_post_state: diff2_post_state.into_sorted(), + ..Default::default() + }; + store.store_trie_updates(block_2, diff2).unwrap(); + + // Prune to block 3 + let block_3 = BlockWithParent::new(block_2.block.hash, NumHash::new(3, B256::random())); + store.prune_earliest_state(block_3).unwrap(); + + let tx = store.env.tx().unwrap(); + + // Verify account history - acc1 at block 1 pruned + let mut acc_cur = tx.new_cursor::().unwrap(); + // Survivor acc2 at block 2 remains + let acc_vv = acc_cur + .seek_by_key_subkey(addr1, 0) + .unwrap() + .expect("Survivor at block 2"); + assert_eq!(acc_vv.value.0, Some(acc2)); + assert_eq!(acc_vv.block_number, 2); + + // Verify account trie history - path1 at block 1 remains because it's the latest for that + // key + let mut trie_cur = tx.cursor_dup_read::().unwrap(); + let trie_vv = trie_cur + .seek_by_key_subkey(StoredNibbles::from(path1), 0) + .unwrap() + .expect("Survivor at block 1"); + assert_eq!(trie_vv.value.0, Some(node1)); + assert_eq!(trie_vv.block_number, 1); + + // Verify storage history - slot1 at block 1 remains + let mut storage_cur = tx.new_cursor::().unwrap(); + let storage_key = HashedStorageKey::new(addr1, slot1); + let storage_vv = storage_cur + .seek_by_key_subkey(storage_key, 0) + .unwrap() + .expect("Survivor at block 1"); + assert_eq!(storage_vv.value.0.as_ref().unwrap().0, U256::from(1234)); + assert_eq!(storage_vv.block_number, 1); + + // Verify storage trie history + let mut storage_trie_cur = tx.cursor_dup_read::().unwrap(); + let storage_trie_key = StorageTrieKey::new(addr1, StoredNibbles::from(storage_path1)); + let storage_trie_vv = storage_trie_cur + .seek_by_key_subkey(storage_trie_key, 0) + .unwrap() + .expect("Survivor at block 1"); + assert_eq!(storage_trie_vv.value.0, Some(storage_node1)); + assert_eq!(storage_trie_vv.block_number, 1); + + // Verify change sets pruned + let mut change_cur = tx.new_cursor::().unwrap(); + assert!(change_cur.seek_exact(1).unwrap().is_none()); + assert!(change_cur.seek_exact(2).unwrap().is_none()); + } + + #[test] + fn test_prune_earliest_state_churn_create_delete_recreate() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + store.set_earliest_block_number(0, B256::ZERO).unwrap(); + + let addr = B256::random(); + + // Block 1: Create + let b1 = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + let acc1 = Account { + nonce: 1, + ..Default::default() + }; + let mut diff1 = HashedPostState::default(); + diff1.accounts.insert(addr, Some(acc1)); + store + .store_trie_updates( + b1, + BlockStateDiff { + sorted_post_state: diff1.into_sorted(), + ..Default::default() + }, + ) + .unwrap(); + + // Block 2: Delete + let b2 = BlockWithParent::new(b1.block.hash, NumHash::new(2, B256::random())); + let mut diff2 = HashedPostState::default(); + diff2.accounts.insert(addr, None); + store + .store_trie_updates( + b2, + BlockStateDiff { + sorted_post_state: diff2.into_sorted(), + ..Default::default() + }, + ) + .unwrap(); + + // Block 3: Recreate + let b3 = BlockWithParent::new(b2.block.hash, NumHash::new(3, B256::random())); + let acc3 = Account { + nonce: 3, + ..Default::default() + }; + let mut diff3 = HashedPostState::default(); + diff3.accounts.insert(addr, Some(acc3)); + store + .store_trie_updates( + b3, + BlockStateDiff { + sorted_post_state: diff3.into_sorted(), + ..Default::default() + }, + ) + .unwrap(); + + // Prune to Block 3 + store.prune_earliest_state(b3).unwrap(); + + let tx = store.env.tx().unwrap(); + let mut cur = tx.new_cursor::().unwrap(); + + // Block 1 (Older than 3) should be deleted. + // Since Block 3 exists, seek(1) will land on Block 3. + if let Some(val) = cur.seek_by_key_subkey(addr, 1).unwrap() { + assert!( + val.block_number >= 3, + "Block 1 should use be pruned, found {}", + val.block_number + ); + } + + // Block 2 (Older than 3) should be deleted + if let Some(val) = cur.seek_by_key_subkey(addr, 2).unwrap() { + assert!( + val.block_number >= 3, + "Block 2 should use be pruned, found {}", + val.block_number + ); + } + + // Block 3 (Survivor) should exist + let val = cur + .seek_by_key_subkey(addr, 3) + .unwrap() + .expect("Block 3 should survive"); + assert_eq!(val.block_number, 3); + assert_eq!(val.value.0, Some(acc3)); + } + + #[test] + fn test_prune_earliest_state_returns_correct_counts() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + store.set_earliest_block_number(0, B256::ZERO).unwrap(); + + let addr = B256::random(); + + // Block 1: Insert + let b1 = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + let acc1 = Account { + nonce: 1, + ..Default::default() + }; + let mut diff1 = HashedPostState::default(); + diff1.accounts.insert(addr, Some(acc1)); + store + .store_trie_updates( + b1, + BlockStateDiff { + sorted_post_state: diff1.into_sorted(), + ..Default::default() + }, + ) + .unwrap(); + + // Block 2: Update + let b2 = BlockWithParent::new(b1.block.hash, NumHash::new(2, B256::random())); + let acc2 = Account { + nonce: 2, + ..Default::default() + }; + let mut diff2 = HashedPostState::default(); + diff2.accounts.insert(addr, Some(acc2)); + store + .store_trie_updates( + b2, + BlockStateDiff { + sorted_post_state: diff2.into_sorted(), + ..Default::default() + }, + ) + .unwrap(); + + // Prune to Block 2. + // Survivor is at Block 2. + // Block 1 should be deleted. + // Count should be 1. + let counts = store.prune_earliest_state(b2).unwrap(); + + assert_eq!(counts.hashed_accounts_written_total, 1); + assert_eq!(counts.account_trie_updates_written_total, 0); + } + + #[test] + fn test_prune_earliest_state_empty_window_updates_pointer() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + store.set_earliest_block_number(0, B256::ZERO).unwrap(); + + let target = BlockWithParent::new(B256::random(), NumHash::new(5, B256::random())); + + // Prune empty + store.prune_earliest_state(target).unwrap(); + + let earliest = store.get_earliest_block_number().unwrap(); + assert_eq!(earliest, Some((5, target.block.hash))); + } + + #[test] + fn test_prune_earliest_state_uninitialized_guard() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // Earliest not set + let target = BlockWithParent::new(B256::random(), NumHash::new(5, B256::random())); + + let counts = store.prune_earliest_state(target).unwrap(); + assert_eq!(counts, WriteCounts::default()); + + // Check earliest is still None + assert_eq!(store.get_earliest_block_number().unwrap(), None); + } + + #[test] + fn test_block_change_set_crud_operations() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + let tx = store.env.tx_mut().expect("rw tx"); + let mut cursor = tx.cursor_write::().expect("cursor"); + + let block_1 = 42u64; + let block_2 = 43u64; + + let entry1 = ChangeSet { + account_trie_keys: vec![StoredNibbles::default()], + storage_trie_keys: vec![], + hashed_account_keys: vec![B256::ZERO], + hashed_storage_keys: vec![], + }; + let entry2 = ChangeSet { + account_trie_keys: vec![], + storage_trie_keys: vec![StorageTrieKey::new(B256::ZERO, StoredNibbles::default())], + hashed_account_keys: vec![], + hashed_storage_keys: vec![HashedStorageKey::new(B256::ZERO, B256::ZERO)], + }; + + // Insert entries + cursor.insert(block_1, &entry1).unwrap(); + cursor.insert(block_2, &entry2).unwrap(); + + // Read entries + let mut walker = cursor.walk(Some(block_1)).unwrap(); + let mut entries = vec![walker.next().unwrap().unwrap().1]; + if let Some(Ok((_, val))) = walker.next() { + entries.push(val); + } + entries.sort(); + let mut expected = vec![entry1.clone(), entry2.clone()]; + expected.sort(); + assert_eq!(entries, expected); + + // Delete entry1 + let mut walker = cursor.walk(Some(block_1)).unwrap(); + while let Some(Ok((_, val))) = walker.next() { + if val == entry1 { + walker.delete_current().unwrap(); + break; + } + } + + // Verify delete + let mut walker = cursor.walk(Some(block_1)).unwrap(); + assert_eq!(walker.next().unwrap().unwrap().1, entry2); + assert!(walker.next().is_none()); + } + + #[test] + fn store_trie_updates_deleted_account_trie() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + const BLOCK: BlockWithParent = + BlockWithParent::new(B256::ZERO, NumHash::new(7, B256::ZERO)); + + // Prepare a BlockStateDiff that removes an account trie node at `acc_path` + let acc_path = Nibbles::from_nibbles_unchecked([0x0A, 0x0B, 0x0C]); + let mut diff_trie_updates = TrieUpdates::default(); + diff_trie_updates.removed_nodes.insert(acc_path); + let diff = BlockStateDiff { + sorted_trie_updates: diff_trie_updates.into_sorted(), + sorted_post_state: HashedPostStateSorted::default(), + }; + store.store_trie_updates(BLOCK, diff).expect("store"); + + // Verify deletion was written at BLOCK + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + let vv = cur + .seek_by_key_subkey(StoredNibbles::from(acc_path), BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv.block_number, BLOCK.block.number); + assert!(vv.value.0.is_none(), "expected account trie deletion"); + } + + #[test] + fn store_trie_updates_deleted_storage_trie() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + const BLOCK: BlockWithParent = + BlockWithParent::new(B256::ZERO, NumHash::new(8, B256::ZERO)); + + // Prepare a BlockStateDiff that removes a storage trie node for `addr` at `st_path` + let addr = B256::from([0xAB; 32]); + let st_path = Nibbles::from_nibbles_unchecked([0x01, 0x02, 0x03]); + + let mut diff_trie_updates = TrieUpdates::default(); + let mut st_updates = reth_trie::updates::StorageTrieUpdates::default(); + // mark this storage trie node as removed + st_updates.removed_nodes.insert(st_path); + diff_trie_updates.storage_tries.insert(addr, st_updates); + let diff = BlockStateDiff { + sorted_trie_updates: diff_trie_updates.into_sorted(), + sorted_post_state: HashedPostStateSorted::default(), + }; + store.store_trie_updates(BLOCK, diff).expect("store"); + + // Verify deletion was written at BLOCK + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + let key = StorageTrieKey::new(addr, StoredNibbles::from(st_path)); + let vv = cur + .seek_by_key_subkey(key, BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv.block_number, BLOCK.block.number); + assert!(vv.value.0.is_none(), "expected storage trie deletion"); + } + + #[test] + fn store_trie_updates_wiped_storage_trie_nodes() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr_wiped = B256::from([0x10; 32]); + let addr_live = B256::from([0xF0; 32]); + + // Seed some storage-trie nodes at block 0 for the address that will be wiped. + let p1 = Nibbles::from_nibbles_unchecked([0x01, 0x02]); + let p2 = Nibbles::from_nibbles_unchecked([0x0A, 0x0B, 0x0C]); + let n1 = BranchNodeCompact::default(); + let n2 = BranchNodeCompact::default(); + + store + .store_storage_branches(addr_wiped, vec![(p1, Some(n1)), (p2, Some(n2))]) + .expect("seed wiped addr trie nodes"); + + // Build a BlockStateDiff that wipes addr_wiped's storage trie, and + // also adds a normal storage-trie node for addr_live. + const BLOCK: BlockWithParent = + BlockWithParent::new(B256::ZERO, NumHash::new(123, B256::ZERO)); + let mut diff_trie_updates = TrieUpdates::default(); + + // Wipe for addr_wiped + let mut wiped_updates = StorageTrieUpdates::default(); + wiped_updates.set_deleted(true); + diff_trie_updates + .storage_tries + .insert(addr_wiped, wiped_updates); + + // Normal update for addr_live + let live_path = Nibbles::from_nibbles_unchecked([0xEE, 0xFF]); + let live_node = BranchNodeCompact::default(); + let mut live_updates = StorageTrieUpdates::default(); + live_updates.storage_nodes.insert(live_path, live_node); + diff_trie_updates + .storage_tries + .insert(addr_live, live_updates); + + // Execute the store + let diff = BlockStateDiff { + sorted_trie_updates: diff_trie_updates.into_sorted(), + sorted_post_state: HashedPostStateSorted::default(), + }; + store.store_trie_updates(BLOCK, diff).expect("store"); + + // Verify: for addr_wiped, each previously existing path now has a deletion tombstone at + // BLOCK. + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + for path in [p1, p2] { + let key = StorageTrieKey::new(addr_wiped, StoredNibbles::from(path)); + let vv = cur + .seek_by_key_subkey(key, BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv.block_number, BLOCK.block.number); + assert!( + vv.value.0.is_none(), + "expected tombstone at wipe block for path {path:?}" + ); + } + } + + // Verify: addr_live got its normal node written at BLOCK (not a deletion). + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + let key = StorageTrieKey::new(addr_live, StoredNibbles::from(live_path)); + let vv = cur + .seek_by_key_subkey(key, BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv.block_number, BLOCK.block.number); + assert!( + vv.value.0.is_some(), + "expected normal node for non-wiped address at BLOCK" + ); + } + } + + /// Mirror of `tests/lib.rs::test_store_trie_updates_with_wiped_storage_and_new_slots` for + /// the storage *trie* path. When `StorageTrieUpdates::is_deleted` is true AND + /// `storage_nodes` is non-empty (the shape revm/reth produce when a contract is destroyed + /// and recreated with a fresh trie in the same block), the wipe branch must tombstone every + /// pre-existing path for the address AND persist the new post-recreation nodes — with new + /// nodes winning on path collision. + #[test] + fn store_trie_updates_wiped_storage_trie_with_new_nodes() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr = B256::from([0x77; 32]); + let pre_path_dropped = Nibbles::from_nibbles_unchecked([0x01, 0x02]); + let pre_path_overwritten = Nibbles::from_nibbles_unchecked([0x0A, 0x0B]); + let new_path_kept = Nibbles::from_nibbles_unchecked([0xCC, 0xDD]); + + store + .store_storage_branches( + addr, + vec![ + (pre_path_dropped, Some(BranchNodeCompact::default())), + (pre_path_overwritten, Some(BranchNodeCompact::default())), + ], + ) + .expect("seed"); + + const BLOCK: BlockWithParent = + BlockWithParent::new(B256::ZERO, NumHash::new(123, B256::ZERO)); + let mut diff_trie_updates = TrieUpdates::default(); + let mut wiped_with_new = StorageTrieUpdates::default(); + wiped_with_new.set_deleted(true); + wiped_with_new + .storage_nodes + .insert(new_path_kept, BranchNodeCompact::default()); + wiped_with_new + .storage_nodes + .insert(pre_path_overwritten, BranchNodeCompact::default()); + diff_trie_updates.storage_tries.insert(addr, wiped_with_new); + + let diff = BlockStateDiff { + sorted_trie_updates: diff_trie_updates.into_sorted(), + sorted_post_state: HashedPostStateSorted::default(), + }; + store.store_trie_updates(BLOCK, diff).expect("store"); + + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + let dropped_key = StorageTrieKey::new(addr, StoredNibbles::from(pre_path_dropped)); + let dropped_vv = cur + .seek_by_key_subkey(dropped_key, BLOCK.block.number) + .expect("seek") + .expect("tombstone exists"); + assert_eq!(dropped_vv.block_number, BLOCK.block.number); + assert!( + dropped_vv.value.0.is_none(), + "pre-existing path with no new node must tombstone" + ); + + let overwritten_key = StorageTrieKey::new(addr, StoredNibbles::from(pre_path_overwritten)); + let overwritten_vv = cur + .seek_by_key_subkey(overwritten_key, BLOCK.block.number) + .expect("seek") + .expect("overlay exists"); + assert_eq!(overwritten_vv.block_number, BLOCK.block.number); + assert!( + overwritten_vv.value.0.is_some(), + "collision path must reflect the new node, not the wipe tombstone", + ); + + let new_key = StorageTrieKey::new(addr, StoredNibbles::from(new_path_kept)); + let new_vv = cur + .seek_by_key_subkey(new_key, BLOCK.block.number) + .expect("seek") + .expect("new node exists"); + assert_eq!(new_vv.block_number, BLOCK.block.number); + assert!( + new_vv.value.0.is_some(), + "new path must be persisted, not dropped by the wipe" + ); + } + + #[test] + fn store_trie_updates_wiped_storage() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // We'll pre-seed storage at block 0, then issue a wipe at BLOCK. + const BLOCK: BlockWithParent = + BlockWithParent::new(B256::ZERO, NumHash::new(42, B256::ZERO)); + + let addr = B256::from([0x55; 32]); + let s1 = B256::from([0x01; 32]); + let s2 = B256::from([0x02; 32]); + let v1 = U256::from(111u64); + let v2 = U256::from(222u64); + + // Seed prior storage (block_number = 0 in store_hashed_storages) + store + .store_hashed_storages(addr, vec![(s1, v1), (s2, v2)]) + .expect("seed"); + + // Build BlockStateDiff that marks this address as wiped at BLOCK + let mut diff_post_state = HashedPostState::default(); + + let wiped = reth_trie::HashedStorage::new(true); + + diff_post_state.storages.insert(addr, wiped); + + // Execute + let diff = BlockStateDiff { + sorted_trie_updates: TrieUpdatesSorted::default(), + sorted_post_state: diff_post_state.into_sorted(), + }; + store.store_trie_updates(BLOCK, diff).expect("store"); + + // Verify: for each pre-existing slot, there should be a tombstone (MaybeDeleted(None)) at + // BLOCK. + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + for slot in [s1, s2] { + let key = HashedStorageKey::new(addr, slot); + let vv = cur + .seek_by_key_subkey(key, BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv.block_number, BLOCK.block.number); + assert!( + vv.value.0.is_none(), + "expected deletion tombstone for slot {:?} at block {}", + slot, + BLOCK.block.number, + ); + } + } + + #[test] + fn store_trie_updates_wiped_and_non_wiped_mixed_order() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // Choose addresses so that wiped < non_wiped in sort order (your impl sorts by address) + let addr_wiped = B256::from([0x01; 32]); // will sort first + let addr_live = B256::from([0xF0; 32]); // will sort later + + // Slots & values + let ws1 = B256::from([0xA1; 32]); + let ws2 = B256::from([0xA2; 32]); + let wv1 = U256::from(111u64); + let wv2 = U256::from(222u64); + + let ls1 = B256::from([0xB1; 32]); + let lv1_old = U256::from(333u64); + let lv1_new = U256::from(999u64); // will be written at BLOCK + + // Seed prior storage at block 0 for BOTH addresses + store + .store_hashed_storages(addr_wiped, vec![(ws1, wv1), (ws2, wv2)]) + .expect("seed wiped addr"); + store + .store_hashed_storages(addr_live, vec![(ls1, lv1_old)]) + .expect("seed live addr"); + + // Build diff: wiped first (by address sort), then non-wiped with a write + const BLOCK: BlockWithParent = + BlockWithParent::new(B256::ZERO, NumHash::new(77, B256::ZERO)); + let mut diff_post_state = HashedPostState::default(); + + // Wiped storage for addr_wiped + let wiped = reth_trie::HashedStorage::new(true); + diff_post_state.storages.insert(addr_wiped, wiped); + + // Non-wiped storage for addr_live (append new value) + let mut live = reth_trie::HashedStorage::default(); + live.storage.insert(ls1, lv1_new); + diff_post_state.storages.insert(addr_live, live); + + // Execute + let diff = BlockStateDiff { + sorted_trie_updates: TrieUpdatesSorted::default(), + sorted_post_state: diff_post_state.into_sorted(), + }; + store.store_trie_updates(BLOCK, diff).expect("store"); + + // Verify: wiped address got tombstones at BLOCK for each pre-existing slot + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + for slot in [ws1, ws2] { + let key = HashedStorageKey::new(addr_wiped, slot); + let vv = cur + .seek_by_key_subkey(key, BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv.block_number, BLOCK.block.number); + assert!( + vv.value.0.is_none(), + "expected deletion tombstone for wiped slot {:?} at block {}", + slot, + BLOCK.block.number, + ); + } + } + + // Verify: non-wiped address got the new value at BLOCK (not a deletion) + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + let key = HashedStorageKey::new(addr_live, ls1); + let vv = cur + .seek_by_key_subkey(key, BLOCK.block.number) + .expect("seek") + .expect("exists"); + assert_eq!(vv.block_number, BLOCK.block.number); + let inner = vv.value.0.as_ref().expect("Some(StorageValue)"); + assert_eq!( + inner.0, lv1_new, + "expected updated value for non-wiped address" + ); + } + } + + #[test] + fn test_proof_window() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // Test initial state (no values set) + let initial_value = store.get_earliest_block_number().expect("get earliest"); + assert_eq!(initial_value, None); + + // Test setting the value + let block_number = 42u64; + let hash = B256::random(); + store + .set_earliest_block_number(block_number, hash) + .expect("set earliest"); + + // Verify value was stored correctly + let retrieved = store.get_earliest_block_number().expect("get earliest"); + assert_eq!(retrieved, Some((block_number, hash))); + + // Test updating with new values + let new_block_number = 100u64; + let new_hash = B256::random(); + store + .set_earliest_block_number(new_block_number, new_hash) + .expect("update earliest"); + + // Verify update worked + let updated = store + .get_earliest_block_number() + .expect("get updated earliest"); + assert_eq!(updated, Some((new_block_number, new_hash))); + + // Verify that latest_block falls back to earliest when not set + let latest = store.get_latest_block_number().expect("get latest"); + assert_eq!( + latest, + Some((new_block_number, new_hash)), + "Latest block should fall back to earliest when not explicitly set" + ); + } + + #[test] + fn replace_updates_prunes_and_adds_new_chain() { + let dir = tempfile::TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // Test address and helper to make diffs with distinct nonces. + let addr = B256::from([0xAB; 32]); + let make_diff = |nonce: u64| { + let mut d_post_state = HashedPostState::default(); + d_post_state.accounts.insert( + addr, + Some(Account { + nonce, + ..Default::default() + }), + ); + BlockStateDiff { + sorted_trie_updates: TrieUpdatesSorted::default(), + sorted_post_state: d_post_state.into_sorted(), + } + }; + + // --- Build initial canonical chain: 1 -> 2 -> 3 --- + let b1 = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + let b2 = BlockWithParent::new(b1.block.hash, NumHash::new(2, B256::random())); + let b3 = BlockWithParent::new(b2.block.hash, NumHash::new(3, B256::random())); + + store + .store_trie_updates(b1, make_diff(10)) + .expect("store b1"); + store + .store_trie_updates(b2, make_diff(20)) + .expect("store b2"); + store + .store_trie_updates(b3, make_diff(30)) + .expect("store b3"); + + // Sanity: entries for 1,2,3 exist with expected nonces. + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + let v1 = cur + .seek_by_key_subkey(addr, 1) + .expect("seek") + .expect("exists"); + let v2 = cur + .seek_by_key_subkey(addr, 2) + .expect("seek") + .expect("exists"); + let v3 = cur + .seek_by_key_subkey(addr, 3) + .expect("seek") + .expect("exists"); + assert_eq!(v1.value.0.unwrap().nonce, 10); + assert_eq!(v2.value.0.unwrap().nonce, 20); + assert_eq!(v3.value.0.unwrap().nonce, 30); + } + + // --- Reorg at LCA = 2: prune >2, then add 3' and 4' --- + let b3p = BlockWithParent::new(b2.block.hash, NumHash::new(3, B256::random())); // 3' + let b4p = BlockWithParent::new(b3p.block.hash, NumHash::new(4, B256::random())); // 4' + + // Build blocks_to_add vec. + let blocks_to_add = vec![(b3p, make_diff(300)), (b4p, make_diff(400))]; + + store + .replace_updates(BlockNumHash::new(2, b2.block.hash), blocks_to_add) + .expect("replace_updates succeeds"); + + // --- Verify post-conditions --- + + // 1) HashedAccountHistory: blocks 1,2 remain; block 3 was replaced (nonce=300); block 4 + // exists (nonce=400). + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + let v1 = cur + .seek_by_key_subkey(addr, 1) + .expect("seek") + .expect("exists"); + assert_eq!(v1.value.0.unwrap().nonce, 10); + + let v2 = cur + .seek_by_key_subkey(addr, 2) + .expect("seek") + .expect("exists"); + assert_eq!(v2.value.0.unwrap().nonce, 20); + + // Old block 3 (nonce=30) should have been pruned; we should now have a fresh entry at 3 + // with nonce=300. + let v3_new = cur + .seek_by_key_subkey(addr, 3) + .expect("seek") + .expect("replaced exists"); + assert_eq!( + v3_new.value.0.unwrap().nonce, + 300, + "block 3 should be replaced by new chain" + ); + + // New block 4 should exist. + let v4 = cur + .seek_by_key_subkey(addr, 4) + .expect("seek") + .expect("exists"); + assert_eq!(v4.value.0.unwrap().nonce, 400); + } + + // 2) BlockChangeSet contains exactly entries for block numbers {1,2,3,4}. + { + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + let mut seen = std::collections::BTreeSet::new(); + let mut it = cur.walk(Some(1)).expect("walk"); + while let Some(Ok((bn, _))) = it.next() { + seen.insert(bn); + } + let expected: std::collections::BTreeSet = [1u64, 2, 3, 4].into_iter().collect(); + assert_eq!( + seen, expected, + "BlockChangeSet should reflect pruned+new chain" + ); + } + } + + #[test] + fn test_unwind_history_basic() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr = B256::random(); + let make_diff = |nonce: u64| { + let mut d_post_state = HashedPostState::default(); + d_post_state.accounts.insert( + addr, + Some(Account { + nonce, + ..Default::default() + }), + ); + BlockStateDiff { + sorted_trie_updates: TrieUpdatesSorted::default(), + sorted_post_state: d_post_state.into_sorted(), + } + }; + + // Build chain: blocks 1 -> 2 -> 3 -> 4 + let b0 = NumHash::new(0, B256::random()); + let b1 = BlockWithParent::new(b0.hash, NumHash::new(1, B256::random())); + let b2 = BlockWithParent::new(b1.block.hash, NumHash::new(2, B256::random())); + let b3 = BlockWithParent::new(b2.block.hash, NumHash::new(3, B256::random())); + let b4 = BlockWithParent::new(b3.block.hash, NumHash::new(4, B256::random())); + + store + .set_earliest_block_number_hash(b0.number, b0.hash) + .expect("set earliest"); + store + .store_trie_updates(b1, make_diff(10)) + .expect("store b1"); + store + .store_trie_updates(b2, make_diff(20)) + .expect("store b2"); + store + .store_trie_updates(b3, make_diff(30)) + .expect("store b3"); + store + .store_trie_updates(b4, make_diff(40)) + .expect("store b4"); + + // Unwind to block 2 + store.unwind_history(b2).expect("unwind"); + + // Verify: blocks 1 and 2 remain, blocks 3 and 4 are removed + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + assert!( + cur.seek_by_key_subkey(addr, 1).unwrap().is_some(), + "Block 1 should remain" + ); + assert!( + cur.seek_by_key_subkey(addr, 2).unwrap().is_none(), + "Block 2 should be removed" + ); + assert!( + cur.seek_by_key_subkey(addr, 3).unwrap().is_none(), + "Block 3 should be removed" + ); + assert!( + cur.seek_by_key_subkey(addr, 4).unwrap().is_none(), + "Block 4 should be removed" + ); + + // Verify ProofWindow LatestBlock is updated + let mut proof_window_cur = tx.cursor_read::().expect("cursor"); + let latest = proof_window_cur + .seek_exact(ProofWindowKey::LatestBlock) + .expect("seek") + .expect("latest exists"); + assert_eq!(latest.1.number(), 1); + assert_eq!(*latest.1.hash(), b2.parent); + } + + #[test] + fn test_unwind_history_to_earliest() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr = B256::random(); + let make_diff = |nonce: u64| { + let mut d_post_state = HashedPostState::default(); + d_post_state.accounts.insert( + addr, + Some(Account { + nonce, + ..Default::default() + }), + ); + BlockStateDiff { + sorted_trie_updates: TrieUpdatesSorted::default(), + sorted_post_state: d_post_state.into_sorted(), + } + }; + + // Build chain: blocks 1 -> 2 -> 3 + let b1 = BlockWithParent::new(B256::ZERO, NumHash::new(1, B256::random())); + let b2 = BlockWithParent::new(b1.block.hash, NumHash::new(2, B256::random())); + let b3 = BlockWithParent::new(b2.block.hash, NumHash::new(3, B256::random())); + + store + .set_earliest_block_number_hash(b1.block.number, b1.block.hash) + .expect("set earliest"); + store + .store_trie_updates(b2, make_diff(20)) + .expect("store b2"); + store + .store_trie_updates(b3, make_diff(30)) + .expect("store b3"); + + // Unwind to block b1 + let res = store.unwind_history(b1); + // should fail as we cannot unwind past earliest block + assert!(res.is_err(), "unwind to earliest block should error"); + assert!(matches!( + res.unwrap_err(), + MorphProofsStorageError::UnwindBeyondEarliest { .. } + )); + } + + #[test] + fn test_unwind_history_with_storage() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr = B256::random(); + let slot = B256::random(); + + let make_diff = |nonce: u64, slot_value: u64| { + let mut d_post_state = HashedPostState::default(); + d_post_state.accounts.insert( + addr, + Some(Account { + nonce, + ..Default::default() + }), + ); + let mut storage = HashedStorage::default(); + storage.storage.insert(slot, U256::from(slot_value)); + d_post_state.storages.insert(addr, storage); + BlockStateDiff { + sorted_trie_updates: TrieUpdatesSorted::default(), + sorted_post_state: d_post_state.into_sorted(), + } + }; + + // Build chain with storage changes + let b0 = BlockWithParent::new(B256::ZERO, NumHash::new(0, B256::random())); + let b1 = BlockWithParent::new(b0.block.hash, NumHash::new(1, B256::random())); + let b2 = BlockWithParent::new(b1.block.hash, NumHash::new(2, B256::random())); + let b3 = BlockWithParent::new(b2.block.hash, NumHash::new(3, B256::random())); + + store + .set_earliest_block_number_hash(b0.block.number, b0.block.hash) + .expect("set earliest"); + store + .store_trie_updates(b1, make_diff(10, 100)) + .expect("store b1"); + store + .store_trie_updates(b2, make_diff(20, 200)) + .expect("store b2"); + store + .store_trie_updates(b3, make_diff(30, 300)) + .expect("store b3"); + + // Unwind to block 1 + store.unwind_history(b2).expect("unwind"); + + // Verify account history + let tx = store.env.tx().expect("tx"); + let mut acc_cur = tx.new_cursor::().expect("cursor"); + assert!( + acc_cur.seek_by_key_subkey(addr, 1).unwrap().is_some(), + "Block 1 should remain" + ); + assert!( + acc_cur.seek_by_key_subkey(addr, 2).unwrap().is_none(), + "Block 2 should be removed" + ); + assert!( + acc_cur.seek_by_key_subkey(addr, 3).unwrap().is_none(), + "Block 3 should be removed" + ); + + // Verify storage history + let mut storage_cur = tx.new_cursor::().expect("cursor"); + let storage_key = HashedStorageKey::new(addr, slot); + assert!( + storage_cur + .seek_by_key_subkey(storage_key.clone(), 1) + .unwrap() + .is_some(), + "Storage at block 1 should remain" + ); + assert!( + storage_cur + .seek_by_key_subkey(storage_key.clone(), 2) + .unwrap() + .is_none(), + "Storage at block 2 should be removed" + ); + assert!( + storage_cur + .seek_by_key_subkey(storage_key, 3) + .unwrap() + .is_none(), + "Storage at block 3 should be removed" + ); + } + + #[test] + fn test_unwind_history_with_trie_nodes() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let path1 = Nibbles::from_nibbles_unchecked([0x01]); + let path2 = Nibbles::from_nibbles_unchecked([0x02]); + let node1 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let node2 = BranchNodeCompact::new(0b10, 0, 0, vec![], Some(B256::random())); + + let make_diff = |path: Nibbles, node: BranchNodeCompact| { + let mut d_trie_updates = TrieUpdates::default(); + d_trie_updates.account_nodes.insert(path, node); + BlockStateDiff { + sorted_trie_updates: d_trie_updates.into_sorted(), + sorted_post_state: HashedPostStateSorted::default(), + } + }; + + // Build chain with trie updates + let b0 = NumHash::new(0, B256::random()); + let b1 = BlockWithParent::new(b0.hash, NumHash::new(1, B256::random())); + let b2 = BlockWithParent::new(b1.block.hash, NumHash::new(2, B256::random())); + let b3 = BlockWithParent::new(b2.block.hash, NumHash::new(3, B256::random())); + + store + .set_earliest_block_number_hash(b0.number, b0.hash) + .expect("set earliest"); + store + .store_trie_updates(b1, make_diff(path1, node1)) + .expect("store b1"); + store + .store_trie_updates(b2, make_diff(path2, node2.clone())) + .expect("store b2"); + store + .store_trie_updates(b3, make_diff(path1, node2)) + .expect("store b3"); + + // Unwind to block 1 + store.unwind_history(b2).expect("unwind"); + + // Verify trie node history + let tx = store.env.tx().expect("tx"); + let mut trie_cur = tx.cursor_dup_read::().expect("cursor"); + + assert!( + trie_cur + .seek_by_key_subkey(StoredNibbles::from(path1), 1) + .unwrap() + .is_some(), + "Trie node at block 1 should remain" + ); + assert!( + trie_cur + .seek_by_key_subkey(StoredNibbles::from(path2), 2) + .unwrap() + .is_none(), + "Trie node at block 2 should be removed" + ); + assert!( + trie_cur + .seek_by_key_subkey(StoredNibbles::from(path1), 3) + .unwrap() + .is_none(), + "Trie node at block 3 should be removed" + ); + } + + #[test] + fn test_unwind_history_comprehensive() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // Setup comprehensive scenario with accounts, storage, and trie nodes + let addr1 = B256::random(); + let addr2 = B256::random(); + let slot1 = B256::random(); + let slot2 = B256::random(); + let path1 = Nibbles::from_nibbles_unchecked([0x01]); + let path2 = Nibbles::from_nibbles_unchecked([0x02]); + let storage_path1 = Nibbles::from_nibbles_unchecked([0x03]); + + let acc1 = Account { + nonce: 1, + balance: U256::from(100), + ..Default::default() + }; + let acc2 = Account { + nonce: 2, + balance: U256::from(200), + ..Default::default() + }; + let node1 = BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::random())); + let node2 = BranchNodeCompact::new(0b10, 0, 0, vec![], Some(B256::random())); + let storage_node1 = BranchNodeCompact::new(0b100, 0, 0, vec![], Some(B256::random())); + + // Block 0: Set earliest block + let b0 = NumHash::new(0, B256::random()); + store + .set_earliest_block_number_hash(b0.number, b0.hash) + .expect("set earliest"); + + // Block 1: Insert multiple types of data + let b1 = BlockWithParent::new(b0.hash, NumHash::new(1, B256::random())); + let mut diff1_trie_updates = TrieUpdates::default(); + let mut diff1_post_state = HashedPostState::default(); + diff1_post_state.accounts.insert(addr1, Some(acc1)); + diff1_trie_updates.account_nodes.insert(path1, node1); + let mut storage1 = HashedStorage::default(); + storage1.storage.insert(slot1, U256::from(1111)); + diff1_post_state.storages.insert(addr1, storage1); + let mut storage_updates1 = StorageTrieUpdates::default(); + storage_updates1 + .storage_nodes + .insert(storage_path1, storage_node1); + diff1_trie_updates + .storage_tries + .insert(addr1, storage_updates1); + let diff1 = BlockStateDiff { + sorted_trie_updates: diff1_trie_updates.into_sorted(), + sorted_post_state: diff1_post_state.into_sorted(), + }; + store.store_trie_updates(b1, diff1).expect("store b1"); + + // Block 2: More updates + let b2 = BlockWithParent::new(b1.block.hash, NumHash::new(2, B256::random())); + let mut diff2_trie_updates = TrieUpdates::default(); + let mut diff2_post_state = HashedPostState::default(); + diff2_post_state.accounts.insert(addr2, Some(acc2)); + diff2_trie_updates.account_nodes.insert(path2, node2); + let mut storage2 = HashedStorage::default(); + storage2.storage.insert(slot2, U256::from(2222)); + diff2_post_state.storages.insert(addr2, storage2); + let diff2 = BlockStateDiff { + sorted_trie_updates: diff2_trie_updates.into_sorted(), + sorted_post_state: diff2_post_state.into_sorted(), + }; + store.store_trie_updates(b2, diff2).expect("store b2"); + + // Block 3: Additional updates + let b3 = BlockWithParent::new(b2.block.hash, NumHash::new(3, B256::random())); + let mut diff3_post_state = HashedPostState::default(); + diff3_post_state.accounts.insert(addr1, Some(acc2)); // update addr1 + let diff3 = BlockStateDiff { + sorted_trie_updates: TrieUpdatesSorted::default(), + sorted_post_state: diff3_post_state.into_sorted(), + }; + store.store_trie_updates(b3, diff3).expect("store b3"); + + // Unwind to block 1 + store.unwind_history(b2).expect("unwind"); + + let tx = store.env.tx().expect("tx"); + + // Verify account history: block 1 remains, blocks 2 and 3 removed + let mut acc_cur = tx.new_cursor::().expect("cursor"); + assert!(acc_cur.seek_by_key_subkey(addr1, 1).unwrap().is_some()); + assert!(acc_cur.seek_by_key_subkey(addr2, 2).unwrap().is_none()); + assert!(acc_cur.seek_by_key_subkey(addr1, 3).unwrap().is_none()); + + // Verify trie history + let mut trie_cur = tx.cursor_dup_read::().expect("cursor"); + assert!( + trie_cur + .seek_by_key_subkey(StoredNibbles::from(path1), 1) + .unwrap() + .is_some() + ); + assert!( + trie_cur + .seek_by_key_subkey(StoredNibbles::from(path2), 2) + .unwrap() + .is_none() + ); + + // Verify storage history + let mut storage_cur = tx.new_cursor::().expect("cursor"); + assert!( + storage_cur + .seek_by_key_subkey(HashedStorageKey::new(addr1, slot1), 1) + .unwrap() + .is_some() + ); + assert!( + storage_cur + .seek_by_key_subkey(HashedStorageKey::new(addr2, slot2), 2) + .unwrap() + .is_none() + ); + + // Verify storage trie history + let mut storage_trie_cur = tx.cursor_dup_read::().expect("cursor"); + assert!( + storage_trie_cur + .seek_by_key_subkey( + StorageTrieKey::new(addr1, StoredNibbles::from(storage_path1)), + 1 + ) + .unwrap() + .is_some() + ); + + // Verify ProofWindow LatestBlock is updated + let mut proof_window_cur = tx.cursor_read::().expect("cursor"); + let latest = proof_window_cur + .seek_exact(ProofWindowKey::LatestBlock) + .expect("seek") + .expect("latest exists"); + assert_eq!(latest.1.number(), 1); + assert_eq!(*latest.1.hash(), b1.block.hash); + + // Verify change sets are removed for blocks > 1 + let mut change_cur = tx.new_cursor::().expect("cursor"); + assert!( + change_cur.seek_exact(1).unwrap().is_some(), + "Block 1 changeset should remain" + ); + assert!( + change_cur.seek_exact(2).unwrap().is_none(), + "Block 2 changeset should be removed" + ); + assert!( + change_cur.seek_exact(3).unwrap().is_none(), + "Block 3 changeset should be removed" + ); + } + + #[test] + fn test_unwind_history_empty_chain() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + // Try to unwind when there's nothing stored yet + let unwind_to = BlockWithParent::new(B256::ZERO, NumHash::new(0, B256::ZERO)); + let result = store.unwind_history(unwind_to); + + // Should succeed (no-op) + assert!(result.is_ok(), "Unwinding empty chain should succeed"); + } + + #[test] + fn test_unwind_history_idempotent() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr = B256::random(); + let make_diff = |nonce: u64| { + let mut d_post_state = HashedPostState::default(); + d_post_state.accounts.insert( + addr, + Some(Account { + nonce, + ..Default::default() + }), + ); + BlockStateDiff { + sorted_trie_updates: TrieUpdatesSorted::default(), + sorted_post_state: d_post_state.into_sorted(), + } + }; + + // Build chain: blocks 1 -> 2 -> 3 + let b0 = NumHash::new(0, B256::random()); + let b1 = BlockWithParent::new(b0.hash, NumHash::new(1, B256::random())); + let b2 = BlockWithParent::new(b1.block.hash, NumHash::new(2, B256::random())); + let b3 = BlockWithParent::new(b2.block.hash, NumHash::new(3, B256::random())); + + store + .set_earliest_block_number_hash(b0.number, b0.hash) + .expect("set earliest"); + store + .store_trie_updates(b1, make_diff(10)) + .expect("store b1"); + store + .store_trie_updates(b2, make_diff(20)) + .expect("store b2"); + store + .store_trie_updates(b3, make_diff(30)) + .expect("store b3"); + + // Unwind to block 1 + store.unwind_history(b2).expect("first unwind"); + + // Unwind again to the same block (should be idempotent) + store.unwind_history(b2).expect("second unwind"); + + // Verify state is still correct + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + assert!( + cur.seek_by_key_subkey(addr, 1).unwrap().is_some(), + "Block 1 should remain" + ); + assert!( + cur.seek_by_key_subkey(addr, 2).unwrap().is_none(), + "Block 2 should be removed" + ); + assert!( + cur.seek_by_key_subkey(addr, 3).unwrap().is_none(), + "Block 3 should be removed" + ); + } + + #[test] + fn test_unwind_history_beyond_latest() { + let dir = TempDir::new().unwrap(); + let store = MdbxProofsStorage::new(dir.path()).expect("env"); + + let addr = B256::random(); + let make_diff = |nonce: u64| { + let mut d_post_state = HashedPostState::default(); + d_post_state.accounts.insert( + addr, + Some(Account { + nonce, + ..Default::default() + }), + ); + BlockStateDiff { + sorted_trie_updates: TrieUpdatesSorted::default(), + sorted_post_state: d_post_state.into_sorted(), + } + }; + + // Build chain: blocks 1 -> 2 -> 3 + let b0 = NumHash::new(0, B256::random()); + let b1 = BlockWithParent::new(b0.hash, NumHash::new(1, B256::random())); + let b2 = BlockWithParent::new(b1.block.hash, NumHash::new(2, B256::random())); + let b3 = BlockWithParent::new(b2.block.hash, NumHash::new(3, B256::random())); + let b4 = BlockWithParent::new(b3.block.hash, NumHash::new(4, B256::random())); + let b5 = BlockWithParent::new(b4.block.hash, NumHash::new(5, B256::random())); + + store + .set_earliest_block_number_hash(b0.number, b0.hash) + .expect("set earliest"); + store + .store_trie_updates(b1, make_diff(10)) + .expect("store b1"); + store + .store_trie_updates(b2, make_diff(20)) + .expect("store b2"); + store + .store_trie_updates(b3, make_diff(30)) + .expect("store b3"); + + // Unwind to block 1 + store.unwind_history(b5).expect("first unwind"); + + // Verify state is still correct + let tx = store.env.tx().expect("tx"); + let mut cur = tx.new_cursor::().expect("cursor"); + + assert!( + cur.seek_by_key_subkey(addr, 1).unwrap().is_some(), + "Block 1 should remain" + ); + assert!( + cur.seek_by_key_subkey(addr, 2).unwrap().is_some(), + "Block 2 should remain" + ); + assert!( + cur.seek_by_key_subkey(addr, 3).unwrap().is_some(), + "Block 3 should remain" + ); + + let mut proof_window_cur = tx.cursor_read::().expect("cursor"); + let latest = proof_window_cur + .seek_exact(ProofWindowKey::LatestBlock) + .expect("seek") + .expect("latest exists"); + assert_eq!(latest.1.number(), b3.block.number); + assert_eq!(*latest.1.hash(), b3.block.hash); + } +} diff --git a/crates/proofs/src/error.rs b/crates/proofs/src/error.rs new file mode 100644 index 00000000..03a3aee6 --- /dev/null +++ b/crates/proofs/src/error.rs @@ -0,0 +1,233 @@ +//! Errors interfacing with [`MorphProofsStore`](crate::MorphProofsStore) type. + +use std::sync::Arc; + +use alloy_primitives::B256; +use reth_codecs::DecompressError; +use reth_db::DatabaseError; +use reth_execution_errors::BlockExecutionError; +use reth_provider::ProviderError; +use reth_trie_common::Nibbles; +use thiserror::Error; + +/// Error type for storage operations +#[derive(Debug, Clone, Error)] +pub enum MorphProofsStorageError { + /// The proof DB belongs to another canonical chain. + #[error("proof database chain identity mismatch: {0}")] + ChainIdentityMismatch(&'static str), + /// The proof DB schema cannot be read by this binary. + #[error("proof database schema mismatch: expected {expected}, found {actual}")] + SchemaMismatch { + /// Schema expected by this binary. + expected: u32, + /// Schema persisted in the database. + actual: u32, + }, + /// Durable metadata is missing or malformed. + #[error("corrupt proof database metadata: {0}")] + CorruptMetadata(&'static str), + /// Only part of the durable identity metadata is present. + #[error( + "proof database identity metadata is incomplete; manually delete the proof database before reinitializing" + )] + IncompleteIdentityMetadata, + /// Proof/history data exists without the metadata needed to validate its identity. + #[error( + "proof database contains proof/history data but identity metadata is missing; manually delete the proof database before reinitializing" + )] + ProofDataWithoutIdentityMetadata, + /// No blocks found + #[error("No blocks found")] + NoBlocksFound, + /// Parent block number is less than earliest stored block number + #[error("Parent block number is less than earliest stored block number")] + UnknownParent, + /// Block is out of order + #[error( + "Block {block_number} is out of order (parent: {parent_block_hash}, latest stored block hash: {latest_block_hash})" + )] + OutOfOrder { + /// The block number being inserted + block_number: u64, + /// The parent hash of the block being inserted + parent_block_hash: B256, + /// block hash of the latest stored block + latest_block_hash: B256, + }, + /// Block update failed since parent state + #[error( + "Cannot execute block updates for block {block_number} without parent state {parent_block_number} (latest stored block number: {latest_block_number})" + )] + MissingParentBlock { + /// The block number being executed + block_number: u64, + /// The parent state of the block being executed + parent_block_number: u64, + /// Latest stored block number + latest_block_number: u64, + }, + /// State root mismatch + #[error( + "State root mismatch for block {block_number} (have: {current_state_hash}, expected: {expected_state_hash})" + )] + StateRootMismatch { + /// Block number + block_number: u64, + /// Have state root + current_state_hash: B256, + /// Expected state root + expected_state_hash: B256, + }, + /// No change set for block + #[error("No change set found for block {0}")] + NoChangeSetForBlock(u64), + /// Missing account trie history for a specific path at a specific block number + #[error("Missing account trie history for path {0:?} at block {1}")] + MissingAccountTrieHistory(Nibbles, u64), + /// Missing storage trie history for a specific address and path at a specific block number + #[error("Missing storage trie history for address {0:?}, path {1:?} at block {2}")] + MissingStorageTrieHistory(B256, Nibbles, u64), + /// Missing hashed account history for a specific key at a specific block number + #[error("Missing hashed account history for key {0:?} at block {1}")] + MissingHashedAccountHistory(B256, u64), + /// Missing hashed storage history for a specific address and key at a specific block number + #[error( + "Missing hashed storage history for address {hashed_address:?}, key {hashed_storage_key:?} at block {block_number}" + )] + MissingHashedStorageHistory { + /// The hashed address + hashed_address: B256, + /// The hashed storage key + hashed_storage_key: B256, + /// The block number + block_number: u64, + }, + /// Attempted to unwind to a block beyond the earliest stored block + #[error( + "Attempted to unwind to block {unwind_block_number} beyond earliest stored block {earliest_block_number}" + )] + UnwindBeyondEarliest { + /// The block number being unwound to + unwind_block_number: u64, + /// The earliest stored block number + earliest_block_number: u64, + }, + /// Error occurred while interacting with the database. + #[error(transparent)] + DatabaseError(DatabaseError), + /// Error occurred while trying to acquire a lock. + #[error("failed lock attempt")] + TryLockError, + /// Error occurred during block execution. + #[error(transparent)] + ExecutionError(Arc), + /// Error occurred while interacting with the provider. + #[error(transparent)] + ProviderError(Arc), + /// Initialization detected inconsistent state between proofs storage and source DB. + #[error( + "Initialization Proofs storage detected inconsistent state. Storage does not match source DB. \ + Please clear proofs data and retry initialization." + )] + InitializeStorageInconsistentState, + /// Batch session used after its underlying transaction has been committed or aborted. + #[error("Batch session used after its underlying transaction was closed")] + BatchSessionClosed, +} + +impl From for MorphProofsStorageError { + fn from(error: BlockExecutionError) -> Self { + Self::ExecutionError(Arc::new(error)) + } +} + +impl From for MorphProofsStorageError { + fn from(error: ProviderError) -> Self { + Self::ProviderError(Arc::new(error)) + } +} + +impl From for DatabaseError { + fn from(error: MorphProofsStorageError) -> Self { + match error { + MorphProofsStorageError::DatabaseError(err) => err, + _ => Self::Custom(Arc::new(error)), + } + } +} + +impl From for MorphProofsStorageError { + fn from(error: DatabaseError) -> Self { + if let DatabaseError::Custom(ref err) = error + && let Some(err) = err.downcast_ref::() + { + return err.clone(); + } + Self::DatabaseError(error) + } +} + +impl From for MorphProofsStorageError { + fn from(error: DecompressError) -> Self { + Self::DatabaseError(DatabaseError::Other(format!("decompress error: {error}"))) + } +} + +/// Result type for storage operations +pub type MorphProofsStorageResult = Result; + +#[cfg(test)] +mod tests { + use reth_execution_errors::BlockValidationError; + + use super::*; + + #[test] + fn test_morph_proofs_store_error_to_db_error() { + let original_error = MorphProofsStorageError::NoBlocksFound; + + let db_error = DatabaseError::from(original_error); + assert!(matches!(db_error, DatabaseError::Custom(_))); + + let converted_error = MorphProofsStorageError::from(db_error); + assert!(matches!( + converted_error, + MorphProofsStorageError::NoBlocksFound + )) + } + + #[test] + fn test_db_error_to_morph_proofs_store_error() { + let original_error = DatabaseError::Decode; + + let morph_proofs_store_error = MorphProofsStorageError::from(original_error); + assert!(matches!( + morph_proofs_store_error, + MorphProofsStorageError::DatabaseError(DatabaseError::Decode) + )); + + let converted_error = DatabaseError::from(morph_proofs_store_error); + assert!(matches!(converted_error, DatabaseError::Decode)) + } + + #[test] + fn test_conversion_from_block_execution_error() { + let block_execution_error = + BlockExecutionError::Validation(BlockValidationError::IncrementBalanceFailed); + + let morph_proofs_store_error = MorphProofsStorageError::from(block_execution_error); + assert!( + matches!(morph_proofs_store_error, MorphProofsStorageError::ExecutionError(err) if matches!(*err, BlockExecutionError::Validation(BlockValidationError::IncrementBalanceFailed))) + ) + } + + #[test] + fn test_conversion_from_provider_error() { + let provider_error = ProviderError::SenderRecoveryError; + let morph_proofs_store_error = MorphProofsStorageError::from(provider_error); + assert!( + matches!(morph_proofs_store_error, MorphProofsStorageError::ProviderError(err) if matches!(*err, ProviderError::SenderRecoveryError)) + ) + } +} diff --git a/crates/proofs/src/in_memory.rs b/crates/proofs/src/in_memory.rs new file mode 100644 index 00000000..08ac6787 --- /dev/null +++ b/crates/proofs/src/in_memory.rs @@ -0,0 +1,1138 @@ +//! In-memory implementation of [`MorphProofsStore`] for testing purposes + +use std::{collections::BTreeMap, sync::Arc}; + +use alloy_eips::{BlockNumHash, NumHash, eip1898::BlockWithParent}; +use alloy_primitives::{B256, U256}; +use parking_lot::RwLock; +use reth_db::DatabaseError; +use reth_primitives_traits::Account; +use reth_trie::{ + hashed_cursor::{HashedCursor, HashedStorageCursor}, + trie_cursor::{TrieCursor, TrieStorageCursor}, +}; +use reth_trie_common::{ + BranchNodeCompact, HashedPostStateSorted, Nibbles, StoredNibbles, updates::TrieUpdatesSorted, +}; + +use crate::{ + BlockStateDiff, MorphProofsStorageError, MorphProofsStorageResult, MorphProofsStore, + api::{ + InitialStateAnchor, InitialStateStatus, MorphProofsBatchSession, MorphProofsBatchStore, + MorphProofsInitialStateStore, WriteCounts, + }, + db::{HashedStorageKey, StorageTrieKey}, +}; + +/// In-memory implementation of [`MorphProofsStore`] for testing purposes +#[derive(Debug, Clone)] +pub struct InMemoryProofsStorage { + /// Shared state across all instances + inner: Arc>, +} + +#[derive(Debug, Default)] +struct InMemoryStorageInner { + /// Account trie branches: (`block_number`, path) -> `branch_node` + account_branches: BTreeMap<(u64, Nibbles), Option>, + + /// Storage trie branches: (`block_number`, `hashed_address`, path) -> `branch_node` + storage_branches: BTreeMap<(u64, B256, Nibbles), Option>, + + /// Hashed accounts: (`block_number`, `hashed_address`) -> account + hashed_accounts: BTreeMap<(u64, B256), Option>, + + /// Hashed storages: (`block_number`, `hashed_address`, `hashed_slot`) -> value + hashed_storages: BTreeMap<(u64, B256, B256), U256>, + + /// Trie updates by block number + trie_updates: BTreeMap, + + /// Post state by block number + post_states: BTreeMap, + + /// Earliest block number and hash + earliest_block: Option<(u64, B256)>, + + /// The anchor block (initial state) of the store. + anchor_block: Option<(u64, B256)>, +} + +impl InMemoryStorageInner { + fn store_trie_updates( + &mut self, + block_number: u64, + block_state_diff: BlockStateDiff, + ) -> WriteCounts { + let mut result = WriteCounts::default(); + + // Store account branch nodes + for (path, branch) in block_state_diff.sorted_trie_updates.account_nodes_ref() { + self.account_branches + .insert((block_number, *path), branch.clone()); + result.account_trie_updates_written_total += 1; + } + + // Store storage branch nodes and removals + for (address, storage_trie_updates) in + block_state_diff.sorted_trie_updates.storage_tries_ref() + { + // Store storage branch nodes + for (path, branch) in storage_trie_updates.storage_nodes_ref() { + self.storage_branches + .insert((block_number, *address, *path), branch.clone()); + result.storage_trie_updates_written_total += 1; + } + } + + for (address, account) in &block_state_diff.sorted_post_state.accounts { + self.hashed_accounts + .insert((block_number, *address), *account); + result.hashed_accounts_written_total += 1; + } + + for (hashed_address, storage) in &block_state_diff.sorted_post_state.storages { + // Handle wiped storage: iterate all existing values and mark them as deleted + // This is an expensive operation and should never happen for blocks going forward. + if storage.wiped { + // Collect latest values for each slot up to the current block + let mut slot_to_latest: std::collections::BTreeMap = + std::collections::BTreeMap::new(); + + for ((block, address, slot), value) in &self.hashed_storages { + if *block < block_number && *address == *hashed_address { + if let Some((existing_block, _)) = slot_to_latest.get(slot) { + if *block > *existing_block { + slot_to_latest.insert(*slot, (*block, *value)); + } + } else { + slot_to_latest.insert(*slot, (*block, *value)); + } + } + } + + // Store zero values for all non-zero slots to mark them as deleted + for (slot, (_, value)) in slot_to_latest { + if !value.is_zero() { + self.hashed_storages + .insert((block_number, *hashed_address, slot), U256::ZERO); + result.hashed_storages_written_total += 1; + } + } + } + // Persist any new slots that arrive in the same block as the wipe (e.g. SELFDESTRUCT + // + same-block re-CREATE2 produces `wiped = true` together with fresh + // `storage_slots_ref()` for the recreated contract). Mirrors the MDBX backend. + for (slot, value) in storage.storage_slots_ref() { + self.hashed_storages + .insert((block_number, *hashed_address, *slot), *value); + result.hashed_storages_written_total += 1; + } + } + + self.trie_updates + .insert(block_number, block_state_diff.sorted_trie_updates.clone()); + self.post_states + .insert(block_number, block_state_diff.sorted_post_state); + + result + } +} + +impl Default for InMemoryProofsStorage { + fn default() -> Self { + Self::new() + } +} + +impl InMemoryProofsStorage { + /// Create a new in-memory Morph proofs storage instance. + pub fn new() -> Self { + Self { + inner: Arc::new(RwLock::new(InMemoryStorageInner::default())), + } + } + + fn get_latest_account_trie_key(&self) -> MorphProofsStorageResult> { + let inner = self.inner.read(); + Ok(inner + .account_branches + .range(( + std::ops::Bound::Included((0, Nibbles::default())), + std::ops::Bound::Excluded((1, Nibbles::default())), + )) + .next_back() + .map(|((_, nibbles), _)| StoredNibbles::from(*nibbles))) + } + + fn get_latest_storage_trie_key(&self) -> MorphProofsStorageResult> { + let inner = self.inner.read(); + Ok(inner + .storage_branches + .range(( + std::ops::Bound::Included((0, B256::ZERO, Nibbles::default())), + std::ops::Bound::Excluded((1, B256::ZERO, Nibbles::default())), + )) + .next_back() + .map(|((_, address, nibbles), _)| { + StorageTrieKey::new(*address, StoredNibbles::from(*nibbles)) + })) + } + + fn get_latest_hashed_account_key(&self) -> MorphProofsStorageResult> { + let inner = self.inner.read(); + Ok(inner + .hashed_accounts + .range(( + std::ops::Bound::Included((0, B256::ZERO)), + std::ops::Bound::Excluded((1, B256::ZERO)), + )) + .next_back() + .map(|((_, address), _)| *address)) + } + + fn get_latest_hashed_storage_key(&self) -> MorphProofsStorageResult> { + let inner = self.inner.read(); + Ok(inner + .hashed_storages + .range(( + std::ops::Bound::Included((0, B256::ZERO, B256::ZERO)), + std::ops::Bound::Excluded((1, B256::ZERO, B256::ZERO)), + )) + .next_back() + .map(|((_, address, slot), _)| HashedStorageKey::new(*address, *slot))) + } +} + +/// In-memory implementation of [`TrieCursor`]. +#[derive(Debug)] +pub struct InMemoryTrieCursor { + /// inner storage reference + inner: Arc>, + /// Hashed address being queried + hashed_address: Option, + /// max block number for the cursor + max_block_number: u64, + /// Current position in the iteration (-1 means not positioned yet) + position: isize, + + /// Whether the entries have been populated + is_populated: bool, + /// Sorted entries that match the query parameters + entries: Vec<(Nibbles, BranchNodeCompact)>, +} + +impl InMemoryTrieCursor { + const fn new( + inner: Arc>, + hashed_address: Option, + max_block_number: u64, + ) -> Self { + Self { + inner, + hashed_address, + max_block_number, + position: -1, + + is_populated: false, + entries: Vec::new(), + } + } + + fn ensure_entries_populated(&mut self) -> Result<(), DatabaseError> { + if self.is_populated { + return Ok(()); + } + + let storage = self + .inner + .try_read() + .ok_or(MorphProofsStorageError::TryLockError)?; + + // Common logic: collect latest values for each path + let mut path_to_latest: std::collections::BTreeMap< + Nibbles, + (u64, Option), + > = std::collections::BTreeMap::new(); + + let mut collected_entries: Vec<(Nibbles, BranchNodeCompact)> = + if let Some(addr) = self.hashed_address { + // Storage trie cursor + for ((block, address, path), branch) in &storage.storage_branches { + if *block <= self.max_block_number && *address == addr { + if let Some((existing_block, _)) = path_to_latest.get(path) { + if *block > *existing_block { + path_to_latest.insert(*path, (*block, branch.clone())); + } + } else { + path_to_latest.insert(*path, (*block, branch.clone())); + } + } + } + + path_to_latest + .into_iter() + .filter_map(|(path, (_, branch))| branch.map(|b| (path, b))) + .collect() + } else { + // Account trie cursor + for ((block, path), branch) in &storage.account_branches { + if *block <= self.max_block_number { + if let Some((existing_block, _)) = path_to_latest.get(path) { + if *block > *existing_block { + path_to_latest.insert(*path, (*block, branch.clone())); + } + } else { + path_to_latest.insert(*path, (*block, branch.clone())); + } + } + } + + path_to_latest + .into_iter() + .filter_map(|(path, (_, branch))| branch.map(|b| (path, b))) + .collect() + }; + + // Sort by path for consistent ordering + collected_entries.sort_by_key(|(a, _)| *a); + self.entries = collected_entries; + self.is_populated = true; + Ok(()) + } +} + +impl TrieCursor for InMemoryTrieCursor { + fn seek_exact( + &mut self, + path: Nibbles, + ) -> Result, DatabaseError> { + self.ensure_entries_populated()?; + + if let Some(pos) = self.entries.iter().position(|(p, _)| *p == path) { + self.position = pos as isize; + Ok(Some(self.entries[pos].clone())) + } else { + Ok(None) + } + } + + fn seek( + &mut self, + path: Nibbles, + ) -> Result, DatabaseError> { + self.ensure_entries_populated()?; + + if let Some(pos) = self.entries.iter().position(|(p, _)| *p >= path) { + self.position = pos as isize; + Ok(Some(self.entries[pos].clone())) + } else { + Ok(None) + } + } + + fn next(&mut self) -> Result, DatabaseError> { + self.ensure_entries_populated()?; + + self.position += 1; + if self.position >= 0 && (self.position as usize) < self.entries.len() { + Ok(Some(self.entries[self.position as usize].clone())) + } else { + Ok(None) + } + } + + fn current(&mut self) -> Result, DatabaseError> { + self.ensure_entries_populated()?; + + if self.position >= 0 && (self.position as usize) < self.entries.len() { + Ok(Some(self.entries[self.position as usize].0)) + } else { + Ok(None) + } + } + + fn reset(&mut self) { + self.position = -1; + } +} + +impl TrieStorageCursor for InMemoryTrieCursor { + fn set_hashed_address(&mut self, hashed_address: B256) { + self.hashed_address = Some(hashed_address); + self.is_populated = false; + self.entries.clear(); + self.reset(); + } +} + +/// In-memory implementation of [`HashedCursor`] for storage slots +#[derive(Debug)] +pub struct InMemoryStorageCursor { + /// inner storage reference + inner: Arc>, + /// hashed address for which the cursor is iterating + hashed_address: B256, + /// max block number for the cursor + max_block_number: u64, + /// Current position in the iteration (-1 means not positioned yet) + position: isize, + + /// Whether the entries have been populated + is_populated: bool, + /// Sorted entries that match the query parameters + entries: Vec<(B256, U256)>, +} + +impl InMemoryStorageCursor { + const fn new( + storage: Arc>, + hashed_address: B256, + max_block_number: u64, + ) -> Self { + Self { + inner: storage, + hashed_address, + max_block_number, + position: -1, + + is_populated: false, + entries: Vec::new(), + } + } + + fn ensure_entries_populated(&mut self) -> Result<(), DatabaseError> { + if self.is_populated { + return Ok(()); + } + + let storage = self + .inner + .try_read() + .ok_or(MorphProofsStorageError::TryLockError)?; + + // Collect latest values for each slot + let mut slot_to_latest: std::collections::BTreeMap = + std::collections::BTreeMap::new(); + + for ((block, address, slot), value) in &storage.hashed_storages { + if *block <= self.max_block_number && *address == self.hashed_address { + if let Some((existing_block, _)) = slot_to_latest.get(slot) { + if *block > *existing_block { + slot_to_latest.insert(*slot, (*block, *value)); + } + } else { + slot_to_latest.insert(*slot, (*block, *value)); + } + } + } + + // Filter out zero values - they represent deleted/empty storage slots + let mut entries: Vec<(B256, U256)> = slot_to_latest + .into_iter() + .filter_map(|(slot, (_, value))| { + if value.is_zero() { + None + } else { + Some((slot, value)) + } + }) + .collect(); + + entries.sort_by_key(|(slot, _)| *slot); + self.entries = entries; + self.is_populated = true; + Ok(()) + } +} + +impl HashedCursor for InMemoryStorageCursor { + type Value = U256; + + fn seek(&mut self, key: B256) -> Result, DatabaseError> { + self.ensure_entries_populated()?; + + if let Some(pos) = self.entries.iter().position(|(k, _)| *k >= key) { + self.position = pos as isize; + Ok(Some(self.entries[pos])) + } else { + Ok(None) + } + } + + fn next(&mut self) -> Result, DatabaseError> { + self.ensure_entries_populated()?; + + self.position += 1; + if self.position >= 0 && (self.position as usize) < self.entries.len() { + Ok(Some(self.entries[self.position as usize])) + } else { + Ok(None) + } + } + + fn reset(&mut self) { + self.position = -1; + } +} + +impl HashedStorageCursor for InMemoryStorageCursor { + fn is_storage_empty(&mut self) -> Result { + Ok(self.seek(B256::ZERO)?.is_none()) + } + + fn set_hashed_address(&mut self, hashed_address: B256) { + self.hashed_address = hashed_address; + self.is_populated = false; + self.entries.clear(); + self.reset(); + } +} + +/// In-memory implementation of [`HashedCursor`] for accounts +#[derive(Debug)] +pub struct InMemoryAccountCursor { + /// Current position in the iteration (-1 means not positioned yet) + position: isize, + /// Sorted entries that match the query parameters + entries: Vec<(B256, Account)>, +} + +impl InMemoryAccountCursor { + fn new(storage: &InMemoryStorageInner, max_block_number: u64) -> Self { + // Collect latest accounts for each address + let mut addr_to_latest: std::collections::BTreeMap)> = + std::collections::BTreeMap::new(); + + for ((block, address), account) in &storage.hashed_accounts { + if *block <= max_block_number { + if let Some((existing_block, _)) = addr_to_latest.get(address) { + if *block > *existing_block { + addr_to_latest.insert(*address, (*block, *account)); + } + } else { + addr_to_latest.insert(*address, (*block, *account)); + } + } + } + + let mut entries: Vec<(B256, Account)> = addr_to_latest + .into_iter() + .filter_map(|(address, (_, account))| account.map(|acc| (address, acc))) + .collect(); + + entries.sort_by_key(|(address, _)| *address); + + Self { + position: -1, + entries, + } + } +} + +impl HashedCursor for InMemoryAccountCursor { + type Value = Account; + + fn seek(&mut self, key: B256) -> Result, DatabaseError> { + if let Some(pos) = self.entries.iter().position(|(k, _)| *k >= key) { + self.position = pos as isize; + Ok(Some(self.entries[pos])) + } else { + Ok(None) + } + } + + fn next(&mut self) -> Result, DatabaseError> { + self.position += 1; + if self.position >= 0 && (self.position as usize) < self.entries.len() { + Ok(Some(self.entries[self.position as usize])) + } else { + Ok(None) + } + } + + fn reset(&mut self) { + // no reset needed + } +} + +impl MorphProofsStore for InMemoryProofsStorage { + type StorageTrieCursor<'tx> = InMemoryTrieCursor; + type AccountTrieCursor<'tx> = InMemoryTrieCursor; + type StorageCursor<'tx> = InMemoryStorageCursor; + type AccountHashedCursor<'tx> = InMemoryAccountCursor; + type Tx<'tx> + = () + where + Self: 'tx; + + fn get_earliest_block_number(&self) -> MorphProofsStorageResult> { + let inner = self.inner.read(); + Ok(inner.earliest_block) + } + + fn get_latest_block_number(&self) -> MorphProofsStorageResult> { + let inner = self.inner.read(); + // Find the latest block number from trie_updates + let latest_block = inner.trie_updates.keys().max().copied(); + Ok(latest_block.map_or(inner.earliest_block, |block| Some((block, B256::ZERO)))) + } + + fn get_earliest_block_number_with_tx<'tx, 'db>( + &self, + _tx: &'tx Self::Tx<'db>, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx, + { + self.get_earliest_block_number() + } + + fn get_latest_block_number_with_tx<'tx, 'db>( + &self, + _tx: &'tx Self::Tx<'db>, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx, + { + self.get_latest_block_number() + } + + fn storage_trie_cursor<'tx>( + &'tx self, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + Ok(InMemoryTrieCursor::new( + Arc::clone(&self.inner), + Some(hashed_address), + max_block_number, + )) + } + + fn account_trie_cursor<'tx>( + &'tx self, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + Ok(InMemoryTrieCursor::new( + Arc::clone(&self.inner), + None, + max_block_number, + )) + } + + fn storage_hashed_cursor<'tx>( + &'tx self, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + Ok(InMemoryStorageCursor::new( + Arc::clone(&self.inner), + hashed_address, + max_block_number, + )) + } + + fn account_hashed_cursor<'tx>( + &'tx self, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + let inner = self + .inner + .try_read() + .ok_or(MorphProofsStorageError::TryLockError)?; + Ok(InMemoryAccountCursor::new(&inner, max_block_number)) + } + + fn ro_tx<'tx>(&'tx self) -> MorphProofsStorageResult> { + Ok(()) + } + + fn storage_trie_cursor_with_tx<'tx, 'db>( + &self, + _tx: &'tx Self::Tx<'db>, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx, + { + self.storage_trie_cursor(hashed_address, max_block_number) + } + + fn account_trie_cursor_with_tx<'tx, 'db>( + &self, + _tx: &'tx Self::Tx<'db>, + max_block_number: u64, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx, + { + self.account_trie_cursor(max_block_number) + } + + fn storage_hashed_cursor_with_tx<'tx, 'db>( + &self, + _tx: &'tx Self::Tx<'db>, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx, + { + self.storage_hashed_cursor(hashed_address, max_block_number) + } + + fn account_hashed_cursor_with_tx<'tx, 'db>( + &self, + _tx: &'tx Self::Tx<'db>, + max_block_number: u64, + ) -> MorphProofsStorageResult> + where + Self: 'db, + 'db: 'tx, + { + self.account_hashed_cursor(max_block_number) + } + + fn store_trie_updates( + &self, + block_ref: BlockWithParent, + block_state_diff: BlockStateDiff, + ) -> MorphProofsStorageResult { + let mut inner = self.inner.write(); + + Ok(inner.store_trie_updates(block_ref.block.number, block_state_diff)) + } + + fn fetch_trie_updates(&self, block_number: u64) -> MorphProofsStorageResult { + let inner = self.inner.read(); + + let trie_updates = inner + .trie_updates + .get(&block_number) + .cloned() + .unwrap_or_default(); + let post_state = inner + .post_states + .get(&block_number) + .cloned() + .unwrap_or_default(); + + Ok(BlockStateDiff { + sorted_trie_updates: trie_updates, + sorted_post_state: post_state, + }) + } + + fn prune_earliest_state( + &self, + new_earliest_block_ref: BlockWithParent, + ) -> MorphProofsStorageResult { + let mut write_counts = WriteCounts::default(); + let mut inner = self.inner.write(); + let new_earliest = new_earliest_block_ref.block.number; + + // 1. Account Branches + // Identify keys to move (blocks <= new_earliest but > 0) + let account_keys: Vec<_> = inner + .account_branches + .keys() + .filter(|(block, _)| *block > 0 && *block <= new_earliest) + .copied() + .collect(); + + for key in account_keys { + if let Some(branch) = inner.account_branches.remove(&key) { + // Determine if we should update the base state (block 0) + // In a simpler model without diff calculation, we just overwrite block 0 + // with the latest version found in the pruned range. + // Since keys are sorted by block, this logic naturally keeps the latest if we + // iterate in order, but here we are just grabbing all of them. + // For correctness in BTreeMap iteration order (which is sorted): + inner.account_branches.insert((0, key.1), branch); + write_counts.account_trie_updates_written_total += 1; + } + } + + // 2. Storage Branches + let storage_keys: Vec<_> = inner + .storage_branches + .keys() + .filter(|(block, _, _)| *block > 0 && *block <= new_earliest) + .copied() + .collect(); + + for key in storage_keys { + if let Some(branch) = inner.storage_branches.remove(&key) { + inner.storage_branches.insert((0, key.1, key.2), branch); + write_counts.storage_trie_updates_written_total += 1; + } + } + + // 3. Hashed Accounts + let acc_keys: Vec<_> = inner + .hashed_accounts + .keys() + .filter(|(block, _)| *block > 0 && *block <= new_earliest) + .copied() + .collect(); + + for key in acc_keys { + if let Some(acc) = inner.hashed_accounts.remove(&key) { + inner.hashed_accounts.insert((0, key.1), acc); + write_counts.hashed_accounts_written_total += 1; + } + } + + // 4. Hashed Storages + let stor_keys: Vec<_> = inner + .hashed_storages + .keys() + .filter(|(block, _, _)| *block > 0 && *block <= new_earliest) + .copied() + .collect(); + + for key in stor_keys { + if let Some(val) = inner.hashed_storages.remove(&key) { + inner.hashed_storages.insert((0, key.1, key.2), val); + write_counts.hashed_storages_written_total += 1; + } + } + + // Update earliest block pointer + if let Some((_, hash)) = inner.earliest_block { + inner.earliest_block = Some((new_earliest, hash)); + } + + // 5. Cleanup Metadata + inner.trie_updates.retain(|block, _| *block > new_earliest); + inner.post_states.retain(|block, _| *block > new_earliest); + + Ok(write_counts) + } + + fn unwind_history(&self, unwind_upto_block: BlockWithParent) -> MorphProofsStorageResult<()> { + let mut inner = self.inner.write(); + let unwind_upto_block_number = unwind_upto_block.block.number - 1; + + // Remove all updates after unwind_upto_block_number + inner + .trie_updates + .retain(|block, _| *block <= unwind_upto_block_number); + inner + .post_states + .retain(|block, _| *block <= unwind_upto_block_number); + inner + .account_branches + .retain(|(block, _), _| *block <= unwind_upto_block_number); + inner + .storage_branches + .retain(|(block, _, _), _| *block <= unwind_upto_block_number); + inner + .hashed_accounts + .retain(|(block, _), _| *block <= unwind_upto_block_number); + inner + .hashed_storages + .retain(|(block, _, _), _| *block <= unwind_upto_block_number); + + Ok(()) + } + + fn replace_updates( + &self, + latest_common_block: BlockNumHash, + blocks_to_add: Vec<(BlockWithParent, BlockStateDiff)>, + ) -> MorphProofsStorageResult<()> { + let mut inner = self.inner.write(); + let latest_common_block_number = latest_common_block.number; + + // Remove all updates after latest_common_block_number + inner + .trie_updates + .retain(|block, _| *block <= latest_common_block_number); + inner + .post_states + .retain(|block, _| *block <= latest_common_block_number); + inner + .account_branches + .retain(|(block, _), _| *block <= latest_common_block_number); + inner + .storage_branches + .retain(|(block, _, _), _| *block <= latest_common_block_number); + inner + .hashed_accounts + .retain(|(block, _), _| *block <= latest_common_block_number); + inner + .hashed_storages + .retain(|(block, _, _), _| *block <= latest_common_block_number); + + for (block, block_state_diff) in blocks_to_add { + inner.store_trie_updates(block.block.number, block_state_diff); + } + + Ok(()) + } + + fn set_earliest_block_number( + &self, + block_number: u64, + hash: B256, + ) -> MorphProofsStorageResult<()> { + let mut inner = self.inner.write(); + inner.earliest_block = Some((block_number, hash)); + Ok(()) + } +} + +/// Degenerate batch session for [`InMemoryProofsStorage`]: writes go directly to shared +/// storage and are visible to subsequent reads. There is no commit/abort distinction — +/// the in-memory store does not provide cross-block atomicity. +#[derive(Debug)] +pub struct InMemoryBatchSession<'a> { + storage: &'a InMemoryProofsStorage, +} + +impl MorphProofsBatchSession for InMemoryBatchSession<'_> { + type StorageTrieCursor<'a> + = InMemoryTrieCursor + where + Self: 'a; + type AccountTrieCursor<'a> + = InMemoryTrieCursor + where + Self: 'a; + type StorageCursor<'a> + = InMemoryStorageCursor + where + Self: 'a; + type AccountHashedCursor<'a> + = InMemoryAccountCursor + where + Self: 'a; + + fn get_earliest_block_number(&self) -> MorphProofsStorageResult> { + self.storage.get_earliest_block_number() + } + + fn get_latest_block_number(&self) -> MorphProofsStorageResult> { + self.storage.get_latest_block_number() + } + + fn storage_trie_cursor( + &self, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + self.storage + .storage_trie_cursor(hashed_address, max_block_number) + } + + fn account_trie_cursor( + &self, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + self.storage.account_trie_cursor(max_block_number) + } + + fn storage_hashed_cursor( + &self, + hashed_address: B256, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + self.storage + .storage_hashed_cursor(hashed_address, max_block_number) + } + + fn account_hashed_cursor( + &self, + max_block_number: u64, + ) -> MorphProofsStorageResult> { + self.storage.account_hashed_cursor(max_block_number) + } + + fn store_trie_updates( + &mut self, + block_ref: BlockWithParent, + block_state_diff: BlockStateDiff, + ) -> MorphProofsStorageResult { + self.storage.store_trie_updates(block_ref, block_state_diff) + } +} + +impl MorphProofsBatchStore for InMemoryProofsStorage { + type BatchSession<'a> + = InMemoryBatchSession<'a> + where + Self: 'a; + + fn with_batch_session(&self, f: F) -> MorphProofsStorageResult + where + F: FnOnce(&mut Self::BatchSession<'_>) -> MorphProofsStorageResult, + { + let mut session = InMemoryBatchSession { storage: self }; + f(&mut session) + } +} + +impl MorphProofsInitialStateStore for InMemoryProofsStorage { + fn initial_state_anchor(&self) -> MorphProofsStorageResult { + let inner = self.inner.read(); + + let Some((block_num, block_hash)) = inner.anchor_block else { + return Ok(InitialStateAnchor::default()); + }; + + let completed = inner.earliest_block.is_some(); + + Ok(InitialStateAnchor { + block: Some(NumHash::new(block_num, block_hash)), + status: if completed { + InitialStateStatus::Completed + } else { + InitialStateStatus::InProgress + }, + latest_account_trie_key: self.get_latest_account_trie_key()?, + latest_storage_trie_key: self.get_latest_storage_trie_key()?, + latest_hashed_account_key: self.get_latest_hashed_account_key()?, + latest_hashed_storage_key: self.get_latest_hashed_storage_key()?, + }) + } + + fn set_initial_state_anchor(&self, anchor: BlockNumHash) -> MorphProofsStorageResult<()> { + let mut inner = self.inner.write(); + inner.anchor_block = Some((anchor.number, anchor.hash)); + Ok(()) + } + + fn store_account_branches( + &self, + updates: Vec<(Nibbles, Option)>, + ) -> MorphProofsStorageResult<()> { + let mut inner = self.inner.write(); + + for (path, branch) in updates { + inner.account_branches.insert((0, path), branch); + } + + Ok(()) + } + + fn store_storage_branches( + &self, + hashed_address: B256, + items: Vec<(Nibbles, Option)>, + ) -> MorphProofsStorageResult<()> { + let mut inner = self.inner.write(); + + for (path, branch) in items { + inner + .storage_branches + .insert((0, hashed_address, path), branch); + } + + Ok(()) + } + + fn store_hashed_accounts( + &self, + accounts: Vec<(B256, Option)>, + ) -> MorphProofsStorageResult<()> { + let mut inner = self.inner.write(); + + for (address, account) in accounts { + inner.hashed_accounts.insert((0, address), account); + } + + Ok(()) + } + + fn store_hashed_storages( + &self, + hashed_address: B256, + storages: Vec<(B256, U256)>, + ) -> MorphProofsStorageResult<()> { + let mut inner = self.inner.write(); + + for (slot, value) in storages { + inner + .hashed_storages + .insert((0, hashed_address, slot), value); + } + + Ok(()) + } + + fn commit_initial_state(&self) -> MorphProofsStorageResult { + let mut inner = self.inner.write(); + if let Some((number, hash)) = inner.anchor_block { + inner.earliest_block = Some((number, hash)); + Ok(BlockNumHash::new(number, hash)) + } else { + Err(MorphProofsStorageError::NoBlocksFound) + } + } +} + +#[cfg(test)] +mod tests { + use alloy_eips::NumHash; + use alloy_primitives::U256; + use reth_primitives_traits::Account; + + use super::*; + use crate::MorphProofsStorageError; + + #[test] + fn test_in_memory_storage_basic_operations() -> Result<(), MorphProofsStorageError> { + let storage = InMemoryProofsStorage::new(); + + // Test setting earliest block + let block_hash = B256::random(); + storage.set_earliest_block_number(1, block_hash)?; + let earliest = storage.get_earliest_block_number()?; + assert_eq!(earliest, Some((1, block_hash))); + + // Test storing and retrieving accounts + let account = Account { + nonce: 1, + balance: U256::from(100), + bytecode_hash: None, + }; + let hashed_address = B256::random(); + + storage.store_hashed_accounts(vec![(hashed_address, Some(account))])?; + + let _cursor = storage.account_hashed_cursor(10)?; + // Note: cursor testing would require more complex setup with proper seek/next operations + + Ok(()) + } + + #[test] + fn test_trie_updates_storage() -> Result<(), MorphProofsStorageError> { + let storage = InMemoryProofsStorage::new(); + + let sorted_trie_updates = TrieUpdatesSorted::default(); + let sorted_post_state = HashedPostStateSorted::default(); + let block_state_diff = BlockStateDiff { + sorted_trie_updates: sorted_trie_updates.clone(), + sorted_post_state: sorted_post_state.clone(), + }; + + const BLOCK: BlockWithParent = + BlockWithParent::new(B256::ZERO, NumHash::new(5, B256::ZERO)); + storage.store_trie_updates(BLOCK, block_state_diff)?; + + let retrieved_diff = storage.fetch_trie_updates(BLOCK.block.number)?; + assert_eq!(retrieved_diff.sorted_trie_updates, sorted_trie_updates); + assert_eq!(retrieved_diff.sorted_post_state, sorted_post_state); + + Ok(()) + } +} diff --git a/crates/proofs/src/initialize.rs b/crates/proofs/src/initialize.rs new file mode 100644 index 00000000..b61277bc --- /dev/null +++ b/crates/proofs/src/initialize.rs @@ -0,0 +1,1357 @@ +//! Initialization job for proofs storage. Handles storing the existing state into the proofs +//! storage. + +use std::{collections::HashMap, time::Instant}; + +use alloy_eips::BlockNumHash; +use alloy_primitives::{B256, U256}; +use derive_more::Constructor; +use reth_db::{ + DatabaseError, + cursor::{DbCursorRO, DbDupCursorRO}, + tables, + transaction::DbTx, +}; +use reth_primitives_traits::{Account, StorageEntry}; +use reth_trie_common::{ + BranchNodeCompact, Nibbles, StorageTrieEntry, StoredNibbles, StoredNibblesSubKey, +}; +use tracing::{debug, info}; + +use crate::{ + MorphProofsStorageError, MorphProofsStore, + api::{InitialStateAnchor, InitialStateStatus, MorphProofsInitialStateStore}, + db::{HashedStorageKey, StorageTrieKey}, +}; + +/// Batch size threshold for storing entries during initialization +const INITIALIZE_STORAGE_THRESHOLD: usize = 100000; + +/// Threshold for logging progress during initialization +const INITIALIZE_LOG_THRESHOLD: usize = 100000; + +/// Initialization job for external storage. +#[derive(Debug, Constructor)] +pub struct InitializationJob { + storage: S, + tx: Tx, +} + +/// Macro to generate simple cursor iterators for tables +macro_rules! define_simple_cursor_iter { + ($iter_name:ident, $table:ty, $key_type:ty, $value_type:ty) => { + struct $iter_name(C); + + impl $iter_name { + const fn new(cursor: C) -> Self { + Self(cursor) + } + } + + impl> Iterator for $iter_name { + type Item = Result<($key_type, $value_type), DatabaseError>; + + fn next(&mut self) -> Option { + self.0.next().transpose() + } + } + }; +} + +/// Macro to generate duplicate cursor iterators for tables with custom logic +macro_rules! define_dup_cursor_iter { + ($iter_name:ident, $table:ty, $key_type:ty, $value_type:ty) => { + struct $iter_name(C); + + impl $iter_name { + const fn new(cursor: C) -> Self { + Self(cursor) + } + } + + impl + DbCursorRO<$table>> Iterator for $iter_name { + type Item = Result<($key_type, $value_type), DatabaseError>; + + fn next(&mut self) -> Option { + // First try to get the next duplicate value + if let Some(res) = self.0.next_dup().transpose() { + return Some(res); + } + + // If no more duplicates, find the next key with values + let Some(Ok((next_key, _))) = self.0.next_no_dup().transpose() else { + // If no more entries, return None + return None; + }; + + // If found, seek to the first duplicate for this key + return self.0.seek(next_key).transpose(); + } + } + }; +} + +// Generate iterators for all 4 table types +define_simple_cursor_iter!(HashedAccountsInit, tables::HashedAccounts, B256, Account); +define_dup_cursor_iter!( + HashedStoragesInit, + tables::HashedStorages, + B256, + StorageEntry +); +define_simple_cursor_iter!( + AccountsTrieInit, + tables::AccountsTrie, + StoredNibbles, + BranchNodeCompact +); +define_dup_cursor_iter!( + StoragesTrieInit, + tables::StoragesTrie, + B256, + StorageTrieEntry +); + +/// Trait to estimate the progress of a initialization job based on the key. +trait CompletionEstimatable { + // Returns a progress estimate as a percentage (0.0 to 1.0) + fn estimate_progress(&self) -> f64; +} + +impl CompletionEstimatable for B256 { + fn estimate_progress(&self) -> f64 { + // use the first 3 bytes as a progress estimate + let progress = self.0[..3].to_vec(); + let mut val: u64 = 0; + for nibble in &progress { + val = (val << 8) | *nibble as u64; + } + val as f64 / (256u64.pow(3)) as f64 + } +} + +impl CompletionEstimatable for StoredNibbles { + fn estimate_progress(&self) -> f64 { + // use the first 6 nibbles as a progress estimate + let progress_nibbles = if self.0.is_empty() { + Nibbles::new() + } else { + self.0.slice(0..(self.0.len().min(6))) + }; + let mut val: u64 = 0; + for nibble in progress_nibbles.iter() { + val = (val << 4) | nibble as u64; + } + val as f64 / (16u64.pow(progress_nibbles.len() as u32)) as f64 + } +} + +impl + InitializationJob +{ + /// Initialize a table from a source iterator to a storage function. Handles batching and + /// logging. + fn initialize< + I: Iterator> + InitTable, + Key: CompletionEstimatable + 'static, + Value: 'static, + >( + &self, + name: &str, + source: I, + storage_threshold: usize, + log_threshold: usize, + ) -> Result { + info!(name = %name, "Starting initialization"); + let start_time = Instant::now(); + + let mut source = source.peekable(); + let Some(first_entry) = source.peek() else { + debug!(target: "reth::cli", "No entries to store for table"); + return Ok(0); + }; + let initial_progress = match first_entry { + Ok(i) => i.0.estimate_progress(), + Err(e) => Err(e.clone())?, + }; + + let storage = &self.storage; + let source_size_hint = source.size_hint().0; + let mut batch = Vec::with_capacity(source_size_hint.min(storage_threshold)); + let mut total_entries: usize = 0; + + for entry in source { + batch.push(entry?); + total_entries += 1; + + if total_entries.is_multiple_of(log_threshold) { + let progress = batch.last().expect("non-empty batch").0.estimate_progress(); + let elapsed = start_time.elapsed(); + let elapsed_secs = elapsed.as_secs_f64(); + + let progress_per_second = if elapsed_secs.is_normal() { + (progress - initial_progress) / elapsed_secs + } else { + 0.0 + }; + let estimated_total_time = if progress_per_second.is_normal() { + (1.0 - progress) / progress_per_second + } else { + 0.0 + }; + let progress_pct = progress * 100.0; + info!( + name = %name, + total_entries, + progress_pct = format_args!("{:.2}", progress_pct), + estimated_total_time = format_args!("{:.1}", estimated_total_time), + "Processed entries" + ); + } + + if batch.len() >= storage_threshold { + info!(name = %name, total_entries = total_entries, "Storing entries"); + I::store_entries(storage, batch)?; + batch = Vec::with_capacity( + (source_size_hint.saturating_sub(total_entries)).min(storage_threshold), + ); + } + } + + if !batch.is_empty() { + info!(name = %name, "Storing final entries"); + I::store_entries(storage, batch)?; + } + + info!(name = %name, total_entries = total_entries, "initialization complete"); + Ok(total_entries as u64) + } + + /// Initialize hashed accounts data + fn initialize_hashed_accounts( + &self, + start_key: Option, + ) -> Result<(), MorphProofsStorageError> { + let mut start_cursor = self.tx.cursor_read::()?; + + if let Some(latest) = start_key { + start_cursor + .seek(latest)? + .filter(|(k, _)| *k == latest) + .ok_or(MorphProofsStorageError::InitializeStorageInconsistentState)?; + } + + let source = HashedAccountsInit::new(start_cursor); + self.initialize( + "hashed accounts", + source, + INITIALIZE_STORAGE_THRESHOLD, + INITIALIZE_LOG_THRESHOLD, + )?; + + Ok(()) + } + + /// Initialize hashed storage data + fn initialize_hashed_storages( + &self, + start_key: Option, + ) -> Result<(), MorphProofsStorageError> { + let mut start_cursor = self.tx.cursor_dup_read::()?; + + if let Some(latest) = start_key { + start_cursor + .seek_by_key_subkey(latest.hashed_address, latest.hashed_storage_key)? + .filter(|v| v.key == latest.hashed_storage_key) + .ok_or(MorphProofsStorageError::InitializeStorageInconsistentState)?; + } + + let source = HashedStoragesInit::new(start_cursor); + self.initialize( + "hashed storage", + source, + INITIALIZE_STORAGE_THRESHOLD, + INITIALIZE_LOG_THRESHOLD, + )?; + + Ok(()) + } + + /// Initialize accounts trie data + fn initialize_accounts_trie( + &self, + start_key: Option, + ) -> Result<(), MorphProofsStorageError> { + let mut start_cursor = self.tx.cursor_read::()?; + + if let Some(latest_key) = start_key { + start_cursor + .seek(latest_key.clone())? + .filter(|(k, _)| *k == latest_key) + .ok_or(MorphProofsStorageError::InitializeStorageInconsistentState)?; + } + + let source = AccountsTrieInit::new(start_cursor); + self.initialize( + "accounts trie", + source, + INITIALIZE_STORAGE_THRESHOLD, + INITIALIZE_LOG_THRESHOLD, + )?; + + Ok(()) + } + + /// Initialize storage trie data + fn initialize_storages_trie( + &self, + start_key: Option, + ) -> Result<(), MorphProofsStorageError> { + let mut start_cursor = self.tx.cursor_dup_read::()?; + + if let Some(latest_key) = start_key { + start_cursor + .seek_by_key_subkey( + latest_key.hashed_address, + StoredNibblesSubKey::from(latest_key.path.0), + )? + .filter(|v| v.nibbles.0 == latest_key.path.0) + .ok_or(MorphProofsStorageError::InitializeStorageInconsistentState)?; + } + + let source = StoragesTrieInit::new(start_cursor); + self.initialize( + "storage trie", + source, + INITIALIZE_STORAGE_THRESHOLD, + INITIALIZE_LOG_THRESHOLD, + )?; + + Ok(()) + } + + /// Run complete initialization of all preimage data + fn initialize_trie(&self, anchor: InitialStateAnchor) -> Result<(), MorphProofsStorageError> { + self.initialize_hashed_accounts(anchor.latest_hashed_account_key)?; + self.initialize_hashed_storages(anchor.latest_hashed_storage_key)?; + self.initialize_storages_trie(anchor.latest_storage_trie_key)?; + self.initialize_accounts_trie(anchor.latest_account_trie_key)?; + Ok(()) + } + + fn validate_anchor_block( + &self, + anchor: &InitialStateAnchor, + best_number: u64, + best_hash: B256, + ) -> Result<(), MorphProofsStorageError> { + let block = anchor + .block + .ok_or(MorphProofsStorageError::InitializeStorageInconsistentState)?; + + if block.number != best_number || block.hash != best_hash { + return Err(MorphProofsStorageError::InitializeStorageInconsistentState); + } + + Ok(()) + } + + /// Run the initialization job. + pub fn run(&self, best_number: u64, best_hash: B256) -> Result<(), MorphProofsStorageError> { + let anchor = self.storage.initial_state_anchor()?; + + match anchor.status { + InitialStateStatus::Completed => return Ok(()), + InitialStateStatus::NotStarted => { + self.storage + .set_initial_state_anchor(BlockNumHash::new(best_number, best_hash))?; + } + InitialStateStatus::InProgress => { + self.validate_anchor_block(&anchor, best_number, best_hash)?; + } + } + + self.initialize_trie(anchor)?; + self.storage.commit_initial_state()?; + + Ok(()) + } +} + +/// Handles storing entries for a particular KV-pair type. +trait InitTable { + /// Key of target table. + type Key: CompletionEstimatable + 'static; + /// Value of target table. + type Value: 'static; + + /// Writes given entries to given storage. + fn store_entries( + store: &impl MorphProofsInitialStateStore, + entries: impl IntoIterator, + ) -> Result<(), MorphProofsStorageError>; +} + +impl InitTable for HashedAccountsInit { + type Key = B256; + type Value = Account; + + /// Save mapping of hashed addresses to accounts to storage. + fn store_entries( + store: &impl MorphProofsInitialStateStore, + entries: impl IntoIterator, + ) -> Result<(), MorphProofsStorageError> { + store.store_hashed_accounts( + entries + .into_iter() + .map(|(address, account)| (address, Some(account))) + .collect(), + )?; + Ok(()) + } +} + +impl InitTable for HashedStoragesInit { + type Key = B256; + type Value = StorageEntry; + + /// Save mapping of hashed addresses to storage entries to storage. + fn store_entries( + store: &impl MorphProofsInitialStateStore, + entries: impl IntoIterator, + ) -> Result<(), MorphProofsStorageError> { + let entries_iter = entries.into_iter(); + let mut by_address: HashMap> = + HashMap::with_capacity(entries_iter.size_hint().0); + + for (address, entry) in entries_iter { + by_address + .entry(address) + .or_default() + .push((entry.key, entry.value)); + } + store.store_hashed_storages_bulk(by_address.into_iter().collect())?; + + Ok(()) + } +} + +impl InitTable for AccountsTrieInit { + type Key = StoredNibbles; + type Value = BranchNodeCompact; + + /// Save mapping of account trie paths to branch nodes to storage. + fn store_entries( + store: &impl MorphProofsInitialStateStore, + entries: impl IntoIterator, + ) -> Result<(), MorphProofsStorageError> { + store.store_account_branches( + entries + .into_iter() + .map(|(path, branch)| (path.0, Some(branch))) + .collect(), + )?; + + Ok(()) + } +} + +impl InitTable for StoragesTrieInit { + type Key = B256; + type Value = StorageTrieEntry; + + /// Save mapping of hashed addresses to storage trie entries to storage. + fn store_entries( + store: &impl MorphProofsInitialStateStore, + entries: impl IntoIterator, + ) -> Result<(), MorphProofsStorageError> { + let entries_iter = entries.into_iter(); + let mut by_address: HashMap)>> = + HashMap::with_capacity(entries_iter.size_hint().0); + + for (hashed_address, storage_entry) in entries_iter { + by_address + .entry(hashed_address) + .or_default() + .push((storage_entry.nibbles.0, Some(storage_entry.node))); + } + store.store_storage_branches_bulk(by_address.into_iter().collect())?; + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + macro_rules! proof_storage_init_tests { + ($mod_name:ident, $storage:ident) => { + mod $mod_name { + use std::sync::Arc; + + use alloy_primitives::{Address, U256, keccak256}; + use reth_db::{ + Database, cursor::DbCursorRW, test_utils::create_test_rw_db, + transaction::DbTxMut, + }; + use reth_primitives_traits::Account; + use reth_trie::{ + BranchNodeCompact, StorageTrieEntry, StoredNibbles, StoredNibblesSubKey, + TrieMask, hashed_cursor::HashedCursor, trie_cursor::TrieCursor, + }; + use tempfile::TempDir; + + use super::super::*; + use crate::$storage; + + /// Helper function to create a key + fn k(b: u8) -> B256 { + let mut bytes = [0u8; 32]; + bytes[0] = b; + B256::from(bytes) + } + + /// Helper function to create a test branch node + fn create_test_branch_node() -> BranchNodeCompact { + let mut state_mask = TrieMask::default(); + state_mask.set_bit(0); + state_mask.set_bit(1); + + BranchNodeCompact { + state_mask, + tree_mask: TrieMask::default(), + hash_mask: TrieMask::default(), + hashes: Arc::new(vec![]), + root_hash: None, + } + } + + #[test] + fn test_initialize_hashed_accounts() { + let db = create_test_rw_db(); + let dir = TempDir::new().unwrap(); + let storage = Arc::new($storage::new(dir.path()).expect("env")); + + // Insert test accounts into database + let tx = db.tx_mut().unwrap(); + let mut cursor = tx.cursor_write::().unwrap(); + + let mut accounts = vec![ + ( + keccak256(Address::repeat_byte(0x01)), + Account { + nonce: 1, + balance: U256::from(100), + bytecode_hash: None, + }, + ), + ( + keccak256(Address::repeat_byte(0x02)), + Account { + nonce: 2, + balance: U256::from(200), + bytecode_hash: None, + }, + ), + ( + keccak256(Address::repeat_byte(0x03)), + Account { + nonce: 3, + balance: U256::from(300), + bytecode_hash: None, + }, + ), + ]; + + // Sort accounts by address for cursor.append (which requires sorted order) + accounts.sort_by_key(|(addr, _)| *addr); + + for (addr, account) in &accounts { + cursor.append(*addr, account).unwrap(); + } + drop(cursor); + tx.commit().unwrap(); + + // Run initialization + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&storage), tx); + job.initialize_hashed_accounts(None).unwrap(); + + // Verify data was stored (will be in sorted order) + let mut account_cursor = storage.account_hashed_cursor(100).unwrap(); + let mut count = 0; + while let Some((key, account)) = account_cursor.next().unwrap() { + // Find matching account in our test data + let expected = accounts.iter().find(|(addr, _)| *addr == key).unwrap(); + assert_eq!((key, account), *expected); + count += 1; + } + assert_eq!(count, 3); + } + + #[test] + fn test_initialize_hashed_storage() { + let db = create_test_rw_db(); + let dir = TempDir::new().unwrap(); + let storage = Arc::new($storage::new(dir.path()).expect("env")); + + // Insert test storage into database + let tx = db.tx_mut().unwrap(); + let mut cursor = tx.cursor_dup_write::().unwrap(); + + let addr1 = keccak256(Address::repeat_byte(0x01)); + let addr2 = keccak256(Address::repeat_byte(0x02)); + + let storage_entries = vec![ + ( + addr1, + StorageEntry { + key: keccak256(B256::repeat_byte(0x10)), + value: U256::from(100), + }, + ), + ( + addr1, + StorageEntry { + key: keccak256(B256::repeat_byte(0x20)), + value: U256::from(200), + }, + ), + ( + addr2, + StorageEntry { + key: keccak256(B256::repeat_byte(0x30)), + value: U256::from(300), + }, + ), + ]; + + for (addr, entry) in &storage_entries { + cursor.upsert(*addr, entry).unwrap(); + } + drop(cursor); + tx.commit().unwrap(); + + // Run initialization + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&storage), tx); + job.initialize_hashed_storages(None).unwrap(); + + // Verify data was stored for addr1 + let mut storage_cursor = storage.storage_hashed_cursor(addr1, 100).unwrap(); + let mut found = vec![]; + while let Some((key, value)) = storage_cursor.next().unwrap() { + found.push((key, value)); + } + assert_eq!(found.len(), 2); + assert_eq!( + found[0], + (storage_entries[0].1.key, storage_entries[0].1.value) + ); + assert_eq!( + found[1], + (storage_entries[1].1.key, storage_entries[1].1.value) + ); + + // Verify data was stored for addr2 + let mut storage_cursor = storage.storage_hashed_cursor(addr2, 100).unwrap(); + let mut found = vec![]; + while let Some((key, value)) = storage_cursor.next().unwrap() { + found.push((key, value)); + } + assert_eq!(found.len(), 1); + assert_eq!( + found[0], + (storage_entries[2].1.key, storage_entries[2].1.value) + ); + } + + #[test] + fn test_initialize_accounts_trie() { + let db = create_test_rw_db(); + let dir = TempDir::new().unwrap(); + let storage = Arc::new($storage::new(dir.path()).expect("env")); + + // Insert test trie nodes into database + let tx = db.tx_mut().unwrap(); + let mut cursor = tx.cursor_write::().unwrap(); + + let branch = create_test_branch_node(); + let nodes = vec![ + ( + StoredNibbles(Nibbles::from_nibbles_unchecked(vec![1])), + branch.clone(), + ), + ( + StoredNibbles(Nibbles::from_nibbles_unchecked(vec![2])), + branch.clone(), + ), + ( + StoredNibbles(Nibbles::from_nibbles_unchecked(vec![3])), + branch, + ), + ]; + + for (path, node) in &nodes { + cursor.append(path.clone(), node).unwrap(); + } + drop(cursor); + tx.commit().unwrap(); + + // Run initialization + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&storage), tx); + job.initialize_accounts_trie(None).unwrap(); + + // Verify data was stored + let mut trie_cursor = storage.account_trie_cursor(100).unwrap(); + let mut count = 0; + while let Some((path, _node)) = trie_cursor.next().unwrap() { + assert_eq!(path, nodes[count].0.0); + count += 1; + } + assert_eq!(count, 3); + } + + #[test] + fn test_initialize_storages_trie() { + let db = create_test_rw_db(); + let dir = TempDir::new().unwrap(); + let storage = Arc::new($storage::new(dir.path()).expect("env")); + + // Insert test storage trie nodes into database + let tx = db.tx_mut().unwrap(); + let mut cursor = tx.cursor_dup_write::().unwrap(); + + let branch = create_test_branch_node(); + let addr1 = keccak256(Address::repeat_byte(0x01)); + let addr2 = keccak256(Address::repeat_byte(0x02)); + + let nodes = vec![ + ( + addr1, + StorageTrieEntry { + nibbles: StoredNibblesSubKey(Nibbles::from_nibbles_unchecked( + vec![1], + )), + node: branch.clone(), + }, + ), + ( + addr1, + StorageTrieEntry { + nibbles: StoredNibblesSubKey(Nibbles::from_nibbles_unchecked( + vec![2], + )), + node: branch.clone(), + }, + ), + ( + addr2, + StorageTrieEntry { + nibbles: StoredNibblesSubKey(Nibbles::from_nibbles_unchecked( + vec![3], + )), + node: branch, + }, + ), + ]; + + for (addr, entry) in &nodes { + cursor.upsert(*addr, entry).unwrap(); + } + drop(cursor); + tx.commit().unwrap(); + + // Run initialization + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&storage), tx); + job.initialize_storages_trie(None).unwrap(); + + // Verify data was stored for addr1 + let mut trie_cursor = storage.storage_trie_cursor(addr1, 100).unwrap(); + let mut found = vec![]; + while let Some((path, _node)) = trie_cursor.next().unwrap() { + found.push(path); + } + assert_eq!(found.len(), 2); + assert_eq!(found[0], nodes[0].1.nibbles.0); + assert_eq!(found[1], nodes[1].1.nibbles.0); + + // Verify data was stored for addr2 + let mut trie_cursor = storage.storage_trie_cursor(addr2, 100).unwrap(); + let mut found = vec![]; + while let Some((path, _node)) = trie_cursor.next().unwrap() { + found.push(path); + } + assert_eq!(found.len(), 1); + assert_eq!(found[0], nodes[2].1.nibbles.0); + } + + #[test] + fn test_full_initialize_run() { + let db = create_test_rw_db(); + let dir = TempDir::new().unwrap(); + let storage = Arc::new($storage::new(dir.path()).expect("env")); + + // Insert some test data + let tx = db.tx_mut().unwrap(); + + // Add accounts + let mut cursor = tx.cursor_write::().unwrap(); + let addr = keccak256(Address::repeat_byte(0x01)); + cursor + .append( + addr, + &Account { + nonce: 1, + balance: U256::from(100), + bytecode_hash: None, + }, + ) + .unwrap(); + drop(cursor); + + // Add storage + let mut cursor = tx.cursor_dup_write::().unwrap(); + cursor + .upsert( + addr, + &StorageEntry { + key: keccak256(B256::repeat_byte(0x10)), + value: U256::from(100), + }, + ) + .unwrap(); + drop(cursor); + + // Add account trie + let mut cursor = tx.cursor_write::().unwrap(); + cursor + .append( + StoredNibbles(Nibbles::from_nibbles_unchecked(vec![1])), + &create_test_branch_node(), + ) + .unwrap(); + drop(cursor); + + // Add storage trie + let mut cursor = tx.cursor_dup_write::().unwrap(); + cursor + .upsert( + addr, + &StorageTrieEntry { + nibbles: StoredNibblesSubKey(Nibbles::from_nibbles_unchecked( + vec![1], + )), + node: create_test_branch_node(), + }, + ) + .unwrap(); + drop(cursor); + + tx.commit().unwrap(); + + // Run full initialization + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&storage), tx); + let best_number = 100; + let best_hash = B256::repeat_byte(0x42); + + // Should be None initially + assert_eq!(storage.initial_state_anchor().unwrap().block, None); + assert_eq!(storage.get_earliest_block_number().unwrap(), None); + + job.run(best_number, best_hash).unwrap(); + + // Should be set after initialization + assert_eq!( + storage.get_earliest_block_number().unwrap(), + Some((best_number, best_hash)) + ); + + // Verify data was initialized + let mut account_cursor = storage.account_hashed_cursor(100).unwrap(); + assert!(account_cursor.next().unwrap().is_some()); + + let mut storage_cursor = storage.storage_hashed_cursor(addr, 100).unwrap(); + assert!(storage_cursor.next().unwrap().is_some()); + + let mut trie_cursor = storage.account_trie_cursor(100).unwrap(); + assert!(trie_cursor.next().unwrap().is_some()); + + let mut storage_trie_cursor = storage.storage_trie_cursor(addr, 100).unwrap(); + assert!(storage_trie_cursor.next().unwrap().is_some()); + } + + #[test] + fn test_initialize_run_skips_if_already_done() { + let db = create_test_rw_db(); + let dir = TempDir::new().unwrap(); + let storage = Arc::new($storage::new(dir.path()).expect("env")); + + // set and commit initial state anchor + storage + .set_initial_state_anchor(BlockNumHash::new(50, B256::repeat_byte(0x01))) + .expect("set anchor"); + storage.commit_initial_state().expect("commit anchor"); + + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&storage), tx); + + // Run initialization - should skip + job.run(100, B256::repeat_byte(0x42)).unwrap(); + + // Should still have the old anchor + let anchor_block = storage + .initial_state_anchor() + .expect("get anchor") + .block + .expect("block"); + assert_eq!( + Some((anchor_block.number, anchor_block.hash)), + Some((50, B256::repeat_byte(0x01))) + ); + + // Should still have the old earliest block + assert_eq!( + storage.get_earliest_block_number().unwrap(), + Some((50, B256::repeat_byte(0x01))) + ); + } + + #[test] + fn test_initialize_resumes_hashed_accounts_with_no_dups() { + let db = create_test_rw_db(); + let dir = TempDir::new().unwrap(); + let store = Arc::new($storage::new(dir.path()).expect("env")); + + store + .set_initial_state_anchor(BlockNumHash::new(0, B256::default())) + .expect("set anchor"); + + // Phase 1 in source: k1, k2 + let k1 = k(1); + let k2 = k(2); + { + let tx = db.tx_mut().unwrap(); + let mut cur = tx.cursor_write::().unwrap(); + cur.append( + k1, + &Account { + nonce: 1, + balance: U256::from(100), + bytecode_hash: None, + }, + ) + .unwrap(); + cur.append( + k2, + &Account { + nonce: 2, + balance: U256::from(200), + bytecode_hash: None, + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + // Initialization #1 + { + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&store), tx); + job.initialize_hashed_accounts(None).unwrap(); + } + + // Resume point must be k2 (max) + assert_eq!( + store + .initial_state_anchor() + .expect("get anchor") + .latest_hashed_account_key, + Some(k2) + ); + + // Phase 2 in source: k3, k4 + let k3 = k(3); + let k4 = k(4); + { + let tx = db.tx_mut().unwrap(); + let mut cur = tx.cursor_write::().unwrap(); + cur.append( + k3, + &Account { + nonce: 3, + balance: U256::from(300), + bytecode_hash: None, + }, + ) + .unwrap(); + cur.append( + k4, + &Account { + nonce: 4, + balance: U256::from(400), + bytecode_hash: None, + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + // Initialization #2 (restart) + { + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&store), tx); + job.initialize_hashed_accounts(Some(k2)).unwrap(); + } + + // Now resume point must be k4 + assert_eq!( + store + .initial_state_anchor() + .expect("get anchor") + .latest_hashed_account_key, + Some(k4) + ); + + // Verify order + no dupes by iterating proofs cursor + let mut cur = store.account_hashed_cursor(0).unwrap(); + let mut got = Vec::new(); + while let Some((key, acct)) = cur.next().unwrap() { + got.push((key, acct)); + } + + // Expect exactly 4, in increasing key order. + assert_eq!(got.len(), 4); + assert_eq!(got[0].0, k1); + assert_eq!(got[1].0, k2); + assert_eq!(got[2].0, k3); + assert_eq!(got[3].0, k4); + + // No dupes + for w in got.windows(2) { + assert!(w[0].0 < w[1].0); + } + } + + #[test] + fn test_initialize_resumes_hashed_storages_with_no_dups() { + let db = create_test_rw_db(); + let dir = TempDir::new().unwrap(); + let store = Arc::new($storage::new(dir.path()).expect("env")); + + store + .set_initial_state_anchor(BlockNumHash::new(0, B256::default())) + .expect("set anchor"); + + let a1 = k(0x10); + let a2 = k(0x20); + + let s11 = k(0x01); + let s12 = k(0x02); + let s21 = k(0x03); + let s22 = k(0x04); + + // Phase 1 source: + // a1: s11,s12 + // a2: s21 + { + let tx = db.tx_mut().unwrap(); + let mut cur = tx.cursor_dup_write::().unwrap(); + cur.upsert( + a1, + &StorageEntry { + key: s11, + value: U256::from(11), + }, + ) + .unwrap(); + cur.upsert( + a1, + &StorageEntry { + key: s12, + value: U256::from(12), + }, + ) + .unwrap(); + cur.upsert( + a2, + &StorageEntry { + key: s21, + value: U256::from(21), + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + // Initialization #1 + { + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&store), tx); + job.initialize_hashed_storages(None).unwrap(); + } + + // Latest key must be (a2, s21) because a2 > a1 + let last1 = store + .initial_state_anchor() + .expect("get anchor") + .latest_hashed_storage_key + .expect("ok"); + assert_eq!(last1.hashed_address, a2); + assert_eq!(last1.hashed_storage_key, s21); + + // Phase 2 source: add s22 to a2 + { + let tx = db.tx_mut().unwrap(); + let mut cur = tx.cursor_dup_write::().unwrap(); + cur.upsert( + a2, + &StorageEntry { + key: s22, + value: U256::from(22), + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + // Initialization #2 + { + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&store), tx); + job.initialize_hashed_storages(Some(HashedStorageKey::new(a2, s21))) + .unwrap(); + } + + // Latest key now must be (a2, s22) + let last2 = store + .initial_state_anchor() + .expect("get anchor") + .latest_hashed_storage_key + .expect("ok"); + assert_eq!(last2.hashed_address, a2); + assert_eq!(last2.hashed_storage_key, s22); + + // Verify no dupes by iterating per-address + { + let mut c = store.storage_hashed_cursor(a1, 0).unwrap(); + let mut got = Vec::new(); + while let Some((slot, val)) = c.next().unwrap() { + got.push((slot, val)); + } + assert_eq!(got.len(), 2); + assert_eq!(got[0].0, s11); + assert_eq!(got[1].0, s12); + } + { + let mut c = store.storage_hashed_cursor(a2, 0).unwrap(); + let mut got = Vec::new(); + while let Some((slot, val)) = c.next().unwrap() { + got.push((slot, val)); + } + assert_eq!(got.len(), 2); + assert_eq!(got[0].0, s21); + assert_eq!(got[1].0, s22); + } + } + + #[test] + fn test_initialize_resumes_accounts_trie_with_no_dups() { + let db = create_test_rw_db(); + let dir = TempDir::new().unwrap(); + let store = Arc::new($storage::new(dir.path()).expect("env")); + + store + .set_initial_state_anchor(BlockNumHash::new(0, B256::default())) + .expect("set anchor"); + + let p1 = StoredNibbles(Nibbles::from_nibbles_unchecked(vec![1])); + let p2 = StoredNibbles(Nibbles::from_nibbles_unchecked(vec![2])); + let p3 = StoredNibbles(Nibbles::from_nibbles_unchecked(vec![3])); + let p4 = StoredNibbles(Nibbles::from_nibbles_unchecked(vec![4])); + + // Phase 1 source: p1,p2 + { + let tx = db.tx_mut().unwrap(); + let mut cur = tx.cursor_write::().unwrap(); + cur.append(p1.clone(), &create_test_branch_node()).unwrap(); + cur.append(p2.clone(), &create_test_branch_node()).unwrap(); + tx.commit().unwrap(); + } + + // Initialization #1 + { + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&store), tx); + job.initialize_accounts_trie(None).unwrap(); + } + + assert_eq!( + store + .initial_state_anchor() + .expect("get anchor") + .latest_account_trie_key, + Some(p2.clone()) + ); + + // Phase 2 source: p3,p4 + { + let tx = db.tx_mut().unwrap(); + let mut cur = tx.cursor_write::().unwrap(); + cur.append(p3.clone(), &create_test_branch_node()).unwrap(); + cur.append(p4.clone(), &create_test_branch_node()).unwrap(); + tx.commit().unwrap(); + } + + // Initialization #2 + { + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&store), tx); + job.initialize_accounts_trie(Some(p2.clone())).unwrap(); + } + + assert_eq!( + store + .initial_state_anchor() + .expect("get anchor") + .latest_account_trie_key, + Some(p4.clone()) + ); + + // Verify 4 ordered, no dupes + let mut c = store.account_trie_cursor(0).unwrap(); + let mut got = Vec::new(); + while let Some((path, _node)) = c.next().unwrap() { + got.push(path); + } + assert_eq!(got.len(), 4); + assert_eq!(got[0], p1.0); + assert_eq!(got[1], p2.0); + assert_eq!(got[2], p3.0); + assert_eq!(got[3], p4.0); + } + + #[test] + fn test_initialize_resumes_storages_trie_with_no_dups() { + let db = create_test_rw_db(); + let dir = TempDir::new().unwrap(); + let store = Arc::new($storage::new(dir.path()).expect("env")); + + store + .set_initial_state_anchor(BlockNumHash::new(0, B256::default())) + .expect("set anchor"); + + let a1 = k(0x10); + let a2 = k(0x20); + + let n1 = StoredNibblesSubKey(Nibbles::from_nibbles_unchecked(vec![1])); + let n2 = StoredNibblesSubKey(Nibbles::from_nibbles_unchecked(vec![2])); + let n3 = StoredNibblesSubKey(Nibbles::from_nibbles_unchecked(vec![3])); + + // Phase 1 source: (a1,n1), (a2,n2) + { + let tx = db.tx_mut().unwrap(); + let mut cur = tx.cursor_dup_write::().unwrap(); + cur.upsert( + a1, + &StorageTrieEntry { + nibbles: n1.clone(), + node: create_test_branch_node(), + }, + ) + .unwrap(); + cur.upsert( + a2, + &StorageTrieEntry { + nibbles: n2.clone(), + node: create_test_branch_node(), + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + // Initialization #1 + { + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&store), tx); + job.initialize_storages_trie(None).unwrap(); + } + + // Latest must be (a2, n2) because a2 > a1 + let last1 = store + .initial_state_anchor() + .expect("get anchor") + .latest_storage_trie_key + .expect("ok"); + assert_eq!(last1.hashed_address, a2); + assert_eq!(last1.path.0, n2.0); + + // Phase 2 source: add (a2,n3) + { + let tx = db.tx_mut().unwrap(); + let mut cur = tx.cursor_dup_write::().unwrap(); + cur.upsert( + a2, + &StorageTrieEntry { + nibbles: n3.clone(), + node: create_test_branch_node(), + }, + ) + .unwrap(); + tx.commit().unwrap(); + } + + // Initialization #2 + { + let tx = db.tx().unwrap(); + let job = InitializationJob::new(Arc::clone(&store), tx); + job.initialize_storages_trie(Some(StorageTrieKey::new( + a2, + StoredNibbles::from(n2.0), + ))) + .unwrap(); + } + + // Latest must now be (a2,n3) + let last2 = store + .initial_state_anchor() + .expect("get anchor") + .latest_storage_trie_key + .expect("ok"); + assert_eq!(last2.hashed_address, a2); + assert_eq!(last2.path.0, n3.0); + + // Verify per-address no dupes and stable ordering + { + let mut c = store.storage_trie_cursor(a1, 0).unwrap(); + + let mut got = Vec::new(); + + // next returns the rest + while let Some((path, _node)) = c.next().unwrap() { + got.push(path); + } + + assert_eq!(got, vec![n1.0]); + } + { + let mut c = store.storage_trie_cursor(a2, 0).unwrap(); + + let mut got = Vec::new(); + + // next returns the rest + while let Some((path, _node)) = c.next().unwrap() { + got.push(path); + } + assert_eq!(got, vec![n2.0, n3.0]); + } + } + } + }; + } + + proof_storage_init_tests!(mdbx_tests, MdbxProofsStorage); +} diff --git a/crates/proofs/src/lib.rs b/crates/proofs/src/lib.rs new file mode 100644 index 00000000..438ec198 --- /dev/null +++ b/crates/proofs/src/lib.rs @@ -0,0 +1,61 @@ +//! Bounded historical MPT storage for Morph `eth_getProof`. +//! +//! The versioned-trie implementation is adapted from Base's proof-history +//! storage at commit `b2673bbd927cb34d7cfad4d448bfbd5bd30eae88` under MIT. +//! Morph intentionally exposes only the MDBX production backend. + +#![cfg_attr(not(test), warn(unused_crate_dependencies))] + +pub mod api; +pub use api::{ + BlockStateDiff, InitialStateAnchor, InitialStateStatus, MorphProofsBatchSession, + MorphProofsBatchStore, MorphProofsInitialStateStore, MorphProofsStore, OperationDurations, + StorageBranchEntries, WriteCounts, +}; + +pub mod initialize; +pub use initialize::InitializationJob; + +#[cfg(any(test, feature = "test-utils"))] +pub mod in_memory; +#[cfg(any(test, feature = "test-utils"))] +pub use in_memory::{ + InMemoryAccountCursor, InMemoryBatchSession, InMemoryProofsStorage, InMemoryStorageCursor, + InMemoryTrieCursor, +}; + +pub mod db; +pub use db::{ + MdbxAccountCursor, MdbxBatchSession, MdbxProofsStorage, MdbxProofsStorageOptions, + MdbxStorageCursor, MdbxTrieCursor, ProofDbIdentity, ProofWindowBounds, +}; + +pub mod metrics; +pub use metrics::{ + MorphProofsHashedAccountCursor, MorphProofsHashedStorageCursor, MorphProofsStorage, + MorphProofsTrieCursor, ProofHistoryMetrics, +}; + +pub mod proof; +pub mod provider; + +mod batch_provider; +pub use batch_provider::MorphProofsBatchStateProviderRef; + +pub mod live; + +pub mod cursor; +pub mod cursor_factory; +pub use cursor_factory::{ + MorphProofsBatchHashedAccountCursorFactory, MorphProofsBatchTrieCursorFactory, + MorphProofsHashedAccountCursorFactory, MorphProofsTrieCursorFactory, +}; + +pub mod error; +pub use error::{MorphProofsStorageError, MorphProofsStorageResult}; + +mod prune; +pub use prune::{ + MorphProofStoragePruner, MorphProofStoragePrunerResult, MorphProofStoragePrunerTask, + PrunerError, PrunerOutput, +}; diff --git a/crates/proofs/src/live.rs b/crates/proofs/src/live.rs new file mode 100644 index 00000000..fafc1023 --- /dev/null +++ b/crates/proofs/src/live.rs @@ -0,0 +1,432 @@ +//! Live trie collector for external proofs storage. + +use std::{sync::Arc, time::Instant}; + +use alloy_eips::{BlockNumHash, NumHash, eip1898::BlockWithParent}; +use derive_more::Constructor; +use reth_evm::{ConfigureEvm, execute::Executor}; +use reth_primitives_traits::{AlloyBlockHeader, BlockTy, NodePrimitives, RecoveredBlock}; +use reth_provider::{ + DatabaseProviderFactory, HashedPostStateProvider, StateProviderFactory, StateRootProvider, +}; +use reth_revm::database::StateProviderDatabase; +use reth_trie_common::{HashedPostStateSorted, updates::TrieUpdatesSorted}; +use tracing::{info, warn}; + +use crate::{ + BlockStateDiff, MorphProofsStorage, MorphProofsStorageError, MorphProofsStore, + api::{MorphProofsBatchSession, MorphProofsBatchStore, OperationDurations, WriteCounts}, + batch_provider::MorphProofsBatchStateProviderRef, + metrics::ProofHistoryMetrics, + provider::MorphProofsStateProviderRef, +}; + +/// Live trie collector for external proofs storage. +#[derive(Debug, Constructor)] +pub struct LiveTrieCollector<'tx, Evm, Provider, PreimageStore> +where + Evm: ConfigureEvm, + Provider: DatabaseProviderFactory + StateProviderFactory, +{ + evm_config: Evm, + provider: Provider, + storage: &'tx MorphProofsStorage, +} + +impl<'tx, Evm, Provider, Store> LiveTrieCollector<'tx, Evm, Provider, Store> +where + Evm: ConfigureEvm, + Provider: DatabaseProviderFactory + StateProviderFactory, + Store: 'tx + MorphProofsStore + Clone + 'static, +{ + fn record_storage_metrics( + operation_durations: &OperationDurations, + write_counts: Option<&WriteCounts>, + ) { + ProofHistoryMetrics::record_block(operation_durations, write_counts); + } + + /// Execute a block and store the updates in the storage. + pub fn execute_and_store_block_updates( + &self, + block: &RecoveredBlock>, + ) -> Result<(), MorphProofsStorageError> { + let mut operation_durations = OperationDurations::default(); + + let start = Instant::now(); + // ensure that we have the state of the parent block + let (Some((earliest, _)), Some((latest, _))) = ( + self.storage.get_earliest_block_number()?, + self.storage.get_latest_block_number()?, + ) else { + return Err(MorphProofsStorageError::NoBlocksFound); + }; + + let parent_block_number = block.number() - 1; + if parent_block_number < earliest { + return Err(MorphProofsStorageError::UnknownParent); + } + + if parent_block_number > latest { + return Err(MorphProofsStorageError::MissingParentBlock { + block_number: block.number(), + parent_block_number, + latest_block_number: latest, + }); + } + + let block_ref = BlockWithParent::new( + block.parent_hash(), + NumHash::new(block.number(), block.hash()), + ); + + let state_provider = MorphProofsStateProviderRef::new( + self.provider.latest()?, + self.storage, + parent_block_number, + ); + + let db = StateProviderDatabase::new(&state_provider); + let block_executor = self.evm_config.batch_executor(db); + + let execution_result = block_executor.execute(&(*block).clone())?; + + operation_durations.execution_duration_seconds = start.elapsed(); + + let hashed_state = state_provider.hashed_post_state(&execution_result.state); + let (state_root, trie_updates) = + state_provider.state_root_with_updates(hashed_state.clone())?; + + operation_durations.state_root_duration_seconds = + start.elapsed() - operation_durations.execution_duration_seconds; + + if state_root != block.state_root() { + return Err(MorphProofsStorageError::StateRootMismatch { + block_number: block.number(), + current_state_hash: state_root, + expected_state_hash: block.state_root(), + }); + } + + let update_result = self.storage.store_trie_updates( + block_ref, + BlockStateDiff { + sorted_trie_updates: trie_updates.into_sorted(), + sorted_post_state: hashed_state.into_sorted(), + }, + )?; + + operation_durations.total_duration_seconds = start.elapsed(); + operation_durations.write_duration_seconds = operation_durations.total_duration_seconds + - operation_durations.state_root_duration_seconds + - operation_durations.execution_duration_seconds; + + Self::record_storage_metrics(&operation_durations, Some(&update_result)); + + info!( + block_number = block.number(), + block_hash = ?block.hash(), + parent_block_number, + parent_block_hash = ?block.parent_hash(), + earliest_proof_before = earliest, + latest_proof_before = latest, + latest_proof_after = block.number(), + state_root = ?state_root, + ?operation_durations, + ?update_result, + "Block executed and trie updates stored successfully", + ); + + Ok(()) + } + + /// Store trie updates for a given block. + pub fn store_block_updates( + &self, + block: BlockWithParent, + sorted_trie_updates: TrieUpdatesSorted, + sorted_post_state: HashedPostStateSorted, + ) -> Result<(), MorphProofsStorageError> { + let start = Instant::now(); + let mut operation_durations = OperationDurations::default(); + + let storage_result = match self.storage.store_trie_updates( + block, + BlockStateDiff { + sorted_trie_updates, + sorted_post_state, + }, + ) { + Ok(res) => res, + Err(MorphProofsStorageError::OutOfOrder { + block_number, + latest_block_hash, + parent_block_hash, + }) => { + warn!( + block_number, + ?latest_block_hash, + ?parent_block_hash, + "Skipping out of order block updates" + ); + return Ok(()); + } + Err(e) => return Err(e), + }; + + let write_duration = start.elapsed(); + operation_durations.total_duration_seconds = write_duration; + operation_durations.write_duration_seconds = write_duration; + + Self::record_storage_metrics(&operation_durations, Some(&storage_result)); + + info!( + block_number = block.block.number, + ?operation_durations, + ?storage_result, + "Trie updates stored successfully", + ); + + Ok(()) + } + + /// Handles chain reorganizations by replacing block updates after a common ancestor. + /// + /// This method removes all block updates after the latest common ancestor (the block before + /// the first block in `new_blocks`) and replaces them with the updates from the provided new + /// chain. + /// + /// # Arguments + /// + /// * `new_blocks` - A vector of references to `RecoveredBlock` instances representing the new + /// blocks to be added to the trie storage. + pub fn unwind_and_store_block_updates( + &self, + block_updates: Vec<( + BlockWithParent, + Arc, + Arc, + )>, + ) -> Result<(), MorphProofsStorageError> { + if block_updates.is_empty() { + return Ok(()); + } + + let start = Instant::now(); + let mut operation_durations = OperationDurations::default(); + let first = &block_updates[0].0; + let latest_common_block = + BlockNumHash::new(first.block.number.saturating_sub(1), first.parent); + let mut block_trie_updates: Vec<(BlockWithParent, BlockStateDiff)> = + Vec::with_capacity(block_updates.len()); + + for (block, trie_updates, hashed_state) in &block_updates { + block_trie_updates.push(( + *block, + BlockStateDiff { + sorted_trie_updates: (**trie_updates).clone(), + sorted_post_state: (**hashed_state).clone(), + }, + )); + } + + self.storage + .replace_updates(latest_common_block, block_trie_updates)?; + let write_duration = start.elapsed(); + operation_durations.total_duration_seconds = write_duration; + operation_durations.write_duration_seconds = write_duration; + + Self::record_storage_metrics(&operation_durations, None); + + info!( + start_block_number = block_updates.first().map(|(b, _, _)| b.block.number), + end_block_number = block_updates.last().map(|(b, _, _)| b.block.number), + ?operation_durations, + "Trie updates rewound and stored successfully", + ); + Ok(()) + } + + /// Remove account, storage and trie updates from historical storage for all blocks from + /// the specified block (inclusive). + pub fn unwind_history(&self, to: BlockWithParent) -> Result<(), MorphProofsStorageError> { + if let Err(error) = self.storage.unwind_history(to) { + ProofHistoryMetrics::record_unwind_error(); + return Err(error); + } + if let Some((earliest, _)) = self.storage.get_earliest_block_number()? + && let Some((latest, _)) = self.storage.get_latest_block_number()? + { + ProofHistoryMetrics::set_window(earliest, latest); + } + Ok(()) + } +} + +/// One block to process inside a batch session: either pre-computed cached trie data (fast +/// path) or a fully recovered block that needs full execution against the session's +/// transaction-local state (cold catch-up path). +#[derive(Debug)] +pub enum BatchBlock { + /// Pre-computed cached trie data; only writes happen. + Cached { + /// Block reference being written. + block_with_parent: BlockWithParent, + /// Pre-computed trie updates from cached notification data. + sorted_trie_updates: Arc, + /// Pre-computed hashed post-state from cached notification data. + sorted_post_state: Arc, + }, + /// Full block requiring execution against session-local state. + Execute(Box>>), +} + +impl<'tx, Evm, Provider, Store> LiveTrieCollector<'tx, Evm, Provider, Store> +where + Evm: ConfigureEvm, + Provider: DatabaseProviderFactory + StateProviderFactory, + Store: 'tx + MorphProofsBatchStore + Clone + 'static, +{ + /// Execute and write a batch of blocks inside a single underlying transaction. + /// Reads during execution of block N observe writes from blocks earlier in the batch, + /// enabling cold catch-up where parent state is staged but not yet committed. The + /// entire batch commits atomically on success and aborts on the first error. + pub fn execute_and_store_batch( + &self, + blocks: Vec>, + ) -> Result<(), MorphProofsStorageError> { + if blocks.is_empty() { + return Ok(()); + } + + let start = Instant::now(); + let mut total_writes = WriteCounts::default(); + let mut block_count: u64 = 0; + let mut last_block_number: u64 = 0; + + self.storage.with_batch_session(|session| { + let (Some((earliest, _)), Some((_, _))) = ( + session.get_earliest_block_number()?, + session.get_latest_block_number()?, + ) else { + return Err(MorphProofsStorageError::NoBlocksFound); + }; + + for entry in blocks { + match entry { + BatchBlock::Cached { + block_with_parent, + sorted_trie_updates, + sorted_post_state, + } => { + let counts = session.store_trie_updates( + block_with_parent, + BlockStateDiff { + sorted_trie_updates: (*sorted_trie_updates).clone(), + sorted_post_state: (*sorted_post_state).clone(), + }, + )?; + total_writes += counts; + block_count += 1; + last_block_number = block_with_parent.block.number; + } + BatchBlock::Execute(block) => { + let counts = self.execute_one_in_session(session, &block, earliest)?; + total_writes += counts; + block_count += 1; + last_block_number = block.number(); + } + } + } + + Ok(()) + })?; + + let total = start.elapsed(); + ProofHistoryMetrics::record_block( + &OperationDurations { + total_duration_seconds: total, + ..Default::default() + }, + Some(&total_writes), + ); + if let Some((earliest, _)) = self.storage.get_earliest_block_number()? { + ProofHistoryMetrics::set_window(earliest, last_block_number); + } + + info!( + block_count, + last_block_number, + ?total_writes, + duration = ?total, + "Batch executed and committed", + ); + + Ok(()) + } + + fn execute_one_in_session( + &self, + session: &mut S, + block: &RecoveredBlock>, + earliest: u64, + ) -> Result + where + S: MorphProofsBatchSession, + { + let latest_in_session = session + .get_latest_block_number()? + .ok_or(MorphProofsStorageError::NoBlocksFound)? + .0; + + let parent_block_number = block.number() - 1; + if parent_block_number < earliest { + return Err(MorphProofsStorageError::UnknownParent); + } + if parent_block_number > latest_in_session { + return Err(MorphProofsStorageError::MissingParentBlock { + block_number: block.number(), + parent_block_number, + latest_block_number: latest_in_session, + }); + } + + let block_ref = BlockWithParent::new( + block.parent_hash(), + NumHash::new(block.number(), block.hash()), + ); + + let state_provider = MorphProofsBatchStateProviderRef::new( + self.provider.latest()?, + session, + parent_block_number, + ); + + let db = StateProviderDatabase::new(&state_provider); + let block_executor = self.evm_config.batch_executor(db); + + let execution_result = block_executor.execute(&(*block).clone())?; + + let hashed_state = state_provider.hashed_post_state(&execution_result.state); + let (state_root, trie_updates) = + state_provider.state_root_with_updates(hashed_state.clone())?; + + if state_root != block.state_root() { + return Err(MorphProofsStorageError::StateRootMismatch { + block_number: block.number(), + current_state_hash: state_root, + expected_state_hash: block.state_root(), + }); + } + + drop(state_provider); + + session.store_trie_updates( + block_ref, + BlockStateDiff { + sorted_trie_updates: trie_updates.into_sorted(), + sorted_post_state: hashed_state.into_sorted(), + }, + ) + } +} diff --git a/crates/proofs/src/metrics.rs b/crates/proofs/src/metrics.rs new file mode 100644 index 00000000..21211c9e --- /dev/null +++ b/crates/proofs/src/metrics.rs @@ -0,0 +1,97 @@ +//! Metrics for bounded historical-proof storage. + +use reth_metrics::{ + Metrics, + metrics::{Counter, Gauge, Histogram}, +}; + +use crate::{OperationDurations, WriteCounts, cursor}; + +/// The production storage type. Kept as an alias so callers remain backend-agnostic +/// without introducing another production backend. +pub type MorphProofsStorage = S; + +/// Trie cursor used by proof generation. +pub type MorphProofsTrieCursor = cursor::MorphProofsTrieCursor; +/// Hashed account cursor used by proof generation. +pub type MorphProofsHashedAccountCursor = cursor::MorphProofsHashedAccountCursor; +/// Hashed storage cursor used by proof generation. +pub type MorphProofsHashedStorageCursor = cursor::MorphProofsHashedStorageCursor; + +/// Proof-history storage and lifecycle metrics. +#[derive(Metrics, Clone)] +#[metrics(scope = "morph.proofs")] +pub struct ProofHistoryMetrics { + /// Total time spent processing a block. + pub block_total_duration_seconds: Histogram, + /// Time spent executing a block when catch-up re-execution is required. + pub block_execution_duration_seconds: Histogram, + /// Time spent calculating a state root. + pub block_state_root_duration_seconds: Histogram, + /// Time spent persisting a block. + pub block_write_duration_seconds: Histogram, + /// Number of account-trie updates written. + pub account_trie_updates_written_total: Counter, + /// Number of storage-trie updates written. + pub storage_trie_updates_written_total: Counter, + /// Number of account leaves written. + pub hashed_accounts_written_total: Counter, + /// Number of storage leaves written. + pub hashed_storages_written_total: Counter, + /// Earliest block currently served by proof history. + pub earliest_block: Gauge, + /// Latest block currently served by proof history. + pub latest_block: Gauge, + /// Number of prune failures. + pub prune_errors_total: Counter, + /// Number of unwind failures. + pub unwind_errors_total: Counter, +} + +impl ProofHistoryMetrics { + /// Records latency and write-count measurements for one processed block or batch. + pub fn record_block(durations: &OperationDurations, counts: Option<&WriteCounts>) { + let metrics = Self::default(); + metrics + .block_total_duration_seconds + .record(durations.total_duration_seconds.as_secs_f64()); + metrics + .block_execution_duration_seconds + .record(durations.execution_duration_seconds.as_secs_f64()); + metrics + .block_state_root_duration_seconds + .record(durations.state_root_duration_seconds.as_secs_f64()); + metrics + .block_write_duration_seconds + .record(durations.write_duration_seconds.as_secs_f64()); + if let Some(counts) = counts { + metrics + .account_trie_updates_written_total + .increment(counts.account_trie_updates_written_total); + metrics + .storage_trie_updates_written_total + .increment(counts.storage_trie_updates_written_total); + metrics + .hashed_accounts_written_total + .increment(counts.hashed_accounts_written_total); + metrics + .hashed_storages_written_total + .increment(counts.hashed_storages_written_total); + } + } + + /// Updates the gauges for the inclusive durable proof window. + pub fn set_window(earliest: u64, latest: u64) { + let metrics = Self::default(); + metrics.earliest_block.set(earliest as f64); + metrics.latest_block.set(latest as f64); + } + + pub(crate) fn record_prune_error() { + Self::default().prune_errors_total.increment(1); + } + + pub(crate) fn record_unwind_error() { + Self::default().unwind_errors_total.increment(1); + } +} diff --git a/crates/proofs/src/proof.rs b/crates/proofs/src/proof.rs new file mode 100644 index 00000000..5a1adeb0 --- /dev/null +++ b/crates/proofs/src/proof.rs @@ -0,0 +1,667 @@ +//! Provides proof operation implementations for [`MorphProofsStorage`]. + +use alloy_primitives::{ + Address, B256, Bytes, keccak256, + map::{B256Map, HashMap}, +}; +use reth_db::DatabaseError; +use reth_execution_errors::{StateProofError, StateRootError, StorageRootError, TrieWitnessError}; +use reth_trie::{ + StateRoot, StorageRoot, TrieType, + hashed_cursor::HashedPostStateCursorFactory, + metrics::TrieRootMetrics, + proof::{self, Proof}, + trie_cursor::InMemoryTrieCursorFactory, + witness::TrieWitness, +}; +use reth_trie_common::{ + AccountProof, ExecutionWitnessMode, HashedPostState, HashedPostStateSorted, HashedStorage, + MultiProof, MultiProofTargets, StorageMultiProof, StorageProof, TrieInput, + updates::TrieUpdates, +}; + +use crate::{ + MorphProofsHashedAccountCursorFactory, MorphProofsStorage, MorphProofsStore, + MorphProofsTrieCursorFactory, +}; + +/// Build the trie + hashed cursor factories sharing one read transaction at the given block. +const fn from_tx<'tx, 'db, S>( + storage: &'db MorphProofsStorage, + tx: &'tx as MorphProofsStore>::Tx<'db>, + block_number: u64, +) -> ( + MorphProofsTrieCursorFactory<'tx, 'db, S>, + MorphProofsHashedAccountCursorFactory<'tx, 'db, S>, +) +where + S: MorphProofsStore + 'db, + 'db: 'tx, +{ + ( + MorphProofsTrieCursorFactory::new(storage, tx, block_number), + MorphProofsHashedAccountCursorFactory::new(storage, tx, block_number), + ) +} + +/// Extends [`Proof`] with operations specific for working with [`MorphProofsStorage`]. +pub trait DatabaseProof<'tx, S: MorphProofsStore + 'tx> { + /// Generates the state proof for target account based on [`TrieInput`]. + fn overlay_account_proof( + storage: &'tx MorphProofsStorage, + block_number: u64, + input: TrieInput, + address: Address, + slots: &[B256], + ) -> Result; + + /// Generates an account proof while reusing an existing read transaction. + fn overlay_account_proof_with_tx<'cursor>( + storage: &'tx MorphProofsStorage, + tx: &'cursor as MorphProofsStore>::Tx<'tx>, + block_number: u64, + input: TrieInput, + address: Address, + slots: &[B256], + ) -> Result + where + 'tx: 'cursor; + + /// Generates the state [`MultiProof`] for target hashed account and storage keys. + fn overlay_multiproof( + storage: &'tx MorphProofsStorage, + block_number: u64, + input: TrieInput, + targets: MultiProofTargets, + ) -> Result; + + /// Generates a state multiproof while reusing an existing read transaction. + fn overlay_multiproof_with_tx<'cursor>( + storage: &'tx MorphProofsStorage, + tx: &'cursor as MorphProofsStore>::Tx<'tx>, + block_number: u64, + input: TrieInput, + targets: MultiProofTargets, + ) -> Result + where + 'tx: 'cursor; +} + +impl<'tx, S> DatabaseProof<'tx, S> + for Proof< + MorphProofsTrieCursorFactory<'tx, 'tx, S>, + MorphProofsHashedAccountCursorFactory<'tx, 'tx, S>, + > +where + S: MorphProofsStore + 'tx + Clone, +{ + fn overlay_account_proof( + storage: &'tx MorphProofsStorage, + block_number: u64, + input: TrieInput, + address: Address, + slots: &[B256], + ) -> Result { + let tx = storage.ro_tx().map_err(Into::::into)?; + Self::overlay_account_proof_with_tx(storage, &tx, block_number, input, address, slots) + } + + fn overlay_account_proof_with_tx<'cursor>( + storage: &'tx MorphProofsStorage, + tx: &'cursor as MorphProofsStore>::Tx<'tx>, + block_number: u64, + input: TrieInput, + address: Address, + slots: &[B256], + ) -> Result + where + 'tx: 'cursor, + { + let nodes_sorted = input.nodes.into_sorted(); + let state_sorted = input.state.into_sorted(); + let (trie_factory, hashed_factory) = from_tx(storage, tx, block_number); + Proof::new(trie_factory.clone(), hashed_factory.clone()) + .with_trie_cursor_factory(InMemoryTrieCursorFactory::new(trie_factory, &nodes_sorted)) + .with_hashed_cursor_factory(HashedPostStateCursorFactory::new( + hashed_factory, + &state_sorted, + )) + .with_prefix_sets_mut(input.prefix_sets) + .account_proof(address, slots) + } + + fn overlay_multiproof( + storage: &'tx MorphProofsStorage, + block_number: u64, + input: TrieInput, + targets: MultiProofTargets, + ) -> Result { + let tx = storage.ro_tx().map_err(Into::::into)?; + Self::overlay_multiproof_with_tx(storage, &tx, block_number, input, targets) + } + + fn overlay_multiproof_with_tx<'cursor>( + storage: &'tx MorphProofsStorage, + tx: &'cursor as MorphProofsStore>::Tx<'tx>, + block_number: u64, + input: TrieInput, + targets: MultiProofTargets, + ) -> Result + where + 'tx: 'cursor, + { + let nodes_sorted = input.nodes.into_sorted(); + let state_sorted = input.state.into_sorted(); + let (trie_factory, hashed_factory) = from_tx(storage, tx, block_number); + Proof::new(trie_factory.clone(), hashed_factory.clone()) + .with_trie_cursor_factory(InMemoryTrieCursorFactory::new(trie_factory, &nodes_sorted)) + .with_hashed_cursor_factory(HashedPostStateCursorFactory::new( + hashed_factory, + &state_sorted, + )) + .with_prefix_sets_mut(input.prefix_sets) + .multiproof(targets) + } +} + +/// Extends [`StorageProof`] with operations specific for working with [`MorphProofsStorage`]. +pub trait DatabaseStorageProof<'tx, S: MorphProofsStore + 'tx> { + /// Generates the storage proof for target slot based on [`TrieInput`]. + fn overlay_storage_proof( + storage: &'tx MorphProofsStorage, + block_number: u64, + address: Address, + slot: B256, + storage: HashedStorage, + ) -> Result; + + /// Generates the storage multiproof for target slots based on [`TrieInput`]. + fn overlay_storage_multiproof( + storage: &'tx MorphProofsStorage, + block_number: u64, + address: Address, + slots: &[B256], + storage: HashedStorage, + ) -> Result; +} + +impl<'tx, S> DatabaseStorageProof<'tx, S> + for proof::StorageProof< + 'static, + MorphProofsTrieCursorFactory<'tx, 'tx, S>, + MorphProofsHashedAccountCursorFactory<'tx, 'tx, S>, + > +where + S: MorphProofsStore + 'tx + Clone, +{ + fn overlay_storage_proof( + storage: &'tx MorphProofsStorage, + block_number: u64, + address: Address, + slot: B256, + hashed_storage: HashedStorage, + ) -> Result { + let hashed_address = keccak256(address); + let prefix_set = hashed_storage.construct_prefix_set(); + let state_sorted = HashedPostStateSorted::new( + Default::default(), + HashMap::from_iter([(hashed_address, hashed_storage.into_sorted())]), + ); + let tx = storage.ro_tx().map_err(Into::::into)?; + let (trie_factory, hashed_factory) = from_tx(storage, &tx, block_number); + proof::StorageProof::new(trie_factory, hashed_factory.clone(), address) + .with_hashed_cursor_factory(HashedPostStateCursorFactory::new( + hashed_factory, + &state_sorted, + )) + .with_prefix_set_mut(prefix_set) + .storage_proof(slot) + } + + fn overlay_storage_multiproof( + storage: &'tx MorphProofsStorage, + block_number: u64, + address: Address, + slots: &[B256], + hashed_storage: HashedStorage, + ) -> Result { + let hashed_address = keccak256(address); + let targets = slots.iter().map(keccak256).collect(); + let prefix_set = hashed_storage.construct_prefix_set(); + let state_sorted = HashedPostStateSorted::new( + Default::default(), + HashMap::from_iter([(hashed_address, hashed_storage.into_sorted())]), + ); + let tx = storage.ro_tx().map_err(Into::::into)?; + let (trie_factory, hashed_factory) = from_tx(storage, &tx, block_number); + proof::StorageProof::new(trie_factory, hashed_factory.clone(), address) + .with_hashed_cursor_factory(HashedPostStateCursorFactory::new( + hashed_factory, + &state_sorted, + )) + .with_prefix_set_mut(prefix_set) + .storage_multiproof(targets) + } +} + +/// Extends [`StateRoot`] with operations specific for working with [`MorphProofsStorage`]. +pub trait DatabaseStateRoot<'tx, S: MorphProofsStore + 'tx + Clone>: Sized { + /// Calculate the state root for this [`HashedPostState`]. + /// Internally, this method retrieves prefixsets and uses them + /// to calculate incremental state root. + /// + /// # Returns + /// + /// The state root for this [`HashedPostState`]. + fn overlay_root( + storage: &'tx MorphProofsStorage, + block_number: u64, + post_state: HashedPostState, + ) -> Result; + + /// Calculates the state root for this [`HashedPostState`] and returns it alongside trie + /// updates. See [`Self::overlay_root`] for more info. + fn overlay_root_with_updates( + storage: &'tx MorphProofsStorage, + block_number: u64, + post_state: HashedPostState, + ) -> Result<(B256, TrieUpdates), StateRootError>; + + /// Calculates the state root for provided [`HashedPostState`] using cached intermediate nodes. + fn overlay_root_from_nodes( + storage: &'tx MorphProofsStorage, + block_number: u64, + input: TrieInput, + ) -> Result; + + /// Calculates the state root and trie updates for provided [`HashedPostState`] using + /// cached intermediate nodes. + fn overlay_root_from_nodes_with_updates( + storage: &'tx MorphProofsStorage, + block_number: u64, + input: TrieInput, + ) -> Result<(B256, TrieUpdates), StateRootError>; +} + +impl<'tx, S> DatabaseStateRoot<'tx, S> + for StateRoot< + MorphProofsTrieCursorFactory<'tx, 'tx, S>, + MorphProofsHashedAccountCursorFactory<'tx, 'tx, S>, + > +where + S: MorphProofsStore + 'tx + Clone, +{ + fn overlay_root( + storage: &'tx MorphProofsStorage, + block_number: u64, + post_state: HashedPostState, + ) -> Result { + let prefix_sets = post_state.construct_prefix_sets().freeze(); + let state_sorted = post_state.into_sorted(); + let tx = storage.ro_tx().map_err(Into::::into)?; + let (trie_factory, hashed_factory) = from_tx(storage, &tx, block_number); + StateRoot::new( + trie_factory, + HashedPostStateCursorFactory::new(hashed_factory, &state_sorted), + ) + .with_prefix_sets(prefix_sets) + .root() + } + + fn overlay_root_with_updates( + storage: &'tx MorphProofsStorage, + block_number: u64, + post_state: HashedPostState, + ) -> Result<(B256, TrieUpdates), StateRootError> { + let prefix_sets = post_state.construct_prefix_sets().freeze(); + let state_sorted = post_state.into_sorted(); + let tx = storage.ro_tx().map_err(Into::::into)?; + let (trie_factory, hashed_factory) = from_tx(storage, &tx, block_number); + StateRoot::new( + trie_factory, + HashedPostStateCursorFactory::new(hashed_factory, &state_sorted), + ) + .with_prefix_sets(prefix_sets) + .root_with_updates() + } + + fn overlay_root_from_nodes( + storage: &'tx MorphProofsStorage, + block_number: u64, + input: TrieInput, + ) -> Result { + let state_sorted = input.state.into_sorted(); + let nodes_sorted = input.nodes.into_sorted(); + let tx = storage.ro_tx().map_err(Into::::into)?; + let (trie_factory, hashed_factory) = from_tx(storage, &tx, block_number); + StateRoot::new( + InMemoryTrieCursorFactory::new(trie_factory, &nodes_sorted), + HashedPostStateCursorFactory::new(hashed_factory, &state_sorted), + ) + .with_prefix_sets(input.prefix_sets.freeze()) + .root() + } + + fn overlay_root_from_nodes_with_updates( + storage: &'tx MorphProofsStorage, + block_number: u64, + input: TrieInput, + ) -> Result<(B256, TrieUpdates), StateRootError> { + let state_sorted = input.state.into_sorted(); + let nodes_sorted = input.nodes.into_sorted(); + let tx = storage.ro_tx().map_err(Into::::into)?; + let (trie_factory, hashed_factory) = from_tx(storage, &tx, block_number); + StateRoot::new( + InMemoryTrieCursorFactory::new(trie_factory, &nodes_sorted), + HashedPostStateCursorFactory::new(hashed_factory, &state_sorted), + ) + .with_prefix_sets(input.prefix_sets.freeze()) + .root_with_updates() + } +} + +/// Extends [`StorageRoot`] with operations specific for working with [`MorphProofsStorage`]. +pub trait DatabaseStorageRoot<'tx, S: MorphProofsStore + 'tx + Clone> { + /// Calculates the storage root for provided [`HashedStorage`]. + fn overlay_root( + storage: &'tx MorphProofsStorage, + block_number: u64, + address: Address, + hashed_storage: HashedStorage, + ) -> Result; +} + +impl<'tx, S> DatabaseStorageRoot<'tx, S> + for StorageRoot< + MorphProofsTrieCursorFactory<'tx, 'tx, S>, + MorphProofsHashedAccountCursorFactory<'tx, 'tx, S>, + > +where + S: MorphProofsStore + 'tx + Clone, +{ + fn overlay_root( + storage: &'tx MorphProofsStorage, + block_number: u64, + address: Address, + hashed_storage: HashedStorage, + ) -> Result { + let prefix_set = hashed_storage.construct_prefix_set().freeze(); + let state_sorted = + HashedPostState::from_hashed_storage(keccak256(address), hashed_storage).into_sorted(); + let tx = storage.ro_tx().map_err(Into::::into)?; + let (trie_factory, hashed_factory) = from_tx(storage, &tx, block_number); + StorageRoot::new( + trie_factory, + HashedPostStateCursorFactory::new(hashed_factory, &state_sorted), + address, + prefix_set, + TrieRootMetrics::new(TrieType::Custom("morph_historical_proofs_storage")), + ) + .root() + } +} + +/// Extends [`TrieWitness`] with operations specific for working with [`MorphProofsStorage`]. +pub trait DatabaseTrieWitness<'tx, S: MorphProofsStore + 'tx + Clone> { + /// Generates the trie witness for the target state based on [`TrieInput`]. + fn overlay_witness( + storage: &'tx MorphProofsStorage, + block_number: u64, + input: TrieInput, + target: HashedPostState, + mode: ExecutionWitnessMode, + ) -> Result, TrieWitnessError>; + + /// Generates the trie witness for the target state, reusing `tx`. + fn overlay_witness_with_tx<'cursor>( + storage: &'tx MorphProofsStorage, + tx: &'cursor as MorphProofsStore>::Tx<'tx>, + block_number: u64, + input: TrieInput, + target: HashedPostState, + mode: ExecutionWitnessMode, + ) -> Result, TrieWitnessError> + where + 'tx: 'cursor; +} + +impl<'tx, S> DatabaseTrieWitness<'tx, S> + for TrieWitness< + MorphProofsTrieCursorFactory<'tx, 'tx, S>, + MorphProofsHashedAccountCursorFactory<'tx, 'tx, S>, + > +where + S: MorphProofsStore + 'tx + Clone, +{ + fn overlay_witness( + storage: &'tx MorphProofsStorage, + block_number: u64, + input: TrieInput, + target: HashedPostState, + mode: ExecutionWitnessMode, + ) -> Result, TrieWitnessError> { + let tx = storage.ro_tx().map_err(|error| { + let error = Into::::into(error); + StateProofError::from(error) + })?; + Self::overlay_witness_with_tx(storage, &tx, block_number, input, target, mode) + } + + fn overlay_witness_with_tx<'cursor>( + storage: &'tx MorphProofsStorage, + tx: &'cursor as MorphProofsStore>::Tx<'tx>, + block_number: u64, + input: TrieInput, + target: HashedPostState, + mode: ExecutionWitnessMode, + ) -> Result, TrieWitnessError> + where + 'tx: 'cursor, + { + let nodes_sorted = input.nodes.into_sorted(); + let state_sorted = input.state.into_sorted(); + let (trie_factory, hashed_factory) = from_tx(storage, tx, block_number); + TrieWitness::new(trie_factory.clone(), hashed_factory.clone()) + .with_trie_cursor_factory(InMemoryTrieCursorFactory::new(trie_factory, &nodes_sorted)) + .with_hashed_cursor_factory(HashedPostStateCursorFactory::new( + hashed_factory, + &state_sorted, + )) + .with_prefix_sets_mut(input.prefix_sets) + .always_include_root_node() + .with_execution_witness_mode(mode) + .compute(target) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use alloy_eips::{BlockNumHash, NumHash, eip1898::BlockWithParent}; + use alloy_primitives::{Address, B256, U256, keccak256}; + use reth_primitives_traits::Account; + use reth_trie::EMPTY_ROOT_HASH; + + use super::*; + use crate::{ + BlockStateDiff, InMemoryProofsStorage, MdbxProofsStorage, MorphProofsInitialStateStore, + MorphProofsStore, + }; + + fn assert_account_and_storage_proof(storage: S) + where + S: MorphProofsStore + MorphProofsInitialStateStore + Clone, + { + let genesis_hash = B256::repeat_byte(0x01); + storage + .set_initial_state_anchor(BlockNumHash::new(0, genesis_hash)) + .unwrap(); + storage.commit_initial_state().unwrap(); + + let address = Address::repeat_byte(0x11); + let slot = B256::repeat_byte(0x22); + let value = U256::from(42); + let account = Account { + nonce: 7, + balance: U256::from(1_000_000), + bytecode_hash: None, + }; + let hashed_address = keccak256(address); + let post_state = HashedPostState::default() + .with_accounts([(hashed_address, Some(account))]) + .with_storages([( + hashed_address, + HashedStorage::from_iter(false, [(keccak256(slot), value)]), + )]); + + let (state_root, trie_updates) = + StateRoot::overlay_root_with_updates(&storage, 0, post_state.clone()).unwrap(); + let block_hash = B256::repeat_byte(0x02); + storage + .store_trie_updates( + BlockWithParent::new(genesis_hash, NumHash::new(1, block_hash)), + BlockStateDiff { + sorted_trie_updates: trie_updates.into_sorted(), + sorted_post_state: post_state.into_sorted(), + }, + ) + .unwrap(); + + let proof = + Proof::overlay_account_proof(&storage, 1, TrieInput::default(), address, &[slot]) + .unwrap(); + + assert_eq!(proof.info, Some(account)); + assert_eq!(proof.storage_proofs.len(), 1); + assert_eq!(proof.storage_proofs[0].value, value); + proof.verify(state_root).unwrap(); + } + + #[test] + fn account_and_storage_proof_verify_in_memory() { + assert_account_and_storage_proof(InMemoryProofsStorage::new()); + } + + #[test] + fn account_and_storage_proof_verify_in_mdbx() { + let directory = tempfile::tempdir().unwrap(); + let storage = Arc::new(MdbxProofsStorage::new(directory.path()).unwrap()); + assert_account_and_storage_proof(storage); + } + + #[test] + fn proof_snapshot_survives_concurrent_prune_boundary_move() { + let directory = tempfile::tempdir().unwrap(); + let storage = Arc::new(MdbxProofsStorage::new(directory.path()).unwrap()); + let genesis_hash = B256::repeat_byte(0x01); + storage + .set_initial_state_anchor(BlockNumHash::new(0, genesis_hash)) + .unwrap(); + storage.commit_initial_state().unwrap(); + + let address = Address::repeat_byte(0x11); + let post_state = HashedPostState::default().with_accounts([( + keccak256(address), + Some(Account { + nonce: 1, + balance: U256::from(1), + bytecode_hash: None, + }), + )]); + let (_, trie_updates) = + StateRoot::overlay_root_with_updates(&storage, 0, post_state.clone()).unwrap(); + let block_one = + BlockWithParent::new(genesis_hash, NumHash::new(1, B256::repeat_byte(0x02))); + storage + .store_trie_updates( + block_one, + BlockStateDiff { + sorted_trie_updates: trie_updates.into_sorted(), + sorted_post_state: post_state.into_sorted(), + }, + ) + .unwrap(); + + let request_tx = storage.ro_tx().unwrap(); + assert_eq!( + storage + .get_earliest_block_number_with_tx(&request_tx) + .unwrap() + .unwrap() + .0, + 0 + ); + + storage.prune_earliest_state(block_one).unwrap(); + assert_eq!(storage.get_earliest_block_number().unwrap().unwrap().0, 1); + assert_eq!( + storage + .get_earliest_block_number_with_tx(&request_tx) + .unwrap() + .unwrap() + .0, + 0, + "request snapshot must retain the validated proof window" + ); + + let proof = Proof::overlay_account_proof_with_tx( + &storage, + &request_tx, + 0, + TrieInput::default(), + address, + &[], + ) + .unwrap(); + assert!(proof.info.is_none()); + proof.verify(EMPTY_ROOT_HASH).unwrap(); + } + + #[test] + fn exclusion_proofs_verify_at_historical_block() { + let storage = InMemoryProofsStorage::new(); + let genesis_hash = B256::repeat_byte(0x01); + storage + .set_initial_state_anchor(BlockNumHash::new(0, genesis_hash)) + .unwrap(); + storage.commit_initial_state().unwrap(); + + let address = Address::repeat_byte(0x11); + let post_state = HashedPostState::default().with_accounts([( + keccak256(address), + Some(Account { + nonce: 1, + balance: U256::from(1), + bytecode_hash: None, + }), + )]); + let (state_root, trie_updates) = + StateRoot::overlay_root_with_updates(&storage, 0, post_state.clone()).unwrap(); + storage + .store_trie_updates( + BlockWithParent::new(genesis_hash, NumHash::new(1, B256::repeat_byte(0x02))), + BlockStateDiff { + sorted_trie_updates: trie_updates.into_sorted(), + sorted_post_state: post_state.into_sorted(), + }, + ) + .unwrap(); + + let missing_address = Address::repeat_byte(0x33); + let missing_slot = B256::repeat_byte(0x44); + let proof = Proof::overlay_account_proof( + &storage, + 1, + TrieInput::default(), + missing_address, + &[missing_slot], + ) + .unwrap(); + + assert!(proof.info.is_none()); + assert_eq!(proof.storage_proofs.len(), 1); + assert_eq!(proof.storage_proofs[0].value, U256::ZERO); + proof.verify(state_root).unwrap(); + } +} diff --git a/crates/proofs/src/provider.rs b/crates/proofs/src/provider.rs new file mode 100644 index 00000000..2d9e2925 --- /dev/null +++ b/crates/proofs/src/provider.rs @@ -0,0 +1,340 @@ +//! Provider for external proofs storage + +use std::fmt::Debug; + +use alloy_primitives::keccak256; +use parking_lot::{MappedMutexGuard, Mutex, MutexGuard}; +use reth_primitives_traits::{Account, Bytecode}; +use reth_provider::{ + AccountReader, BlockHashReader, BytecodeReader, HashedPostStateProvider, ProviderError, + ProviderResult, StateProofProvider, StateProvider, StateRootProvider, StorageRootProvider, +}; +use reth_revm::{ + db::BundleState, + primitives::{Address, B256, Bytes, StorageValue, alloy_primitives::BlockNumber}, +}; +use reth_trie::{ + StateRoot, StorageRoot, + hashed_cursor::HashedCursor, + proof::{self, Proof}, + witness::TrieWitness, +}; +use reth_trie_common::{ + AccountProof, ExecutionWitnessMode, HashedPostState, HashedStorage, KeccakKeyHasher, + MultiProof, MultiProofTargets, StorageMultiProof, StorageProof, TrieInput, + updates::TrieUpdates, +}; + +use crate::{ + MorphProofsStorage, MorphProofsStorageError, MorphProofsStore, + proof::{ + DatabaseProof, DatabaseStateRoot, DatabaseStorageProof, DatabaseStorageRoot, + DatabaseTrieWitness, + }, +}; + +/// State provider for external proofs storage. +pub struct MorphProofsStateProviderRef<'a, Storage: MorphProofsStore> { + /// Historical state provider for non-state related tasks. + latest: Box, + + /// Storage provider for state lookups. + storage: &'a MorphProofsStorage, + + /// Max block number that can be used for state lookups. + block_number: BlockNumber, + + /// Lazily-acquired read-only transaction shared across all EVM state reads. + /// + /// Acquired once on the first [`basic_account`](AccountReader::basic_account) or + /// [`storage`](StateProvider::storage) call and reused for the lifetime of this provider, + /// so that all EVM state reads within a single execution context share one database snapshot + /// and avoid per-call transaction-acquisition contention. + lazy_tx: Mutex>>, +} + +impl<'a, Storage: MorphProofsStore> MorphProofsStateProviderRef<'a, Storage> { + /// Creates a new state provider. + pub fn new( + latest: Box, + storage: &'a MorphProofsStorage, + block_number: BlockNumber, + ) -> Self { + Self { + latest, + storage, + block_number, + lazy_tx: Mutex::new(None), + } + } + + /// Creates a state provider pinned to an existing proof-database read transaction. + pub fn new_with_tx( + latest: Box, + storage: &'a MorphProofsStorage, + block_number: BlockNumber, + tx: Storage::Tx<'a>, + ) -> Self { + Self { + latest, + storage, + block_number, + lazy_tx: Mutex::new(Some(tx)), + } + } + + fn ensure_tx(&self) -> ProviderResult>> { + let mut guard = self.lazy_tx.lock(); + if guard.is_none() { + *guard = Some(self.storage.ro_tx().map_err(Into::::into)?); + } + + Ok(MutexGuard::map(guard, |tx| { + tx.as_mut().expect("read-only transaction initialized") + })) + } +} + +impl<'a, Storage> Debug for MorphProofsStateProviderRef<'a, Storage> +where + Storage: MorphProofsStore + 'a + Debug, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MorphProofsStateProviderRef") + .field("storage", &self.storage) + .field("block_number", &self.block_number) + .finish() + } +} + +impl<'a, Storage: MorphProofsStore + Clone> MorphProofsStateProviderRef<'a, Storage> { + fn storage_by_hashed_key( + &self, + address: Address, + hashed_key: B256, + ) -> ProviderResult> { + let tx = self.ensure_tx()?; + Ok(self + .storage + .storage_hashed_cursor_with_tx(&tx, keccak256(address.0), self.block_number) + .map_err(Into::::into)? + .seek(hashed_key) + .map_err(Into::::into)? + .and_then(|(key, val)| (key == hashed_key).then_some(val))) + } +} + +impl From for ProviderError { + fn from(error: MorphProofsStorageError) -> Self { + Self::other(error) + } +} + +impl<'a, Storage: MorphProofsStore> BlockHashReader for MorphProofsStateProviderRef<'a, Storage> { + fn block_hash(&self, number: BlockNumber) -> ProviderResult> { + self.latest.block_hash(number) + } + + fn canonical_hashes_range( + &self, + start: BlockNumber, + end: BlockNumber, + ) -> ProviderResult> { + self.latest.canonical_hashes_range(start, end) + } +} + +impl<'a, Storage: MorphProofsStore + Clone> StateRootProvider + for MorphProofsStateProviderRef<'a, Storage> +{ + fn state_root(&self, state: HashedPostState) -> ProviderResult { + Ok(StateRoot::overlay_root( + self.storage, + self.block_number, + state, + )?) + } + + fn state_root_from_nodes(&self, input: TrieInput) -> ProviderResult { + Ok(StateRoot::overlay_root_from_nodes( + self.storage, + self.block_number, + input, + )?) + } + + fn state_root_with_updates( + &self, + state: HashedPostState, + ) -> ProviderResult<(B256, TrieUpdates)> { + Ok(StateRoot::overlay_root_with_updates( + self.storage, + self.block_number, + state, + )?) + } + + fn state_root_from_nodes_with_updates( + &self, + input: TrieInput, + ) -> ProviderResult<(B256, TrieUpdates)> { + Ok(StateRoot::overlay_root_from_nodes_with_updates( + self.storage, + self.block_number, + input, + )?) + } +} + +impl<'a, Storage: MorphProofsStore + Clone> StorageRootProvider + for MorphProofsStateProviderRef<'a, Storage> +{ + fn storage_root(&self, address: Address, storage: HashedStorage) -> ProviderResult { + StorageRoot::overlay_root(self.storage, self.block_number, address, storage) + .map_err(|err| ProviderError::Database(err.into())) + } + + fn storage_proof( + &self, + address: Address, + slot: B256, + storage: HashedStorage, + ) -> ProviderResult { + proof::StorageProof::overlay_storage_proof( + self.storage, + self.block_number, + address, + slot, + storage, + ) + .map_err(ProviderError::from) + } + + fn storage_multiproof( + &self, + address: Address, + slots: &[B256], + storage: HashedStorage, + ) -> ProviderResult { + proof::StorageProof::overlay_storage_multiproof( + self.storage, + self.block_number, + address, + slots, + storage, + ) + .map_err(ProviderError::from) + } +} + +impl<'a, Storage: MorphProofsStore + Clone> StateProofProvider + for MorphProofsStateProviderRef<'a, Storage> +{ + fn proof( + &self, + input: TrieInput, + address: Address, + slots: &[B256], + ) -> ProviderResult { + let tx = self.ensure_tx()?; + Proof::overlay_account_proof_with_tx( + self.storage, + &tx, + self.block_number, + input, + address, + slots, + ) + .map_err(ProviderError::from) + } + + fn multiproof( + &self, + input: TrieInput, + targets: MultiProofTargets, + ) -> ProviderResult { + let tx = self.ensure_tx()?; + Proof::overlay_multiproof_with_tx(self.storage, &tx, self.block_number, input, targets) + .map_err(ProviderError::from) + } + + fn witness( + &self, + input: TrieInput, + target: HashedPostState, + mode: ExecutionWitnessMode, + ) -> ProviderResult> { + let tx = self.ensure_tx()?; + TrieWitness::overlay_witness_with_tx( + self.storage, + &tx, + self.block_number, + input, + target, + mode, + ) + .map_err(ProviderError::from) + .map(|hm| hm.into_values().collect()) + } +} + +impl<'a, Storage: MorphProofsStore> HashedPostStateProvider + for MorphProofsStateProviderRef<'a, Storage> +{ + fn hashed_post_state(&self, bundle_state: &BundleState) -> HashedPostState { + HashedPostState::from_bundle_state::(bundle_state.state()) + } +} + +impl<'a, Storage: MorphProofsStore> AccountReader for MorphProofsStateProviderRef<'a, Storage> { + fn basic_account(&self, address: &Address) -> ProviderResult> { + let hashed_key = keccak256(address.0); + let tx = self.ensure_tx()?; + Ok(self + .storage + .account_hashed_cursor_with_tx(&tx, self.block_number) + .map_err(Into::::into)? + .seek(hashed_key) + .map_err(Into::::into)? + .and_then(|(key, account)| (key == hashed_key).then_some(account))) + } +} + +impl<'a, Storage> StateProvider for MorphProofsStateProviderRef<'a, Storage> +where + Storage: MorphProofsStore + Clone, +{ + fn storage(&self, address: Address, storage_key: B256) -> ProviderResult> { + let hashed_key = keccak256(storage_key); + self.storage_by_hashed_key(address, hashed_key) + } +} + +impl<'a, Storage: MorphProofsStore> BytecodeReader for MorphProofsStateProviderRef<'a, Storage> { + fn bytecode_by_hash(&self, code_hash: &B256) -> ProviderResult> { + self.latest.bytecode_by_hash(code_hash) + } +} + +#[cfg(test)] +mod tests { + use reth_provider::noop::NoopProvider; + + use super::*; + use crate::InMemoryProofsStorage; + + #[test] + fn test_morph_proofs_state_provider_ref_debug() { + let latest: Box = Box::::default(); + let storage: crate::MorphProofsStorage = + InMemoryProofsStorage::new(); + let block_number = 42u64; + + let provider = MorphProofsStateProviderRef::new(latest, &storage, block_number); + + assert_eq!( + format!("{provider:?}"), + "MorphProofsStateProviderRef { storage: InMemoryProofsStorage { inner: RwLock { data: InMemoryStorageInner { account_branches: {}, storage_branches: {}, hashed_accounts: {}, hashed_storages: {}, trie_updates: {}, post_states: {}, earliest_block: None, anchor_block: None } } }, block_number: 42 }" + ); + } +} diff --git a/crates/proofs/src/prune/error.rs b/crates/proofs/src/prune/error.rs new file mode 100644 index 00000000..56992c93 --- /dev/null +++ b/crates/proofs/src/prune/error.rs @@ -0,0 +1,102 @@ +use std::{ + fmt, + fmt::{Display, Formatter}, + time::Duration, +}; + +use reth_provider::ProviderError; +use strum::Display; +use thiserror::Error; + +use crate::{MorphProofsStorageError, api::WriteCounts}; + +/// Result of [`MorphProofStoragePruner::run`](crate::MorphProofStoragePruner::run) execution. +pub type MorphProofStoragePrunerResult = Result; + +/// Successful prune summary. +#[derive(Debug, Clone, Default, Eq, PartialEq)] +pub struct PrunerOutput { + /// Total elapsed wall time for this run (fetch + apply). + pub duration: Duration, + /// Earliest block at the start of the run. + pub start_block: u64, + /// New earliest block at the end of the run. + pub end_block: u64, + /// Number of entries updated/removed per table. + pub write_counts: WriteCounts, +} + +impl Display for PrunerOutput { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let blocks = self.end_block.saturating_sub(self.start_block); + let total_entries = self.write_counts.hashed_accounts_written_total + + self.write_counts.hashed_storages_written_total + + self.write_counts.account_trie_updates_written_total + + self.write_counts.storage_trie_updates_written_total; + write!( + f, + "Pruned {}→{} ({} blocks), entries={}, elapsed={:.3}s", + self.start_block, + self.end_block, + blocks, + total_entries, + self.duration.as_secs_f64(), + ) + } +} + +impl PrunerOutput { + /// extend the current [`PrunerOutput`] with another [`PrunerOutput`] + pub fn extend_ref(&mut self, other: Self) { + self.duration += other.duration; + // take the earliest start block + if self.start_block > other.start_block { + self.start_block = other.start_block; + } + // take the latest end block + if self.end_block < other.end_block { + self.end_block = other.end_block; + } + self.write_counts += other.write_counts; + } +} + +/// Error returned by the pruner. +#[derive(Debug, Error, Display)] +pub enum PrunerError { + /// Wrapped error from the underlying `MorphProofStorage` layer. + Storage(#[from] MorphProofsStorageError), + + /// Wrapped error from the reth db provider. + Provider(#[from] ProviderError), + + /// Block not found in the underlying reth storage provider. + BlockNotFound(u64), + + /// The pruner timed out before finishing the prune + TimedOut(Duration), +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::PrunerOutput; + use crate::api::WriteCounts; + + #[test] + fn test_pruner_output_display() { + let pruner_output = PrunerOutput { + duration: Duration::from_secs(10), + start_block: 1, + end_block: 2, + write_counts: WriteCounts::new(1, 2, 3, 4), + }; + let formatted_pruner_output = format!("{pruner_output}"); + + assert_eq!( + formatted_pruner_output, + "Pruned 1→2 (1 blocks), entries=10, elapsed=10.000s" + ); + } +} diff --git a/crates/proofs/src/prune/metrics.rs b/crates/proofs/src/prune/metrics.rs new file mode 100644 index 00000000..2ddc2a46 --- /dev/null +++ b/crates/proofs/src/prune/metrics.rs @@ -0,0 +1,54 @@ +//! Pruner metrics. + +use reth_metrics::{ + Metrics, + metrics::{Gauge, Histogram}, +}; + +use crate::PrunerOutput; + +#[derive(Metrics, Clone)] +#[metrics(scope = "morph.proofs.pruner")] +pub(super) struct PrunerMetrics { + /// Pruning duration. + total_duration_seconds: Histogram, + /// Number of pruned blocks. + pruned_blocks: Gauge, + /// Number of account trie updates written. + account_trie_updates_written: Gauge, + /// Number of storage trie updates written. + storage_trie_updates_written: Gauge, + /// Number of hashed accounts written. + hashed_accounts_written: Gauge, + /// Number of hashed storages written. + hashed_storages_written: Gauge, +} + +impl PrunerMetrics { + pub(super) fn record_prune_result(result: PrunerOutput) { + let blocks_pruned = result.end_block.saturating_sub(result.start_block); + if blocks_pruned == 0 { + return; + } + + let metrics = Self::default(); + metrics + .total_duration_seconds + .record(result.duration.as_secs_f64()); + metrics.pruned_blocks.set(blocks_pruned as f64); + + let counts = &result.write_counts; + metrics + .account_trie_updates_written + .set(counts.account_trie_updates_written_total as f64); + metrics + .storage_trie_updates_written + .set(counts.storage_trie_updates_written_total as f64); + metrics + .hashed_accounts_written + .set(counts.hashed_accounts_written_total as f64); + metrics + .hashed_storages_written + .set(counts.hashed_storages_written_total as f64); + } +} diff --git a/crates/proofs/src/prune/mod.rs b/crates/proofs/src/prune/mod.rs new file mode 100644 index 00000000..4a5781ae --- /dev/null +++ b/crates/proofs/src/prune/mod.rs @@ -0,0 +1,12 @@ +//! Proof storage pruner for removing stale trie data. + +mod error; +pub use error::{MorphProofStoragePrunerResult, PrunerError, PrunerOutput}; + +mod pruner; +pub use pruner::MorphProofStoragePruner; + +mod metrics; + +mod task; +pub use task::MorphProofStoragePrunerTask; diff --git a/crates/proofs/src/prune/pruner.rs b/crates/proofs/src/prune/pruner.rs new file mode 100644 index 00000000..4f64a787 --- /dev/null +++ b/crates/proofs/src/prune/pruner.rs @@ -0,0 +1,735 @@ +use std::cmp; + +use alloy_eips::{BlockNumHash, eip1898::BlockWithParent}; +use reth_provider::BlockHashReader; +use tokio::time::Instant; +use tracing::{error, info, trace}; + +use crate::{ + MorphProofsStorage, MorphProofsStore, ProofHistoryMetrics, + prune::{ + error::{MorphProofStoragePrunerResult, PrunerError, PrunerOutput}, + metrics::PrunerMetrics, + }, +}; + +/// Prunes the proof storage by calling `prune_earliest_state` on the storage provider. +#[derive(Debug)] +pub struct MorphProofStoragePruner { + /// Database provider for the prune + provider: MorphProofsStorage

, + /// Reader to fetch block hash by block number + block_hash_reader: H, + /// Keep at least these many recent blocks. + retention_blocks: u64, + /// Maximum number of blocks to prune in one database transaction + prune_batch_size: u64, +} + +impl MorphProofStoragePruner { + /// Create a new pruner. + pub const fn new( + provider: MorphProofsStorage

, + block_hash_reader: H, + retention_blocks: u64, + prune_batch_size: u64, + ) -> Self { + assert!( + prune_batch_size > 0, + "prune batch size must be greater than zero" + ); + Self { + provider, + block_hash_reader, + retention_blocks, + prune_batch_size, + } + } + + const fn target_earliest_block(&self, latest_block: u64) -> u64 { + latest_block.saturating_sub(self.retention_blocks) + } +} + +impl MorphProofStoragePruner +where + P: MorphProofsStore, + H: BlockHashReader, +{ + fn run_inner(&self) -> MorphProofStoragePrunerResult { + let Some(((earliest_block, _), (latest_block, _))) = self + .provider + .get_earliest_block_number()? + .zip(self.provider.get_latest_block_number()?) + else { + trace!(target: "trie::pruner", "No earliest or latest block in the proof storage"); + return Ok(PrunerOutput::default()); + }; + + let target_earliest_block = self.target_earliest_block(latest_block); + info!( + target: "trie::pruner", + earliest_block, + latest_block, + target_earliest_block, + retention_blocks = self.retention_blocks, + prune_batch_size = self.prune_batch_size, + "Calculated proof storage pruning target", + ); + if earliest_block >= target_earliest_block { + trace!(target: "trie::pruner", "Nothing to prune"); + return Ok(PrunerOutput::default()); + } + + info!( + target: "trie::pruner", + from_block = earliest_block, + to_block = target_earliest_block, + latest_block, + retention_blocks = self.retention_blocks, + "Starting pruning proof storage", + ); + + let mut current_earliest_block = earliest_block; + let mut prune_output = PrunerOutput { + start_block: earliest_block, + end_block: target_earliest_block, + ..Default::default() + }; + + // Prune in batches + while current_earliest_block < target_earliest_block { + // Calculate the end of this batch + let batch_end_block = cmp::min( + current_earliest_block.saturating_add(self.prune_batch_size), + target_earliest_block, + ); + info!( + target: "trie::pruner", + start_block = current_earliest_block, + end_block = batch_end_block, + target_earliest_block, + batch_size = batch_end_block.saturating_sub(current_earliest_block), + "Starting proof storage prune batch", + ); + + let batch_output = self.prune_batch(current_earliest_block, batch_end_block)?; + + prune_output.extend_ref(batch_output); + + // Update loop state + current_earliest_block = batch_end_block; + } + + Ok(prune_output) + } + + /// Prunes a single batch of blocks. + fn prune_batch(&self, start_block: u64, end_block: u64) -> Result { + let batch_start_time = Instant::now(); + info!( + target: "trie::pruner", + start_block, + end_block, + "Resolving proof storage prune batch block hashes", + ); + if end_block == 0 { + trace!( + target: "trie::pruner", + start_block, + end_block, + "Skipping proof storage prune batch at genesis block", + ); + return Ok(PrunerOutput { + duration: batch_start_time.elapsed(), + start_block, + end_block, + ..Default::default() + }); + } + + // Fetch the block hash for the new earliest block of this batch. + // + // The parent hash is intentionally not fetched: `prune_earliest_state` only reads + // `block.number` and `block.hash` from the supplied `BlockWithParent`. Fetching the + // parent hash here was wasted I/O proportional to the number of batches. + let new_earliest_block_hash = self + .block_hash_reader + .block_hash(end_block) + .inspect_err(|err| { + error!( + target: "trie::pruner", + block = end_block, + ?err, + "Failed to fetch block hash for new earliest block during pruning" + ) + })? + .ok_or(PrunerError::BlockNotFound(end_block))?; + + let block_with_parent = BlockWithParent { + parent: Default::default(), + block: BlockNumHash { + number: end_block, + hash: new_earliest_block_hash, + }, + }; + + info!( + target: "trie::pruner", + start_block, + end_block, + block_hash = ?new_earliest_block_hash, + "Resolved proof storage prune batch block hashes", + ); + + info!( + target: "trie::pruner", + start_block, + end_block, + "Applying proof storage prune batch", + ); + let write_counts = self.provider.prune_earliest_state(block_with_parent)?; + + let duration = batch_start_time.elapsed(); + let batch_output = PrunerOutput { + duration, + start_block, + end_block, + write_counts, + }; + + PrunerMetrics::record_prune_result(batch_output.clone()); + + info!( + target: "trie::pruner", + ?batch_output, + "Finished pruning batch of proof storage", + ); + Ok(batch_output) + } + + /// Run the pruner + pub fn run(&self) { + let res = self.try_run(); + match res { + Err(e) => { + error!(target: "trie::pruner", err=%e, "Pruner failed"); + } + Ok(res) => { + info!(target: "trie::pruner", result = %res, "Finished pruning proof storage"); + } + } + } + + /// Runs the pruner and returns failures to command-style callers. + pub fn try_run(&self) -> MorphProofStoragePrunerResult { + let result = self.run_inner(); + match &result { + Ok(_) => { + if let Ok(Some((earliest, _))) = self.provider.get_earliest_block_number() + && let Ok(Some((latest, _))) = self.provider.get_latest_block_number() + { + ProofHistoryMetrics::set_window(earliest, latest); + } + } + Err(_) => ProofHistoryMetrics::record_prune_error(), + } + result + } +} + +#[cfg(test)] +mod tests { + use alloy_eips::{BlockHashOrNumber, NumHash}; + use alloy_primitives::{B256, BlockNumber, keccak256}; + use mockall::mock; + use reth_storage_errors::provider::ProviderResult; + + use super::*; + + mock! ( + #[derive(Debug)] + pub BlockHashReader {} + + impl BlockHashReader for BlockHashReader { + fn block_hash(&self, number: BlockNumber) -> ProviderResult>; + + fn convert_block_hash( + &self, + _hash_or_number: BlockHashOrNumber, + ) -> ProviderResult>; + + fn canonical_hashes_range( + &self, + _start: BlockNumber, + _end: BlockNumber, + ) -> ProviderResult>; + } + ); + + fn b256(n: u64) -> B256 { + keccak256(n.to_be_bytes()) + } + + /// Build a block-with-parent for number `n` with deterministic hash. + fn block(n: u64, parent: B256) -> BlockWithParent { + BlockWithParent::new(parent, NumHash::new(n, b256(n))) + } + + macro_rules! proof_storage_pruner_tests { + ($mod_name:ident, $storage:ident) => { + mod $mod_name { + use std::sync::Arc; + + use alloy_primitives::{B256, U256}; + use reth_primitives_traits::Account; + use reth_trie::{ + BranchNodeCompact, HashedPostState, HashedStorage, Nibbles, + hashed_cursor::HashedCursor, + trie_cursor::TrieCursor, + updates::{StorageTrieUpdates, TrieUpdates, TrieUpdatesSorted}, + }; + use tempfile::TempDir; + + use super::{super::*, MockBlockHashReader, b256, block}; + use crate::{BlockStateDiff, $storage}; + + #[tokio::test] + async fn run_inner_and_and_verify_updated_state() { + // --- env/store --- + let dir = TempDir::new().unwrap(); + let store: MorphProofsStorage> = + MorphProofsStorage::from(Arc::new($storage::new(dir.path()).expect("env"))); + + store + .set_earliest_block_number(0, B256::ZERO) + .expect("set earliest"); + + // --- entities --- + // accounts + let a1 = B256::from([0xA1; 32]); + let a2 = B256::from([0xA2; 32]); + let a3 = B256::from([0xA3; 32]); // introduced later + + // one storage address with 3 slots + let stor_addr = B256::from([0x10; 32]); + let s1 = B256::from([0xB1; 32]); + let s2 = B256::from([0xB2; 32]); + let s3 = B256::from([0xB3; 32]); + + // account-trie paths (p1 gets removed by block 3; p2 remains; p3 added later) + let p1 = Nibbles::from_nibbles_unchecked([0x01, 0x02]); + let p2 = Nibbles::from_nibbles_unchecked([0x03, 0x04]); + let p3 = Nibbles::from_nibbles_unchecked([0x05, 0x06]); + + let node_p1 = + BranchNodeCompact::new(0b1, 0, 0, vec![], Some(B256::from([0x11; 32]))); + let node_p2 = + BranchNodeCompact::new(0b10, 0, 0, vec![], Some(B256::from([0x22; 32]))); + let node_p3 = + BranchNodeCompact::new(0b11, 0, 0, vec![], Some(B256::from([0x33; 32]))); + + // storage-trie paths (st1 removed by block 3; st2 remains; st3 added later) + let st1 = Nibbles::from_nibbles_unchecked([0x0A]); + let st2 = Nibbles::from_nibbles_unchecked([0x0B]); + let st3 = Nibbles::from_nibbles_unchecked([0x0C]); + + let node_st2 = + BranchNodeCompact::new(0b101, 0, 0, vec![], Some(B256::from([0x44; 32]))); + let node_st3 = + BranchNodeCompact::new(0b110, 0, 0, vec![], Some(B256::from([0x55; 32]))); + + // --- write 5 blocks manually --- + let mut parent = B256::ZERO; + + // Block 1: add a1,a2; s1=100, s2=200; add p1, st1 + { + let b1 = block(1, parent); + + let mut d_trie_updates = TrieUpdates::default(); + let mut d_post_state = HashedPostState::default(); + + d_post_state.accounts.insert( + a1, + Some(Account { + nonce: 1, + balance: U256::from(1_001), + ..Default::default() + }), + ); + d_post_state.accounts.insert( + a2, + Some(Account { + nonce: 1, + balance: U256::from(1_002), + ..Default::default() + }), + ); + + let mut hs = HashedStorage::default(); + hs.storage.insert(s1, U256::from(100)); + hs.storage.insert(s2, U256::from(200)); + d_post_state.storages.insert(stor_addr, hs); + + d_trie_updates.account_nodes.insert(p1, node_p1); + let e = d_trie_updates.storage_tries.entry(stor_addr).or_default(); + e.storage_nodes.insert(st1, BranchNodeCompact::default()); + + let d = BlockStateDiff { + sorted_post_state: d_post_state.into_sorted(), + sorted_trie_updates: d_trie_updates.into_sorted(), + }; + store.store_trie_updates(b1, d).expect("b1"); + parent = b256(1); + } + + // Block 2: update a2; add a3; s2=220, s3=300; add p2, st2 + { + let b2 = block(2, parent); + + let mut d_trie_updates = TrieUpdates::default(); + let mut d_post_state = HashedPostState::default(); + + d_post_state.accounts.insert( + a2, + Some(Account { + nonce: 2, + balance: U256::from(2_002), + ..Default::default() + }), + ); + d_post_state.accounts.insert( + a3, + Some(Account { + nonce: 1, + balance: U256::from(1_003), + ..Default::default() + }), + ); + + let mut hs = HashedStorage::default(); + hs.storage.insert(s2, U256::from(220)); + hs.storage.insert(s3, U256::from(300)); + d_post_state.storages.insert(stor_addr, hs); + + d_trie_updates.account_nodes.insert(p2, node_p2.clone()); + let e = d_trie_updates.storage_tries.entry(stor_addr).or_default(); + e.storage_nodes.insert(st2, node_st2.clone()); + + let d = BlockStateDiff { + sorted_post_state: d_post_state.into_sorted(), + sorted_trie_updates: d_trie_updates.into_sorted(), + }; + store.store_trie_updates(b2, d).expect("b2"); + parent = b256(2); + } + + // Block 3: delete a1; leave a2,a3; remove p1; remove st1 (storage-trie) + { + let b3 = block(3, parent); + + let mut d_trie_updates = TrieUpdates::default(); + let mut d_post_state = HashedPostState::default(); + + // delete a1, keep a2 & a3 values unchanged for this block + d_post_state.accounts.insert(a1, None); + + // remove account trie node p1 + d_trie_updates.removed_nodes.insert(p1); + + // remove storage-trie node st1 + let mut st_upd = StorageTrieUpdates::default(); + st_upd.removed_nodes.insert(st1); + d_trie_updates.storage_tries.insert(stor_addr, st_upd); + + let d = BlockStateDiff { + sorted_post_state: d_post_state.into_sorted(), + sorted_trie_updates: d_trie_updates.into_sorted(), + }; + store.store_trie_updates(b3, d).expect("b3"); + parent = b256(3); + } + + // Block 4 (kept): update a2; s1=140; add p3, st3 + { + let b4 = block(4, parent); + + let mut d_trie_updates = TrieUpdates::default(); + let mut d_post_state = HashedPostState::default(); + + d_post_state.accounts.insert( + a2, + Some(Account { + nonce: 3, + balance: U256::from(3_002), + ..Default::default() + }), + ); + + let mut hs = HashedStorage::default(); + hs.storage.insert(s1, U256::from(140)); + d_post_state.storages.insert(stor_addr, hs); + d_trie_updates.account_nodes.insert(p3, node_p3.clone()); + let e = d_trie_updates.storage_tries.entry(stor_addr).or_default(); + e.storage_nodes.insert(st3, node_st3.clone()); + + let d = BlockStateDiff { + sorted_post_state: d_post_state.into_sorted(), + sorted_trie_updates: d_trie_updates.into_sorted(), + }; + store.store_trie_updates(b4, d).expect("b4"); + parent = b256(4); + } + + // Block 5 (kept): update a3; s3=330 + { + let b5 = block(5, parent); + + let mut d_post_state = HashedPostState::default(); + + d_post_state.accounts.insert( + a3, + Some(Account { + nonce: 2, + balance: U256::from(2_003), + ..Default::default() + }), + ); + + let mut hs = HashedStorage::default(); + hs.storage.insert(s3, U256::from(330)); + d_post_state.storages.insert(stor_addr, hs); + + let d = BlockStateDiff { + sorted_post_state: d_post_state.into_sorted(), + sorted_trie_updates: TrieUpdatesSorted::default(), + }; + store.store_trie_updates(b5, d).expect("b5"); + } + + // sanity: earliest=0, latest=5 + { + let e = store + .get_earliest_block_number() + .expect("earliest") + .expect("some"); + let l = store + .get_latest_block_number() + .expect("latest") + .expect("some"); + assert_eq!(e.0, 0); + assert_eq!(l.0, 5); + } + + // --- prune: remove the first 3 blocks, keep 4 and 5 + // new_earliest = 5-1 = 4 + let mut block_hash_reader = MockBlockHashReader::new(); + block_hash_reader + .expect_block_hash() + .withf(move |block_num| *block_num == 4) + .returning(move |_| Ok(Some(b256(4)))); + + let pruner = + MorphProofStoragePruner::new(store.clone(), block_hash_reader, 1, 1000); + let out = pruner.run_inner().expect("pruner ok"); + assert_eq!(out.start_block, 0); + assert_eq!( + out.end_block, 4, + "pruned up to 4 (inclusive); new earliest is 4" + ); + + // proof window moved: earliest=4, latest=5 + { + let e = store + .get_earliest_block_number() + .expect("earliest") + .expect("some"); + let l = store + .get_latest_block_number() + .expect("latest") + .expect("some"); + assert_eq!(e.0, 4); + assert_eq!(e.1, b256(4)); + assert_eq!(l.0, 5); + assert_eq!(l.1, b256(5)); + } + + // --- DB checks + let mut acc_cur = store.account_hashed_cursor(4).expect("acc cur"); + let mut stor_cur = store.storage_hashed_cursor(stor_addr, 4).expect("stor cur"); + let mut acc_trie_cur = store.account_trie_cursor(4).expect("acc trie cur"); + let mut stor_trie_cur = store + .storage_trie_cursor(stor_addr, 4) + .expect("stor trie cur"); + + // Check these histories have been removed + let pruned_hashed_account = a1; + let pruned_trie_accounts = p1; + let pruned_trie_storage = st1; + + assert_ne!( + acc_cur + .seek(pruned_hashed_account) + .expect("seek") + .unwrap() + .0, + pruned_hashed_account, + "deleted account must not exist in earliest snapshot" + ); + assert_ne!( + acc_trie_cur + .seek(pruned_trie_accounts) + .expect("seek") + .unwrap() + .0, + pruned_trie_accounts, + "deleted account trie must not exist in earliest snapshot" + ); + assert_ne!( + stor_trie_cur + .seek(pruned_trie_storage) + .expect("seek") + .unwrap() + .0, + pruned_trie_storage, + "deleted storage trie must not exist in earliest snapshot" + ); + + // Check these histories have been updated - till block 4 + let updated_hashed_accounts = vec![ + ( + a2, + Account { + nonce: 3, + balance: U256::from(3_002), + ..Default::default() + }, + ), + ( + a3, + Account { + nonce: 1, + balance: U256::from(1_003), + ..Default::default() + }, + ), + ]; + let updated_hashed_storage = vec![ + (s1, U256::from(140)), + (s2, U256::from(220)), + (s3, U256::from(300)), + ]; + let updated_trie_accounts = vec![(p2, node_p2), (p3, node_p3)]; + let updated_trie_storage = vec![(st2, node_st2), (st3, node_st3)]; + + for (key, val) in updated_hashed_accounts { + let (k, vv) = acc_cur.seek(key).expect("seek").unwrap(); + assert_eq!(key, k, "key must exist"); + assert_eq!(val, vv, "value must be updated"); + } + + for (key, val) in updated_hashed_storage { + let (k, vv) = stor_cur.seek(key).expect("seek").unwrap(); + assert_eq!(key, k, "key must exist"); + assert_eq!(val, vv, "value must be updated"); + } + + for (key, val) in updated_trie_accounts { + let (k, vv) = acc_trie_cur.seek(key).expect("seek").unwrap(); + assert_eq!(key, k, "key must exist"); + assert_eq!(val, vv, "value must be updated"); + } + for (key, val) in updated_trie_storage { + let (k, vv) = stor_trie_cur.seek(key).expect("seek").unwrap(); + assert_eq!(key, k, "key must exist"); + assert_eq!(val, vv, "value must be updated"); + } + } + + // Both latest and earliest blocks are None -> early return default; DB untouched. + #[tokio::test] + async fn run_inner_where_latest_block_is_none() { + let dir = TempDir::new().unwrap(); + let store: MorphProofsStorage> = + MorphProofsStorage::from(Arc::new($storage::new(dir.path()).expect("env"))); + + let earliest = store.get_earliest_block_number().unwrap(); + let latest = store.get_latest_block_number().unwrap(); + assert!(earliest.is_none()); + assert!(latest.is_none()); + + let block_hash_reader = MockBlockHashReader::new(); + let pruner = MorphProofStoragePruner::new(store, block_hash_reader, 10, 1000); + let out = pruner.run_inner().expect("ok"); + assert_eq!( + out, + PrunerOutput::default(), + "should early-return default output" + ); + } + + // The earliest block is None, but the latest block exists -> early return default. + #[tokio::test] + async fn run_inner_earliest_none_real_db() { + let dir = TempDir::new().unwrap(); + let store: MorphProofsStorage> = + MorphProofsStorage::from(Arc::new($storage::new(dir.path()).expect("env"))); + + // Write a single block to set *latest* only. + store + .store_trie_updates(block(3, B256::ZERO), BlockStateDiff::default()) + .expect("store b1"); + + let earliest = store.get_earliest_block_number().unwrap(); + let latest = store.get_latest_block_number().unwrap(); + assert!(earliest.is_none(), "earliest must remain None"); + assert_eq!(latest.unwrap().0, 3); + + let block_hash_reader = MockBlockHashReader::new(); + let pruner = MorphProofStoragePruner::new(store, block_hash_reader, 1, 1000); + let out = pruner.run_inner().expect("ok"); + assert_eq!( + out, + PrunerOutput::default(), + "should early-return default output" + ); + } + + // interval < min_block_interval -> "Nothing to prune" path; default output. + #[tokio::test] + async fn run_inner_interval_too_small_real_db() { + let dir = TempDir::new().unwrap(); + let store: MorphProofsStorage> = + MorphProofsStorage::from(Arc::new($storage::new(dir.path()).expect("env"))); + + // Set earliest=4 explicitly + let earliest_num = 4u64; + let h4 = b256(4); + store + .set_earliest_block_number(earliest_num, h4) + .expect("set earliest"); + + // Set latest=5 by storing block 5 + let b5 = block(5, h4); + store + .store_trie_updates(b5, BlockStateDiff::default()) + .expect("store b5"); + + // Sanity: earliest=4, latest=5 => interval=1 + let e = store.get_earliest_block_number().unwrap().unwrap(); + let l = store.get_latest_block_number().unwrap().unwrap(); + assert_eq!(e.0, 4); + assert_eq!(l.0, 5); + + // Require min_block_interval=2 (or greater) so interval < min + let block_hash_reader = MockBlockHashReader::new(); + let pruner = MorphProofStoragePruner::new(store, block_hash_reader, 2, 1000); + let out = pruner.run_inner().expect("ok"); + assert_eq!(out, PrunerOutput::default(), "no pruning should occur"); + } + } + }; + } + + proof_storage_pruner_tests!(mdbx_tests, MdbxProofsStorage); +} diff --git a/crates/proofs/src/prune/task.rs b/crates/proofs/src/prune/task.rs new file mode 100644 index 00000000..d9f4fb84 --- /dev/null +++ b/crates/proofs/src/prune/task.rs @@ -0,0 +1,80 @@ +use std::sync::Arc; + +use reth_provider::BlockHashReader; +use reth_tasks::shutdown::GracefulShutdown; +use tokio::{ + time, + time::{Duration, MissedTickBehavior}, +}; +use tracing::{info, warn}; + +use crate::{MorphProofsStorage, MorphProofsStore, prune::MorphProofStoragePruner}; + +/// Number of blocks pruned per MDBX write transaction. +/// +/// Each batch is its own write tx, so the per-tx overhead (page allocation, freelist mgmt, +/// fsync at commit) amortizes over this many blocks. The previous value of 200 caused tx +/// overhead to dominate catch-up pruning runs; 2000 amortizes the fixed cost ~10x while +/// keeping per-tx dirty-page sets and free-page reclamation lag bounded. +const PRUNE_BATCH_SIZE: u64 = 2000; + +/// Periodic pruner task: constructs the pruner and runs it every interval. +#[derive(Debug)] +pub struct MorphProofStoragePrunerTask { + pruner: MorphProofStoragePruner, + retention_blocks: u64, + task_run_interval: Duration, +} + +impl MorphProofStoragePrunerTask +where + P: MorphProofsStore + Send + Sync + 'static, + H: BlockHashReader + Send + Sync + 'static, +{ + /// Initialize a new [`MorphProofStoragePrunerTask`] + pub const fn new( + provider: MorphProofsStorage

, + hash_reader: H, + retention_blocks: u64, + task_run_interval: Duration, + ) -> Self { + let pruner = + MorphProofStoragePruner::new(provider, hash_reader, retention_blocks, PRUNE_BATCH_SIZE); + Self { + pruner, + retention_blocks, + task_run_interval, + } + } + + /// Run forever (until `cancel`), executing one prune pass per `task_run_interval`. + pub async fn run(self, mut signal: GracefulShutdown) { + info!( + target: "trie::pruner_task", + retention_blocks = self.retention_blocks, + interval_secs = self.task_run_interval.as_secs(), + "Starting pruner task" + ); + + let mut interval = time::interval(self.task_run_interval); + interval.set_missed_tick_behavior(MissedTickBehavior::Delay); + + let pruner = Arc::new(self.pruner); + loop { + tokio::select! { + _ = &mut signal => { + info!(target: "trie::pruner_task", "Pruner task cancelled; exiting"); + break; + } + _ = interval.tick() => { + let pruner = Arc::clone(&pruner); + if let Err(e) = tokio::task::spawn_blocking(move || pruner.run()).await { + warn!(target: "trie::pruner_task", err=%e, "Pruner blocking task failed"); + } + } + } + } + + info!(target: "trie::pruner_task", "Pruner task stopped"); + } +} diff --git a/crates/proofs/tests/identity.rs b/crates/proofs/tests/identity.rs new file mode 100644 index 00000000..c45cb71e --- /dev/null +++ b/crates/proofs/tests/identity.rs @@ -0,0 +1,24 @@ +use alloy_primitives::B256; +use morph_proofs::{MdbxProofsStorage, MorphProofsStorageError, ProofDbIdentity}; + +#[test] +fn opening_a_store_persists_and_validates_chain_identity() { + let directory = tempfile::tempdir().unwrap(); + let identity = ProofDbIdentity::new(2818, B256::repeat_byte(0x11)); + + let store = MdbxProofsStorage::open(directory.path(), identity).unwrap(); + assert_eq!(store.proof_window().unwrap(), None); + drop(store); + + MdbxProofsStorage::open(directory.path(), identity).unwrap(); + let error = MdbxProofsStorage::open( + directory.path(), + ProofDbIdentity::new(2819, identity.genesis_hash), + ) + .unwrap_err(); + + assert!(matches!( + error, + MorphProofsStorageError::ChainIdentityMismatch("chain_id") + )); +} diff --git a/crates/rpc/Cargo.toml b/crates/rpc/Cargo.toml index 356a7cb7..e7539dc9 100644 --- a/crates/rpc/Cargo.toml +++ b/crates/rpc/Cargo.toml @@ -16,6 +16,7 @@ morph-chainspec.workspace = true morph-reference-index.workspace = true morph-revm = { workspace = true, features = ["rpc"] } morph-evm = { workspace = true, features = ["rpc"] } +morph-proofs.workspace = true # Reth reth-chainspec.workspace = true @@ -23,9 +24,13 @@ reth-node-api.workspace = true reth-node-builder.workspace = true reth-primitives-traits.workspace = true reth-rpc.workspace = true +reth-rpc-api.workspace = true reth-rpc-convert.workspace = true reth-rpc-eth-api.workspace = true reth-rpc-eth-types.workspace = true +reth-rpc-server-types.workspace = true +reth-metrics.workspace = true +metrics.workspace = true reth-evm.workspace = true reth-provider.workspace = true reth-revm.workspace = true @@ -45,6 +50,7 @@ alloy-eips.workspace = true # JSON-RPC jsonrpsee.workspace = true +async-trait.workspace = true # Serialization serde.workspace = true diff --git a/crates/rpc/src/eth/mod.rs b/crates/rpc/src/eth/mod.rs index cb7331a1..9e325b62 100644 --- a/crates/rpc/src/eth/mod.rs +++ b/crates/rpc/src/eth/mod.rs @@ -32,6 +32,7 @@ use reth_tasks::{ use std::{fmt, marker::PhantomData, sync::Arc, time::Duration}; pub mod call; +pub mod proofs; pub mod receipt; pub mod transaction; diff --git a/crates/rpc/src/eth/proofs.rs b/crates/rpc/src/eth/proofs.rs new file mode 100644 index 00000000..f63c6ca2 --- /dev/null +++ b/crates/rpc/src/eth/proofs.rs @@ -0,0 +1,146 @@ +//! Historical EIP-1186 proof RPC override. + +use std::time::Instant; + +use alloy_eips::BlockId; +use alloy_primitives::Address; +use alloy_rpc_types_eth::EIP1186AccountProofResponse; +use alloy_serde::JsonStorageKey; +use async_trait::async_trait; +use jsonrpsee::{ + core::RpcResult, + proc_macros::rpc, + types::{ErrorCode, ErrorObject}, +}; +use morph_proofs::{MorphProofsStorage, MorphProofsStore}; +use reth_metrics::{ + Metrics, + metrics::{Counter, Histogram}, +}; +use reth_provider::StateProofProvider; +use reth_rpc_api::eth::helpers::FullEthApi; +use reth_rpc_eth_types::EthApiError; + +use crate::state::MorphProofStateProviderFactory; + +/// Maximum number of storage keys accepted by one `eth_getProof` request. +pub const MAX_PROOF_KEYS: usize = 1024; + +/// Validates the storage-key count before any database work starts. +#[derive(Debug)] +pub struct ProofKeyLimit; + +impl ProofKeyLimit { + /// Rejects requests above [`MAX_PROOF_KEYS`]. + pub fn check(keys_len: usize) -> Result<(), ErrorObject<'static>> { + if keys_len > MAX_PROOF_KEYS { + return Err(ErrorObject::owned( + ErrorCode::InvalidParams.code(), + format!("too many storage keys: max {MAX_PROOF_KEYS}, got {keys_len}"), + None::<()>, + )); + } + Ok(()) + } +} + +#[derive(Metrics, Clone)] +#[metrics(scope = "morph.rpc.proofs")] +struct ProofRpcMetrics { + /// Total `eth_getProof` requests. + requests_total: Counter, + /// Successful `eth_getProof` responses. + successful_responses_total: Counter, + /// Failed `eth_getProof` responses. + failures_total: Counter, + /// Successful request latency in seconds. + latency_seconds: Histogram, +} + +#[cfg_attr(not(test), rpc(server, namespace = "eth"))] +#[cfg_attr(test, rpc(server, client, namespace = "eth"))] +pub trait EthProofApiOverride { + /// Returns EIP-1186 account and storage proofs at a retained canonical block. + #[method(name = "getProof")] + async fn get_proof( + &self, + address: Address, + keys: Vec, + block: Option, + ) -> RpcResult; +} + +/// `eth_getProof` implementation backed exclusively by Morph proof history. +#[derive(Debug)] +pub struct EthProofApiExt { + state_provider_factory: MorphProofStateProviderFactory, +} + +impl EthProofApiExt +where + Eth: FullEthApi + Send + Sync + 'static, + P: MorphProofsStore + Clone + 'static, +{ + /// Creates the historical proof RPC override. + pub const fn new(eth_api: Eth, storage: MorphProofsStorage

) -> Self { + Self { + state_provider_factory: MorphProofStateProviderFactory::new(eth_api, storage), + } + } +} + +#[async_trait] +impl EthProofApiOverrideServer for EthProofApiExt +where + Eth: FullEthApi + Send + Sync + 'static, + P: MorphProofsStore + Clone + 'static, +{ + async fn get_proof( + &self, + address: Address, + keys: Vec, + block: Option, + ) -> RpcResult { + ProofKeyLimit::check(keys.len())?; + + let metrics = ProofRpcMetrics::default(); + let start = Instant::now(); + metrics.requests_total.increment(1); + let storage_keys = keys.iter().map(JsonStorageKey::as_b256).collect::>(); + + let result = (|| { + let proof = self + .state_provider_factory + .state_provider(block) + .map_err(|error| ErrorObject::from(EthApiError::from(error)))? + .proof(Default::default(), address, &storage_keys) + .map_err(|error| ErrorObject::from(EthApiError::from(error)))?; + Ok(proof.into_eip1186_response(keys)) + })(); + + match &result { + Ok(_) => { + metrics + .latency_seconds + .record(start.elapsed().as_secs_f64()); + metrics.successful_responses_total.increment(1); + } + Err(_) => metrics.failures_total.increment(1), + } + result + } +} + +#[cfg(test)] +mod tests { + use jsonrpsee::types::ErrorCode; + + use super::{MAX_PROOF_KEYS, ProofKeyLimit}; + + #[test] + fn enforces_get_proof_key_limit() { + assert!(ProofKeyLimit::check(MAX_PROOF_KEYS).is_ok()); + let error = ProofKeyLimit::check(MAX_PROOF_KEYS + 1).expect_err("must reject"); + assert_eq!(error.code(), ErrorCode::InvalidParams.code()); + } +} diff --git a/crates/rpc/src/lib.rs b/crates/rpc/src/lib.rs index e7efe377..eff5e953 100644 --- a/crates/rpc/src/lib.rs +++ b/crates/rpc/src/lib.rs @@ -6,10 +6,13 @@ pub mod error; pub mod eth; pub mod eth_config; pub mod morph; +pub mod proof_status; +pub mod state; pub mod types; pub use error::MorphEthApiError; pub use eth::{MorphEthApi, MorphEthApiBuilder, MorphRpcConverter, MorphRpcTypes}; pub use eth_config::{MorphEthConfigApiServer, MorphEthConfigHandler}; pub use morph::{MorphRpc, MorphRpcHandler, MorphRpcServer, ReferenceQueryArgs}; +pub use proof_status::{ProofStatusApiExt, ProofStatusApiOverrideServer, ProofsSyncStatus}; pub use types::*; diff --git a/crates/rpc/src/proof_status.rs b/crates/rpc/src/proof_status.rs new file mode 100644 index 00000000..23424a65 --- /dev/null +++ b/crates/rpc/src/proof_status.rs @@ -0,0 +1,130 @@ +//! Proof-history synchronization status RPC. + +use async_trait::async_trait; +use jsonrpsee::{core::RpcResult, proc_macros::rpc}; +use morph_proofs::{MorphProofsStorage, MorphProofsStorageResult, MorphProofsStore}; +use reth_rpc_server_types::result::internal_rpc_err; +use serde::{Deserialize, Serialize}; + +/// Inclusive proof-history range currently available to RPC requests. +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +pub struct ProofsSyncStatus { + /// Earliest retained canonical block. + pub earliest: Option, + /// Latest durably indexed canonical block. + pub latest: Option, +} + +#[cfg_attr(not(test), rpc(server, namespace = "debug"))] +#[cfg_attr(test, rpc(server, client, namespace = "debug"))] +pub trait ProofStatusApiOverride { + /// Returns the current durable proof-history range. + #[method(name = "proofsSyncStatus")] + async fn proofs_sync_status(&self) -> RpcResult; +} + +/// `debug_proofsSyncStatus` backed by the proof database. +#[derive(Debug)] +pub struct ProofStatusApiExt

{ + storage: MorphProofsStorage

, +} + +impl

ProofStatusApiExt

{ + /// Creates a proof-status handler. + pub const fn new(storage: MorphProofsStorage

) -> Self { + Self { storage } + } +} + +fn proofs_sync_status_with_tx<'tx, 'db, P>( + storage: &P, + tx: &'tx P::Tx<'db>, +) -> MorphProofsStorageResult +where + P: MorphProofsStore + 'db, + 'db: 'tx, +{ + let earliest = storage + .get_earliest_block_number_with_tx(tx)? + .map(|(number, _)| number); + let latest = storage + .get_latest_block_number_with_tx(tx)? + .map(|(number, _)| number); + Ok(ProofsSyncStatus { earliest, latest }) +} + +#[async_trait] +impl

ProofStatusApiOverrideServer for ProofStatusApiExt

+where + P: MorphProofsStore + Clone + Send + Sync + 'static, +{ + async fn proofs_sync_status(&self) -> RpcResult { + // Keep both bounds on the same MDBX snapshot. A concurrent prune may advance the live + // earliest block, but it must not produce a status range assembled from two generations. + let tx = self + .storage + .ro_tx() + .map_err(|error| internal_rpc_err(error.to_string()))?; + proofs_sync_status_with_tx(&self.storage, &tx) + .map_err(|error| internal_rpc_err(error.to_string())) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use alloy_eips::{BlockNumHash, NumHash, eip1898::BlockWithParent}; + use alloy_primitives::B256; + use morph_proofs::{ + BlockStateDiff, MdbxProofsStorage, MorphProofsInitialStateStore, MorphProofsStore, + }; + + use super::{ProofsSyncStatus, proofs_sync_status_with_tx}; + + #[test] + fn empty_status_uses_json_nulls() { + let json = serde_json::to_value(ProofsSyncStatus { + earliest: None, + latest: None, + }) + .expect("serialize status"); + assert!(json["earliest"].is_null()); + assert!(json["latest"].is_null()); + } + + #[test] + fn status_bounds_share_one_snapshot_during_prune() { + let directory = tempfile::tempdir().expect("tempdir"); + let storage = Arc::new(MdbxProofsStorage::new(directory.path()).expect("open storage")); + let genesis_hash = B256::repeat_byte(0x01); + storage + .set_initial_state_anchor(BlockNumHash::new(0, genesis_hash)) + .expect("set initial anchor"); + storage + .commit_initial_state() + .expect("commit initial state"); + + let block_one = + BlockWithParent::new(genesis_hash, NumHash::new(1, B256::repeat_byte(0x02))); + storage + .store_trie_updates(block_one, BlockStateDiff::default()) + .expect("store block one"); + + let request_tx = storage.ro_tx().expect("open status snapshot"); + storage + .prune_earliest_state(block_one) + .expect("advance live proof window"); + + let request_status = proofs_sync_status_with_tx(&storage, &request_tx) + .expect("read status from pinned snapshot"); + assert_eq!(request_status.earliest, Some(0)); + assert_eq!(request_status.latest, Some(1)); + + let current_tx = storage.ro_tx().expect("open current snapshot"); + let current_status = + proofs_sync_status_with_tx(&storage, ¤t_tx).expect("read current status"); + assert_eq!(current_status.earliest, Some(1)); + assert_eq!(current_status.latest, Some(1)); + } +} diff --git a/crates/rpc/src/state.rs b/crates/rpc/src/state.rs new file mode 100644 index 00000000..657fc7dc --- /dev/null +++ b/crates/rpc/src/state.rs @@ -0,0 +1,185 @@ +//! State-provider routing for bounded historical proofs. + +use alloy_eips::BlockId; +use alloy_primitives::B256; +use morph_proofs::{MorphProofsStorage, MorphProofsStore, provider::MorphProofsStateProviderRef}; +use reth_provider::{ + BlockHashReader, BlockIdReader, ProviderError, ProviderResult, StateProvider, + StateProviderFactory, +}; +use reth_rpc_api::eth::helpers::FullEthApi; +use reth_rpc_eth_types::EthApiError; +use thiserror::Error; + +/// A request cannot be served by the current durable proof window. +#[derive(Debug, Error, PartialEq, Eq)] +enum ProofWindowError { + #[error("historical proof database is not initialized")] + Uninitialized, + #[error("block {requested} is outside the historical proof window [{earliest}, {latest}]")] + Outside { + requested: u64, + earliest: u64, + latest: u64, + }, + #[error( + "historical proof database is not canonical at block {block}: stored {stored}, canonical {canonical:?}" + )] + CanonicalMismatch { + block: u64, + stored: B256, + canonical: Option, + }, +} + +fn validate_canonical_anchor( + block: u64, + stored: B256, + canonical: Option, +) -> Result<(), ProofWindowError> { + if canonical != Some(stored) { + return Err(ProofWindowError::CanonicalMismatch { + block, + stored, + canonical, + }); + } + Ok(()) +} + +fn validate_window( + requested: u64, + earliest: Option, + latest: Option, +) -> Result<(), ProofWindowError> { + let (Some(earliest), Some(latest)) = (earliest, latest) else { + return Err(ProofWindowError::Uninitialized); + }; + if requested < earliest || requested > latest { + return Err(ProofWindowError::Outside { + requested, + earliest, + latest, + }); + } + Ok(()) +} + +/// Builds providers that source account/storage state only from Morph proof history. +#[derive(Debug)] +pub struct MorphProofStateProviderFactory { + eth_api: Eth, + storage: MorphProofsStorage

, +} + +impl MorphProofStateProviderFactory { + /// Creates a proof state-provider factory. + pub const fn new(eth_api: Eth, storage: MorphProofsStorage

) -> Self { + Self { eth_api, storage } + } +} + +impl<'a, Eth, P> MorphProofStateProviderFactory +where + Eth: FullEthApi + Send + Sync + 'static, + P: MorphProofsStore + Clone + 'a, +{ + /// Returns a provider only when `block_id` is inside the durable proof window. + pub fn state_provider( + &'a self, + block_id: Option, + ) -> ProviderResult> { + let block_id = block_id.unwrap_or_default(); + let block_number = self + .eth_api + .provider() + .block_number_for_id(block_id)? + .ok_or(EthApiError::HeaderNotFound(block_id)) + .map_err(ProviderError::other)?; + if let BlockId::Hash(requested) = block_id { + let canonical_hash = self.eth_api.provider().block_hash(block_number)?; + if canonical_hash != Some(requested.block_hash) { + return Err(ProviderError::other(EthApiError::HeaderNotFound(block_id))); + } + } + + // Pin window validation and all proof cursors to one MDBX snapshot. Otherwise a prune + // committed between the bounds check and proof generation could remap version zero to a + // newer baseline and produce a proof for the wrong state root. + let proof_tx = self + .storage + .ro_tx() + .map_err(|error| ProviderError::Database(error.into()))?; + let earliest = self + .storage + .get_earliest_block_number_with_tx(&proof_tx) + .map_err(|error| ProviderError::Database(error.into()))? + .map(|(number, _)| number); + let latest = self + .storage + .get_latest_block_number_with_tx(&proof_tx) + .map_err(|error| ProviderError::Database(error.into()))?; + validate_window(block_number, earliest, latest.map(|(number, _)| number)) + .map_err(ProviderError::other)?; + + let (latest_number, latest_hash) = + latest.ok_or_else(|| ProviderError::other(ProofWindowError::Uninitialized))?; + let canonical_latest_hash = self.eth_api.provider().block_hash(latest_number)?; + validate_canonical_anchor(latest_number, latest_hash, canonical_latest_hash) + .map_err(ProviderError::other)?; + + // Bytecode is content-addressed and block hashes are canonical-chain data, so a latest + // provider is sufficient for those auxiliary reads. Account/storage trie reads below are + // always routed to proof history; no Reth historical overlay is constructed. + let auxiliary = self.eth_api.provider().latest()?; + Ok(Box::new(MorphProofsStateProviderRef::new_with_tx( + auxiliary, + &self.storage, + block_number, + proof_tx, + ))) + } +} + +#[cfg(test)] +mod tests { + use alloy_primitives::B256; + + use super::{ProofWindowError, validate_canonical_anchor, validate_window}; + + #[test] + fn rejects_an_empty_window() { + assert_eq!( + validate_window(10, None, None), + Err(ProofWindowError::Uninitialized) + ); + } + + #[test] + fn accepts_only_inclusive_window_bounds() { + assert!(validate_window(10, Some(10), Some(20)).is_ok()); + assert!(validate_window(20, Some(10), Some(20)).is_ok()); + assert!(matches!( + validate_window(9, Some(10), Some(20)), + Err(ProofWindowError::Outside { .. }) + )); + assert!(matches!( + validate_window(21, Some(10), Some(20)), + Err(ProofWindowError::Outside { .. }) + )); + } + + #[test] + fn fails_closed_while_proof_history_is_on_a_stale_fork() { + let stored = B256::repeat_byte(0x11); + assert!(validate_canonical_anchor(20, stored, Some(stored)).is_ok()); + assert!(matches!( + validate_canonical_anchor(20, stored, Some(B256::repeat_byte(0x22))), + Err(ProofWindowError::CanonicalMismatch { block: 20, .. }) + )); + assert!(matches!( + validate_canonical_anchor(20, stored, None), + Err(ProofWindowError::CanonicalMismatch { block: 20, .. }) + )); + } +} diff --git a/local-test/README.md b/local-test/README.md index e07d35a7..bc48e407 100644 --- a/local-test/README.md +++ b/local-test/README.md @@ -98,7 +98,7 @@ To wipe chain data and start syncing from scratch: ./local-test/reset.sh hoodi --yes # Reset hoodi (no confirmation) ``` -This removes reth's canonical DB, RocksDB history indices, static files, ExEx state, Morph reference index state, and `node-data/data/` for the specified network. Config files (genesis, keys), `reth.toml`, discovery secrets, and logs are preserved. +This removes reth's canonical DB, RocksDB history indices, static files, ExEx state, Morph reference index state, historical proof MDBX state, and `node-data/data/` for the specified network. Config files (genesis, keys), `reth.toml`, discovery secrets, and logs are preserved. ## Storage & engine tuning diff --git a/local-test/reset.sh b/local-test/reset.sh index babc3e9d..68298e4a 100755 --- a/local-test/reset.sh +++ b/local-test/reset.sh @@ -21,6 +21,7 @@ echo " - ${RETH_DATA_DIR}/rocksdb" echo " - ${RETH_DATA_DIR}/static_files" echo " - ${RETH_DATA_DIR}/exex" echo " - ${RETH_DATA_DIR}/morph" +echo " - ${RETH_DATA_DIR}/historical-proofs" echo " - ${NODE_HOME}/data" echo echo "This keeps:" @@ -45,6 +46,7 @@ rm -rf \ "${RETH_DATA_DIR}/static_files" \ "${RETH_DATA_DIR}/exex" \ "${RETH_DATA_DIR}/morph" \ + "${RETH_DATA_DIR}/historical-proofs" \ "${NODE_HOME}/data" mkdir -p "${RETH_DATA_DIR}" "${NODE_HOME}/data" diff --git a/local-test/reth-start.sh b/local-test/reth-start.sh index 3634f579..2cbc3c0a 100755 --- a/local-test/reth-start.sh +++ b/local-test/reth-start.sh @@ -37,7 +37,6 @@ args=( --authrpc.jwtsecret "${JWT_SECRET}" --log.file.directory "$(dirname "${RETH_LOG_FILE}")" --log.file.filter info - --rpc.eth-proof-window 1209600 # Local testing: explicitly avoid external IP probes such as icanhazip.com. --nat none # Batch MDBX writes so they don't compete with Tendermint's LevelDB fsyncs From a2537ffb96299525bf6f39514deb087236ce81f7 Mon Sep 17 00:00:00 2001 From: panos-xyz Date: Wed, 15 Jul 2026 17:39:33 +0800 Subject: [PATCH 06/12] refactor(proofs): require storage v2 and retain seven days --- README.md | 2 +- bin/morph-reth/src/proofs.rs | 6 +- crates/node/src/args.rs | 6 +- crates/proofs-exex/src/lib.rs | 6 +- crates/proofs/src/initialize.rs | 181 +++++++++++++++++--------------- crates/proofs/src/lib.rs | 3 + 6 files changed, 110 insertions(+), 94 deletions(-) diff --git a/README.md b/README.md index 697f15ff..0dd12cf8 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,7 @@ openssl rand -hex 32 > jwt.hex | `--morph.max-tx-per-block` | None (unlimited) | Maximum number of transactions per block | | `--proofs-history` | false | Enable historical `eth_getProof` and proof-history accumulation | | `--proofs-history.storage-path` | `/historical-proofs` | Override the proof MDBX directory | -| `--proofs-history.window` | 1296000 | Number of canonical blocks retained | +| `--proofs-history.window` | 604800 | Number of canonical blocks retained (7 days at 1s/block) | | `--proofs-history.verification-interval` | 0 | Re-execute every Nth indexed block; 0 disables verification | | `--rpc.eth-proof-window` | 0 | Reth overlay limit used only while proof history is disabled | diff --git a/bin/morph-reth/src/proofs.rs b/bin/morph-reth/src/proofs.rs index e7865c07..5b9b9994 100644 --- a/bin/morph-reth/src/proofs.rs +++ b/bin/morph-reth/src/proofs.rs @@ -5,10 +5,10 @@ use std::{path::PathBuf, sync::Arc}; use alloy_consensus::BlockHeader; use clap::{Parser, Subcommand}; use morph_chainspec::{MorphChainSpec, MorphChainSpecParser}; -use morph_node::{MorphNode, args::DEFAULT_PROOFS_HISTORY_WINDOW}; +use morph_node::MorphNode; use morph_proofs::{ - InitializationJob, MdbxProofsStorage, MorphProofStoragePruner, MorphProofsStorage, - MorphProofsStore, ProofDbIdentity, + DEFAULT_PROOFS_HISTORY_WINDOW, InitializationJob, MdbxProofsStorage, MorphProofStoragePruner, + MorphProofsStorage, MorphProofsStore, ProofDbIdentity, }; use reth_chainspec::EthChainSpec; use reth_cli_commands::common::{AccessRights, Environment, EnvironmentArgs}; diff --git a/crates/node/src/args.rs b/crates/node/src/args.rs index 16846daa..b68657c0 100644 --- a/crates/node/src/args.rs +++ b/crates/node/src/args.rs @@ -3,15 +3,13 @@ use std::path::PathBuf; use clap::Args; +use morph_proofs::DEFAULT_PROOFS_HISTORY_WINDOW; /// Default maximum transaction payload bytes per block (120KB). /// /// This matches Morph's go-ethereum configuration. pub const MORPH_DEFAULT_MAX_TX_PAYLOAD_BYTES: u64 = 122_880; -/// Default proof-history retention window: 30 days at a two-second block time. -pub const DEFAULT_PROOFS_HISTORY_WINDOW: u64 = 1_296_000; - /// Morph-specific CLI arguments. /// /// These arguments extend the standard reth CLI with Morph-specific options @@ -101,7 +99,7 @@ mod tests { assert_eq!(args.max_tx_per_block, None); assert!(!args.proofs_history); assert_eq!(args.proofs_history_storage_path, None); - assert_eq!(args.proofs_history_window, 1_296_000); + assert_eq!(args.proofs_history_window, DEFAULT_PROOFS_HISTORY_WINDOW); assert_eq!(args.proofs_history_verification_interval, 0); } diff --git a/crates/proofs-exex/src/lib.rs b/crates/proofs-exex/src/lib.rs index e1ad5d98..03b51315 100644 --- a/crates/proofs-exex/src/lib.rs +++ b/crates/proofs-exex/src/lib.rs @@ -12,7 +12,8 @@ use alloy_consensus::BlockHeader; use alloy_eips::eip1898::BlockWithParent; use futures::TryStreamExt; use morph_proofs::{ - MorphProofStoragePrunerTask, MorphProofsBatchStore, MorphProofsStorage, + DEFAULT_PROOFS_HISTORY_WINDOW, MorphProofStoragePrunerTask, MorphProofsBatchStore, + MorphProofsStorage, live::{BatchBlock, LiveTrieCollector}, metrics::ProofHistoryMetrics, }; @@ -35,9 +36,6 @@ const DEFAULT_MAX_PRUNE_BLOCKS_STARTUP: u64 = 1_000; /// How many blocks to process in a single sync turn before yielding. const SYNC_BLOCKS_PER_TURN: usize = 50; -/// Default proofs history window: 1 month of blocks at 2s block time -const DEFAULT_PROOFS_HISTORY_WINDOW: u64 = 1_296_000; - /// Default interval between proof-storage prune runs. Default is 15 seconds. const DEFAULT_PRUNE_INTERVAL: Duration = Duration::from_secs(15); diff --git a/crates/proofs/src/initialize.rs b/crates/proofs/src/initialize.rs index b61277bc..4f906dfd 100644 --- a/crates/proofs/src/initialize.rs +++ b/crates/proofs/src/initialize.rs @@ -14,7 +14,8 @@ use reth_db::{ }; use reth_primitives_traits::{Account, StorageEntry}; use reth_trie_common::{ - BranchNodeCompact, Nibbles, StorageTrieEntry, StoredNibbles, StoredNibblesSubKey, + BranchNodeCompact, Nibbles, PackedStorageTrieEntry, PackedStoredNibbles, + PackedStoredNibblesSubKey, StoredNibbles, }; use tracing::{debug, info}; @@ -30,7 +31,7 @@ const INITIALIZE_STORAGE_THRESHOLD: usize = 100000; /// Threshold for logging progress during initialization const INITIALIZE_LOG_THRESHOLD: usize = 100000; -/// Initialization job for external storage. +/// Initialization job for external storage backed by Reth Storage V2 tables. #[derive(Debug, Constructor)] pub struct InitializationJob { storage: S, @@ -91,7 +92,7 @@ macro_rules! define_dup_cursor_iter { }; } -// Generate iterators for all 4 table types +// Generate iterators for the hashed-state and Storage V2 trie tables. define_simple_cursor_iter!(HashedAccountsInit, tables::HashedAccounts, B256, Account); define_dup_cursor_iter!( HashedStoragesInit, @@ -100,16 +101,16 @@ define_dup_cursor_iter!( StorageEntry ); define_simple_cursor_iter!( - AccountsTrieInit, - tables::AccountsTrie, - StoredNibbles, + PackedAccountsTrieInit, + tables::PackedAccountsTrie, + PackedStoredNibbles, BranchNodeCompact ); define_dup_cursor_iter!( - StoragesTrieInit, - tables::StoragesTrie, + PackedStoragesTrieInit, + tables::PackedStoragesTrie, B256, - StorageTrieEntry + PackedStorageTrieEntry ); /// Trait to estimate the progress of a initialization job based on the key. @@ -130,13 +131,13 @@ impl CompletionEstimatable for B256 { } } -impl CompletionEstimatable for StoredNibbles { +impl CompletionEstimatable for Nibbles { fn estimate_progress(&self) -> f64 { // use the first 6 nibbles as a progress estimate - let progress_nibbles = if self.0.is_empty() { - Nibbles::new() + let progress_nibbles = if self.is_empty() { + Self::new() } else { - self.0.slice(0..(self.0.len().min(6))) + self.slice(0..(self.len().min(6))) }; let mut val: u64 = 0; for nibble in progress_nibbles.iter() { @@ -146,6 +147,12 @@ impl CompletionEstimatable for StoredNibbles { } } +impl CompletionEstimatable for PackedStoredNibbles { + fn estimate_progress(&self) -> f64 { + self.0.estimate_progress() + } +} + impl InitializationJob { @@ -282,19 +289,19 @@ impl &self, start_key: Option, ) -> Result<(), MorphProofsStorageError> { - let mut start_cursor = self.tx.cursor_read::()?; + let mut start_cursor = self.tx.cursor_read::()?; if let Some(latest_key) = start_key { + let packed_key = PackedStoredNibbles::from(latest_key.0); start_cursor - .seek(latest_key.clone())? - .filter(|(k, _)| *k == latest_key) + .seek(packed_key.clone())? + .filter(|(key, _)| *key == packed_key) .ok_or(MorphProofsStorageError::InitializeStorageInconsistentState)?; } - let source = AccountsTrieInit::new(start_cursor); self.initialize( "accounts trie", - source, + PackedAccountsTrieInit::new(start_cursor), INITIALIZE_STORAGE_THRESHOLD, INITIALIZE_LOG_THRESHOLD, )?; @@ -307,22 +314,19 @@ impl &self, start_key: Option, ) -> Result<(), MorphProofsStorageError> { - let mut start_cursor = self.tx.cursor_dup_read::()?; + let mut start_cursor = self.tx.cursor_dup_read::()?; if let Some(latest_key) = start_key { + let packed_subkey = PackedStoredNibblesSubKey::from(latest_key.path.0); start_cursor - .seek_by_key_subkey( - latest_key.hashed_address, - StoredNibblesSubKey::from(latest_key.path.0), - )? - .filter(|v| v.nibbles.0 == latest_key.path.0) + .seek_by_key_subkey(latest_key.hashed_address, packed_subkey)? + .filter(|value| value.nibbles.0 == latest_key.path.0) .ok_or(MorphProofsStorageError::InitializeStorageInconsistentState)?; } - let source = StoragesTrieInit::new(start_cursor); self.initialize( "storage trie", - source, + PackedStoragesTrieInit::new(start_cursor), INITIALIZE_STORAGE_THRESHOLD, INITIALIZE_LOG_THRESHOLD, )?; @@ -436,11 +440,10 @@ impl InitTable for HashedStoragesInit { } } -impl InitTable for AccountsTrieInit { - type Key = StoredNibbles; +impl InitTable for PackedAccountsTrieInit { + type Key = PackedStoredNibbles; type Value = BranchNodeCompact; - /// Save mapping of account trie paths to branch nodes to storage. fn store_entries( store: &impl MorphProofsInitialStateStore, entries: impl IntoIterator, @@ -451,16 +454,14 @@ impl InitTable for AccountsTrieInit { .map(|(path, branch)| (path.0, Some(branch))) .collect(), )?; - Ok(()) } } -impl InitTable for StoragesTrieInit { +impl InitTable for PackedStoragesTrieInit { type Key = B256; - type Value = StorageTrieEntry; + type Value = PackedStorageTrieEntry; - /// Save mapping of hashed addresses to storage trie entries to storage. fn store_entries( store: &impl MorphProofsInitialStateStore, entries: impl IntoIterator, @@ -476,7 +477,6 @@ impl InitTable for StoragesTrieInit { .push((storage_entry.nibbles.0, Some(storage_entry.node))); } store.store_storage_branches_bulk(by_address.into_iter().collect())?; - Ok(()) } } @@ -495,8 +495,9 @@ mod tests { }; use reth_primitives_traits::Account; use reth_trie::{ - BranchNodeCompact, StorageTrieEntry, StoredNibbles, StoredNibblesSubKey, - TrieMask, hashed_cursor::HashedCursor, trie_cursor::TrieCursor, + BranchNodeCompact, PackedStorageTrieEntry, PackedStoredNibbles, + PackedStoredNibblesSubKey, TrieMask, hashed_cursor::HashedCursor, + trie_cursor::TrieCursor, }; use tempfile::TempDir; @@ -673,20 +674,20 @@ mod tests { // Insert test trie nodes into database let tx = db.tx_mut().unwrap(); - let mut cursor = tx.cursor_write::().unwrap(); + let mut cursor = tx.cursor_write::().unwrap(); let branch = create_test_branch_node(); let nodes = vec![ ( - StoredNibbles(Nibbles::from_nibbles_unchecked(vec![1])), + PackedStoredNibbles::from(Nibbles::from_nibbles_unchecked(vec![1])), branch.clone(), ), ( - StoredNibbles(Nibbles::from_nibbles_unchecked(vec![2])), + PackedStoredNibbles::from(Nibbles::from_nibbles_unchecked(vec![2])), branch.clone(), ), ( - StoredNibbles(Nibbles::from_nibbles_unchecked(vec![3])), + PackedStoredNibbles::from(Nibbles::from_nibbles_unchecked(vec![3])), branch, ), ]; @@ -720,7 +721,7 @@ mod tests { // Insert test storage trie nodes into database let tx = db.tx_mut().unwrap(); - let mut cursor = tx.cursor_dup_write::().unwrap(); + let mut cursor = tx.cursor_dup_write::().unwrap(); let branch = create_test_branch_node(); let addr1 = keccak256(Address::repeat_byte(0x01)); @@ -729,28 +730,28 @@ mod tests { let nodes = vec![ ( addr1, - StorageTrieEntry { - nibbles: StoredNibblesSubKey(Nibbles::from_nibbles_unchecked( - vec![1], - )), + PackedStorageTrieEntry { + nibbles: PackedStoredNibblesSubKey::from( + Nibbles::from_nibbles_unchecked(vec![1]), + ), node: branch.clone(), }, ), ( addr1, - StorageTrieEntry { - nibbles: StoredNibblesSubKey(Nibbles::from_nibbles_unchecked( - vec![2], - )), + PackedStorageTrieEntry { + nibbles: PackedStoredNibblesSubKey::from( + Nibbles::from_nibbles_unchecked(vec![2]), + ), node: branch.clone(), }, ), ( addr2, - StorageTrieEntry { - nibbles: StoredNibblesSubKey(Nibbles::from_nibbles_unchecked( - vec![3], - )), + PackedStorageTrieEntry { + nibbles: PackedStoredNibblesSubKey::from( + Nibbles::from_nibbles_unchecked(vec![3]), + ), node: branch, }, ), @@ -825,24 +826,24 @@ mod tests { drop(cursor); // Add account trie - let mut cursor = tx.cursor_write::().unwrap(); + let mut cursor = tx.cursor_write::().unwrap(); cursor .append( - StoredNibbles(Nibbles::from_nibbles_unchecked(vec![1])), + PackedStoredNibbles::from(Nibbles::from_nibbles_unchecked(vec![1])), &create_test_branch_node(), ) .unwrap(); drop(cursor); // Add storage trie - let mut cursor = tx.cursor_dup_write::().unwrap(); + let mut cursor = tx.cursor_dup_write::().unwrap(); cursor .upsert( addr, - &StorageTrieEntry { - nibbles: StoredNibblesSubKey(Nibbles::from_nibbles_unchecked( - vec![1], - )), + &PackedStorageTrieEntry { + nibbles: PackedStoredNibblesSubKey::from( + Nibbles::from_nibbles_unchecked(vec![1]), + ), node: create_test_branch_node(), }, ) @@ -1175,9 +1176,17 @@ mod tests { // Phase 1 source: p1,p2 { let tx = db.tx_mut().unwrap(); - let mut cur = tx.cursor_write::().unwrap(); - cur.append(p1.clone(), &create_test_branch_node()).unwrap(); - cur.append(p2.clone(), &create_test_branch_node()).unwrap(); + let mut cur = tx.cursor_write::().unwrap(); + cur.append( + PackedStoredNibbles::from(p1.0.clone()), + &create_test_branch_node(), + ) + .unwrap(); + cur.append( + PackedStoredNibbles::from(p2.0.clone()), + &create_test_branch_node(), + ) + .unwrap(); tx.commit().unwrap(); } @@ -1199,9 +1208,17 @@ mod tests { // Phase 2 source: p3,p4 { let tx = db.tx_mut().unwrap(); - let mut cur = tx.cursor_write::().unwrap(); - cur.append(p3.clone(), &create_test_branch_node()).unwrap(); - cur.append(p4.clone(), &create_test_branch_node()).unwrap(); + let mut cur = tx.cursor_write::().unwrap(); + cur.append( + PackedStoredNibbles::from(p3.0.clone()), + &create_test_branch_node(), + ) + .unwrap(); + cur.append( + PackedStoredNibbles::from(p4.0.clone()), + &create_test_branch_node(), + ) + .unwrap(); tx.commit().unwrap(); } @@ -1246,26 +1263,26 @@ mod tests { let a1 = k(0x10); let a2 = k(0x20); - let n1 = StoredNibblesSubKey(Nibbles::from_nibbles_unchecked(vec![1])); - let n2 = StoredNibblesSubKey(Nibbles::from_nibbles_unchecked(vec![2])); - let n3 = StoredNibblesSubKey(Nibbles::from_nibbles_unchecked(vec![3])); + let n1 = Nibbles::from_nibbles_unchecked(vec![1]); + let n2 = Nibbles::from_nibbles_unchecked(vec![2]); + let n3 = Nibbles::from_nibbles_unchecked(vec![3]); // Phase 1 source: (a1,n1), (a2,n2) { let tx = db.tx_mut().unwrap(); - let mut cur = tx.cursor_dup_write::().unwrap(); + let mut cur = tx.cursor_dup_write::().unwrap(); cur.upsert( a1, - &StorageTrieEntry { - nibbles: n1.clone(), + &PackedStorageTrieEntry { + nibbles: PackedStoredNibblesSubKey::from(n1.clone()), node: create_test_branch_node(), }, ) .unwrap(); cur.upsert( a2, - &StorageTrieEntry { - nibbles: n2.clone(), + &PackedStorageTrieEntry { + nibbles: PackedStoredNibblesSubKey::from(n2.clone()), node: create_test_branch_node(), }, ) @@ -1287,16 +1304,16 @@ mod tests { .latest_storage_trie_key .expect("ok"); assert_eq!(last1.hashed_address, a2); - assert_eq!(last1.path.0, n2.0); + assert_eq!(last1.path.0, n2); // Phase 2 source: add (a2,n3) { let tx = db.tx_mut().unwrap(); - let mut cur = tx.cursor_dup_write::().unwrap(); + let mut cur = tx.cursor_dup_write::().unwrap(); cur.upsert( a2, - &StorageTrieEntry { - nibbles: n3.clone(), + &PackedStorageTrieEntry { + nibbles: PackedStoredNibblesSubKey::from(n3.clone()), node: create_test_branch_node(), }, ) @@ -1310,7 +1327,7 @@ mod tests { let job = InitializationJob::new(Arc::clone(&store), tx); job.initialize_storages_trie(Some(StorageTrieKey::new( a2, - StoredNibbles::from(n2.0), + StoredNibbles::from(n2.clone()), ))) .unwrap(); } @@ -1322,7 +1339,7 @@ mod tests { .latest_storage_trie_key .expect("ok"); assert_eq!(last2.hashed_address, a2); - assert_eq!(last2.path.0, n3.0); + assert_eq!(last2.path.0, n3); // Verify per-address no dupes and stable ordering { @@ -1335,7 +1352,7 @@ mod tests { got.push(path); } - assert_eq!(got, vec![n1.0]); + assert_eq!(got, vec![n1]); } { let mut c = store.storage_trie_cursor(a2, 0).unwrap(); @@ -1346,7 +1363,7 @@ mod tests { while let Some((path, _node)) = c.next().unwrap() { got.push(path); } - assert_eq!(got, vec![n2.0, n3.0]); + assert_eq!(got, vec![n2, n3]); } } } diff --git a/crates/proofs/src/lib.rs b/crates/proofs/src/lib.rs index 438ec198..953632df 100644 --- a/crates/proofs/src/lib.rs +++ b/crates/proofs/src/lib.rs @@ -16,6 +16,9 @@ pub use api::{ pub mod initialize; pub use initialize::InitializationJob; +/// Default proof-history retention window: 7 days at a one-second block time. +pub const DEFAULT_PROOFS_HISTORY_WINDOW: u64 = 604_800; + #[cfg(any(test, feature = "test-utils"))] pub mod in_memory; #[cfg(any(test, feature = "test-utils"))] From 4a96ad444208c80e471ecdf4da648c93daf39b51 Mon Sep 17 00:00:00 2001 From: panos Date: Thu, 13 Aug 2026 10:31:30 +0800 Subject: [PATCH 07/12] chore: refresh Cargo.lock after v1.1.0 workspace bump Keep morph-proofs and morph-proofs-exex lockfile versions aligned with the merged workspace package version. --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ca2963c7..fea5b089 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5210,7 +5210,7 @@ dependencies = [ [[package]] name = "morph-proofs" -version = "1.0.0" +version = "1.1.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -5244,7 +5244,7 @@ dependencies = [ [[package]] name = "morph-proofs-exex" -version = "1.0.0" +version = "1.1.0" dependencies = [ "alloy-consensus", "alloy-eips", From e74898ab5b1b903165d6fea0cf0ffa5bc366fc7c Mon Sep 17 00:00:00 2001 From: panos Date: Tue, 18 Aug 2026 11:42:11 +0800 Subject: [PATCH 08/12] test(proofs): cover historical-proof correctness end to end The previous proof-history test only proved RPC wiring: it mined one empty block and checked that `eth_getProof` for the zero address returned the same response on the normal and auth ports. It could not tell a correct historical proof from a latest-state proof. Node e2e (crates/node/tests/it/proof_history.rs): - Deploy an SSTORE setter and mutate state across blocks, then query each height and assert the value that block actually held. - Verify every successful proof against that block's canonical stateRoot via `AccountProof::verify`, which also checks storage proofs against the account storage root. Assert a proof does not verify against a different block's root. - Drive the fork case purely through a reorg instead of poking `unwind_history` first, so a broken ExEx reorg path can no longer pass. The height is unchanged across the reorg, so wait on the exact (number, hash). - Read requested slots by key and panic on a missing entry. Defaulting to zero let a response carrying no storage proof pass an `== 0` assertion. Add a multi-slot case including an unset slot with its exclusion proof. - Tighten the future-block assertion to the window error; numeric block ids resolve without an existence check, so the window bounds reject them. - Add a canonical-block-hash lookup alongside the height lookup. - Rename `survives_node_restart_and_continues` to `db_survives_node_restart`: it never appended after the restart. Assert per-block change sets survive the reopen, since pointers alone would outlive lost rows. This also pins that the earliest block is a baseline snapshot rather than a diff. - Rename the verification-interval test to reflect that it asserts correctness on that path, not that the interval selects it. ExEx unit tests (crates/proofs-exex/src/lib.rs): - Cover `build_batch_entry` directly, asserting the chosen `BatchBlock` variant for interval 0/1/N on and off the interval, with and without cached data. Stubbing `should_verify` to false turns two of these red, whereas the e2e stayed green because the cached path also yields correct proofs. - Add an `ensure_initialized` case for a latest hash that is not canonical, the shape of a proofs DB restored beside a mismatched chain snapshot. - Assert error messages in the existing `ensure_initialized` failure cases; the prune-threshold fixture also trips the later canonical-hash check, so a bare `expect_err` did not identify which guard fired. --- crates/node/Cargo.toml | 4 + crates/node/tests/it/proof_history.rs | 868 ++++++++++++++++++++++++-- crates/proofs-exex/src/lib.rs | 172 ++++- 3 files changed, 977 insertions(+), 67 deletions(-) diff --git a/crates/node/Cargo.toml b/crates/node/Cargo.toml index 620180dd..514d81b6 100644 --- a/crates/node/Cargo.toml +++ b/crates/node/Cargo.toml @@ -89,8 +89,12 @@ reth-payload-primitives.workspace = true reth-rpc-server-types.workspace = true reth-tasks.workspace = true reth-tracing.workspace = true +reth-trie.workspace = true alloy-consensus.workspace = true +alloy-eips.workspace = true alloy-primitives.workspace = true +alloy-signer.workspace = true +alloy-signer-local.workspace = true alloy-rpc-types-engine.workspace = true alloy-rpc-types-eth.workspace = true jsonrpsee.workspace = true diff --git a/crates/node/tests/it/proof_history.rs b/crates/node/tests/it/proof_history.rs index 2e8a95f3..f4b8e7d8 100644 --- a/crates/node/tests/it/proof_history.rs +++ b/crates/node/tests/it/proof_history.rs @@ -1,31 +1,135 @@ -//! End-to-end coexistence coverage for proof history and the reference index. +//! Historical-proof correctness tests. +//! +//! These exercise `eth_getProof` against the durable proof-history index, not +//! just RPC wiring. Each successful proof is checked against that block's +//! canonical `stateRoot`. Reference-index coverage lives in `reference_index.rs`. use std::{sync::Arc, time::Duration}; +use alloy_consensus::{SignableTransaction, TxEip1559}; +use alloy_eips::{Encodable2718, NumHash, eip1898::BlockWithParent}; use alloy_genesis::Genesis; -use alloy_primitives::{Address, B256}; -use alloy_rpc_types_eth::EIP1186AccountProofResponse; +use alloy_primitives::{Address, B256, Bytes, TxKind, U256}; +use alloy_rpc_types_eth::{Block, EIP1186AccountProofResponse}; +use alloy_signer::SignerSync; +use alloy_signer_local::PrivateKeySigner; use jsonrpsee::{core::client::ClientT, http_client::HttpClient, rpc_params}; use morph_chainspec::MorphChainSpec; use morph_node::{ MorphAddOns, MorphNode, - test_utils::{MorphTestNode, advance_empty_block, morph_payload_attributes}, + test_utils::{ + MorphTestNode, MorphTxBuilder, make_deploy_tx, morph_payload_attributes, wallet_at_index, + }, +}; +use morph_payload_types::ExecutableL2Data; +use morph_primitives::MorphHeader; +use morph_proofs::{ + DEFAULT_PROOFS_HISTORY_WINDOW, InitializationJob, MdbxProofsStorage, MorphProofsStorage, + MorphProofsStore, ProofDbIdentity, }; -use morph_proofs::{InitializationJob, MdbxProofsStorage, MorphProofsStorage, ProofDbIdentity}; use morph_proofs_exex::MorphProofsExEx; -use morph_reference_index::ReferenceTransactionResult; -use morph_rpc::{ProofsSyncStatus, ReferenceQueryArgs}; +use morph_rpc::ProofsSyncStatus; use reth_chainspec::EthChainSpec; use reth_e2e_test_utils::node::NodeTestContext; use reth_node_builder::{EngineNodeLauncher, Node, NodeBuilder, NodeConfig, NodeHandle}; use reth_node_core::args::{DiscoveryArgs, NetworkArgs, RpcServerArgs}; -use reth_provider::{DBProvider, DatabaseProviderFactory, providers::BlockchainProvider}; +use reth_payload_primitives::BuiltPayload; +use reth_provider::{ + BlockReaderIdExt, DBProvider, DatabaseProviderFactory, ReceiptProvider, + providers::BlockchainProvider, +}; use reth_rpc_server_types::RpcModuleSelection; use reth_tasks::Runtime; +use reth_trie::AccountProof; +use serde::Serialize; + +const CHAIN_ID: u64 = 2910; +const ACCOUNT0: Address = alloy_primitives::address!("f39Fd6e51aad88F6F4ce6aB8827279cffFb92266"); +const ACCOUNT1: Address = alloy_primitives::address!("70997970C51812dc3A010C7d01b50e0d17dc79C8"); +const ACCOUNT2: Address = alloy_primitives::address!("3C44CdDdB6a900fa2b585dd299e03d12FA4293BC"); +const GENESIS_CONTRACT: Address = + alloy_primitives::address!("530000000000000000000000000000000000000f"); +const GENESIS_SLOT: B256 = B256::with_last_byte(0x06); + +/// Runtime: `PUSH1 0; CALLDATALOAD; PUSH1 0; SSTORE; STOP`. +/// +/// Init code copies those 7 bytes and returns them as the deployed account. +const SETTER_INIT: &[u8] = &[ + 0x60, 0x07, // PUSH1 7 + 0x60, 0x0c, // PUSH1 12 + 0x60, 0x00, // PUSH1 0 + 0x39, // CODECOPY + 0x60, 0x07, // PUSH1 7 + 0x60, 0x00, // PUSH1 0 + 0xf3, // RETURN + 0x60, 0x00, 0x35, 0x60, 0x00, 0x55, 0x00, +]; + +#[derive(Clone, Copy)] +struct LaunchOpts { + window: u64, + prune_interval: Duration, + verification_interval: u64, +} + +impl Default for LaunchOpts { + fn default() -> Self { + Self { + window: DEFAULT_PROOFS_HISTORY_WINDOW, + // Keep the background pruner idle unless a test is specifically + // asserting prune behavior. + prune_interval: Duration::from_secs(3600), + verification_interval: 0, + } + } +} + +struct ProofHistoryHarness { + node: MorphTestNode, + storage: MorphProofsStorage>, + _proofs_dir: tempfile::TempDir, + chain_spec: Arc, +} + +fn signer(index: u32) -> PrivateKeySigner { + wallet_at_index(index, CHAIN_ID) +} -async fn launch_node_with_proof_history( +fn signed_transfer(from: u32, nonce: u64, to: Address, value: u64) -> eyre::Result { + let tx = TxEip1559 { + chain_id: CHAIN_ID, + nonce, + gas_limit: 21_000, + max_fee_per_gas: 20_000_000_000u128, + max_priority_fee_per_gas: 20_000_000_000u128, + to: TxKind::Call(to), + value: U256::from(value), + access_list: Default::default(), + input: Bytes::new(), + }; + let sig = signer(from) + .sign_hash_sync(&tx.signature_hash()) + .map_err(|error| eyre::eyre!("signing transfer failed: {error}"))?; + Ok(tx.into_signed(sig).encoded_2718().into()) +} + +fn signed_call(from: u32, nonce: u64, to: Address, data: Bytes) -> eyre::Result { + MorphTxBuilder::new(CHAIN_ID, signer(from), nonce) + .with_v1_eth_fee() + .with_to(to) + .with_data(data) + .with_gas_limit(100_000) + .build_signed() +} + +fn slot_calldata(value: u64) -> Bytes { + Bytes::from(B256::from(U256::from(value)).to_vec()) +} + +async fn launch_node( chain_spec: Arc, storage: MorphProofsStorage>, + opts: LaunchOpts, ) -> eyre::Result { let runtime = Runtime::test(); let network = NetworkArgs { @@ -64,7 +168,11 @@ async fn launch_node_with_proof_history( InitializationJob::new(exex_storage.clone(), provider.into_tx()) .run(head.number, head.hash)?; - let exex = MorphProofsExEx::builder(ctx, exex_storage).build(); + let exex = MorphProofsExEx::builder(ctx, exex_storage) + .with_proofs_history_window(opts.window) + .with_proofs_history_prune_interval(opts.prune_interval) + .with_verification_interval(opts.verification_interval) + .build(); Ok(async move { exex.run().await }) }) .launch_with_fn(|builder| { @@ -83,15 +191,41 @@ async fn launch_node_with_proof_history( Ok(node) } -async fn wait_for_proof_tip(client: &HttpClient, expected: u64) -> eyre::Result { - let deadline = tokio::time::Instant::now() + Duration::from_secs(10); +async fn setup(opts: LaunchOpts) -> eyre::Result { + let genesis: Genesis = serde_json::from_str(include_str!("../assets/test-genesis.json"))?; + let chain_spec = Arc::new(MorphChainSpec::from_genesis(genesis)); + let proofs_dir = tempfile::tempdir()?; + let storage = Arc::new(MdbxProofsStorage::open( + proofs_dir.path(), + ProofDbIdentity::new(chain_spec.chain().id(), chain_spec.genesis_hash()), + )?); + let node = launch_node(chain_spec.clone(), storage.clone(), opts).await?; + Ok(ProofHistoryHarness { + node, + storage, + _proofs_dir: proofs_dir, + chain_spec, + }) +} + +fn rpc_client(node: &MorphTestNode) -> eyre::Result { + node.rpc_client() + .ok_or_else(|| eyre::eyre!("HTTP RPC client not available")) +} + +async fn wait_for_proof_status( + client: &HttpClient, + expected: impl Fn(&ProofsSyncStatus) -> bool, + context: &str, +) -> eyre::Result { + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); let mut last_status = None; while tokio::time::Instant::now() < deadline { if let Ok(status) = client .request::("debug_proofsSyncStatus", rpc_params![]) .await { - if status.latest == Some(expected) { + if expected(&status) { return Ok(status); } last_status = Some(status); @@ -99,80 +233,686 @@ async fn wait_for_proof_tip(client: &HttpClient, expected: u64) -> eyre::Result< tokio::time::sleep(Duration::from_millis(25)).await; } Err(eyre::eyre!( - "proof history did not reach block {expected}: {last_status:?}" + "proof history did not reach {context}: {last_status:?}" )) } -async fn wait_for_reference_index(client: &HttpClient) -> eyre::Result<()> { - let args = ReferenceQueryArgs { - reference: B256::repeat_byte(0x99), - offset: None, - limit: None, - }; +async fn wait_for_proof_tip(client: &HttpClient, expected: u64) -> eyre::Result { + wait_for_proof_status( + client, + |status| status.latest == Some(expected), + &format!("tip {expected}"), + ) + .await +} + +async fn wait_for_window( + client: &HttpClient, + earliest: u64, + latest: u64, +) -> eyre::Result { + wait_for_proof_status( + client, + |status| status.earliest == Some(earliest) && status.latest == Some(latest), + &format!("window [{earliest}, {latest}]"), + ) + .await +} + +/// Waits for proof history to anchor on an exact `(number, hash)` tip. +/// +/// `debug_proofsSyncStatus` only reports heights, so a reorg that keeps the height +/// but swaps the branch is invisible to [`wait_for_proof_tip`]. +async fn wait_for_proof_latest( + storage: &MorphProofsStorage>, + number: u64, + hash: B256, +) -> eyre::Result<()> { + let deadline = tokio::time::Instant::now() + Duration::from_secs(15); + let mut last = None; + while tokio::time::Instant::now() < deadline { + let latest = storage.get_latest_block_number()?; + if latest == Some((number, hash)) { + return Ok(()); + } + last = latest; + tokio::time::sleep(Duration::from_millis(25)).await; + } + Err(eyre::eyre!( + "proof history latest never became ({number}, {hash}); last seen {last:?}" + )) +} + +async fn get_block(client: &HttpClient, block: B) -> eyre::Result { + client + .request("eth_getBlockByNumber", rpc_params![block, false]) + .await + .map_err(|error| eyre::eyre!("eth_getBlockByNumber failed: {error}")) +} + +async fn get_proof( + client: &HttpClient, + address: Address, + slots: Vec, + block: B, +) -> eyre::Result { + client + .request("eth_getProof", rpc_params![address, slots, block]) + .await + .map_err(|error| eyre::eyre!("eth_getProof failed: {error}")) +} + +fn verify_against_state_root( + proof: &EIP1186AccountProofResponse, + state_root: B256, +) -> eyre::Result<()> { + AccountProof::from_eip1186_proof(proof.clone()) + .verify(state_root) + .map_err(|error| { + eyre::eyre!( + "proof for {} does not verify against state root {state_root}: {error}", + proof.address + ) + }) +} + +async fn get_verified_proof( + client: &HttpClient, + address: Address, + slots: Vec, + block: B, +) -> eyre::Result { + let header_block = get_block(client, block.clone()).await?; + let proof = get_proof(client, address, slots, block).await?; + verify_against_state_root(&proof, header_block.header.state_root)?; + Ok(proof) +} + +async fn proof_error( + client: &HttpClient, + address: Address, + block: B, +) -> eyre::Result { + match client + .request::( + "eth_getProof", + rpc_params![address, Vec::::new(), block], + ) + .await + { + Ok(proof) => Err(eyre::eyre!( + "expected eth_getProof to fail, got proof for {}", + proof.address + )), + Err(error) => Ok(error.to_string()), + } +} + +fn assert_outside_window(error: &str, requested: u64) { + assert!( + error.contains("outside the historical proof window") + && error.contains(&requested.to_string()), + "expected window error for block {requested}, got: {error}" + ); +} + +async fn include_tx( + node: &mut MorphTestNode, + raw_tx: Bytes, +) -> eyre::Result { + use alloy_consensus::TxReceipt; + use alloy_consensus::transaction::TxHashRef; + + node.rpc.inject_tx(raw_tx).await?; + // The payload builder can emit an empty block if it races the pool insert. + tokio::time::sleep(Duration::from_millis(200)).await; + let payload = node.advance_block().await?; + eyre::ensure!( + payload.block().body().transactions.len() == 1, + "expected the injected transaction to be the sole tx in the block" + ); + let tx_hash = *payload + .block() + .body() + .transactions + .first() + .expect("checked non-empty") + .tx_hash(); + let receipt = node + .inner + .provider + .receipt_by_hash(tx_hash)? + .ok_or_else(|| eyre::eyre!("missing receipt for included tx {tx_hash}"))?; + eyre::ensure!(receipt.status(), "included transaction reverted: {tx_hash}"); + Ok(payload.block().hash()) +} + +/// Reads one requested slot out of a proof response. +/// +/// Deliberately panics on a missing entry: defaulting to zero would let a response +/// that carries no storage proof at all pass an `== U256::ZERO` assertion. +fn storage_entry(proof: &EIP1186AccountProofResponse, slot: B256) -> U256 { + let entry = proof + .storage_proof + .iter() + .find(|entry| entry.key.as_b256() == slot) + .unwrap_or_else(|| { + panic!( + "no storage proof entry for slot {slot} on {} (got {} entries)", + proof.address, + proof.storage_proof.len() + ) + }); + entry.value +} + +fn unix_now() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("time went backwards") + .as_secs() +} + +async fn reopen_proofs( + path: &std::path::Path, + identity: ProofDbIdentity, +) -> eyre::Result> { let deadline = tokio::time::Instant::now() + Duration::from_secs(10); let mut last_error = None; while tokio::time::Instant::now() < deadline { - match client - .request::, _>( - "morph_getTransactionHashesByReference", - (args.clone(),), - ) - .await - { - Ok(results) if results.is_empty() => return Ok(()), - Ok(results) => { - return Err(eyre::eyre!( - "unexpected reference-index results: {results:?}" - )); + match MdbxProofsStorage::open(path, identity) { + Ok(storage) => return Ok(Arc::new(storage)), + Err(error) => { + last_error = Some(error.to_string()); + tokio::time::sleep(Duration::from_millis(200)).await; } - Err(error) => last_error = Some(error.to_string()), } - tokio::time::sleep(Duration::from_millis(25)).await; } Err(eyre::eyre!( - "reference index did not become ready: {}", - last_error.as_deref().unwrap_or("no RPC response") + "failed to reopen proofs db: {}", + last_error.unwrap_or_else(|| "no attempt".to_string()) )) } +// ----------------------------------------------------------------------------- +// 1 + 2. Multi-block historical queries and independent cryptographic verification +// ----------------------------------------------------------------------------- + #[tokio::test(flavor = "multi_thread")] -async fn proof_history_and_reference_index_advance_together() -> eyre::Result<()> { +async fn proof_history_multi_block_account_and_storage() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - let genesis: Genesis = serde_json::from_str(include_str!("../assets/test-genesis.json"))?; - let chain_spec = Arc::new(MorphChainSpec::from_genesis(genesis)); - let proof_directory = tempfile::tempdir()?; - let proof_storage = Arc::new(MdbxProofsStorage::open( - proof_directory.path(), - ProofDbIdentity::new(chain_spec.chain().id(), chain_spec.genesis_hash()), - )?); - let mut node = launch_node_with_proof_history(chain_spec, proof_storage).await?; + let mut env = setup(LaunchOpts::default()).await?; + let client = rpc_client(&env.node)?; - advance_empty_block(&mut node).await?; - let client = node - .rpc_client() - .ok_or_else(|| eyre::eyre!("HTTP RPC client not available"))?; + wait_for_proof_tip(&client, 0).await?; - let status = wait_for_proof_tip(&client, 1).await?; - assert_eq!(status.earliest, Some(0)); - wait_for_reference_index(&client).await?; + let genesis_sender = get_verified_proof(&client, ACCOUNT0, vec![], "0x0").await?; + let genesis_recipient = get_verified_proof(&client, ACCOUNT1, vec![], "0x0").await?; + let genesis_storage = + get_verified_proof(&client, GENESIS_CONTRACT, vec![GENESIS_SLOT], "0x0").await?; + assert_eq!(storage_entry(&genesis_storage, GENESIS_SLOT), U256::from(1)); - let normal_proof: EIP1186AccountProofResponse = client - .request( - "eth_getProof", - rpc_params![Address::ZERO, Vec::::new(), "0x0"], + include_tx(&mut env.node, signed_transfer(0, 0, ACCOUNT1, 1_000)?).await?; + wait_for_proof_tip(&client, 1).await?; + + let setter = Address::create(&ACCOUNT0, 1); + include_tx( + &mut env.node, + make_deploy_tx(CHAIN_ID, signer(0), 1, SETTER_INIT)?, + ) + .await?; + wait_for_proof_tip(&client, 2).await?; + + include_tx(&mut env.node, signed_call(0, 2, setter, slot_calldata(11))?).await?; + wait_for_proof_tip(&client, 3).await?; + + include_tx(&mut env.node, signed_call(0, 3, setter, slot_calldata(22))?).await?; + wait_for_proof_tip(&client, 4).await?; + + let at_1_sender = get_verified_proof(&client, ACCOUNT0, vec![], "0x1").await?; + let at_1_recipient = get_verified_proof(&client, ACCOUNT1, vec![], "0x1").await?; + assert_eq!(at_1_sender.nonce, 1); + assert!(at_1_sender.balance < genesis_sender.balance); + assert_eq!( + at_1_recipient.balance, + genesis_recipient.balance + U256::from(1_000) + ); + + let at_2_setter = get_verified_proof(&client, setter, vec![B256::ZERO], "0x2").await?; + assert_ne!( + at_2_setter.code_hash, + alloy_primitives::KECCAK256_EMPTY, + "setter must be deployed at block 2" + ); + assert_eq!(storage_entry(&at_2_setter, B256::ZERO), U256::ZERO); + + let at_3_setter = get_verified_proof(&client, setter, vec![B256::ZERO], "0x3").await?; + let at_4_setter = get_verified_proof(&client, setter, vec![B256::ZERO], "0x4").await?; + assert_eq!(storage_entry(&at_3_setter, B256::ZERO), U256::from(11)); + assert_eq!(storage_entry(&at_4_setter, B256::ZERO), U256::from(22)); + + // Historical queries must not leak later state. + let historical = get_verified_proof(&client, setter, vec![B256::ZERO], "0x3").await?; + assert_eq!(storage_entry(&historical, B256::ZERO), U256::from(11)); + let still_genesis = + get_verified_proof(&client, GENESIS_CONTRACT, vec![GENESIS_SLOT], "0x4").await?; + assert_eq!(storage_entry(&still_genesis, GENESIS_SLOT), U256::from(1)); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn proof_history_proofs_verify_against_block_state_root() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let mut env = setup(LaunchOpts::default()).await?; + let client = rpc_client(&env.node)?; + + include_tx(&mut env.node, signed_transfer(0, 0, ACCOUNT1, 5_000)?).await?; + wait_for_proof_tip(&client, 1).await?; + + let block_0 = get_block(&client, "0x0").await?; + let block_1 = get_block(&client, "0x1").await?; + assert_ne!(block_0.header.state_root, block_1.header.state_root); + + let proof = get_proof(&client, ACCOUNT0, vec![], "0x1").await?; + verify_against_state_root(&proof, block_1.header.state_root)?; + assert!( + AccountProof::from_eip1186_proof(proof.clone()) + .verify(block_0.header.state_root) + .is_err(), + "account proof must not verify against a different block's state root" + ); + + let storage_proof = get_proof(&client, GENESIS_CONTRACT, vec![GENESIS_SLOT], "0x1").await?; + verify_against_state_root(&storage_proof, block_1.header.state_root)?; + assert_eq!(storage_entry(&storage_proof, GENESIS_SLOT), U256::from(1)); + + Ok(()) +} + +// ----------------------------------------------------------------------------- +// 3. earliest / latest / future / missing-block bounds +// ----------------------------------------------------------------------------- + +#[tokio::test(flavor = "multi_thread")] +async fn proof_history_rpc_block_bounds() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let mut env = setup(LaunchOpts::default()).await?; + let client = rpc_client(&env.node)?; + + include_tx(&mut env.node, signed_transfer(0, 0, ACCOUNT1, 1)?).await?; + include_tx(&mut env.node, signed_transfer(0, 1, ACCOUNT2, 2)?).await?; + wait_for_proof_tip(&client, 2).await?; + + let earliest = get_verified_proof(&client, ACCOUNT0, vec![], "earliest").await?; + let latest = get_verified_proof(&client, ACCOUNT0, vec![], "latest").await?; + assert_eq!(earliest.nonce, 0); + assert_eq!(latest.nonce, 2); + + let by_number = get_verified_proof(&client, ACCOUNT0, vec![], "0x2").await?; + assert_eq!(by_number.balance, latest.balance); + + // A canonical hash must resolve to the same proof as its height. + let head_hash = get_block(&client, "0x2").await?.header.hash; + let by_hash = get_proof(&client, ACCOUNT0, vec![], head_hash).await?; + assert_eq!(by_hash, by_number); + + // Numeric block ids resolve without an existence check, so a future height is + // rejected by the window bounds rather than by header lookup. + let future = proof_error(&client, ACCOUNT0, "0x64").await?; + assert_outside_window(&future, 0x64); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn proof_history_returns_every_requested_storage_slot() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let mut env = setup(LaunchOpts::default()).await?; + let client = rpc_client(&env.node)?; + + include_tx(&mut env.node, signed_transfer(0, 0, ACCOUNT1, 7)?).await?; + wait_for_proof_tip(&client, 1).await?; + + // Three populated genesis slots plus one that was never written. + let populated = [ + (GENESIS_SLOT, U256::from(1u64)), + (B256::with_last_byte(0x07), U256::from(0x35ba5d7b55u64)), + (B256::with_last_byte(0x09), U256::from(1u64)), + ]; + let empty = B256::with_last_byte(0x42); + let slots = populated + .iter() + .map(|(slot, _)| *slot) + .chain(std::iter::once(empty)) + .collect::>(); + + let proof = get_verified_proof(&client, GENESIS_CONTRACT, slots.clone(), "0x1").await?; + assert_eq!( + proof.storage_proof.len(), + slots.len(), + "every requested slot must come back with its own proof" + ); + for (slot, expected) in populated { + assert_eq!(storage_entry(&proof, slot), expected, "slot {slot}"); + } + // An unset slot must still be proven, as a zero value rather than a missing entry. + assert_eq!(storage_entry(&proof, empty), U256::ZERO); + assert!( + !proof + .storage_proof + .iter() + .find(|entry| entry.key.as_b256() == empty) + .expect("entry for the unset slot") + .proof + .is_empty(), + "an unset slot needs a non-empty exclusion proof" + ); + + Ok(()) +} + +// ----------------------------------------------------------------------------- +// 4. Prune: earliest advances, in-window succeeds, out-of-window fails +// ----------------------------------------------------------------------------- + +#[tokio::test(flavor = "multi_thread")] +async fn proof_history_prune_updates_window_bounds() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let mut env = setup(LaunchOpts { + window: 2, + prune_interval: Duration::from_millis(100), + verification_interval: 0, + }) + .await?; + let client = rpc_client(&env.node)?; + + for nonce in 0..5 { + include_tx( + &mut env.node, + signed_transfer(0, nonce, ACCOUNT1, 100 + nonce)?, ) .await?; - let auth_proof: EIP1186AccountProofResponse = node - .auth_server_handle() - .http_client() - .request( + } + wait_for_proof_tip(&client, 5).await?; + let status = wait_for_window(&client, 3, 5).await?; + assert_eq!(status.earliest, Some(3)); + assert_eq!(status.latest, Some(5)); + + let pruned = proof_error(&client, ACCOUNT0, "0x2").await?; + assert_outside_window(&pruned, 2); + + let earliest_tag = proof_error(&client, ACCOUNT0, "earliest").await?; + assert_outside_window(&earliest_tag, 0); + + let in_window = get_verified_proof(&client, ACCOUNT0, vec![], "0x3").await?; + let latest = get_verified_proof(&client, ACCOUNT0, vec![], "latest").await?; + assert_eq!(in_window.nonce, 3); + assert_eq!(latest.nonce, 5); + + Ok(()) +} + +// ----------------------------------------------------------------------------- +// 5. Unwind: delete target and later, latest becomes parent, hash is correct. +// This pokes storage directly to pin the `unwind_history` contract in a live node. +// Reorg-driven unwind is covered by `proof_history_reorg_replaces_old_branch`. +// ----------------------------------------------------------------------------- + +#[tokio::test(flavor = "multi_thread")] +async fn proof_history_unwind_restores_parent_tip() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let mut env = setup(LaunchOpts::default()).await?; + let client = rpc_client(&env.node)?; + + let block1 = include_tx(&mut env.node, signed_transfer(0, 0, ACCOUNT1, 10)?).await?; + let block2 = include_tx(&mut env.node, signed_transfer(0, 1, ACCOUNT2, 20)?).await?; + include_tx(&mut env.node, signed_transfer(0, 2, ACCOUNT1, 30)?).await?; + wait_for_proof_tip(&client, 3).await?; + + let before = get_verified_proof(&client, ACCOUNT0, vec![], "0x1").await?; + env.storage + .unwind_history(BlockWithParent::new(block1, NumHash::new(2, block2)))?; + + let status = wait_for_proof_tip(&client, 1).await?; + assert_eq!(status.earliest, Some(0)); + assert_eq!(status.latest, Some(1)); + assert_eq!( + env.storage.get_latest_block_number()?, + Some((1, block1)), + "unwind must set latest to the target parent hash" + ); + + let after = get_verified_proof(&client, ACCOUNT0, vec![], "0x1").await?; + assert_eq!(after, before); + + let unwound = proof_error(&client, ACCOUNT0, "0x2").await?; + assert_outside_window(&unwound, 2); + + Ok(()) +} + +// ----------------------------------------------------------------------------- +// 6. Reorg-driven revert: the ExEx must drop the old branch and serve the new one +// ----------------------------------------------------------------------------- + +#[tokio::test(flavor = "multi_thread")] +async fn proof_history_reorg_replaces_old_branch() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let env = setup(LaunchOpts::default()).await?; + let client = rpc_client(&env.node)?; + let auth = env.node.auth_server_handle().http_client(); + let now = unix_now(); + let genesis_hash = env.chain_spec.genesis_hash(); + + let tx_b1 = signed_transfer(0, 0, ACCOUNT1, 1_000)?; + let tx_b2a = signed_transfer(0, 1, ACCOUNT1, 2_000)?; + let tx_b2b = signed_transfer(0, 1, ACCOUNT2, 3_000)?; + + let mut p1 = morph_payload_types::AssembleL2BlockV2Params::new(genesis_hash, vec![tx_b1]); + p1.timestamp = Some(now - 6); + let block1: ExecutableL2Data = auth.request("engine_assembleL2BlockV2", (p1,)).await?; + let block1_hash = block1.hash; + let _: MorphHeader = auth.request("engine_newL2BlockV2", (block1,)).await?; + + let mut p2a = morph_payload_types::AssembleL2BlockV2Params::new(block1_hash, vec![tx_b2a]); + p2a.timestamp = Some(now - 4); + let block2a: ExecutableL2Data = auth.request("engine_assembleL2BlockV2", (p2a,)).await?; + let block2a_hash = block2a.hash; + let _: MorphHeader = auth.request("engine_newL2BlockV2", (block2a,)).await?; + wait_for_proof_tip(&client, 2).await?; + + let balance_at_1 = get_verified_proof(&client, ACCOUNT1, vec![], "0x1") + .await? + .balance; + let balance_at_2a = get_verified_proof(&client, ACCOUNT1, vec![], "0x2") + .await? + .balance; + assert_eq!(balance_at_2a, balance_at_1 + U256::from(2_000)); + + // No manual `unwind_history` here on purpose: the reorg notification alone has to + // drive the revert. Poking storage first would let this test pass even if the ExEx + // reorg path were broken. + let mut p2b = morph_payload_types::AssembleL2BlockV2Params::new(block1_hash, vec![tx_b2b]); + p2b.timestamp = Some(now - 2); + let block2b: ExecutableL2Data = auth.request("engine_assembleL2BlockV2", (p2b,)).await?; + assert_ne!(block2b.hash, block2a_hash); + let block2b_hash = block2b.hash; + let _: MorphHeader = auth.request("engine_newL2BlockV2", (block2b,)).await?; + + let head = env + .node + .inner + .provider + .sealed_header_by_number_or_tag(alloy_rpc_types_eth::BlockNumberOrTag::Latest)? + .expect("head after fork import"); + assert_eq!(head.hash(), block2b_hash); + // The height stays 2 across the reorg, so only the hash proves the branch swapped. + wait_for_proof_latest(&env.storage, 2, block2b_hash).await?; + + let account1_on_fork = get_verified_proof(&client, ACCOUNT1, vec![], "0x2").await?; + let account2_on_fork = get_verified_proof(&client, ACCOUNT2, vec![], "0x2").await?; + assert_eq!( + account1_on_fork.balance, balance_at_1, + "old-branch credit to ACCOUNT1 must not survive the fork" + ); + assert_eq!( + account2_on_fork.balance, + get_verified_proof(&client, ACCOUNT2, vec![], "0x1") + .await? + .balance + + U256::from(3_000) + ); + + match auth + .request::( "eth_getProof", - rpc_params![Address::ZERO, Vec::::new(), "0x0"], + rpc_params![ACCOUNT1, Vec::::new(), block2a_hash], ) - .await?; - assert_eq!(normal_proof.address, Address::ZERO); - assert_eq!(normal_proof, auth_proof); + .await + { + Ok(_) => { + return Err(eyre::eyre!( + "old-branch block hash {block2a_hash} must not serve a proof" + )); + } + Err(error) => { + let message = error.to_string(); + assert!( + message.to_lowercase().contains("not found") + || message.contains("HeaderNotFound") + || message.contains("not canonical"), + "old fork hash should be rejected, got: {message}" + ); + } + } + + Ok(()) +} + +// ----------------------------------------------------------------------------- +// 7. verification-interval = 1: proofs stay correct on the forced re-execution path. +// That the interval actually *selects* that path is pinned by the +// `build_batch_entry_*` tests in `morph-proofs-exex`, which can observe the +// chosen `BatchBlock` variant directly. +// ----------------------------------------------------------------------------- + +#[tokio::test(flavor = "multi_thread")] +async fn proof_history_stays_correct_with_verification_enabled() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let mut env = setup(LaunchOpts { + window: DEFAULT_PROOFS_HISTORY_WINDOW, + prune_interval: Duration::from_secs(3600), + verification_interval: 1, + }) + .await?; + let client = rpc_client(&env.node)?; + + include_tx(&mut env.node, signed_transfer(0, 0, ACCOUNT1, 111)?).await?; + include_tx(&mut env.node, signed_transfer(0, 1, ACCOUNT2, 222)?).await?; + include_tx(&mut env.node, signed_transfer(0, 2, ACCOUNT1, 333)?).await?; + wait_for_proof_tip(&client, 3).await?; + + let at_1 = get_verified_proof(&client, ACCOUNT1, vec![], "0x1").await?; + let at_2 = get_verified_proof(&client, ACCOUNT2, vec![], "0x2").await?; + let at_3 = get_verified_proof(&client, ACCOUNT1, vec![], "0x3").await?; + assert_eq!(at_1.balance, at_3.balance - U256::from(333)); + assert!(at_2.balance > U256::ZERO); + + // Cached-path corruption would fail this: each height must still match that + // block's independently fetched canonical state root. + for block in ["0x1", "0x2", "0x3"] { + let header = get_block(&client, block).await?; + let proof = get_proof(&client, ACCOUNT0, vec![], block).await?; + verify_against_state_root(&proof, header.header.state_root)?; + } + + Ok(()) +} + +// ----------------------------------------------------------------------------- +// 8. Restart: the proofs MDBX reopens with its window and per-block rows intact +// ----------------------------------------------------------------------------- + +#[tokio::test(flavor = "multi_thread")] +async fn proof_history_db_survives_node_restart() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let genesis: Genesis = serde_json::from_str(include_str!("../assets/test-genesis.json"))?; + let chain_spec = Arc::new(MorphChainSpec::from_genesis(genesis)); + let proofs_dir = tempfile::tempdir()?; + let identity = ProofDbIdentity::new(chain_spec.chain().id(), chain_spec.genesis_hash()); + let proofs_path = proofs_dir.path().to_path_buf(); + + // First process on an owned runtime. `Runtime::test()` inside `#[tokio::test]` + // attaches to the test handle and would keep the ExEx (and its MDBX lock) alive. + let first_tip = std::thread::spawn(move || { + let rt = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .expect("owned runtime"); + let result = rt.block_on(async move { + let storage = Arc::new(MdbxProofsStorage::open(&proofs_path, identity)?); + let mut node = launch_node(chain_spec, storage.clone(), LaunchOpts::default()).await?; + let client = rpc_client(&node)?; + include_tx(&mut node, signed_transfer(0, 0, ACCOUNT1, 10)?).await?; + include_tx(&mut node, signed_transfer(0, 1, ACCOUNT2, 20)?).await?; + wait_for_proof_tip(&client, 2).await?; + let first_tip = node + .inner + .provider + .sealed_header_by_number_or_tag(alloy_rpc_types_eth::BlockNumberOrTag::Latest)? + .expect("tip before restart") + .hash(); + drop(node); + drop(storage); + Ok::<_, eyre::Error>(first_tip) + }); + rt.shutdown_timeout(Duration::from_secs(3)); + result + }) + .join() + .map_err(|_| eyre::eyre!("first node thread panicked"))??; + + let storage = reopen_proofs(proofs_dir.path(), identity).await?; + assert_eq!( + storage.get_latest_block_number()?, + Some((2, first_tip)), + "proof MDBX must keep the pre-restart tip after reopen" + ); + assert_eq!( + storage.get_earliest_block_number()?.map(|(n, _)| n), + Some(0) + ); + + // Pointers alone would survive even if the per-block rows were lost, so read the + // change sets back. The earliest block is the baseline snapshot rather than a diff, + // so change sets start above it. + assert!( + storage.fetch_trie_updates(0).is_err(), + "the initialization anchor is a baseline snapshot, not a change set" + ); + for block in 1..=2 { + storage + .fetch_trie_updates(block) + .unwrap_or_else(|error| panic!("block {block} trie updates must persist: {error}")); + } + assert!( + storage.fetch_trie_updates(3).is_err(), + "no history may exist past the reopened tip" + ); + // Not covered here: continuing to append after the restart. That needs a chain DB + // that outlives the node, and this harness drops its reth TempDatabase on node + // drop. The ExEx-side restart guards are covered by `ensure_initialized*` tests in + // `morph-proofs-exex`. Ok(()) } diff --git a/crates/proofs-exex/src/lib.rs b/crates/proofs-exex/src/lib.rs index 03b51315..edd6e670 100644 --- a/crates/proofs-exex/src/lib.rs +++ b/crates/proofs-exex/src/lib.rs @@ -677,7 +677,7 @@ mod tests { use alloy_eips::{BlockNumHash, NumHash, eip1898::BlockWithParent}; use morph_proofs::{BlockStateDiff, MdbxProofsStorage, MorphProofsStorage, MorphProofsStore}; use reth_db::test_utils::tempdir_path; - use reth_ethereum_primitives::{Block, Receipt}; + use reth_ethereum_primitives::{Block, EthPrimitives, Receipt}; use reth_execution_types::{Chain, ExecutionOutcome}; use reth_primitives_traits::RecoveredBlock; use reth_trie::{ @@ -1093,7 +1093,11 @@ mod tests { .expect("exex test context"); let exex = build_test_exex(ctx, proofs.clone()); - let _ = exex.ensure_initialized().expect_err("should return error"); + let error = exex.ensure_initialized().expect_err("should return error"); + assert!( + error.to_string().contains("not initialized"), + "expected an uninitialized-storage error, got: {error}" + ); } #[tokio::test] @@ -1122,7 +1126,38 @@ mod tests { .expect("exex test context"); let exex = build_test_exex(ctx, proofs.clone()); - let _ = exex.ensure_initialized().expect_err("should return error"); + let error = exex.ensure_initialized().expect_err("should return error"); + // Without this the test would also pass on the later canonical-hash check, + // which fires for the same fixture. + assert!( + error.to_string().contains("exceeds the safety threshold"), + "expected the startup prune-threshold error, got: {error}" + ); + } + + #[tokio::test] + async fn ensure_initialized_errors_when_latest_is_not_canonical() { + let dir = tempdir_path(); + let store = Arc::new(MdbxProofsStorage::new(dir.as_path()).expect("env")); + let proofs: MorphProofsStorage> = Arc::clone(&store); + + let (ctx, handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + + // Anchor proof history at the right height but the wrong hash: this is the + // shape of a proofs DB restored next to a mismatched chain snapshot, and it + // must refuse to serve rather than emit proofs for a foreign chain. + let genesis = handle.genesis.num_hash(); + assert_ne!(genesis.hash, b256(0x00), "fixture must not collide"); + init_storage_at(proofs.clone(), NumHash::new(genesis.number, b256(0x00))); + + let exex = build_test_exex(ctx, proofs.clone()); + let error = exex.ensure_initialized().expect_err("should return error"); + assert!( + error.to_string().contains("latest block hash mismatch"), + "expected a canonical-hash mismatch error, got: {error}" + ); } #[tokio::test] @@ -1187,4 +1222,135 @@ mod tests { "Main thread should not have processed the blocks synchronously" ); } + + // ------------------------------------------------------------------------- + // build_batch_entry: cached fast path vs forced full re-execution + // ------------------------------------------------------------------------- + + /// Concrete `MorphProofsExEx` used to reach the private `build_batch_entry`. + type TestProofsExEx = MorphProofsExEx>; + + fn cached_trie_data(num: u64) -> CachedBlockTrieData { + CachedBlockTrieData { + block_with_parent: BlockWithParent::new( + hash_for_num(num.saturating_sub(1)), + NumHash::new(num, hash_for_num(num)), + ), + trie_data: LazyTrieData::ready(ComputedTrieData::new( + Arc::new(HashedPostStateSorted::default()), + Arc::new(TrieUpdatesSorted::default()), + )), + } + } + + /// The test provider only holds genesis, so any block above it that reaches the + /// execution branch fails with `Missing block`. That error is therefore a precise + /// witness that the cached fast path was *not* taken. + fn assert_took_execution_path(result: eyre::Result>, num: u64) { + let error = result.expect_err("execution path must fail on the genesis-only test provider"); + assert!( + error.to_string().contains(&format!("Missing block {num}")), + "expected a missing-block error proving full execution was attempted, got: {error}" + ); + } + + #[tokio::test] + async fn build_batch_entry_uses_cache_when_verification_is_disabled() { + let (ctx, _handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + + let entry = + TestProofsExEx::build_batch_entry(5, Some(cached_trie_data(5)), ctx.provider(), 0) + .expect("cached entry"); + assert!( + matches!(entry, BatchBlock::Cached { .. }), + "interval 0 must never force re-execution" + ); + } + + #[tokio::test] + async fn build_batch_entry_uses_cache_off_the_verification_interval() { + let (ctx, _handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + + // 5 % 3 != 0 and 7 % 3 != 0, so both stay on the cached path. + for num in [5, 7] { + let entry = TestProofsExEx::build_batch_entry( + num, + Some(cached_trie_data(num)), + ctx.provider(), + 3, + ) + .expect("cached entry"); + assert!( + matches!(entry, BatchBlock::Cached { .. }), + "block {num} is not a multiple of 3 and must use the cache" + ); + } + } + + #[tokio::test] + async fn build_batch_entry_forces_execution_on_the_verification_interval() { + let (ctx, _handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + + // 6 % 3 == 0 and 5 % 1 == 0: cached data must be discarded in favour of execution. + assert_took_execution_path( + TestProofsExEx::build_batch_entry(6, Some(cached_trie_data(6)), ctx.provider(), 3), + 6, + ); + assert_took_execution_path( + TestProofsExEx::build_batch_entry(5, Some(cached_trie_data(5)), ctx.provider(), 1), + 5, + ); + } + + #[tokio::test] + async fn build_batch_entry_falls_back_to_execution_without_cache() { + let (ctx, _handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + + assert_took_execution_path( + TestProofsExEx::build_batch_entry(5, None, ctx.provider(), 0), + 5, + ); + } + + #[tokio::test] + async fn build_batch_entry_returns_execute_variant_for_an_available_block() { + let (ctx, handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + let genesis = handle.genesis.number; + + // Genesis is the one block the test provider can recover, so this pins the + // positive `Execute` variant rather than only the missing-block error. + let entry = TestProofsExEx::build_batch_entry( + genesis, + Some(cached_trie_data(genesis)), + ctx.provider(), + 1, + ) + .expect("genesis is recoverable"); + match entry { + BatchBlock::Execute(block) => assert_eq!(block.number, genesis), + BatchBlock::Cached { .. } => { + panic!("verification interval 1 must discard cached data for every block") + } + } + + // Same block, verification disabled: back on the cached path. + let cached = TestProofsExEx::build_batch_entry( + genesis, + Some(cached_trie_data(genesis)), + ctx.provider(), + 0, + ) + .expect("cached entry"); + assert!(matches!(cached, BatchBlock::Cached { .. })); + } } From 106d41e7d70bd45df155c4926ec8fc43e09aaa28 Mon Sep 17 00:00:00 2001 From: panos Date: Tue, 18 Aug 2026 14:51:04 +0800 Subject: [PATCH 09/12] fix(proofs): retry stalled history sync and clear review scan findings Forward-sync now re-arms after transient errors instead of waiting forever for the next notification. Tests read account nonces from state so CodeQL no longer flags them as hardcoded cryptographic IVs. --- Cargo.lock | 1 - crates/node/Cargo.toml | 1 + crates/node/tests/it/proof_history.rs | 91 ++++++++++++++++++--------- crates/proofs-exex/src/lib.rs | 46 +++++++++++--- crates/proofs/Cargo.toml | 1 - crates/proofs/src/db/store.rs | 18 +++--- crates/proofs/src/in_memory.rs | 12 +++- crates/proofs/src/live.rs | 4 +- crates/proofs/src/prune/error.rs | 21 ++++++- 9 files changed, 141 insertions(+), 54 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fea5b089..3f0219fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5235,7 +5235,6 @@ dependencies = [ "reth-trie", "reth-trie-common", "serde", - "strum", "tempfile", "thiserror 2.0.18", "tokio", diff --git a/crates/node/Cargo.toml b/crates/node/Cargo.toml index 514d81b6..ced41d3f 100644 --- a/crates/node/Cargo.toml +++ b/crates/node/Cargo.toml @@ -89,6 +89,7 @@ reth-payload-primitives.workspace = true reth-rpc-server-types.workspace = true reth-tasks.workspace = true reth-tracing.workspace = true +reth-transaction-pool.workspace = true reth-trie.workspace = true alloy-consensus.workspace = true alloy-eips.workspace = true diff --git a/crates/node/tests/it/proof_history.rs b/crates/node/tests/it/proof_history.rs index f4b8e7d8..b8d9656b 100644 --- a/crates/node/tests/it/proof_history.rs +++ b/crates/node/tests/it/proof_history.rs @@ -35,11 +35,12 @@ use reth_node_builder::{EngineNodeLauncher, Node, NodeBuilder, NodeConfig, NodeH use reth_node_core::args::{DiscoveryArgs, NetworkArgs, RpcServerArgs}; use reth_payload_primitives::BuiltPayload; use reth_provider::{ - BlockReaderIdExt, DBProvider, DatabaseProviderFactory, ReceiptProvider, + AccountReader, BlockReaderIdExt, DBProvider, DatabaseProviderFactory, ReceiptProvider, providers::BlockchainProvider, }; use reth_rpc_server_types::RpcModuleSelection; use reth_tasks::Runtime; +use reth_transaction_pool::TransactionPool; use reth_trie::AccountProof; use serde::Serialize; @@ -356,6 +357,32 @@ fn assert_outside_window(error: &str, requested: u64) { ); } +fn sender_nonce(node: &MorphTestNode) -> eyre::Result { + node.inner + .provider + .basic_account(&ACCOUNT0)? + .map(|account| account.nonce) + .ok_or_else(|| eyre::eyre!("missing genesis sender {ACCOUNT0}")) +} + +async fn include_transfer( + node: &mut MorphTestNode, + to: Address, + value: u64, +) -> eyre::Result { + let nonce = sender_nonce(node)?; + include_tx(node, signed_transfer(0, nonce, to, value)?).await +} + +async fn include_call( + node: &mut MorphTestNode, + to: Address, + data: Bytes, +) -> eyre::Result { + let nonce = sender_nonce(node)?; + include_tx(node, signed_call(0, nonce, to, data)?).await +} + async fn include_tx( node: &mut MorphTestNode, raw_tx: Bytes, @@ -363,9 +390,16 @@ async fn include_tx( use alloy_consensus::TxReceipt; use alloy_consensus::transaction::TxHashRef; - node.rpc.inject_tx(raw_tx).await?; + let tx_hash = node.rpc.inject_tx(raw_tx).await?; // The payload builder can emit an empty block if it races the pool insert. - tokio::time::sleep(Duration::from_millis(200)).await; + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + while !node.inner.pool.contains(&tx_hash) { + eyre::ensure!( + tokio::time::Instant::now() < deadline, + "injected transaction {tx_hash} never became visible in the pool" + ); + tokio::time::sleep(Duration::from_millis(25)).await; + } let payload = node.advance_block().await?; eyre::ensure!( payload.block().body().transactions.len() == 1, @@ -453,21 +487,22 @@ async fn proof_history_multi_block_account_and_storage() -> eyre::Result<()> { get_verified_proof(&client, GENESIS_CONTRACT, vec![GENESIS_SLOT], "0x0").await?; assert_eq!(storage_entry(&genesis_storage, GENESIS_SLOT), U256::from(1)); - include_tx(&mut env.node, signed_transfer(0, 0, ACCOUNT1, 1_000)?).await?; + include_transfer(&mut env.node, ACCOUNT1, 1_000).await?; wait_for_proof_tip(&client, 1).await?; - let setter = Address::create(&ACCOUNT0, 1); + let nonce = sender_nonce(&env.node)?; + let setter = Address::create(&ACCOUNT0, nonce); include_tx( &mut env.node, - make_deploy_tx(CHAIN_ID, signer(0), 1, SETTER_INIT)?, + make_deploy_tx(CHAIN_ID, signer(0), nonce, SETTER_INIT)?, ) .await?; wait_for_proof_tip(&client, 2).await?; - include_tx(&mut env.node, signed_call(0, 2, setter, slot_calldata(11))?).await?; + include_call(&mut env.node, setter, slot_calldata(11)).await?; wait_for_proof_tip(&client, 3).await?; - include_tx(&mut env.node, signed_call(0, 3, setter, slot_calldata(22))?).await?; + include_call(&mut env.node, setter, slot_calldata(22)).await?; wait_for_proof_tip(&client, 4).await?; let at_1_sender = get_verified_proof(&client, ACCOUNT0, vec![], "0x1").await?; @@ -509,7 +544,7 @@ async fn proof_history_proofs_verify_against_block_state_root() -> eyre::Result< let mut env = setup(LaunchOpts::default()).await?; let client = rpc_client(&env.node)?; - include_tx(&mut env.node, signed_transfer(0, 0, ACCOUNT1, 5_000)?).await?; + include_transfer(&mut env.node, ACCOUNT1, 5_000).await?; wait_for_proof_tip(&client, 1).await?; let block_0 = get_block(&client, "0x0").await?; @@ -543,8 +578,8 @@ async fn proof_history_rpc_block_bounds() -> eyre::Result<()> { let mut env = setup(LaunchOpts::default()).await?; let client = rpc_client(&env.node)?; - include_tx(&mut env.node, signed_transfer(0, 0, ACCOUNT1, 1)?).await?; - include_tx(&mut env.node, signed_transfer(0, 1, ACCOUNT2, 2)?).await?; + include_transfer(&mut env.node, ACCOUNT1, 1).await?; + include_transfer(&mut env.node, ACCOUNT2, 2).await?; wait_for_proof_tip(&client, 2).await?; let earliest = get_verified_proof(&client, ACCOUNT0, vec![], "earliest").await?; @@ -575,7 +610,7 @@ async fn proof_history_returns_every_requested_storage_slot() -> eyre::Result<() let mut env = setup(LaunchOpts::default()).await?; let client = rpc_client(&env.node)?; - include_tx(&mut env.node, signed_transfer(0, 0, ACCOUNT1, 7)?).await?; + include_transfer(&mut env.node, ACCOUNT1, 7).await?; wait_for_proof_tip(&client, 1).await?; // Three populated genesis slots plus one that was never written. @@ -632,12 +667,8 @@ async fn proof_history_prune_updates_window_bounds() -> eyre::Result<()> { .await?; let client = rpc_client(&env.node)?; - for nonce in 0..5 { - include_tx( - &mut env.node, - signed_transfer(0, nonce, ACCOUNT1, 100 + nonce)?, - ) - .await?; + for i in 0..5 { + include_transfer(&mut env.node, ACCOUNT1, 100 + i).await?; } wait_for_proof_tip(&client, 5).await?; let status = wait_for_window(&client, 3, 5).await?; @@ -671,9 +702,9 @@ async fn proof_history_unwind_restores_parent_tip() -> eyre::Result<()> { let mut env = setup(LaunchOpts::default()).await?; let client = rpc_client(&env.node)?; - let block1 = include_tx(&mut env.node, signed_transfer(0, 0, ACCOUNT1, 10)?).await?; - let block2 = include_tx(&mut env.node, signed_transfer(0, 1, ACCOUNT2, 20)?).await?; - include_tx(&mut env.node, signed_transfer(0, 2, ACCOUNT1, 30)?).await?; + let block1 = include_transfer(&mut env.node, ACCOUNT1, 10).await?; + let block2 = include_transfer(&mut env.node, ACCOUNT2, 20).await?; + include_transfer(&mut env.node, ACCOUNT1, 30).await?; wait_for_proof_tip(&client, 3).await?; let before = get_verified_proof(&client, ACCOUNT0, vec![], "0x1").await?; @@ -712,9 +743,11 @@ async fn proof_history_reorg_replaces_old_branch() -> eyre::Result<()> { let now = unix_now(); let genesis_hash = env.chain_spec.genesis_hash(); - let tx_b1 = signed_transfer(0, 0, ACCOUNT1, 1_000)?; - let tx_b2a = signed_transfer(0, 1, ACCOUNT1, 2_000)?; - let tx_b2b = signed_transfer(0, 1, ACCOUNT2, 3_000)?; + let nonce0 = sender_nonce(&env.node)?; + let nonce1 = nonce0 + 1; + let tx_b1 = signed_transfer(0, nonce0, ACCOUNT1, 1_000)?; + let tx_b2a = signed_transfer(0, nonce1, ACCOUNT1, 2_000)?; + let tx_b2b = signed_transfer(0, nonce1, ACCOUNT2, 3_000)?; let mut p1 = morph_payload_types::AssembleL2BlockV2Params::new(genesis_hash, vec![tx_b1]); p1.timestamp = Some(now - 6); @@ -816,9 +849,9 @@ async fn proof_history_stays_correct_with_verification_enabled() -> eyre::Result .await?; let client = rpc_client(&env.node)?; - include_tx(&mut env.node, signed_transfer(0, 0, ACCOUNT1, 111)?).await?; - include_tx(&mut env.node, signed_transfer(0, 1, ACCOUNT2, 222)?).await?; - include_tx(&mut env.node, signed_transfer(0, 2, ACCOUNT1, 333)?).await?; + include_transfer(&mut env.node, ACCOUNT1, 111).await?; + include_transfer(&mut env.node, ACCOUNT2, 222).await?; + include_transfer(&mut env.node, ACCOUNT1, 333).await?; wait_for_proof_tip(&client, 3).await?; let at_1 = get_verified_proof(&client, ACCOUNT1, vec![], "0x1").await?; @@ -863,8 +896,8 @@ async fn proof_history_db_survives_node_restart() -> eyre::Result<()> { let storage = Arc::new(MdbxProofsStorage::open(&proofs_path, identity)?); let mut node = launch_node(chain_spec, storage.clone(), LaunchOpts::default()).await?; let client = rpc_client(&node)?; - include_tx(&mut node, signed_transfer(0, 0, ACCOUNT1, 10)?).await?; - include_tx(&mut node, signed_transfer(0, 1, ACCOUNT2, 20)?).await?; + include_transfer(&mut node, ACCOUNT1, 10).await?; + include_transfer(&mut node, ACCOUNT2, 20).await?; wait_for_proof_tip(&client, 2).await?; let first_tip = node .inner diff --git a/crates/proofs-exex/src/lib.rs b/crates/proofs-exex/src/lib.rs index edd6e670..1636cd23 100644 --- a/crates/proofs-exex/src/lib.rs +++ b/crates/proofs-exex/src/lib.rs @@ -36,6 +36,10 @@ const DEFAULT_MAX_PRUNE_BLOCKS_STARTUP: u64 = 1_000; /// How many blocks to process in a single sync turn before yielding. const SYNC_BLOCKS_PER_TURN: usize = 50; +/// Backoff after a failed forward-sync turn. The work is re-armed so the loop +/// retries instead of waiting indefinitely for the next notification. +const SYNC_RETRY_DELAY: Duration = Duration::from_secs(1); + /// Default interval between proof-storage prune runs. Default is 15 seconds. const DEFAULT_PRUNE_INTERVAL: Duration = Duration::from_secs(15); @@ -409,10 +413,12 @@ where Ok(Some((n, _))) => n, Ok(None) => { error!(target: "morph::proofs_exex", "No blocks stored in proofs storage during sync"); + Self::reschedule_sync_up_to(sync_target, target).await; return; } Err(e) => { error!(target: "morph::proofs_exex", error = ?e, "Failed to get latest block"); + Self::reschedule_sync_up_to(sync_target, target).await; return; } }; @@ -441,6 +447,7 @@ where Ok(entry) => batch.push(entry), Err(e) => { error!(target: "morph::proofs_exex", block_number = block_num, error = ?e, "Preparing block for batch failed"); + Self::reschedule_sync_up_to(sync_target, target).await; return; } } @@ -448,6 +455,7 @@ where if let Err(e) = collector.execute_and_store_batch(batch) { error!(target: "morph::proofs_exex", start = latest + 1, end, error = ?e, "Batch processing failed"); + Self::reschedule_sync_up_to(sync_target, target).await; return; } @@ -456,6 +464,20 @@ where } } + /// Re-arm a forward-sync target after a transient failure. + /// + /// `take_state()` already consumed the original `SyncUpTo`, so returning + /// without this would leave the loop blocked on `notified()` until the next + /// chain event. If a revert or a newer tip arrived during the backoff, leave + /// that pending work in place instead of overwriting it with a stale target. + async fn reschedule_sync_up_to(sync_target: &SyncTarget, target: u64) { + tokio::time::sleep(SYNC_RETRY_DELAY).await; + if sync_target.has_pending_state() { + return; + } + sync_target.update_state(SyncTargetState::SyncUpTo { to: target }); + } + fn build_batch_entry( block_number: u64, cached: Option, @@ -693,22 +715,17 @@ mod tests { B256::new([byte; 32]) } - // deterministic hash from block number: 0 -> 0x00.., 1 -> 0x01.., etc. fn hash_for_num(num: u64) -> B256 { - // if you only care about small test numbers, this is enough: - b256(num as u8) - - // If you want to avoid wrapping when num > 255, use something like: - // let mut out = [0u8; 32]; - // out[0..8].copy_from_slice(&num.to_be_bytes()); - // B256::new(out) + let mut out = [0u8; 32]; + out[0..8].copy_from_slice(&num.to_be_bytes()); + B256::new(out) } fn mk_block(num: u64) -> RecoveredBlock { let mut b: RecoveredBlock = Default::default(); b.set_block_number(num); b.set_hash(hash_for_num(num)); - b.set_parent_hash(hash_for_num(num - 1)); + b.set_parent_hash(hash_for_num(num.saturating_sub(1))); b } @@ -1353,4 +1370,15 @@ mod tests { .expect("cached entry"); assert!(matches!(cached, BatchBlock::Cached { .. })); } + + #[test] + fn hash_for_num_encodes_full_block_number() { + assert_eq!(hash_for_num(0), b256(0x00)); + assert_ne!( + hash_for_num(0), + hash_for_num(256), + "block numbers must not wrap on a single byte" + ); + let _genesis = mk_block(0); + } } diff --git a/crates/proofs/Cargo.toml b/crates/proofs/Cargo.toml index db4b1603..ac5196ce 100644 --- a/crates/proofs/Cargo.toml +++ b/crates/proofs/Cargo.toml @@ -20,7 +20,6 @@ derive_more.workspace = true metrics.workspace = true parking_lot.workspace = true serde.workspace = true -strum = { workspace = true, features = ["derive"] } thiserror.workspace = true tokio = { workspace = true, features = ["sync"] } tracing.workspace = true diff --git a/crates/proofs/src/db/store.rs b/crates/proofs/src/db/store.rs index 0f7064d5..bf4460fc 100644 --- a/crates/proofs/src/db/store.rs +++ b/crates/proofs/src/db/store.rs @@ -698,18 +698,20 @@ impl MdbxProofsStorage { walker.delete_current()?; } - // Delete using the simplified API: iterator of (key, subkey) - self.delete_dup_sorted::(tx, history.clone().account_trie)?; - self.delete_dup_sorted::(tx, history.clone().storage_trie)?; - self.delete_dup_sorted::(tx, history.clone().hashed_account)?; - self.delete_dup_sorted::(tx, history.clone().hashed_storage)?; - - Ok(WriteCounts { + let counts = WriteCounts { account_trie_updates_written_total: history.account_trie.len() as u64, storage_trie_updates_written_total: history.storage_trie.len() as u64, hashed_accounts_written_total: history.hashed_account.len() as u64, hashed_storages_written_total: history.hashed_storage.len() as u64, - }) + }; + + // Delete using the simplified API: iterator of (key, subkey) + self.delete_dup_sorted::(tx, history.account_trie)?; + self.delete_dup_sorted::(tx, history.storage_trie)?; + self.delete_dup_sorted::(tx, history.hashed_account)?; + self.delete_dup_sorted::(tx, history.hashed_storage)?; + + Ok(counts) } /// Write trie/state history for `block_number` from `block_state_diff`. diff --git a/crates/proofs/src/in_memory.rs b/crates/proofs/src/in_memory.rs index 08ac6787..be9be05d 100644 --- a/crates/proofs/src/in_memory.rs +++ b/crates/proofs/src/in_memory.rs @@ -819,7 +819,7 @@ impl MorphProofsStore for InMemoryProofsStorage { fn unwind_history(&self, unwind_upto_block: BlockWithParent) -> MorphProofsStorageResult<()> { let mut inner = self.inner.write(); - let unwind_upto_block_number = unwind_upto_block.block.number - 1; + let unwind_upto_block_number = unwind_upto_block.block.number.saturating_sub(1); // Remove all updates after unwind_upto_block_number inner @@ -1135,4 +1135,14 @@ mod tests { Ok(()) } + + #[test] + fn unwind_history_at_genesis_does_not_underflow() -> Result<(), MorphProofsStorageError> { + let storage = InMemoryProofsStorage::new(); + storage.unwind_history(BlockWithParent::new( + B256::ZERO, + NumHash::new(0, B256::ZERO), + ))?; + Ok(()) + } } diff --git a/crates/proofs/src/live.rs b/crates/proofs/src/live.rs index fafc1023..c3bc6eb6 100644 --- a/crates/proofs/src/live.rs +++ b/crates/proofs/src/live.rs @@ -89,7 +89,7 @@ where let db = StateProviderDatabase::new(&state_provider); let block_executor = self.evm_config.batch_executor(db); - let execution_result = block_executor.execute(&(*block).clone())?; + let execution_result = block_executor.execute(block)?; operation_durations.execution_duration_seconds = start.elapsed(); @@ -405,7 +405,7 @@ where let db = StateProviderDatabase::new(&state_provider); let block_executor = self.evm_config.batch_executor(db); - let execution_result = block_executor.execute(&(*block).clone())?; + let execution_result = block_executor.execute(block)?; let hashed_state = state_provider.hashed_post_state(&execution_result.state); let (state_root, trie_updates) = diff --git a/crates/proofs/src/prune/error.rs b/crates/proofs/src/prune/error.rs index 56992c93..ae31c595 100644 --- a/crates/proofs/src/prune/error.rs +++ b/crates/proofs/src/prune/error.rs @@ -5,7 +5,6 @@ use std::{ }; use reth_provider::ProviderError; -use strum::Display; use thiserror::Error; use crate::{MorphProofsStorageError, api::WriteCounts}; @@ -62,18 +61,22 @@ impl PrunerOutput { } /// Error returned by the pruner. -#[derive(Debug, Error, Display)] +#[derive(Debug, Error)] pub enum PrunerError { /// Wrapped error from the underlying `MorphProofStorage` layer. + #[error(transparent)] Storage(#[from] MorphProofsStorageError), /// Wrapped error from the reth db provider. + #[error(transparent)] Provider(#[from] ProviderError), /// Block not found in the underlying reth storage provider. + #[error("block {0} not found in the underlying reth storage provider")] BlockNotFound(u64), /// The pruner timed out before finishing the prune + #[error("pruner timed out after {0:?}")] TimedOut(Duration), } @@ -81,7 +84,7 @@ pub enum PrunerError { mod tests { use std::time::Duration; - use super::PrunerOutput; + use super::{PrunerError, PrunerOutput}; use crate::api::WriteCounts; #[test] @@ -99,4 +102,16 @@ mod tests { "Pruned 1→2 (1 blocks), entries=10, elapsed=10.000s" ); } + + #[test] + fn test_pruner_error_display_preserves_context() { + assert_eq!( + PrunerError::BlockNotFound(42).to_string(), + "block 42 not found in the underlying reth storage provider" + ); + assert_eq!( + PrunerError::TimedOut(Duration::from_secs(5)).to_string(), + "pruner timed out after 5s" + ); + } } From 0df541757a26eab29d6f3e1efb31582be1dd502a Mon Sep 17 00:00:00 2001 From: panos Date: Fri, 21 Aug 2026 14:12:10 +0800 Subject: [PATCH 10/12] feat(rpc): add eth_getMultiProof and harden the historical proof RPCs Port reth's `eth_getMultiProof` (paradigmxyz/reth#26555, which landed after the pinned v2.4.0) onto the bounded proof-history storage: one consolidated multiproof per request, expanded back into one EIP-1186 response per target in request order. Duplicate addresses are consolidated for proof generation and each response carries only the slots its own target asked for. Move both proof handlers off the runtime workers. They walked MDBX and rebuilt trie nodes synchronously inside the async fn, so a handful of concurrent requests could starve the Engine API, and `eth_getMultiProof` widens the blast radius from one account to hundreds. Both now acquire the shared proof permit (`--rpc.proof-permits`) and run on reth's blocking pool, matching upstream `EthState::get_proof`. The permit is moved into the blocking task so a cancelled RPC future cannot release it while the computation is still running, which would otherwise let the concurrency ceiling be exceeded. Bound requests along two independent dimensions instead of one combined budget, because an account target costs several times a storage slot: it retains its own account-trie path and opens a storage-trie cursor, while slots share one already-open trie. At most `--proofs-history.max-multi-proof-targets` accounts (default 256) and 1024 storage keys, the latter fixed to match go-ethereum's `eth_getProof`. Sizing evidence: Morph mainnet blocks touch at most ~36 accounts, but a Hoodi load test peaked near ~200 accounts in a single block, which a combined 128-unit budget would have split into two round trips. Report both methods under one set of metric names distinguished by a `method` label, so a dashboard can split per method or sum across both; separate `get_proof_*` / `get_multi_proof_*` names allowed neither. Add `rejected_total` and count size-limit rejections after `requests_total`, so `requests_total == rejected_total + successful_responses_total + failures_total` now holds; rejections were previously counted nowhere. Tests cover the paths that differ from a naive implementation: - Non-existence proofs. `AccountProof::verify` only accepts a missing leaf when `info` is None and the storage root is the empty root, so a response that invented either field fails. `eth_getMultiProof` must return the same proof as `eth_getProof`, including when batched beside a present account. - A contract target with no requested slots. `Proof::multiproof` pre-seeds every target with an empty `StorageMultiProof` and only overwrites it at the account leaf, so a regression would silently report the empty storage root for contracts that do have storage. - Duplicate-address expansion back into request order. - Both request limits, launched with a deliberately tiny target limit so the configured value is proven to reach the handler rather than the default. --- Cargo.lock | 21 +- README.md | 32 ++- bin/morph-reth/src/main.rs | 7 +- crates/node/src/add_ons.rs | 34 ++- crates/node/src/args.rs | 36 +++ crates/node/tests/it/proof_history.rs | 292 ++++++++++++++++++++- crates/proofs/src/lib.rs | 2 +- crates/rpc/Cargo.toml | 1 + crates/rpc/src/eth/proofs.rs | 356 +++++++++++++++++++++++--- crates/rpc/src/state.rs | 16 ++ 10 files changed, 729 insertions(+), 68 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3f0219fa..f31f355f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -945,7 +945,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -956,7 +956,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -2829,7 +2829,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3201,7 +3201,7 @@ dependencies = [ "libc", "log", "rustversion", - "windows-link 0.1.3", + "windows-link 0.2.1", "windows-result 0.4.1", ] @@ -5382,6 +5382,7 @@ dependencies = [ "reth-storage-api", "reth-tasks", "reth-transaction-pool", + "reth-trie-common", "revm", "serde", "serde_json", @@ -5545,7 +5546,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -9880,7 +9881,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -9960,7 +9961,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs 1.0.7", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -10492,7 +10493,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -10694,7 +10695,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -11850,7 +11851,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/README.md b/README.md index 0dd12cf8..d0127a10 100644 --- a/README.md +++ b/README.md @@ -121,10 +121,11 @@ openssl rand -hex 32 > jwt.hex |------|---------|-------------| | `--morph.max-tx-payload-bytes` | 122880 (120KB) | Maximum transaction payload bytes per block | | `--morph.max-tx-per-block` | None (unlimited) | Maximum number of transactions per block | -| `--proofs-history` | false | Enable historical `eth_getProof` and proof-history accumulation | +| `--proofs-history` | false | Enable historical `eth_getProof` / `eth_getMultiProof` and proof-history accumulation | | `--proofs-history.storage-path` | `/historical-proofs` | Override the proof MDBX directory | | `--proofs-history.window` | 604800 | Number of canonical blocks retained (7 days at 1s/block) | | `--proofs-history.verification-interval` | 0 | Re-execute every Nth indexed block; 0 disables verification | +| `--proofs-history.max-multi-proof-targets` | 256 | Maximum account targets per `eth_getMultiProof` request | | `--rpc.eth-proof-window` | 0 | Reth overlay limit used only while proof history is disabled | #### Initializing Historical Proofs @@ -143,10 +144,31 @@ initialization so the source state remains fixed: --authrpc.jwtsecret jwt.hex ``` -The standard `eth_getProof` method is then served only for the inclusive range reported by -`debug_proofsSyncStatus`; requests outside that range never fall back to Reth's historical overlay. -The same `eth_getProof` override is installed on the authenticated RPC server. For manual -maintenance use `morph-reth proofs prune` and `morph-reth proofs unwind --target `. +The standard `eth_getProof` method and reth-compatible `eth_getMultiProof` extension are then +served only for the inclusive range reported by `debug_proofsSyncStatus`; requests outside that +range never fall back to Reth's historical overlay. Both overrides are also installed on the +authenticated RPC server. For manual maintenance use `morph-reth proofs prune` and +`morph-reth proofs unwind --target `. + +`eth_getMultiProof` accepts an ordered list of `[address, storageKeys]` targets, computes one +consolidated proof, and returns one EIP-1186 response per target in request order. Duplicate +addresses are consolidated internally and expanded back, each response carrying only the slots its +own target requested. Its storage keys must be full 32-byte values, unlike the short form +`eth_getProof` accepts. Two independent limits apply: at most +`--proofs-history.max-multi-proof-targets` account targets (default 256) and at most 1024 storage +keys in total. They are counted separately because an account target costs several times a storage +slot -- it retains its own account-trie path and opens a storage-trie cursor, while slots share one +already-open trie. Raise the target limit when a prover needs blocks that touch more accounts than +the default in a single round trip; the 1024-key cap is fixed to match go-ethereum's `eth_getProof`. + +Both overrides report under the `morph.rpc.proofs` metric scope, distinguished by a `method` +label (`eth_getProof` / `eth_getMultiProof`) rather than by separate metric names, so the counters +can be split per method or summed. `requests_total` is counted before the request-size limits are +applied and equals `rejected_total + successful_responses_total + failures_total`. + +Note that `eth_getMultiProof` exists **only** while `--proofs-history` is enabled, because Reth +`v2.4.0` has no native implementation to fall back to. `eth_getProof` remains available either way, +served by Reth itself when proof history is off. For cold copies, stop the source node and copy the complete chain data directory, including `historical-proofs`, as one consistent unit. Startup validates the proof database schema, chain ID, diff --git a/bin/morph-reth/src/main.rs b/bin/morph-reth/src/main.rs index 207597fd..eacf9cbb 100644 --- a/bin/morph-reth/src/main.rs +++ b/bin/morph-reth/src/main.rs @@ -122,6 +122,7 @@ fn main() { storage, morph_args.proofs_history_window, morph_args.proofs_history_verification_interval, + morph_args.proofs_history_max_multi_proof_targets, )) } else { None @@ -129,8 +130,8 @@ fn main() { let node = MorphNode::new(morph_args); let mut add_ons = MorphAddOns::new(); - if let Some((storage, _, _)) = &proof_history { - add_ons = add_ons.with_proof_history(storage.clone()); + if let Some((storage, _, _, max_multi_proof_targets)) = &proof_history { + add_ons = add_ons.with_proof_history(storage.clone(), *max_multi_proof_targets); } let mut node_builder = builder @@ -138,7 +139,7 @@ fn main() { .with_components(node.components_builder()) .with_add_ons(add_ons); - if let Some((storage, window, verification_interval)) = proof_history { + if let Some((storage, window, verification_interval, _)) = proof_history { node_builder = node_builder.install_exex( "morph-proof-history", async move |ctx| { diff --git a/crates/node/src/add_ons.rs b/crates/node/src/add_ons.rs index 7dba06d6..6e857113 100644 --- a/crates/node/src/add_ons.rs +++ b/crates/node/src/add_ons.rs @@ -52,8 +52,12 @@ pub struct MorphAddOns< > { /// Inner RPC add-ons from reth. inner: RpcAddOns, - /// Optional proof-history storage used to replace `eth_getProof` on normal and auth RPC. - proof_history: Option>>, + /// Optional storage plus the `eth_getMultiProof` account-target limit, used to replace the + /// historical proof RPCs on the normal and auth servers. + /// + /// Held as one unit because the limit is only ever enforced by the RPC that this storage + /// installs; there is no way to configure one without the other. + proof_history: Option<(MorphProofsStorage>, usize)>, } impl MorphAddOns, MorphEthApiBuilder> @@ -79,12 +83,14 @@ where } } - /// Attach initialized proof-history storage to the RPC add-ons. + /// Attach initialized proof-history storage, and the account-target limit its + /// `eth_getMultiProof` override enforces, to the RPC add-ons. pub fn with_proof_history( mut self, storage: MorphProofsStorage>, + max_multi_proof_targets: usize, ) -> Self { - self.proof_history = Some(storage); + self.proof_history = Some((storage, max_multi_proof_targets)); self } } @@ -201,21 +207,31 @@ where .. } = container; - if let Some(storage) = proof_history { + if let Some((storage, max_multi_proof_targets)) = proof_history { let eth_api = registry.eth_api().clone(); modules .replace_configured( - EthProofApiExt::new(eth_api.clone(), storage.clone()).into_rpc(), + EthProofApiExt::new( + eth_api.clone(), + storage.clone(), + max_multi_proof_targets, + ) + .into_rpc(), ) .map_err(|error| { - eyre::eyre!("Failed to replace normal RPC eth_getProof: {error}") + eyre::eyre!("Failed to replace normal historical proof RPCs: {error}") })?; auth_module .replace_auth_methods( - EthProofApiExt::new(eth_api, storage.clone()).into_rpc(), + EthProofApiExt::new( + eth_api, + storage.clone(), + max_multi_proof_targets, + ) + .into_rpc(), ) .map_err(|error| { - eyre::eyre!("Failed to replace auth RPC eth_getProof: {error}") + eyre::eyre!("Failed to replace auth historical proof RPCs: {error}") })?; modules .replace_configured(ProofStatusApiExt::new(storage).into_rpc()) diff --git a/crates/node/src/args.rs b/crates/node/src/args.rs index b68657c0..2b8888f8 100644 --- a/crates/node/src/args.rs +++ b/crates/node/src/args.rs @@ -4,6 +4,7 @@ use std::path::PathBuf; use clap::Args; use morph_proofs::DEFAULT_PROOFS_HISTORY_WINDOW; +use morph_rpc::eth::proofs::DEFAULT_MAX_MULTI_PROOF_TARGETS; /// Default maximum transaction payload bytes per block (120KB). /// @@ -63,6 +64,20 @@ pub struct MorphArgs { default_value_t = 0 )] pub proofs_history_verification_interval: u64, + + /// Maximum account targets accepted by one `eth_getMultiProof` request. + /// + /// Each target costs an account-trie path plus a storage-trie cursor, so this bounds the most + /// expensive dimension of a batch. Raise it when a prover needs blocks that touch more + /// accounts than the default in one round trip; the separate 1024 storage-key cap is fixed to + /// match go-ethereum's `eth_getProof`. + #[arg( + long = "proofs-history.max-multi-proof-targets", + value_name = "COUNT", + default_value_t = DEFAULT_MAX_MULTI_PROOF_TARGETS, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..) + )] + pub proofs_history_max_multi_proof_targets: usize, } impl Default for MorphArgs { @@ -74,6 +89,7 @@ impl Default for MorphArgs { proofs_history_storage_path: None, proofs_history_window: DEFAULT_PROOFS_HISTORY_WINDOW, proofs_history_verification_interval: 0, + proofs_history_max_multi_proof_targets: DEFAULT_MAX_MULTI_PROOF_TARGETS, } } } @@ -101,6 +117,10 @@ mod tests { assert_eq!(args.proofs_history_storage_path, None); assert_eq!(args.proofs_history_window, DEFAULT_PROOFS_HISTORY_WINDOW); assert_eq!(args.proofs_history_verification_interval, 0); + assert_eq!( + args.proofs_history_max_multi_proof_targets, + DEFAULT_MAX_MULTI_PROOF_TARGETS + ); } #[test] @@ -154,6 +174,8 @@ mod tests { "2592000", "--proofs-history.verification-interval", "64", + "--proofs-history.max-multi-proof-targets", + "512", ]) .args; @@ -164,6 +186,20 @@ mod tests { ); assert_eq!(args.proofs_history_window, 2_592_000); assert_eq!(args.proofs_history_verification_interval, 64); + assert_eq!(args.proofs_history_max_multi_proof_targets, 512); + } + + #[test] + fn rejects_a_zero_multi_proof_target_limit() { + // Zero would reject every non-empty batch, so clap must refuse it up front. + assert!( + CommandParser::::try_parse_from([ + "test", + "--proofs-history.max-multi-proof-targets", + "0", + ]) + .is_err() + ); } #[test] diff --git a/crates/node/tests/it/proof_history.rs b/crates/node/tests/it/proof_history.rs index b8d9656b..a29bbd35 100644 --- a/crates/node/tests/it/proof_history.rs +++ b/crates/node/tests/it/proof_history.rs @@ -1,12 +1,13 @@ //! Historical-proof correctness tests. //! -//! These exercise `eth_getProof` against the durable proof-history index, not -//! just RPC wiring. Each successful proof is checked against that block's -//! canonical `stateRoot`. Reference-index coverage lives in `reference_index.rs`. +//! These exercise `eth_getProof` and `eth_getMultiProof` against the durable +//! proof-history index, not just RPC wiring. Each successful proof is checked +//! against that block's canonical `stateRoot`. Reference-index coverage lives +//! in `reference_index.rs`. use std::{sync::Arc, time::Duration}; -use alloy_consensus::{SignableTransaction, TxEip1559}; +use alloy_consensus::{SignableTransaction, TxEip1559, constants::KECCAK_EMPTY}; use alloy_eips::{Encodable2718, NumHash, eip1898::BlockWithParent}; use alloy_genesis::Genesis; use alloy_primitives::{Address, B256, Bytes, TxKind, U256}; @@ -28,7 +29,7 @@ use morph_proofs::{ MorphProofsStore, ProofDbIdentity, }; use morph_proofs_exex::MorphProofsExEx; -use morph_rpc::ProofsSyncStatus; +use morph_rpc::{ProofsSyncStatus, eth::proofs::DEFAULT_MAX_MULTI_PROOF_TARGETS}; use reth_chainspec::EthChainSpec; use reth_e2e_test_utils::node::NodeTestContext; use reth_node_builder::{EngineNodeLauncher, Node, NodeBuilder, NodeConfig, NodeHandle}; @@ -41,7 +42,7 @@ use reth_provider::{ use reth_rpc_server_types::RpcModuleSelection; use reth_tasks::Runtime; use reth_transaction_pool::TransactionPool; -use reth_trie::AccountProof; +use reth_trie::{AccountProof, EMPTY_ROOT_HASH}; use serde::Serialize; const CHAIN_ID: u64 = 2910; @@ -51,6 +52,8 @@ const ACCOUNT2: Address = alloy_primitives::address!("3C44CdDdB6a900fa2b585dd299 const GENESIS_CONTRACT: Address = alloy_primitives::address!("530000000000000000000000000000000000000f"); const GENESIS_SLOT: B256 = B256::with_last_byte(0x06); +/// Never funded and never a sender, so it has no account-trie leaf at any tested height. +const ABSENT: Address = alloy_primitives::address!("00000000000000000000000000000000000abbe1"); /// Runtime: `PUSH1 0; CALLDATALOAD; PUSH1 0; SSTORE; STOP`. /// @@ -71,6 +74,7 @@ struct LaunchOpts { window: u64, prune_interval: Duration, verification_interval: u64, + max_multi_proof_targets: usize, } impl Default for LaunchOpts { @@ -81,6 +85,7 @@ impl Default for LaunchOpts { // asserting prune behavior. prune_interval: Duration::from_secs(3600), verification_interval: 0, + max_multi_proof_targets: DEFAULT_MAX_MULTI_PROOF_TARGETS, } } } @@ -159,7 +164,7 @@ async fn launch_node( .testing_node(runtime.clone()) .with_types_and_provider::>() .with_components(node.components_builder()) - .with_add_ons(MorphAddOns::new().with_proof_history(storage)) + .with_add_ons(MorphAddOns::new().with_proof_history(storage, opts.max_multi_proof_targets)) .install_exex("morph-proof-history", async move |ctx| { let head = ctx.head; let provider = ctx @@ -303,6 +308,17 @@ async fn get_proof( .map_err(|error| eyre::eyre!("eth_getProof failed: {error}")) } +async fn get_multi_proof( + client: &HttpClient, + targets: Vec<(Address, Vec)>, + block: B, +) -> eyre::Result> { + client + .request("eth_getMultiProof", rpc_params![targets, block]) + .await + .map_err(|error| eyre::eyre!("eth_getMultiProof failed: {error}")) +} + fn verify_against_state_root( proof: &EIP1186AccountProofResponse, state_root: B256, @@ -349,6 +365,26 @@ async fn proof_error( } } +async fn multi_proof_error( + client: &HttpClient, + targets: Vec<(Address, Vec)>, + block: B, +) -> eyre::Result { + match client + .request::, _>( + "eth_getMultiProof", + rpc_params![targets, block], + ) + .await + { + Ok(proofs) => Err(eyre::eyre!( + "expected eth_getMultiProof to fail, got {} proofs", + proofs.len() + )), + Err(error) => Ok(error.to_string()), + } +} + fn assert_outside_window(error: &str, requested: u64) { assert!( error.contains("outside the historical proof window") @@ -567,6 +603,243 @@ async fn proof_history_proofs_verify_against_block_state_root() -> eyre::Result< Ok(()) } +#[tokio::test(flavor = "multi_thread")] +async fn proof_history_multi_proof_uses_one_historical_state() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let mut env = setup(LaunchOpts::default()).await?; + let client = rpc_client(&env.node)?; + + let recipient_at_genesis = get_verified_proof(&client, ACCOUNT1, vec![], "0x0").await?; + include_transfer(&mut env.node, ACCOUNT1, 1_234).await?; + wait_for_proof_tip(&client, 1).await?; + include_transfer(&mut env.node, ACCOUNT2, 5_678).await?; + wait_for_proof_tip(&client, 2).await?; + + // Query block 1 after block 2 is canonical so the request cannot be served from latest state. + let block_1 = get_block(&client, "0x1").await?; + let empty_slot = B256::with_last_byte(0x42); + let expected_addresses = [ACCOUNT1, GENESIS_CONTRACT, ACCOUNT0]; + let proofs = get_multi_proof( + &client, + vec![ + (ACCOUNT1, vec![]), + (GENESIS_CONTRACT, vec![GENESIS_SLOT, empty_slot]), + (ACCOUNT0, vec![]), + ], + "0x1", + ) + .await?; + + assert_eq!(proofs.len(), expected_addresses.len()); + assert_eq!( + proofs.iter().map(|proof| proof.address).collect::>(), + expected_addresses + ); + for proof in &proofs { + verify_against_state_root(proof, block_1.header.state_root)?; + } + assert_eq!( + proofs[0].balance, + recipient_at_genesis.balance + U256::from(1_234) + ); + assert_eq!(storage_entry(&proofs[1], GENESIS_SLOT), U256::from(1)); + assert_eq!(storage_entry(&proofs[1], empty_slot), U256::ZERO); + assert_eq!(proofs[2].nonce, 1); + + // Match upstream semantics for empty batches while still validating the requested block. + assert!(get_multi_proof(&client, vec![], "0x1").await?.is_empty()); + let outside = multi_proof_error(&client, vec![(ACCOUNT0, Vec::::new())], "0x64").await?; + assert_outside_window(&outside, 0x64); + + Ok(()) +} + +/// Absent accounts must produce a valid *non-existence* proof. +/// +/// This is a distinct code path: `MultiProof::account_proof` leaves `info` at `None` and falls +/// back to `EMPTY_ROOT_HASH`, and `AccountProof::verify` only accepts a missing leaf when `info` +/// is `None` **and** `storage_root` is the empty root. A response that invented either field, or +/// an implementation that returned an existing neighbour's leaf, would fail verification here. +/// +/// `KECCAK_EMPTY` / `EMPTY_ROOT_HASH` are asserted because they are the values the absent account +/// *would* hash to, which is what keeps the response self-consistent: a verifier can recompute the +/// leaf that must be missing. Zero hashes would carry no such meaning, and only signal absence +/// out-of-band. +#[tokio::test(flavor = "multi_thread")] +async fn proof_history_proves_absent_accounts() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let mut env = setup(LaunchOpts::default()).await?; + let client = rpc_client(&env.node)?; + + include_transfer(&mut env.node, ACCOUNT1, 1_000).await?; + wait_for_proof_tip(&client, 1).await?; + include_transfer(&mut env.node, ACCOUNT2, 2_000).await?; + wait_for_proof_tip(&client, 2).await?; + + let block_1 = get_block(&client, "0x1").await?; + let absent_slot = B256::with_last_byte(0x07); + + let single = get_proof(&client, ABSENT, vec![absent_slot], "0x1").await?; + verify_against_state_root(&single, block_1.header.state_root)?; + assert_eq!(single.nonce, 0); + assert_eq!(single.balance, U256::ZERO); + assert_eq!(single.code_hash, KECCAK_EMPTY); + assert_eq!(single.storage_hash, EMPTY_ROOT_HASH); + // A slot on a missing account still gets an entry, proving zero rather than omitting it. + assert_eq!(storage_entry(&single, absent_slot), U256::ZERO); + + // Batching an absent target next to a present one must not contaminate either response. + let batched = get_multi_proof( + &client, + vec![(ACCOUNT0, vec![]), (ABSENT, vec![absent_slot])], + "0x1", + ) + .await?; + assert_eq!(batched.len(), 2); + for proof in &batched { + verify_against_state_root(proof, block_1.header.state_root)?; + } + assert_eq!(batched[0].address, ACCOUNT0); + assert_eq!(batched[0].nonce, 1); + assert_ne!(batched[0].balance, U256::ZERO); + assert_eq!( + batched[1], single, + "eth_getMultiProof must return the same non-existence proof as eth_getProof" + ); + + Ok(()) +} + +/// A contract target with no requested slots must still report its real storage root. +/// +/// `Proof::multiproof` pre-seeds every target with an empty `StorageMultiProof` and only +/// overwrites it once the account leaf is reached, so a regression here would silently return +/// `EMPTY_ROOT_HASH` for contracts that do have storage. +#[tokio::test(flavor = "multi_thread")] +async fn proof_history_multi_proof_reports_contract_storage_root_without_slots() -> eyre::Result<()> +{ + reth_tracing::init_test_tracing(); + + let mut env = setup(LaunchOpts::default()).await?; + let client = rpc_client(&env.node)?; + + include_transfer(&mut env.node, ACCOUNT1, 1).await?; + wait_for_proof_tip(&client, 1).await?; + + let block_1 = get_block(&client, "0x1").await?; + let with_slot = get_proof(&client, GENESIS_CONTRACT, vec![GENESIS_SLOT], "0x1").await?; + let proofs = get_multi_proof(&client, vec![(GENESIS_CONTRACT, vec![])], "0x1").await?; + + assert_eq!(proofs.len(), 1); + verify_against_state_root(&proofs[0], block_1.header.state_root)?; + assert!(proofs[0].storage_proof.is_empty()); + assert_ne!( + proofs[0].storage_hash, EMPTY_ROOT_HASH, + "genesis contract has storage, so its storage root must not be the empty root" + ); + assert_eq!(proofs[0].storage_hash, with_slot.storage_hash); + assert_eq!(proofs[0].code_hash, with_slot.code_hash); + + Ok(()) +} + +/// Repeating an address must yield one response per request entry, each scoped to its own slots. +/// +/// Proof generation consolidates the duplicates into a single account target, so the expansion +/// back to request order is the part that can regress. +#[tokio::test(flavor = "multi_thread")] +async fn proof_history_multi_proof_expands_duplicate_targets() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let mut env = setup(LaunchOpts::default()).await?; + let client = rpc_client(&env.node)?; + + include_transfer(&mut env.node, ACCOUNT1, 1).await?; + wait_for_proof_tip(&client, 1).await?; + + let block_1 = get_block(&client, "0x1").await?; + let empty_slot = B256::with_last_byte(0x42); + let proofs = get_multi_proof( + &client, + vec![ + (GENESIS_CONTRACT, vec![GENESIS_SLOT]), + (GENESIS_CONTRACT, vec![empty_slot]), + (GENESIS_CONTRACT, vec![]), + ], + "0x1", + ) + .await?; + + assert_eq!(proofs.len(), 3); + for proof in &proofs { + assert_eq!(proof.address, GENESIS_CONTRACT); + verify_against_state_root(proof, block_1.header.state_root)?; + } + // Each entry carries exactly the slots it asked for, not the consolidated union. + assert_eq!(proofs[0].storage_proof.len(), 1); + assert_eq!(storage_entry(&proofs[0], GENESIS_SLOT), U256::from(1)); + assert_eq!(proofs[1].storage_proof.len(), 1); + assert_eq!(storage_entry(&proofs[1], empty_slot), U256::ZERO); + assert!(proofs[2].storage_proof.is_empty()); + + Ok(()) +} + +/// The configured account-target limit and the fixed storage-key limit are both enforced at RPC. +/// +/// Launching with a deliberately tiny target limit also proves the CLI value reaches the handler +/// instead of the compiled-in default. +#[tokio::test(flavor = "multi_thread")] +async fn proof_history_multi_proof_enforces_configured_limits() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let mut env = setup(LaunchOpts { + max_multi_proof_targets: 2, + ..LaunchOpts::default() + }) + .await?; + let client = rpc_client(&env.node)?; + + include_transfer(&mut env.node, ACCOUNT1, 1).await?; + wait_for_proof_tip(&client, 1).await?; + + // At the configured limit the request still succeeds. + let at_limit = + get_multi_proof(&client, vec![(ACCOUNT0, vec![]), (ACCOUNT1, vec![])], "0x1").await?; + assert_eq!(at_limit.len(), 2); + + let too_many_targets = multi_proof_error( + &client, + vec![(ACCOUNT0, vec![]), (ACCOUNT1, vec![]), (ACCOUNT2, vec![])], + "0x1", + ) + .await?; + assert!( + too_many_targets.contains("too many proof targets") && too_many_targets.contains("max 2"), + "expected configured target-limit error, got: {too_many_targets}" + ); + + // The storage-key cap is fixed at 1024 and is independent of the target limit. + let too_many_keys = multi_proof_error( + &client, + vec![(GENESIS_CONTRACT, vec![B256::ZERO; 1025])], + "0x1", + ) + .await?; + assert!( + too_many_keys.contains("too many storage keys") && too_many_keys.contains("got 1025"), + "expected storage-key-limit error, got: {too_many_keys}" + ); + + // An empty batch is not short-circuited: it still range-checks the requested block. + let empty_outside = multi_proof_error(&client, vec![], "0x64").await?; + assert_outside_window(&empty_outside, 0x64); + + Ok(()) +} + // ----------------------------------------------------------------------------- // 3. earliest / latest / future / missing-block bounds // ----------------------------------------------------------------------------- @@ -662,7 +935,7 @@ async fn proof_history_prune_updates_window_bounds() -> eyre::Result<()> { let mut env = setup(LaunchOpts { window: 2, prune_interval: Duration::from_millis(100), - verification_interval: 0, + ..LaunchOpts::default() }) .await?; let client = rpc_client(&env.node)?; @@ -842,9 +1115,8 @@ async fn proof_history_stays_correct_with_verification_enabled() -> eyre::Result reth_tracing::init_test_tracing(); let mut env = setup(LaunchOpts { - window: DEFAULT_PROOFS_HISTORY_WINDOW, - prune_interval: Duration::from_secs(3600), verification_interval: 1, + ..LaunchOpts::default() }) .await?; let client = rpc_client(&env.node)?; diff --git a/crates/proofs/src/lib.rs b/crates/proofs/src/lib.rs index 953632df..a015d830 100644 --- a/crates/proofs/src/lib.rs +++ b/crates/proofs/src/lib.rs @@ -1,4 +1,4 @@ -//! Bounded historical MPT storage for Morph `eth_getProof`. +//! Bounded historical MPT storage for Morph proof RPCs. //! //! The versioned-trie implementation is adapted from Base's proof-history //! storage at commit `b2673bbd927cb34d7cfad4d448bfbd5bd30eae88` under MIT. diff --git a/crates/rpc/Cargo.toml b/crates/rpc/Cargo.toml index e7539dc9..0fd1e93f 100644 --- a/crates/rpc/Cargo.toml +++ b/crates/rpc/Cargo.toml @@ -37,6 +37,7 @@ reth-revm.workspace = true reth-storage-api.workspace = true reth-tasks.workspace = true reth-errors.workspace = true +reth-trie-common.workspace = true reth-transaction-pool.workspace = true # Alloy diff --git a/crates/rpc/src/eth/proofs.rs b/crates/rpc/src/eth/proofs.rs index f63c6ca2..4f1abd67 100644 --- a/crates/rpc/src/eth/proofs.rs +++ b/crates/rpc/src/eth/proofs.rs @@ -1,9 +1,9 @@ //! Historical EIP-1186 proof RPC override. -use std::time::Instant; +use std::{sync::LazyLock, time::Instant}; use alloy_eips::BlockId; -use alloy_primitives::Address; +use alloy_primitives::{Address, B256, keccak256}; use alloy_rpc_types_eth::EIP1186AccountProofResponse; use alloy_serde::JsonStorageKey; use async_trait::async_trait; @@ -13,20 +13,33 @@ use jsonrpsee::{ types::{ErrorCode, ErrorObject}, }; use morph_proofs::{MorphProofsStorage, MorphProofsStore}; +use reth_errors::RethError; use reth_metrics::{ Metrics, - metrics::{Counter, Histogram}, + metrics::{Counter, Histogram, Label}, }; use reth_provider::StateProofProvider; use reth_rpc_api::eth::helpers::FullEthApi; +use reth_rpc_eth_api::FromEthApiError; use reth_rpc_eth_types::EthApiError; +use reth_trie_common::MultiProofTargets; use crate::state::MorphProofStateProviderFactory; -/// Maximum number of storage keys accepted by one `eth_getProof` request. +/// Maximum storage keys accepted by one proof request. +/// +/// Matches go-ethereum's `eth_getProof` cap, so it stays a constant rather than a CLI knob. pub const MAX_PROOF_KEYS: usize = 1024; -/// Validates the storage-key count before any database work starts. +/// Default maximum account targets accepted by one `eth_getMultiProof` request. +/// +/// Chosen so neither request dimension dominates proof cost, and so realistic demand fits in one +/// round trip: Morph mainnet blocks touch at most ~36 accounts, while a Hoodi load test peaked +/// near ~200 accounts in a single block. Override with +/// `--proofs-history.max-multi-proof-targets`. +pub const DEFAULT_MAX_MULTI_PROOF_TARGETS: usize = 256; + +/// Validates request sizes before any database work starts. #[derive(Debug)] pub struct ProofKeyLimit; @@ -42,21 +55,138 @@ impl ProofKeyLimit { } Ok(()) } + + /// Rejects a multiproof batch that exceeds `max_targets` accounts or [`MAX_PROOF_KEYS`] keys. + /// + /// The two dimensions are counted separately because an account target costs several times a + /// storage slot: every account target retains its own account-trie path and opens a + /// storage-trie cursor, while slots share one already-open trie. A single combined budget + /// would therefore let an all-accounts batch cost several times an all-slots batch of the + /// same nominal size. + /// + /// Duplicate addresses are charged once per occurrence even though proof generation + /// consolidates them. That over-counts, which is the safe direction for a pre-flight check. + pub fn check_multi( + targets: &[(Address, Vec)], + max_targets: usize, + ) -> Result<(), ErrorObject<'static>> { + if targets.len() > max_targets { + return Err(ErrorObject::owned( + ErrorCode::InvalidParams.code(), + format!( + "too many proof targets: max {max_targets}, got {}", + targets.len() + ), + None::<()>, + )); + } + + let storage_keys = targets + .iter() + .try_fold(0usize, |total, (_, slots)| total.checked_add(slots.len())) + .unwrap_or(usize::MAX); + if storage_keys > MAX_PROOF_KEYS { + return Err(ErrorObject::owned( + ErrorCode::InvalidParams.code(), + format!("too many storage keys: max {MAX_PROOF_KEYS}, got {storage_keys}"), + None::<()>, + )); + } + Ok(()) + } } +/// Per-method counters, distinguished by a `method` label rather than by metric name. +/// +/// One name per measurement lets a dashboard split by method or sum across both; separate +/// `get_proof_*` / `get_multi_proof_*` names would allow neither without string surgery. #[derive(Metrics, Clone)] #[metrics(scope = "morph.rpc.proofs")] struct ProofRpcMetrics { - /// Total `eth_getProof` requests. + /// Requests received, counted before the request-size limits are applied. requests_total: Counter, - /// Successful `eth_getProof` responses. + /// Requests refused by a request-size limit before any database work started. + rejected_total: Counter, + /// Requests answered with a proof. successful_responses_total: Counter, - /// Failed `eth_getProof` responses. + /// Requests that failed after passing the request-size limits. failures_total: Counter, /// Successful request latency in seconds. latency_seconds: Histogram, } +// `new_with_labels` re-registers on every call, unlike the cached `default()`, so build each +// method's handles once and share them. +static GET_PROOF_METRICS: LazyLock = + LazyLock::new(|| ProofRpcMetrics::new_with_labels(vec![Label::new("method", "eth_getProof")])); +static GET_MULTI_PROOF_METRICS: LazyLock = LazyLock::new(|| { + ProofRpcMetrics::new_with_labels(vec![Label::new("method", "eth_getMultiProof")]) +}); + +impl ProofRpcMetrics { + /// Counts a request refused by a size limit. + /// + /// Kept separate from `failures_total` so that + /// `requests_total == rejected_total + successful_responses_total + failures_total` holds; + /// previously a rejected request was counted nowhere at all. + fn record_rejection(&self) { + self.rejected_total.increment(1); + } + + /// Records the outcome of a request that passed the size limits. + /// + /// Latency is only sampled on success, so a flood of failures cannot skew the histogram. + fn record_response(&self, start: Instant, result: &RpcResult) { + match result { + Ok(_) => { + self.latency_seconds.record(start.elapsed().as_secs_f64()); + self.successful_responses_total.increment(1); + } + Err(_) => self.failures_total.increment(1), + } + } +} + +/// Runs a proof computation on the blocking pool. +/// +/// Proof generation walks MDBX and rebuilds trie nodes synchronously with no await points, so +/// leaving it on a runtime worker would let a handful of concurrent requests starve the Engine API. +async fn spawn_proof_task(eth_api: Eth, task: F) -> RpcResult +where + Eth: FullEthApi + Send + Sync + 'static, + F: FnOnce() -> Result + Send + 'static, + T: Send + 'static, +{ + let permit = eth_api + .acquire_owned_tracing() + .await + .map_err(RethError::other) + .map_err(EthApiError::Internal) + .map_err(Eth::Error::from) + .map_err(Into::into)?; + + eth_api + .spawn_blocking_io(move |_| { + // Keep the shared proof permit until the blocking computation itself has stopped, + // even if the requesting RPC future is cancelled while this task is running. + let _permit = permit; + task() + }) + .await + .map_err(Into::into) +} + +fn multiproof_targets(targets: &[(Address, Vec)]) -> MultiProofTargets { + let mut proof_targets = MultiProofTargets::with_capacity(targets.len()); + for (address, slots) in targets { + proof_targets + .entry(keccak256(address)) + .or_default() + .extend(slots.iter().map(keccak256)); + } + proof_targets +} + #[cfg_attr(not(test), rpc(server, namespace = "eth"))] #[cfg_attr(test, rpc(server, client, namespace = "eth"))] pub trait EthProofApiOverride { @@ -68,12 +198,30 @@ pub trait EthProofApiOverride { keys: Vec, block: Option, ) -> RpcResult; + + /// Returns EIP-1186 proofs for multiple targets at a retained canonical block. + /// + /// Storage keys are full 32-byte values, not the short form `eth_getProof` accepts. + /// + /// Duplicate addresses are consolidated for proof generation and expanded back into their + /// original request order, each response carrying only the slots its own target requested. + /// + /// An empty batch still resolves and range-checks `block`, so it fails the same way a + /// non-empty batch would outside the retained window; it returns an empty response only once + /// the block is accepted. + #[method(name = "getMultiProof")] + async fn get_multi_proof( + &self, + targets: Vec<(Address, Vec)>, + block: Option, + ) -> RpcResult>; } -/// `eth_getProof` implementation backed exclusively by Morph proof history. +/// Historical proof RPC implementation backed exclusively by Morph proof history. #[derive(Debug)] pub struct EthProofApiExt { state_provider_factory: MorphProofStateProviderFactory, + max_multi_proof_targets: usize, } impl EthProofApiExt @@ -82,9 +230,14 @@ where P: MorphProofsStore + Clone + 'static, { /// Creates the historical proof RPC override. - pub const fn new(eth_api: Eth, storage: MorphProofsStorage

) -> Self { + pub const fn new( + eth_api: Eth, + storage: MorphProofsStorage

, + max_multi_proof_targets: usize, + ) -> Self { Self { state_provider_factory: MorphProofStateProviderFactory::new(eth_api, storage), + max_multi_proof_targets, } } } @@ -92,7 +245,7 @@ where #[async_trait] impl EthProofApiOverrideServer for EthProofApiExt where - Eth: FullEthApi + Send + Sync + 'static, + Eth: FullEthApi + Clone + Send + Sync + 'static, P: MorphProofsStore + Clone + 'static, { async fn get_proof( @@ -101,41 +254,104 @@ where keys: Vec, block: Option, ) -> RpcResult { - ProofKeyLimit::check(keys.len())?; - - let metrics = ProofRpcMetrics::default(); + let metrics = &*GET_PROOF_METRICS; let start = Instant::now(); metrics.requests_total.increment(1); - let storage_keys = keys.iter().map(JsonStorageKey::as_b256).collect::>(); + if let Err(error) = ProofKeyLimit::check(keys.len()) { + metrics.record_rejection(); + return Err(error); + } - let result = (|| { - let proof = self - .state_provider_factory + let factory = self.state_provider_factory.clone(); + let eth_api = factory.eth_api().clone(); + let result = spawn_proof_task(eth_api, move || { + let storage_keys = keys.iter().map(JsonStorageKey::as_b256).collect::>(); + let proof = factory .state_provider(block) - .map_err(|error| ErrorObject::from(EthApiError::from(error)))? + .map_err(Eth::Error::from_eth_err)? .proof(Default::default(), address, &storage_keys) - .map_err(|error| ErrorObject::from(EthApiError::from(error)))?; + .map_err(Eth::Error::from_eth_err)?; Ok(proof.into_eip1186_response(keys)) - })(); + }) + .await; - match &result { - Ok(_) => { - metrics - .latency_seconds - .record(start.elapsed().as_secs_f64()); - metrics.successful_responses_total.increment(1); - } - Err(_) => metrics.failures_total.increment(1), + metrics.record_response(start, &result); + result + } + + async fn get_multi_proof( + &self, + targets: Vec<(Address, Vec)>, + block: Option, + ) -> RpcResult> { + let metrics = &*GET_MULTI_PROOF_METRICS; + let start = Instant::now(); + metrics.requests_total.increment(1); + if let Err(error) = ProofKeyLimit::check_multi(&targets, self.max_multi_proof_targets) { + metrics.record_rejection(); + return Err(error); } + + let factory = self.state_provider_factory.clone(); + let eth_api = factory.eth_api().clone(); + let result = spawn_proof_task(eth_api, move || { + let state = factory + .state_provider(block) + .map_err(Eth::Error::from_eth_err)?; + let multiproof = state + .multiproof(Default::default(), multiproof_targets(&targets)) + .map_err(Eth::Error::from_eth_err)?; + + targets + .into_iter() + .map(|(address, slots)| { + let proof = multiproof + .account_proof(address, &slots) + .map_err(RethError::other) + .map_err(EthApiError::Internal) + .map_err(Eth::Error::from)?; + let storage_keys = slots.into_iter().map(JsonStorageKey::from).collect(); + Ok(proof.into_eip1186_response(storage_keys)) + }) + .collect() + }) + .await; + + metrics.record_response(start, &result); result } } #[cfg(test)] mod tests { + use std::sync::LazyLock; + + use alloy_primitives::{Address, B256, U256, keccak256}; use jsonrpsee::types::ErrorCode; - use super::{MAX_PROOF_KEYS, ProofKeyLimit}; + use super::{ + DEFAULT_MAX_MULTI_PROOF_TARGETS, GET_MULTI_PROOF_METRICS, GET_PROOF_METRICS, + MAX_PROOF_KEYS, ProofKeyLimit, multiproof_targets, + }; + + #[test] + fn builds_per_method_metric_handles() { + // Unlike the cached `default()`, `new_with_labels` runs registration lazily on first use, + // so a bad label or scope would otherwise only panic while serving a real request. + LazyLock::force(&GET_PROOF_METRICS); + LazyLock::force(&GET_MULTI_PROOF_METRICS); + } + + fn empty_targets(count: usize) -> Vec<(Address, Vec)> { + (0..count) + .map(|index| { + ( + Address::from_word(B256::from(U256::from(index))), + Vec::::new(), + ) + }) + .collect() + } #[test] fn enforces_get_proof_key_limit() { @@ -143,4 +359,84 @@ mod tests { let error = ProofKeyLimit::check(MAX_PROOF_KEYS + 1).expect_err("must reject"); assert_eq!(error.code(), ErrorCode::InvalidParams.code()); } + + #[test] + fn enforces_get_multi_proof_storage_key_limit() { + let at_limit = [ + (Address::ZERO, vec![B256::ZERO; 512]), + (Address::repeat_byte(1), vec![B256::ZERO; 512]), + ]; + assert!(ProofKeyLimit::check_multi(&at_limit, DEFAULT_MAX_MULTI_PROOF_TARGETS).is_ok()); + + let over_limit = [ + (Address::ZERO, vec![B256::ZERO; 512]), + (Address::repeat_byte(1), vec![B256::ZERO; 513]), + ]; + let error = ProofKeyLimit::check_multi(&over_limit, DEFAULT_MAX_MULTI_PROOF_TARGETS) + .expect_err("must reject"); + assert_eq!(error.code(), ErrorCode::InvalidParams.code()); + assert!(error.message().contains("too many storage keys")); + assert!(error.message().contains("got 1025")); + } + + #[test] + fn enforces_get_multi_proof_target_limit() { + let at_limit = empty_targets(DEFAULT_MAX_MULTI_PROOF_TARGETS); + assert!(ProofKeyLimit::check_multi(&at_limit, DEFAULT_MAX_MULTI_PROOF_TARGETS).is_ok()); + + let over_limit = empty_targets(DEFAULT_MAX_MULTI_PROOF_TARGETS + 1); + let error = ProofKeyLimit::check_multi(&over_limit, DEFAULT_MAX_MULTI_PROOF_TARGETS) + .expect_err("must reject"); + assert_eq!(error.code(), ErrorCode::InvalidParams.code()); + assert!(error.message().contains("too many proof targets")); + assert!( + error + .message() + .contains(&format!("got {}", DEFAULT_MAX_MULTI_PROOF_TARGETS + 1)) + ); + } + + #[test] + fn honors_a_configured_target_limit_below_the_default() { + // The configured value, not the default, decides: 8 targets pass at 8 and fail at 7. + let targets = empty_targets(8); + assert!(ProofKeyLimit::check_multi(&targets, 8).is_ok()); + let error = ProofKeyLimit::check_multi(&targets, 7).expect_err("must reject"); + assert!(error.message().contains("max 7, got 8")); + } + + #[test] + fn charges_duplicate_addresses_once_per_occurrence() { + // Consolidation happens during proof generation, but the pre-flight check counts entries, + // so a duplicated address still consumes two target slots. + let address = Address::repeat_byte(0x11); + let targets = [(address, Vec::::new()), (address, Vec::::new())]; + assert!(ProofKeyLimit::check_multi(&targets, 2).is_ok()); + assert!(ProofKeyLimit::check_multi(&targets, 1).is_err()); + } + + #[test] + fn consolidates_duplicate_targets_for_proof_generation() { + let address = Address::repeat_byte(0x11); + let slot_a = B256::repeat_byte(0x22); + let slot_b = B256::repeat_byte(0x33); + let targets = + multiproof_targets(&[(address, vec![slot_a]), (address, vec![slot_b, slot_a])]); + + assert_eq!(targets.len(), 1); + let slots = targets + .get(&keccak256(address)) + .expect("consolidated account target"); + assert_eq!(slots.len(), 2); + assert!(slots.contains(&keccak256(slot_a))); + assert!(slots.contains(&keccak256(slot_b))); + } + + #[test] + fn accepts_an_empty_multi_proof_batch() { + assert!(multiproof_targets(&[]).is_empty()); + assert!(ProofKeyLimit::check_multi(&[], DEFAULT_MAX_MULTI_PROOF_TARGETS).is_ok()); + // Even a zero limit admits an empty batch, so the block check still runs. + assert!(ProofKeyLimit::check_multi(&[], 0).is_ok()); + } } diff --git a/crates/rpc/src/state.rs b/crates/rpc/src/state.rs index 657fc7dc..6e4a9930 100644 --- a/crates/rpc/src/state.rs +++ b/crates/rpc/src/state.rs @@ -77,6 +77,22 @@ impl MorphProofStateProviderFactory { pub const fn new(eth_api: Eth, storage: MorphProofsStorage

) -> Self { Self { eth_api, storage } } + + /// Returns the underlying Eth API used for proof task scheduling and chain reads. + pub const fn eth_api(&self) -> &Eth { + &self.eth_api + } +} + +// Hand-written so the `Clone` bounds land on the impl rather than the struct: proof handlers clone +// the factory into a blocking task, where the returned provider only borrows the local clone. +impl Clone for MorphProofStateProviderFactory { + fn clone(&self) -> Self { + Self { + eth_api: self.eth_api.clone(), + storage: self.storage.clone(), + } + } } impl<'a, Eth, P> MorphProofStateProviderFactory From ce4fd0e84cf509f41261b8352df9aad59f539d19 Mon Sep 17 00:00:00 2001 From: panos Date: Wed, 26 Aug 2026 08:32:30 +0800 Subject: [PATCH 11/12] chore(grafana): expose proof history runtime metrics (#170) Bring the Morph dashboard in line with compatible reth v2.4 metrics so proof-history and engine internals are observable. --- etc/README.md | 10 +- etc/docker-compose.yml | 2 +- etc/grafana/dashboards/morph-reth.json | 3903 +++++++++++++++++++----- etc/prometheus/prometheus.yml | 2 +- 4 files changed, 3220 insertions(+), 697 deletions(-) diff --git a/etc/README.md b/etc/README.md index b2669a01..4d88fe98 100644 --- a/etc/README.md +++ b/etc/README.md @@ -14,10 +14,13 @@ up to date. The [`docker-compose.yml`](./docker-compose.yml) in this directory brings up a local morph-reth node together with Prometheus and Grafana. The `morph-reth` -service runs with `--chain morph` and exposes metrics on port `9001`, +service runs against Morph mainnet with `--chain mainnet` and exposes metrics on port `9001`, which Prometheus scrapes with the labels defined in [`prometheus/prometheus.yml`](./prometheus/prometheus.yml). +To monitor Hoodi instead, change the service argument to `--chain hoodi` and +set the Prometheus `chain` label to `hoodi`. + ```bash ./etc/generate-jwt.sh docker compose -f etc/docker-compose.yml up -d @@ -27,6 +30,11 @@ Grafana is available at (default admin/admin) and will auto-provision the unified Morph dashboard from [`grafana/dashboards/`](./grafana/dashboards/). +The dashboard's **Proof History** and **Execution Extension Runtime** rows +require the node to run with `--proofs-history`. The example Compose stack does +not enable proof history by default, so those panels are expected to be empty +unless the flag is added to the `morph-reth` service command. + ### Grafana #### Adding a new metric to Grafana diff --git a/etc/docker-compose.yml b/etc/docker-compose.yml index 570aeaf2..aa2f9a8c 100644 --- a/etc/docker-compose.yml +++ b/etc/docker-compose.yml @@ -19,7 +19,7 @@ services: pid: host command: > node - --chain morph + --chain mainnet --datadir /var/lib/morph-reth/data --metrics 0.0.0.0:9001 --log.file.directory /var/lib/morph-reth/logs diff --git a/etc/grafana/dashboards/morph-reth.json b/etc/grafana/dashboards/morph-reth.json index 2f538627..6a11b8b3 100644 --- a/etc/grafana/dashboards/morph-reth.json +++ b/etc/grafana/dashboards/morph-reth.json @@ -144,6 +144,18 @@ "legendFormat": "Target: {{target_triple}}", "range": false, "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": false, + "expr": "max by (cargo_features) (reth_info{chain=\"$chain\", role=~\"$role\", service=~\"$service\"})", + "instant": true, + "legendFormat": "Features: {{cargo_features}}", + "range": false, + "refId": "F" } ], "title": "Build Info", @@ -656,7 +668,7 @@ }, "editorMode": "code", "expr": "reth_sync_execution_gas_per_second{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", - "legendFormat": "{{service}} · Gas/s", + "legendFormat": "{{service}} \u00b7 Gas/s", "range": true, "refId": "A" }, @@ -671,7 +683,7 @@ "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, - "legendFormat": "{{service}} · Avg Gas/s (1m)", + "legendFormat": "{{service}} \u00b7 Avg Gas/s (1m)", "range": true, "refId": "B", "useBackend": false @@ -687,7 +699,7 @@ "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, - "legendFormat": "{{service}} · Avg Gas/s (5m)", + "legendFormat": "{{service}} \u00b7 Avg Gas/s (5m)", "range": true, "refId": "C", "useBackend": false @@ -703,7 +715,7 @@ "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, - "legendFormat": "{{service}} · Avg Gas/s (10m)", + "legendFormat": "{{service}} \u00b7 Avg Gas/s (10m)", "range": true, "refId": "D", "useBackend": false @@ -719,7 +731,7 @@ "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, - "legendFormat": "{{service}} · Avg Gas/s (30m)", + "legendFormat": "{{service}} \u00b7 Avg Gas/s (30m)", "range": true, "refId": "E", "useBackend": false @@ -735,7 +747,7 @@ "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, - "legendFormat": "{{service}} · Avg Gas/s (1h)", + "legendFormat": "{{service}} \u00b7 Avg Gas/s (1h)", "range": true, "refId": "F", "useBackend": false @@ -751,7 +763,7 @@ "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, - "legendFormat": "{{service}} · Avg Gas/s (24h)", + "legendFormat": "{{service}} \u00b7 Avg Gas/s (24h)", "range": true, "refId": "G", "useBackend": false @@ -918,7 +930,7 @@ "type": "timeseries", "targets": [ { - "expr": "reth_morph_engine_head_block_timegap_seconds{chain=\"$chain\", role=~\"$role\"}", + "expr": "reth_morph_engine_head_block_timegap_seconds{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", "legendFormat": "{{service}}" } ] @@ -1265,24 +1277,11 @@ "type": "timeseries", "targets": [ { - "expr": "sum by (service) (rate(reth_morph_engine_assemble_l2_block_failures_total{chain=\"$chain\", role=~\"$role\"}[$__rate_interval]) + rate(reth_morph_engine_validate_l2_block_failures_total{chain=\"$chain\", role=~\"$role\"}[$__rate_interval]) + rate(reth_morph_engine_new_l2_block_failures_total{chain=\"$chain\", role=~\"$role\"}[$__rate_interval]) + rate(reth_morph_engine_new_safe_l2_block_failures_total{chain=\"$chain\", role=~\"$role\"}[$__rate_interval]))", + "expr": "sum by (service) (rate(reth_morph_engine_assemble_l2_block_failures_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval]) + rate(reth_morph_engine_validate_l2_block_failures_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval]) + rate(reth_morph_engine_new_l2_block_failures_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval]) + rate(reth_morph_engine_new_safe_l2_block_failures_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval]))", "legendFormat": "{{service}}" } ] }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 61 - }, - "id": 16, - "panels": [], - "title": "Payload Builder", - "type": "row" - }, { "datasource": { "type": "prometheus", @@ -1294,30 +1293,30 @@ "mode": "palette-classic-by-name" }, "custom": { + "axisCenteredZero": false, "drawStyle": "line", "fillOpacity": 0, "lineWidth": 1, "pointSize": 5, "showPoints": "auto", - "spanNulls": false, - "axisCenteredZero": false + "spanNulls": false }, - "unit": "s", - "min": 0 + "min": 0, + "unit": "ops" }, "overrides": [] }, "gridPos": { - "h": 9, + "h": 8, "w": 12, "x": 0, - "y": 62 + "y": 61 }, - "id": 17, + "id": 300, "options": { "legend": { "displayMode": "list", - "placement": "right", + "placement": "bottom", "showLegend": true }, "tooltip": { @@ -1325,19 +1324,35 @@ "sort": "none" } }, - "title": "Payload Build Duration", - "description": "End-to-end payload build duration (state setup + tx execution + finalize + seal).", - "type": "timeseries", "targets": [ { - "expr": "rate(reth_morph_payload_builder_payload_build_duration_seconds_sum{role=~\"$role\", chain=\"$chain\", service=~\"$service\"}[$__rate_interval]) / rate(reth_morph_payload_builder_payload_build_duration_seconds_count{role=~\"$role\", chain=\"$chain\", service=~\"$service\"}[$__rate_interval]) > 0", - "legendFormat": "{{service}} avg" + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(reth_consensus_engine_beacon_new_payload_messages{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "instant": false, + "legendFormat": "{{service}} \u00b7 newPayload", + "range": true, + "refId": "A" }, { - "expr": "reth_morph_payload_builder_payload_build_duration_seconds{role=~\"$role\", chain=\"$chain\", service=~\"$service\", quantile=\"$quantile\"} > 0", - "legendFormat": "{{service}} p{{quantile}}" + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(reth_consensus_engine_beacon_forkchoice_updated_messages{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "instant": false, + "legendFormat": "{{service}} \u00b7 forkchoiceUpdated", + "range": true, + "refId": "B" } - ] + ], + "title": "Engine Tree Messages", + "type": "timeseries", + "description": "Rates of newPayload and forkchoiceUpdated messages received by the internal engine tree." }, { "datasource": { @@ -1350,30 +1365,30 @@ "mode": "palette-classic-by-name" }, "custom": { + "axisCenteredZero": false, "drawStyle": "line", "fillOpacity": 0, "lineWidth": 1, "pointSize": 5, "showPoints": "auto", - "spanNulls": false, - "axisCenteredZero": false + "spanNulls": false }, - "unit": "s", - "min": 0 + "min": 0, + "unit": "ops" }, "overrides": [] }, "gridPos": { - "h": 9, + "h": 8, "w": 12, "x": 12, - "y": 62 + "y": 61 }, - "id": 18, + "id": 301, "options": { "legend": { "displayMode": "list", - "placement": "right", + "placement": "bottom", "showLegend": true }, "tooltip": { @@ -1381,19 +1396,35 @@ "sort": "none" } }, - "title": "Tx Execution Duration", - "description": "Total time to execute all transactions (L1 messages + pool) for one block.", - "type": "timeseries", "targets": [ { - "expr": "rate(reth_morph_payload_builder_commit_txs_all_duration_seconds_sum{role=~\"$role\", chain=\"$chain\", service=~\"$service\"}[$__rate_interval]) / rate(reth_morph_payload_builder_commit_txs_all_duration_seconds_count{role=~\"$role\", chain=\"$chain\", service=~\"$service\"}[$__rate_interval]) > 0", - "legendFormat": "{{service}} avg" + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(reth_consensus_engine_beacon_failed_new_payload_response_deliveries{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "instant": false, + "legendFormat": "{{service}} \u00b7 newPayload", + "range": true, + "refId": "A" }, { - "expr": "reth_morph_payload_builder_commit_txs_all_duration_seconds{role=~\"$role\", chain=\"$chain\", service=~\"$service\", quantile=\"$quantile\"} > 0", - "legendFormat": "{{service}} p{{quantile}}" + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(reth_consensus_engine_beacon_failed_forkchoice_updated_response_deliveries{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "instant": false, + "legendFormat": "{{service}} \u00b7 forkchoiceUpdated", + "range": true, + "refId": "B" } - ] + ], + "title": "Engine Tree Failed Response Deliveries", + "type": "timeseries", + "description": "Failed internal response deliveries, typically because the requester terminated before the engine tree replied." }, { "datasource": { @@ -1406,30 +1437,30 @@ "mode": "palette-classic-by-name" }, "custom": { + "axisCenteredZero": false, "drawStyle": "line", "fillOpacity": 0, "lineWidth": 1, "pointSize": 5, "showPoints": "auto", - "spanNulls": false, - "axisCenteredZero": false + "spanNulls": false }, - "unit": "s", - "min": 0 + "min": 0, + "unit": "s" }, "overrides": [] }, "gridPos": { - "h": 9, + "h": 8, "w": 12, "x": 0, - "y": 71 + "y": 69 }, - "id": 19, + "id": 302, "options": { "legend": { "displayMode": "list", - "placement": "right", + "placement": "bottom", "showLegend": true }, "tooltip": { @@ -1437,19 +1468,23 @@ "sort": "none" } }, - "title": "Per-Tx Apply Duration", - "description": "Time for the EVM execution of a single transaction.", - "type": "timeseries", "targets": [ { - "expr": "rate(reth_morph_payload_builder_commit_tx_apply_duration_seconds_sum{role=~\"$role\", chain=\"$chain\", service=~\"$service\"}[$__rate_interval]) / rate(reth_morph_payload_builder_commit_tx_apply_duration_seconds_count{role=~\"$role\", chain=\"$chain\", service=~\"$service\"}[$__rate_interval]) > 0", - "legendFormat": "{{service}} avg" - }, - { - "expr": "reth_morph_payload_builder_commit_tx_apply_duration_seconds{role=~\"$role\", chain=\"$chain\", service=~\"$service\", quantile=\"$quantile\"} > 0", - "legendFormat": "{{service}} p{{quantile}}" + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_consensus_engine_beacon_new_payload_forkchoice_updated_time_diff{chain=\"$chain\", role=~\"$role\", service=~\"$service\", quantile=\"$quantile\"}", + "instant": false, + "legendFormat": "{{service}} \u00b7 {{role}} \u00b7 p{{quantile}}", + "range": true, + "refId": "A" } - ] + ], + "title": "Engine Tree newPayload to FCU Gap", + "type": "timeseries", + "description": "Time from a newPayload response to the next forkchoiceUpdated request." }, { "datasource": { @@ -1462,30 +1497,30 @@ "mode": "palette-classic-by-name" }, "custom": { + "axisCenteredZero": false, "drawStyle": "line", "fillOpacity": 0, "lineWidth": 1, "pointSize": 5, "showPoints": "auto", - "spanNulls": false, - "axisCenteredZero": false + "spanNulls": false }, - "unit": "short", - "min": 0 + "min": 0, + "unit": "s" }, "overrides": [] }, "gridPos": { - "h": 9, + "h": 8, "w": 12, "x": 12, - "y": 71 + "y": 69 }, - "id": 20, + "id": 303, "options": { "legend": { "displayMode": "list", - "placement": "right", + "placement": "bottom", "showLegend": true }, "tooltip": { @@ -1493,27 +1528,23 @@ "sort": "none" } }, - "title": "Block Transactions", - "type": "timeseries", "targets": [ { - "expr": "reth_morph_payload_builder_block_transactions{role=~\"$role\", chain=\"$chain\", service=~\"$service\"} > 0", - "legendFormat": "{{service}}" + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_consensus_engine_beacon_new_payload_latency{chain=\"$chain\", role=~\"$role\", service=~\"$service\", quantile=\"$quantile\"}", + "instant": false, + "legendFormat": "{{service}} \u00b7 {{role}} \u00b7 p{{quantile}}", + "range": true, + "refId": "A" } - ] - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 80 - }, - "id": 23, - "panels": [], - "title": "Execution & State Root", - "type": "row" + ], + "title": "Engine Tree newPayload Processing Latency", + "type": "timeseries", + "description": "Internal engine-tree processing latency for newly executed valid newPayload calls; early errors and already-seen blocks are excluded." }, { "datasource": { @@ -1526,55 +1557,16 @@ "mode": "palette-classic-by-name" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 24, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", + "fillOpacity": 0, "lineWidth": 1, "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] + "spanNulls": false }, - "unit": "s", - "min": 0 + "min": 0, + "unit": "s" }, "overrides": [] }, @@ -1582,200 +1574,106 @@ "h": 8, "w": 12, "x": 0, - "y": 81 + "y": 77 }, - "id": 24, + "id": 304, "options": { "legend": { - "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { - "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "12.2.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_sync_block_validation_state_root_duration{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{service}} · State Root Duration", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_sync_execution_execution_duration{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, + "editorMode": "code", + "expr": "reth_consensus_engine_beacon_forkchoice_updated_latency{chain=\"$chain\", role=~\"$role\", service=~\"$service\", quantile=\"$quantile\"}", "instant": false, - "legendFormat": "{{service}} · Execution Duration", + "legendFormat": "{{service}} \u00b7 {{role}} \u00b7 p{{quantile}}", "range": true, - "refId": "B", - "useBackend": false + "refId": "A" } ], - "title": "Block Processing Latency", - "type": "timeseries" + "title": "Engine Tree FCU Processing Latency", + "type": "timeseries", + "description": "Internal engine-tree processing latency for forkchoiceUpdated." }, { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 85 }, - "fieldConfig": { + "id": 16, + "panels": [], + "title": "Payload Builder", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { "defaults": { "color": { "mode": "palette-classic-by-name" }, "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, "showPoints": "auto", - "showValues": false, "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] + "axisCenteredZero": false }, - "unit": "percentunit", + "unit": "s", "min": 0 }, "overrides": [] }, "gridPos": { - "h": 8, + "h": 9, "w": 12, - "x": 12, - "y": 81 + "x": 0, + "y": 86 }, - "id": 25, + "id": 17, "options": { "legend": { - "calcs": [], "displayMode": "list", - "placement": "bottom", + "placement": "right", "showLegend": true }, "tooltip": { - "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "12.2.1", + "title": "Payload Build Duration", + "description": "End-to-end payload build duration (state setup + tx execution + finalize + seal).", + "type": "timeseries", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "reth_sync_caching_account_cache_hits{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} / (reth_sync_caching_account_cache_hits{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} + reth_sync_caching_account_cache_misses{chain=\"$chain\", role=~\"$role\", service=~\"$service\"})", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{service}} · Account cache hits", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "reth_sync_caching_storage_cache_hits{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} / (reth_sync_caching_storage_cache_hits{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} + reth_sync_caching_storage_cache_misses{chain=\"$chain\", role=~\"$role\", service=~\"$service\"})", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{service}} · Storage cache hits", - "range": true, - "refId": "B", - "useBackend": false + "expr": "rate(reth_morph_payload_builder_payload_build_duration_seconds_sum{role=~\"$role\", chain=\"$chain\", service=~\"$service\"}[$__rate_interval]) / rate(reth_morph_payload_builder_payload_build_duration_seconds_count{role=~\"$role\", chain=\"$chain\", service=~\"$service\"}[$__rate_interval]) > 0", + "legendFormat": "{{service}} avg" }, { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "reth_sync_caching_code_cache_hits{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} / (reth_sync_caching_code_cache_hits{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} + reth_sync_caching_code_cache_misses{chain=\"$chain\", role=~\"$role\", service=~\"$service\"})", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{service}} · Code cache hits", - "range": true, - "refId": "C", - "useBackend": false + "expr": "reth_morph_payload_builder_payload_build_duration_seconds{role=~\"$role\", chain=\"$chain\", service=~\"$service\", quantile=\"$quantile\"} > 0", + "legendFormat": "{{service}} p{{quantile}}" } - ], - "title": "Execution cache hitrate", - "type": "timeseries" + ] }, { "datasource": { @@ -1788,126 +1686,50 @@ "mode": "palette-classic-by-name" }, "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, "showPoints": "auto", - "showValues": false, "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 1, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] + "axisCenteredZero": false }, - "unit": "percentunit", + "unit": "s", "min": 0 }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "Precompile cache hits" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": true, - "viz": true - } - } - ] - } - ] + "overrides": [] }, "gridPos": { - "h": 8, + "h": 9, "w": 12, "x": 12, - "y": 89 + "y": 86 }, - "id": 27, + "id": 18, "options": { "legend": { - "calcs": [], "displayMode": "list", - "placement": "bottom", + "placement": "right", "showLegend": true }, "tooltip": { - "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "12.2.1", + "title": "Tx Execution Duration", + "description": "Total time to execute all transactions (L1 messages + pool) for one block.", + "type": "timeseries", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "disableTextWrap": false, - "editorMode": "code", - "expr": "reth_sync_caching_precompile_cache_hits{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} / (reth_sync_caching_precompile_cache_hits{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} + reth_sync_caching_precompile_cache_misses{chain=\"$chain\", role=~\"$role\", service=~\"$service\"})", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{service}} · Precompile cache hits", - "range": true, - "refId": "A", - "useBackend": false + "expr": "rate(reth_morph_payload_builder_commit_txs_all_duration_seconds_sum{role=~\"$role\", chain=\"$chain\", service=~\"$service\"}[$__rate_interval]) / rate(reth_morph_payload_builder_commit_txs_all_duration_seconds_count{role=~\"$role\", chain=\"$chain\", service=~\"$service\"}[$__rate_interval]) > 0", + "legendFormat": "{{service}} avg" + }, + { + "expr": "reth_morph_payload_builder_commit_txs_all_duration_seconds{role=~\"$role\", chain=\"$chain\", service=~\"$service\", quantile=\"$quantile\"} > 0", + "legendFormat": "{{service}} p{{quantile}}" } - ], - "title": "Precompile cache hitrate", - "type": "timeseries" + ] }, { "datasource": { @@ -1920,79 +1742,144 @@ "mode": "palette-classic-by-name" }, "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, "showPoints": "auto", - "showValues": false, "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "axisCenteredZero": false }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 95 + }, + "id": 19, + "options": { + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "title": "Per-Tx Apply Duration", + "description": "Time for the EVM execution of a single transaction.", + "type": "timeseries", + "targets": [ + { + "expr": "rate(reth_morph_payload_builder_commit_tx_apply_duration_seconds_sum{role=~\"$role\", chain=\"$chain\", service=~\"$service\"}[$__rate_interval]) / rate(reth_morph_payload_builder_commit_tx_apply_duration_seconds_count{role=~\"$role\", chain=\"$chain\", service=~\"$service\"}[$__rate_interval]) > 0", + "legendFormat": "{{service}} avg" + }, + { + "expr": "reth_morph_payload_builder_commit_tx_apply_duration_seconds{role=~\"$role\", chain=\"$chain\", service=~\"$service\", quantile=\"$quantile\"} > 0", + "legendFormat": "{{service}} p{{quantile}}" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" }, - "unit": "none", + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false, + "axisCenteredZero": false + }, + "unit": "short", "min": 0 }, "overrides": [] }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 95 + }, + "id": 20, + "options": { + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "title": "Block Transactions", + "type": "timeseries", + "targets": [ + { + "expr": "reth_morph_payload_builder_block_transactions{role=~\"$role\", chain=\"$chain\", service=~\"$service\"} > 0", + "legendFormat": "{{service}}" + } + ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisCenteredZero": false, + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false + }, + "min": 0, + "unit": "short" + }, + "overrides": [] + }, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 89 + "y": 104 }, - "id": 32, + "id": 305, "options": { "legend": { - "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { - "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "12.2.1", "targets": [ { "datasource": { @@ -2000,15 +1887,16 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "reth_sparse_state_trie_multiproof_total_account_nodes{chain=\"$chain\", role=~\"$role\", service=~\"$service\",quantile=~\"(0|0.5|0.9|0.95|1)\"}", + "expr": "reth_payloads_active_jobs{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", "instant": false, - "legendFormat": "{{service}} · Account {{quantile}} percentile", + "legendFormat": "{{service}}", "range": true, - "refId": "Branch Nodes" + "refId": "A" } ], - "title": "Total multiproof account nodes", - "type": "timeseries" + "title": "Active Jobs", + "type": "timeseries", + "description": "Number of active standard reth payload jobs." }, { "datasource": { @@ -2021,55 +1909,76 @@ "mode": "palette-classic-by-name" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "spanNulls": false }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 80 - } - ] + "min": 0, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 104 + }, + "id": 306, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" }, - "unit": "none", - "min": 0 + "editorMode": "code", + "expr": "reth_payloads_initiated_jobs{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "instant": false, + "legendFormat": "{{service}}", + "range": true, + "refId": "A" + } + ], + "title": "Initiated Jobs", + "type": "timeseries", + "description": "Total number of standard reth payload jobs initiated." + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisCenteredZero": false, + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false + }, + "min": 0, + "unit": "short" }, "overrides": [] }, @@ -2077,23 +1986,20 @@ "h": 8, "w": 12, "x": 0, - "y": 97 + "y": 112 }, - "id": 33, + "id": 307, "options": { "legend": { - "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { - "hideZeros": false, "mode": "multi", "sort": "none" } }, - "pluginVersion": "12.2.1", "targets": [ { "datasource": { @@ -2101,22 +2007,35 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "reth_sparse_state_trie_multiproof_total_storage_nodes{chain=\"$chain\", role=~\"$role\", service=~\"$service\",quantile=~\"(0|0.5|0.9|0.95|1)\"}", + "expr": "reth_payloads_failed_jobs{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", "instant": false, - "legendFormat": "{{service}} · Storage {{quantile}} percentile", + "legendFormat": "{{service}}", "range": true, - "refId": "Branch Nodes" + "refId": "A" } ], - "title": "Total multiproof storage nodes", - "type": "timeseries" + "title": "Failed Jobs", + "type": "timeseries", + "description": "Total number of standard reth payload jobs that failed." + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 120 + }, + "id": 23, + "panels": [], + "title": "Execution & State Root", + "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "description": "Histogram for state root latency, the time spent blocked waiting for the state root.", "fieldConfig": { "defaults": { "color": { @@ -2131,7 +2050,7 @@ "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 24, "gradientMode": "none", "hideFrom": { "legend": false, @@ -2178,10 +2097,10 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 97 + "x": 0, + "y": 121 }, - "id": 37, + "id": 24, "options": { "legend": { "calcs": [], @@ -2202,20 +2121,2376 @@ "type": "prometheus", "uid": "${datasource}" }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "reth_sync_block_validation_state_root_histogram{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{service}} · State Root Duration p{{quantile}}", - "range": true, - "refId": "A", - "useBackend": false + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_sync_block_validation_state_root_duration{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} \u00b7 State Root Duration", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "reth_sync_execution_execution_duration{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} \u00b7 Execution Duration", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Block Processing Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 121 + }, + "id": 25, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "reth_sync_caching_account_cache_hits{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} / (reth_sync_caching_account_cache_hits{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} + reth_sync_caching_account_cache_misses{chain=\"$chain\", role=~\"$role\", service=~\"$service\"})", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} \u00b7 Account cache hits", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "reth_sync_caching_storage_cache_hits{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} / (reth_sync_caching_storage_cache_hits{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} + reth_sync_caching_storage_cache_misses{chain=\"$chain\", role=~\"$role\", service=~\"$service\"})", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} \u00b7 Storage cache hits", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "reth_sync_caching_code_cache_hits{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} / (reth_sync_caching_code_cache_hits{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} + reth_sync_caching_code_cache_misses{chain=\"$chain\", role=~\"$role\", service=~\"$service\"})", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} \u00b7 Code cache hits", + "range": true, + "refId": "C", + "useBackend": false + } + ], + "title": "Execution cache hitrate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit", + "min": 0 + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Precompile cache hits" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": true, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 129 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "reth_sync_caching_precompile_cache_hits{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} / (reth_sync_caching_precompile_cache_hits{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} + reth_sync_caching_precompile_cache_misses{chain=\"$chain\", role=~\"$role\", service=~\"$service\"})", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} \u00b7 Precompile cache hits", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Precompile cache hitrate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 129 + }, + "id": 32, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_sparse_state_trie_multiproof_total_account_nodes{chain=\"$chain\", role=~\"$role\", service=~\"$service\",quantile=~\"(0|0.5|0.9|0.95|1)\"}", + "instant": false, + "legendFormat": "{{service}} \u00b7 Account {{quantile}} percentile", + "range": true, + "refId": "Branch Nodes" + } + ], + "title": "Total multiproof account nodes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 137 + }, + "id": 33, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_sparse_state_trie_multiproof_total_storage_nodes{chain=\"$chain\", role=~\"$role\", service=~\"$service\",quantile=~\"(0|0.5|0.9|0.95|1)\"}", + "instant": false, + "legendFormat": "{{service}} \u00b7 Storage {{quantile}} percentile", + "range": true, + "refId": "Branch Nodes" + } + ], + "title": "Total multiproof storage nodes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Histogram for state root latency, the time spent blocked waiting for the state root.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 137 + }, + "id": 37, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "reth_sync_block_validation_state_root_histogram{chain=\"$chain\", role=~\"$role\", service=~\"$service\", quantile=\"$quantile\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{service}} \u00b7 {{role}} \u00b7 p{{quantile}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "State root latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisCenteredZero": false, + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false + }, + "min": 0, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 145 + }, + "id": 308, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_tree_root_sparse_trie_account_cache_hits_sum{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} / (reth_tree_root_sparse_trie_account_cache_hits_sum{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} + reth_tree_root_sparse_trie_account_cache_misses_sum{chain=\"$chain\", role=~\"$role\", service=~\"$service\"})", + "instant": false, + "legendFormat": "{{service}} \u00b7 {{role}} \u00b7 hit %", + "range": true, + "refId": "A" + } + ], + "title": "Account Trie Cache Hit %", + "type": "timeseries", + "description": "Process-lifetime percent of account leaf updates served without requiring proofs." + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisCenteredZero": false, + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false + }, + "min": 0, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 145 + }, + "id": 309, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_tree_root_sparse_trie_storage_cache_hits_sum{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} / (reth_tree_root_sparse_trie_storage_cache_hits_sum{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} + reth_tree_root_sparse_trie_storage_cache_misses_sum{chain=\"$chain\", role=~\"$role\", service=~\"$service\"})", + "instant": false, + "legendFormat": "{{service}} \u00b7 {{role}} \u00b7 hit %", + "range": true, + "refId": "A" + } + ], + "title": "Storage Trie Cache Hit %", + "type": "timeseries", + "description": "Process-lifetime percent of storage leaf updates served without requiring proofs." + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisCenteredZero": false, + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false + }, + "min": 0, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 153 + }, + "id": 310, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_trie_proof_task_account_worker_idle_time_seconds{chain=\"$chain\", role=~\"$role\", service=~\"$service\", quantile=\"$quantile\"}", + "instant": false, + "legendFormat": "{{service}} \u00b7 {{role}} \u00b7 p{{quantile}}", + "range": true, + "refId": "A" + } + ], + "title": "Account Proof Worker Idle Time", + "type": "timeseries", + "description": "Time account proof workers spend idle waiting for messages." + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisCenteredZero": false, + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false + }, + "min": 0, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 153 + }, + "id": 311, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_trie_proof_task_storage_worker_idle_time_seconds{chain=\"$chain\", role=~\"$role\", service=~\"$service\", quantile=\"$quantile\"}", + "instant": false, + "legendFormat": "{{service}} \u00b7 {{role}} \u00b7 p{{quantile}}", + "range": true, + "refId": "A" + } + ], + "title": "Storage Proof Worker Idle Time", + "type": "timeseries", + "description": "Time storage proof workers spend idle waiting for messages." + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisCenteredZero": false, + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false + }, + "min": 0, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 153 + }, + "id": 312, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_tree_root_sparse_trie_idle_time_seconds{chain=\"$chain\", role=~\"$role\", service=~\"$service\", quantile=\"$quantile\"}", + "instant": false, + "legendFormat": "{{service}} \u00b7 {{role}} \u00b7 p{{quantile}}", + "range": true, + "refId": "A" + } + ], + "title": "Sparse Trie Task Idle Time", + "type": "timeseries", + "description": "Time the sparse trie task spends idle waiting for messages." + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisCenteredZero": false, + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false + }, + "min": 0, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 153 + }, + "id": 313, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_tree_root_hashing_task_idle_time_seconds{chain=\"$chain\", role=~\"$role\", service=~\"$service\", quantile=\"$quantile\"}", + "instant": false, + "legendFormat": "{{service}} \u00b7 {{role}} \u00b7 p{{quantile}}", + "range": true, + "refId": "A" + } + ], + "title": "Hashing Task Idle Time", + "type": "timeseries", + "description": "Time the hashing task spends idle waiting for execution messages." + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisCenteredZero": false, + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false + }, + "min": 0, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 161 + }, + "id": 314, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (service) (rate(reth_sync_block_validation_state_root_task_timeout_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{service}} \u00b7 timeout", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (service) (rate(reth_sync_block_validation_state_root_task_fallback_success_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{service}} \u00b7 fallback success", + "range": true, + "refId": "B" + } + ], + "title": "State Root Task Fallback Outcomes", + "type": "timeseries", + "description": "Rates of state-root task timeouts and successful sequential fallbacks." + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisCenteredZero": false, + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false + }, + "min": 0, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 161 + }, + "id": 315, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_sync_block_validation_deferred_trie_compute_duration{chain=\"$chain\", role=~\"$role\", service=~\"$service\", quantile=\"$quantile\"}", + "instant": false, + "legendFormat": "{{service}} \u00b7 {{role}} \u00b7 p{{quantile}}", + "range": true, + "refId": "A" + } + ], + "title": "Block Validation Deferred Trie Compute Duration", + "type": "timeseries", + "description": "Duration of deferred trie computation performed as block-validation overhead; this uses the v2.4 deferred_trie_compute_duration metric." + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 169 + }, + "id": 316, + "panels": [], + "title": "Proof History", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisCenteredZero": false, + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false + }, + "min": 0, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 170 + }, + "id": 317, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_morph_proofs_earliest_block{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "instant": false, + "legendFormat": "{{service}} \u00b7 earliest", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_morph_proofs_latest_block{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "instant": false, + "legendFormat": "{{service}} \u00b7 latest", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_morph_proofs_latest_block{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} - reth_morph_proofs_earliest_block{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} + 1", + "instant": false, + "legendFormat": "{{service}} \u00b7 inclusive window", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "clamp_min(reth_blockchain_tree_canonical_chain_height{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} - reth_morph_proofs_latest_block{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}, 0)", + "instant": false, + "legendFormat": "{{service}} \u00b7 canonical lag", + "range": true, + "refId": "D" + } + ], + "title": "Proof History Window Health", + "type": "timeseries", + "description": "Durable proof window bounds, inclusive size (latest - earliest + 1), and lag behind the canonical chain." + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisCenteredZero": false, + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false + }, + "min": 0, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 178 + }, + "id": 318, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_morph_proofs_block_total_duration_seconds{chain=\"$chain\", role=~\"$role\", service=~\"$service\", quantile=\"$quantile\"}", + "instant": false, + "legendFormat": "{{service}} \u00b7 total p{{quantile}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_morph_proofs_block_execution_duration_seconds{chain=\"$chain\", role=~\"$role\", service=~\"$service\", quantile=\"$quantile\"}", + "instant": false, + "legendFormat": "{{service}} \u00b7 execution p{{quantile}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_morph_proofs_block_state_root_duration_seconds{chain=\"$chain\", role=~\"$role\", service=~\"$service\", quantile=\"$quantile\"}", + "instant": false, + "legendFormat": "{{service}} \u00b7 state root p{{quantile}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_morph_proofs_block_write_duration_seconds{chain=\"$chain\", role=~\"$role\", service=~\"$service\", quantile=\"$quantile\"}", + "instant": false, + "legendFormat": "{{service}} \u00b7 write p{{quantile}}", + "range": true, + "refId": "D" + } + ], + "title": "Proof History Ingest Operation Duration", + "type": "timeseries", + "description": "Live-path observations are single-block samples. During catch-up, total duration can represent an entire batch while execution, state-root, and write are recorded as zero placeholders; do not interpret catch-up samples as per-block or per-stage latency." + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisCenteredZero": false, + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false + }, + "min": 0, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 186 + }, + "id": 319, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(reth_morph_proofs_account_trie_updates_written_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "instant": false, + "legendFormat": "{{service}} \u00b7 account trie updates", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(reth_morph_proofs_storage_trie_updates_written_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "instant": false, + "legendFormat": "{{service}} \u00b7 storage trie updates", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(reth_morph_proofs_hashed_accounts_written_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "instant": false, + "legendFormat": "{{service}} \u00b7 hashed accounts", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(reth_morph_proofs_hashed_storages_written_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "instant": false, + "legendFormat": "{{service}} \u00b7 hashed storages", + "range": true, + "refId": "D" + } + ], + "title": "Proof History Write Volume", + "type": "timeseries", + "description": "Rates of proof-history trie updates and hashed leaves written." + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisCenteredZero": false, + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false + }, + "min": 0, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 186 + }, + "id": 320, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(reth_morph_proofs_prune_errors_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "instant": false, + "legendFormat": "{{service}} \u00b7 prune", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(reth_morph_proofs_unwind_errors_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "instant": false, + "legendFormat": "{{service}} \u00b7 unwind", + "range": true, + "refId": "B" + } + ], + "title": "Proof History Errors", + "type": "timeseries", + "description": "Rates of proof-history prune and unwind failures." + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisCenteredZero": false, + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false + }, + "min": 0, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 194 + }, + "id": 321, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (service, method) (rate(reth_morph_rpc_proofs_requests_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{service}} \u00b7 {{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Proof RPC Request Rate by Method", + "type": "timeseries", + "description": "Proof RPC requests received, split by eth_getProof and eth_getMultiProof." + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisCenteredZero": false, + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false + }, + "min": 0, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 194 + }, + "id": 322, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (service, method) (rate(reth_morph_rpc_proofs_successful_responses_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{service}} \u00b7 {{method}} success", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (service, method) (rate(reth_morph_rpc_proofs_failures_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{service}} \u00b7 {{method}} failure", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (service, method) (rate(reth_morph_rpc_proofs_rejected_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{service}} \u00b7 {{method}} rejected", + "range": true, + "refId": "C" + } + ], + "title": "Proof RPC Outcomes by Method", + "type": "timeseries", + "description": "Successful, failed, and size-limit-rejected proof RPC rates by method." + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisCenteredZero": false, + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false + }, + "min": 0, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 202 + }, + "id": 323, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_morph_rpc_proofs_latency_seconds{chain=\"$chain\", role=~\"$role\", service=~\"$service\", quantile=\"$quantile\"}", + "instant": false, + "legendFormat": "{{service}} \u00b7 {{role}} \u00b7 {{method}} p{{quantile}}", + "range": true, + "refId": "A" + } + ], + "title": "Proof RPC Successful Latency by Method", + "type": "timeseries", + "description": "Latency histogram records successful proof RPC responses only; failures and rejections are excluded." + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisCenteredZero": false, + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false + }, + "min": 0, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 202 + }, + "id": 324, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_morph_proofs_pruner_total_duration_seconds{chain=\"$chain\", role=~\"$role\", service=~\"$service\", quantile=\"$quantile\"}", + "instant": false, + "legendFormat": "{{service}} \u00b7 {{role}} \u00b7 p{{quantile}}", + "range": true, + "refId": "A" + } + ], + "title": "Proof Pruner Batch Duration", + "type": "timeseries", + "description": "Duration of non-empty proof-history prune batches." + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisCenteredZero": false, + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false + }, + "min": 0, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 210 + }, + "id": 325, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_morph_proofs_pruner_pruned_blocks{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "instant": false, + "legendFormat": "{{service}} \u00b7 pruned blocks", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_morph_proofs_pruner_account_trie_updates_written{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "instant": false, + "legendFormat": "{{service}} \u00b7 account trie updates", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_morph_proofs_pruner_storage_trie_updates_written{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "instant": false, + "legendFormat": "{{service}} \u00b7 storage trie updates", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_morph_proofs_pruner_hashed_accounts_written{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "instant": false, + "legendFormat": "{{service}} \u00b7 hashed accounts", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_morph_proofs_pruner_hashed_storages_written{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "instant": false, + "legendFormat": "{{service}} \u00b7 hashed storages", + "range": true, + "refId": "E" + } + ], + "title": "Latest Proof Prune Batch Stats", + "type": "timeseries", + "description": "Gauges describe the latest completed proof-prune batch; they are not cumulative totals." + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 218 + }, + "id": 326, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisCenteredZero": false, + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false + }, + "min": 0, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 219 + }, + "id": 327, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_exex_notifications_sent_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "instant": false, + "legendFormat": "{{service}} \u00b7 {{exex}}", + "range": true, + "refId": "A" + } + ], + "title": "Total Notifications Sent", + "type": "timeseries", + "description": "Proof History runtime when its execution extension and write-ahead log are enabled. Total canonical-state notifications sent to the ExEx." + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisCenteredZero": false, + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false + }, + "min": 0, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 219 + }, + "id": 328, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_exex_events_sent_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "instant": false, + "legendFormat": "{{service}} \u00b7 {{exex}}", + "range": true, + "refId": "A" + } + ], + "title": "Total Events Sent", + "type": "timeseries", + "description": "Proof History runtime when its execution extension and write-ahead log are enabled. Total events sent from the ExEx to its manager." + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisCenteredZero": false, + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false + }, + "min": 0, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 227 + }, + "id": 329, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_exex_manager_current_capacity{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "instant": false, + "legendFormat": "{{service}} \u00b7 current", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max_over_time(reth_exex_manager_max_capacity{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[1h])", + "instant": false, + "legendFormat": "{{service}} \u00b7 max", + "range": true, + "refId": "B" + } + ], + "title": "Current and Max Capacity", + "type": "timeseries", + "description": "Proof History runtime when its execution extension and write-ahead log are enabled. Current and maximum notification-buffer capacity." + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisCenteredZero": false, + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false + }, + "min": 0, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 227 + }, + "id": 330, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_exex_manager_buffer_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "instant": false, + "legendFormat": "{{service}}", + "range": true, + "refId": "A" + } + ], + "title": "Buffer Size", + "type": "timeseries", + "description": "Proof History runtime when its execution extension and write-ahead log are enabled. Current number of buffered state notifications." + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 235 + }, + "id": 331, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_exex_manager_num_exexs{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "instant": false, + "legendFormat": "{{service}}", + "range": true, + "refId": "A" + } + ], + "title": "Number of ExExes", + "type": "stat", + "description": "Proof History runtime when its execution extension and write-ahead log are enabled. Number of execution extensions installed in the node." + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisCenteredZero": false, + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false + }, + "min": 0, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 235 + }, + "id": 332, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_exex_wal_lowest_committed_block_height{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} and on (chain, role, service, instance) (reth_exex_wal_committed_blocks_count{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0)", + "instant": false, + "legendFormat": "{{service}} \u00b7 lowest", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_exex_wal_highest_committed_block_height{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} and on (chain, role, service, instance) (reth_exex_wal_committed_blocks_count{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0)", + "instant": false, + "legendFormat": "{{service}} \u00b7 highest", + "range": true, + "refId": "B" + } + ], + "title": "Current Committed Block Heights", + "type": "timeseries", + "description": "Proof History runtime when its execution extension and write-ahead log are enabled. Lowest and highest committed block heights retained by the WAL." + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisCenteredZero": false, + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false + }, + "min": 0, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 243 + }, + "id": 333, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_exex_wal_committed_blocks_count{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "instant": false, + "legendFormat": "{{service}} \u00b7 committed blocks", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_exex_wal_notifications_count{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "instant": false, + "legendFormat": "{{service}} \u00b7 notifications", + "range": true, + "refId": "B" + } + ], + "title": "Number of entities", + "type": "timeseries", + "description": "Proof History runtime when its execution extension and write-ahead log are enabled. Current committed-block and notification entity counts in the WAL." + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisCenteredZero": false, + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false + }, + "min": 0, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 243 + }, + "id": 334, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_exex_wal_size_bytes{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "instant": false, + "legendFormat": "{{service}}", + "range": true, + "refId": "A" + } + ], + "title": "Total size of all notifications", + "type": "timeseries", + "description": "Proof History runtime when its execution extension and write-ahead log are enabled. Total size of all notifications retained in the WAL." } ], - "title": "State root latency", - "type": "timeseries" + "title": "Execution Extension Runtime", + "type": "row", + "description": "Proof History runtime when its execution extension and write-ahead log are enabled." }, { "collapsed": true, @@ -2223,7 +4498,7 @@ "h": 1, "w": 24, "x": 0, - "y": 105 + "y": 219 }, "id": 38, "panels": [ @@ -2292,7 +4567,7 @@ "h": 8, "w": 12, "x": 0, - "y": 138 + "y": 252 }, "id": 39, "options": { @@ -2320,7 +4595,7 @@ "expr": "avg by (service) (rate(reth_database_transaction_close_duration_seconds_sum{chain=\"$chain\", role=~\"$role\", service=~\"$service\", outcome=\"commit\"}[$__rate_interval]) / clamp_min(rate(reth_database_transaction_close_duration_seconds_count{chain=\"$chain\", role=~\"$role\", service=~\"$service\", outcome=\"commit\"}[$__rate_interval]), 1))", "format": "time_series", "instant": false, - "legendFormat": "{{service}} · Commit time", + "legendFormat": "{{service}} \u00b7 Commit time", "range": true, "refId": "A" } @@ -2353,7 +4628,7 @@ "h": 8, "w": 12, "x": 12, - "y": 138 + "y": 252 }, "id": 40, "maxDataPoints": 25, @@ -2377,7 +4652,7 @@ "color": "rgba(255,0,255,0.7)" }, "filterValues": { - "le": 1E-9 + "le": 1e-09 }, "legend": { "show": true @@ -2411,7 +4686,7 @@ "expr": "avg by (service, quantile) (max_over_time(reth_database_transaction_close_duration_seconds{chain=\"$chain\", role=~\"$role\", service=~\"$service\", outcome=\"commit\"}[$__rate_interval]))", "format": "time_series", "instant": false, - "legendFormat": "{{service}} · {{quantile}}", + "legendFormat": "{{service}} \u00b7 {{quantile}}", "range": true, "refId": "A" } @@ -2484,7 +4759,7 @@ "h": 8, "w": 12, "x": 0, - "y": 146 + "y": 260 }, "id": 41, "options": { @@ -2512,7 +4787,7 @@ "expr": "sum by (service, outcome, mode) (rate(reth_database_transaction_open_duration_seconds_sum{chain=\"$chain\", role=~\"$role\", service=~\"$service\", outcome!=\"\"}[$__rate_interval]) / rate(reth_database_transaction_open_duration_seconds_count{chain=\"$chain\", role=~\"$role\", service=~\"$service\", outcome!=\"\"}[$__rate_interval])) > 0", "format": "time_series", "instant": false, - "legendFormat": "{{service}} · {{mode}}, {{outcome}}", + "legendFormat": "{{service}} \u00b7 {{mode}}, {{outcome}}", "range": true, "refId": "A" } @@ -2584,7 +4859,7 @@ "h": 8, "w": 12, "x": 12, - "y": 146 + "y": 260 }, "id": 42, "options": { @@ -2612,7 +4887,7 @@ "expr": "max by (service, outcome, mode) (max_over_time(reth_database_transaction_open_duration_seconds{chain=\"$chain\", role=~\"$role\", service=~\"$service\", outcome!=\"\", quantile=\"1\"}[$__interval])) > 0", "format": "time_series", "instant": false, - "legendFormat": "{{service}} · {{mode}}, {{outcome}}", + "legendFormat": "{{service}} \u00b7 {{mode}}, {{outcome}}", "range": true, "refId": "A" } @@ -2712,7 +4987,7 @@ }, { "id": "displayName", - "value": "${__field.labels.service} · Diff (opened − closed)" + "value": "${__field.labels.service} \u00b7 Diff (opened \u2212 closed)" } ] } @@ -2722,7 +4997,7 @@ "h": 8, "w": 12, "x": 0, - "y": 154 + "y": 268 }, "id": 43, "options": { @@ -2753,7 +5028,7 @@ "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{service}} · Opened", + "legendFormat": "{{service}} \u00b7 Opened", "range": true, "refId": "A", "useBackend": false @@ -2768,7 +5043,7 @@ "expr": "sum by (service) (reth_database_transaction_closed_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\", mode=\"read-write\"})", "format": "time_series", "instant": false, - "legendFormat": "{{service}} · Closed", + "legendFormat": "{{service}} \u00b7 Closed", "range": true, "refId": "B" }, @@ -2878,7 +5153,7 @@ }, { "id": "displayName", - "value": "${__field.labels.service} · Diff (opened − closed)" + "value": "${__field.labels.service} \u00b7 Diff (opened \u2212 closed)" } ] } @@ -2888,7 +5163,7 @@ "h": 8, "w": 12, "x": 12, - "y": 154 + "y": 268 }, "id": 44, "options": { @@ -2919,7 +5194,7 @@ "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{service}} · Opened", + "legendFormat": "{{service}} \u00b7 Opened", "range": true, "refId": "A", "useBackend": false @@ -2937,7 +5212,7 @@ "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{service}} · Closed", + "legendFormat": "{{service}} \u00b7 Closed", "range": true, "refId": "B", "useBackend": false @@ -2983,7 +5258,7 @@ "h": 8, "w": 12, "x": 0, - "y": 162 + "y": 276 }, "id": 45, "options": { @@ -3023,7 +5298,7 @@ "editorMode": "builder", "expr": "reth_db_table_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", "interval": "", - "legendFormat": "{{service}} · {{table}}", + "legendFormat": "{{service}} \u00b7 {{table}}", "range": true, "refId": "A" } @@ -3099,7 +5374,7 @@ "h": 8, "w": 12, "x": 12, - "y": 162 + "y": 276 }, "id": 46, "options": { @@ -3127,7 +5402,7 @@ "expr": "max(max_over_time(reth_database_operation_large_value_duration_seconds{chain=\"$chain\", role=~\"$role\", service=~\"$service\", quantile=\"1\"}[$__interval])) by (service, table) > 0", "format": "time_series", "instant": false, - "legendFormat": "{{service}} · {{table}}", + "legendFormat": "{{service}} \u00b7 {{table}}", "range": true, "refId": "A" } @@ -3162,7 +5437,7 @@ "h": 8, "w": 12, "x": 0, - "y": 170 + "y": 284 }, "id": 47, "options": { @@ -3198,7 +5473,7 @@ }, "editorMode": "builder", "expr": "sum by (service, type) ( reth_db_table_pages{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} )", - "legendFormat": "{{service}} · {{type}}", + "legendFormat": "{{service}} \u00b7 {{type}}", "range": true, "refId": "A" } @@ -3275,7 +5550,7 @@ "h": 8, "w": 12, "x": 12, - "y": 170 + "y": 284 }, "id": 48, "options": { @@ -3376,7 +5651,7 @@ "h": 8, "w": 12, "x": 0, - "y": 178 + "y": 292 }, "id": 49, "options": { @@ -3401,7 +5676,7 @@ }, "editorMode": "code", "expr": "sum(reth_db_freelist{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}) by (service, job)", - "legendFormat": "{{service}} · Pages ({{job}})", + "legendFormat": "{{service}} \u00b7 Pages ({{job}})", "range": true, "refId": "A" } @@ -3539,7 +5814,7 @@ "h": 8, "w": 12, "x": 12, - "y": 178 + "y": 292 }, "id": 50, "options": { @@ -3613,7 +5888,7 @@ "h": 8, "w": 8, "x": 0, - "y": 186 + "y": 300 }, "id": 51, "options": { @@ -3653,7 +5928,7 @@ "editorMode": "code", "expr": "reth_static_files_segment_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", "interval": "", - "legendFormat": "{{service}} · {{segment}}", + "legendFormat": "{{service}} \u00b7 {{segment}}", "range": true, "refId": "A" } @@ -3779,7 +6054,7 @@ "h": 8, "w": 8, "x": 8, - "y": 186 + "y": 300 }, "id": 52, "options": { @@ -3944,7 +6219,7 @@ "h": 8, "w": 8, "x": 16, - "y": 186 + "y": 300 }, "id": 53, "options": { @@ -4059,7 +6334,7 @@ "h": 8, "w": 12, "x": 0, - "y": 194 + "y": 308 }, "id": 54, "options": { @@ -4160,7 +6435,7 @@ "h": 8, "w": 12, "x": 12, - "y": 194 + "y": 308 }, "id": 55, "options": { @@ -4185,7 +6460,7 @@ }, "editorMode": "code", "expr": "max by (service, segment) (max_over_time(reth_static_files_jar_provider_write_duration_seconds{chain=\"$chain\", role=~\"$role\", service=~\"$service\", operation=\"commit-writer\", quantile=\"1\"}[$__interval])) or (0 * max by (service, segment) (reth_static_files_segment_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}))", - "legendFormat": "{{service}} · {{segment}}", + "legendFormat": "{{service}} \u00b7 {{segment}}", "range": true, "refId": "A" } @@ -4260,7 +6535,7 @@ "h": 8, "w": 12, "x": 0, - "y": 202 + "y": 316 }, "id": 56, "options": { @@ -4286,7 +6561,7 @@ "editorMode": "builder", "expr": "reth_blockchain_tree_canonical_chain_height{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", "hide": false, - "legendFormat": "{{service}} · Canonical chain height", + "legendFormat": "{{service}} \u00b7 Canonical chain height", "range": true, "refId": "B" } @@ -4361,7 +6636,7 @@ "h": 8, "w": 12, "x": 12, - "y": 202 + "y": 316 }, "id": 57, "options": { @@ -4387,7 +6662,7 @@ "editorMode": "builder", "expr": "reth_blockchain_tree_block_buffer_blocks{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", "hide": false, - "legendFormat": "{{service}} · Buffered blocks", + "legendFormat": "{{service}} \u00b7 Buffered blocks", "range": true, "refId": "B" } @@ -4462,7 +6737,7 @@ "h": 8, "w": 12, "x": 0, - "y": 210 + "y": 324 }, "id": 58, "options": { @@ -4564,7 +6839,7 @@ "h": 8, "w": 12, "x": 12, - "y": 210 + "y": 324 }, "id": 59, "options": { @@ -4609,7 +6884,7 @@ "h": 1, "w": 24, "x": 0, - "y": 138 + "y": 252 }, "id": 60, "panels": [ @@ -4706,7 +6981,7 @@ "h": 8, "w": 12, "x": 0, - "y": 219 + "y": 333 }, "id": 61, "options": { @@ -4735,7 +7010,7 @@ "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, - "legendFormat": "{{service}} · {{transport}}", + "legendFormat": "{{service}} \u00b7 {{transport}}", "range": true, "refId": "A", "useBackend": false @@ -4769,7 +7044,7 @@ "h": 8, "w": 12, "x": 12, - "y": 219 + "y": 333 }, "id": 62, "maxDataPoints": 25, @@ -4793,7 +7068,7 @@ "color": "rgba(255,0,255,0.7)" }, "filterValues": { - "le": 1E-9 + "le": 1e-09 }, "legend": { "show": true @@ -4902,7 +7177,7 @@ "h": 8, "w": 12, "x": 0, - "y": 227 + "y": 341 }, "id": 63, "options": { @@ -4928,7 +7203,7 @@ "editorMode": "code", "expr": "max by (service, method) (max_over_time(reth_rpc_server_calls_time_seconds{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])) > 0", "instant": false, - "legendFormat": "{{service}} · {{method}}", + "legendFormat": "{{service}} \u00b7 {{method}}", "range": true, "refId": "A" } @@ -4961,7 +7236,7 @@ "h": 8, "w": 12, "x": 12, - "y": 227 + "y": 341 }, "id": 64, "maxDataPoints": 25, @@ -4985,7 +7260,7 @@ "color": "rgba(255,0,255,0.7)" }, "filterValues": { - "le": 1E-9 + "le": 1e-09 }, "legend": { "show": true @@ -5130,7 +7405,7 @@ "h": 8, "w": 12, "x": 0, - "y": 235 + "y": 349 }, "id": 65, "options": { @@ -5159,7 +7434,7 @@ "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{service}} · Headers cache cached items", + "legendFormat": "{{service}} \u00b7 Headers cache cached items", "range": true, "refId": "A", "useBackend": false @@ -5176,7 +7451,7 @@ "hide": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{service}} · Receipts cache queued consumers", + "legendFormat": "{{service}} \u00b7 Receipts cache queued consumers", "range": true, "refId": "B", "useBackend": false @@ -5193,7 +7468,7 @@ "hide": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{service}} · Headers cache queued consumers", + "legendFormat": "{{service}} \u00b7 Headers cache queued consumers", "range": true, "refId": "C", "useBackend": false @@ -5210,7 +7485,7 @@ "hide": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{service}} · Block cache queued consumers", + "legendFormat": "{{service}} \u00b7 Block cache queued consumers", "range": true, "refId": "D", "useBackend": false @@ -5227,7 +7502,7 @@ "hide": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{service}} · Blocks cache memory usage", + "legendFormat": "{{service}} \u00b7 Blocks cache memory usage", "range": true, "refId": "E", "useBackend": false @@ -5244,7 +7519,7 @@ "hide": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{service}} · Receipts cache cached items", + "legendFormat": "{{service}} \u00b7 Receipts cache cached items", "range": true, "refId": "F", "useBackend": false @@ -5261,7 +7536,7 @@ "hide": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{service}} · Receipts cache memory usage", + "legendFormat": "{{service}} \u00b7 Receipts cache memory usage", "range": true, "refId": "G", "useBackend": false @@ -5278,7 +7553,7 @@ "hide": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{service}} · Block cache cached items", + "legendFormat": "{{service}} \u00b7 Block cache cached items", "range": true, "refId": "H", "useBackend": false @@ -5354,7 +7629,7 @@ "h": 8, "w": 12, "x": 12, - "y": 235 + "y": 349 }, "id": 66, "options": { @@ -5380,7 +7655,7 @@ "editorMode": "code", "expr": "sum by (service, method) (rate(reth_rpc_server_calls_successful_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])) > 0", "instant": false, - "legendFormat": "{{service}} · {{method}}", + "legendFormat": "{{service}} \u00b7 {{method}}", "range": true, "refId": "A" } @@ -5398,7 +7673,7 @@ "h": 1, "w": 24, "x": 0, - "y": 139 + "y": 253 }, "id": 117, "panels": [ @@ -5470,7 +7745,7 @@ "h": 8, "w": 12, "x": 0, - "y": 441 + "y": 555 }, "id": 119, "options": { @@ -5572,7 +7847,7 @@ "h": 8, "w": 12, "x": 12, - "y": 441 + "y": 555 }, "id": 120, "options": { @@ -5674,7 +7949,7 @@ "h": 8, "w": 12, "x": 0, - "y": 449 + "y": 563 }, "id": 121, "options": { @@ -5700,7 +7975,7 @@ "editorMode": "builder", "expr": "reth_process_open_fds{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", "instant": false, - "legendFormat": "{{service}} · Open", + "legendFormat": "{{service}} \u00b7 Open", "range": true, "refId": "A" } @@ -5776,7 +8051,7 @@ "h": 8, "w": 12, "x": 12, - "y": 449 + "y": 563 }, "id": 122, "options": { @@ -5803,7 +8078,7 @@ "expr": "(reth_executor_spawn_critical_tasks_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} or (0 * reth_process_open_fds{chain=\"$chain\", role=~\"$role\", service=~\"$service\"})) - (reth_executor_spawn_finished_critical_tasks_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} or (0 * reth_process_open_fds{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}))", "hide": false, "instant": false, - "legendFormat": "{{service}} · Tasks running", + "legendFormat": "{{service}} \u00b7 Tasks running", "range": true, "refId": "C" } @@ -5892,7 +8167,7 @@ "h": 8, "w": 12, "x": 0, - "y": 457 + "y": 571 }, "id": 123, "options": { @@ -5923,7 +8198,7 @@ "hide": false, "includeNullMetadata": false, "instant": false, - "legendFormat": "{{service}} · Tasks started", + "legendFormat": "{{service}} \u00b7 Tasks started", "range": true, "refId": "A", "useBackend": false @@ -5937,7 +8212,7 @@ "expr": "(reth_executor_spawn_regular_tasks_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} or (0 * reth_process_open_fds{chain=\"$chain\", role=~\"$role\", service=~\"$service\"})) - (reth_executor_spawn_finished_regular_tasks_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} or (0 * reth_process_open_fds{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}))", "hide": false, "instant": false, - "legendFormat": "{{service}} · Tasks running", + "legendFormat": "{{service}} \u00b7 Tasks running", "range": true, "refId": "C" } @@ -6026,7 +8301,7 @@ "h": 8, "w": 12, "x": 12, - "y": 457 + "y": 571 }, "id": 124, "options": { @@ -6057,7 +8332,7 @@ "hide": false, "includeNullMetadata": false, "instant": false, - "legendFormat": "{{service}} · Tasks started", + "legendFormat": "{{service}} \u00b7 Tasks started", "range": true, "refId": "A", "useBackend": false @@ -6071,7 +8346,7 @@ "expr": "(reth_executor_spawn_regular_blocking_tasks_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} or (0 * reth_process_open_fds{chain=\"$chain\", role=~\"$role\", service=~\"$service\"})) - (reth_executor_spawn_finished_regular_blocking_tasks_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} or (0 * reth_process_open_fds{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}))", "hide": false, "instant": false, - "legendFormat": "{{service}} · Tasks running", + "legendFormat": "{{service}} \u00b7 Tasks running", "range": true, "refId": "C" } @@ -6146,7 +8421,7 @@ "h": 8, "w": 24, "x": 0, - "y": 465 + "y": 579 }, "id": 125, "options": { @@ -6179,6 +8454,246 @@ ], "title": "Pruner duration, total", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisCenteredZero": false, + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false + }, + "min": 0, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 587 + }, + "id": 335, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_jemalloc_active{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "instant": false, + "legendFormat": "{{service}} \u00b7 active", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_jemalloc_allocated{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "instant": false, + "legendFormat": "{{service}} \u00b7 allocated", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_jemalloc_mapped{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "instant": false, + "legendFormat": "{{service}} \u00b7 mapped", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_jemalloc_metadata{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "instant": false, + "legendFormat": "{{service}} \u00b7 metadata", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_jemalloc_resident{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "instant": false, + "legendFormat": "{{service}} \u00b7 resident", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_jemalloc_retained{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "instant": false, + "legendFormat": "{{service}} \u00b7 retained", + "range": true, + "refId": "F" + } + ], + "title": "Jemalloc Memory", + "type": "timeseries", + "description": "Memory reported by jemalloc in default-feature builds." + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisCenteredZero": false, + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false + }, + "min": 0, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 587 + }, + "id": 336, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(reth_pruner_segments_duration_seconds_sum{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval]) / rate(reth_pruner_segments_duration_seconds_count{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", + "instant": false, + "legendFormat": "{{service}} \u00b7 {{segment}}", + "range": true, + "refId": "A" + } + ], + "title": "Pruner duration, per segment", + "type": "timeseries", + "description": "Average standard reth pruning duration by segment." + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic-by-name" + }, + "custom": { + "axisCenteredZero": false, + "drawStyle": "line", + "fillOpacity": 0, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false + }, + "min": 0, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 595 + }, + "id": 337, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "reth_pruner_segments_highest_pruned_block{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", + "instant": false, + "legendFormat": "{{service}} \u00b7 {{segment}}", + "range": true, + "refId": "A" + } + ], + "title": "Highest pruned block, per segment", + "type": "timeseries", + "description": "Highest block pruned by each standard reth pruning segment." } ], "title": "Process & Maintenance", @@ -6190,7 +8705,7 @@ "h": 1, "w": 24, "x": 0, - "y": 140 + "y": 254 }, "id": 80, "panels": [ @@ -6261,7 +8776,7 @@ "h": 8, "w": 12, "x": 0, - "y": 293 + "y": 407 }, "id": 82, "options": { @@ -6284,7 +8799,7 @@ }, "editorMode": "builder", "expr": "reth_transaction_pool_basefee_pool_size_bytes{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", - "legendFormat": "{{service}} · Base Fee Pool Size", + "legendFormat": "{{service}} \u00b7 Base Fee Pool Size", "range": true, "refId": "A" }, @@ -6296,7 +8811,7 @@ "editorMode": "builder", "expr": "reth_transaction_pool_pending_pool_size_bytes{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", "hide": false, - "legendFormat": "{{service}} · Pending Pool Size", + "legendFormat": "{{service}} \u00b7 Pending Pool Size", "range": true, "refId": "B" }, @@ -6308,7 +8823,7 @@ "editorMode": "builder", "expr": "reth_transaction_pool_queued_pool_size_bytes{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", "hide": false, - "legendFormat": "{{service}} · Queued Pool Size", + "legendFormat": "{{service}} \u00b7 Queued Pool Size", "range": true, "refId": "C" }, @@ -6319,7 +8834,7 @@ }, "editorMode": "builder", "expr": "reth_transaction_pool_blob_pool_size_bytes{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", - "legendFormat": "{{service}} · Blob Pool Size", + "legendFormat": "{{service}} \u00b7 Blob Pool Size", "range": true, "refId": "D" } @@ -6394,7 +8909,7 @@ "h": 8, "w": 12, "x": 12, - "y": 293 + "y": 407 }, "id": 83, "options": { @@ -6421,7 +8936,7 @@ "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{service}} · Dirty Accounts", + "legendFormat": "{{service}} \u00b7 Dirty Accounts", "range": true, "refId": "A", "useBackend": false @@ -6438,7 +8953,7 @@ "hide": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{service}} · Drift Count", + "legendFormat": "{{service}} \u00b7 Drift Count", "range": true, "refId": "B", "useBackend": false @@ -6455,7 +8970,7 @@ "hide": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{service}} · Reinserted Transactions", + "legendFormat": "{{service}} \u00b7 Reinserted Transactions", "range": true, "refId": "C", "useBackend": false @@ -6472,7 +8987,7 @@ "hide": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{service}} · Deleted Tracked Finalized Blobs", + "legendFormat": "{{service}} \u00b7 Deleted Tracked Finalized Blobs", "range": true, "refId": "D", "useBackend": false @@ -6547,7 +9062,7 @@ "h": 8, "w": 12, "x": 0, - "y": 301 + "y": 415 }, "id": 84, "options": { @@ -6570,7 +9085,7 @@ }, "editorMode": "builder", "expr": "reth_transaction_pool_basefee_pool_transactions{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", - "legendFormat": "{{service}} · Base Fee Pool Transactions", + "legendFormat": "{{service}} \u00b7 Base Fee Pool Transactions", "range": true, "refId": "A" }, @@ -6582,7 +9097,7 @@ "editorMode": "builder", "expr": "reth_transaction_pool_pending_pool_transactions{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", "hide": false, - "legendFormat": "{{service}} · Pending Pool Transactions", + "legendFormat": "{{service}} \u00b7 Pending Pool Transactions", "range": true, "refId": "B" }, @@ -6594,7 +9109,7 @@ "editorMode": "builder", "expr": "reth_transaction_pool_queued_pool_transactions{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", "hide": false, - "legendFormat": "{{service}} · Queued Pool Transactions", + "legendFormat": "{{service}} \u00b7 Queued Pool Transactions", "range": true, "refId": "C" }, @@ -6605,7 +9120,7 @@ }, "editorMode": "builder", "expr": "reth_transaction_pool_blob_pool_transactions{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", - "legendFormat": "{{service}} · Blob Pool Transactions", + "legendFormat": "{{service}} \u00b7 Blob Pool Transactions", "range": true, "refId": "D" } @@ -6693,7 +9208,7 @@ "h": 8, "w": 12, "x": 12, - "y": 301 + "y": 415 }, "id": 85, "options": { @@ -6719,7 +9234,7 @@ "expr": "rate(reth_transaction_pool_inserted_transactions{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", "fullMetaSearch": false, "includeNullMetadata": true, - "legendFormat": "{{service}} · Inserted Transactions", + "legendFormat": "{{service}} \u00b7 Inserted Transactions", "range": true, "refId": "A", "useBackend": false @@ -6735,7 +9250,7 @@ "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, - "legendFormat": "{{service}} · Removed Transactions", + "legendFormat": "{{service}} \u00b7 Removed Transactions", "range": true, "refId": "B", "useBackend": false @@ -6751,7 +9266,7 @@ "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, - "legendFormat": "{{service}} · Invalid Transactions", + "legendFormat": "{{service}} \u00b7 Invalid Transactions", "range": true, "refId": "C", "useBackend": false @@ -6768,7 +9283,7 @@ "hide": false, "includeNullMetadata": false, "instant": false, - "legendFormat": "{{service}} · Bad Transactions", + "legendFormat": "{{service}} \u00b7 Bad Transactions", "range": true, "refId": "D", "useBackend": false @@ -6843,7 +9358,7 @@ "h": 8, "w": 12, "x": 0, - "y": 309 + "y": 423 }, "id": 86, "options": { @@ -6867,7 +9382,7 @@ "editorMode": "builder", "expr": "reth_network_pending_pool_imports{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", "hide": false, - "legendFormat": "{{service}} · Transactions Pending Import", + "legendFormat": "{{service}} \u00b7 Transactions Pending Import", "range": true, "refId": "C" } @@ -6967,7 +9482,7 @@ "h": 8, "w": 12, "x": 12, - "y": 309 + "y": 423 }, "id": 87, "options": { @@ -6992,7 +9507,7 @@ "expr": "rate(reth_network_pool_transactions_messages_sent_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", "hide": false, "instant": false, - "legendFormat": "{{service}} · Tx", + "legendFormat": "{{service}} \u00b7 Tx", "range": true, "refId": "A" }, @@ -7004,7 +9519,7 @@ "editorMode": "builder", "expr": "rate(reth_network_pool_transactions_messages_received_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", "hide": false, - "legendFormat": "{{service}} · Rx", + "legendFormat": "{{service}} \u00b7 Rx", "range": true, "refId": "B" }, @@ -7016,7 +9531,7 @@ "editorMode": "code", "expr": "reth_network_pool_transactions_messages_sent_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} - reth_network_pool_transactions_messages_received_total{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", "hide": false, - "legendFormat": "{{service}} · Messages in Channel", + "legendFormat": "{{service}} \u00b7 Messages in Channel", "range": true, "refId": "C" } @@ -7091,7 +9606,7 @@ "h": 8, "w": 12, "x": 0, - "y": 317 + "y": 431 }, "id": 88, "options": { @@ -7118,7 +9633,7 @@ "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{service}} · Queued Messages per Second", + "legendFormat": "{{service}} \u00b7 Queued Messages per Second", "range": true, "refId": "A", "useBackend": false @@ -7193,7 +9708,7 @@ "h": 8, "w": 12, "x": 12, - "y": 317 + "y": 431 }, "id": 89, "options": { @@ -7220,7 +9735,7 @@ "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{service}} · All transactions by hash", + "legendFormat": "{{service}} \u00b7 All transactions by hash", "range": true, "refId": "A", "useBackend": false @@ -7237,7 +9752,7 @@ "hide": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{service}} · All transactions by id", + "legendFormat": "{{service}} \u00b7 All transactions by id", "range": true, "refId": "B", "useBackend": false @@ -7254,7 +9769,7 @@ "hide": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{service}} · All transactions by all senders", + "legendFormat": "{{service}} \u00b7 All transactions by all senders", "range": true, "refId": "C", "useBackend": false @@ -7271,7 +9786,7 @@ "hide": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{service}} · Blob transactions nonce gaps", + "legendFormat": "{{service}} \u00b7 Blob transactions nonce gaps", "range": true, "refId": "D", "useBackend": false @@ -7347,7 +9862,7 @@ "h": 8, "w": 12, "x": 0, - "y": 325 + "y": 439 }, "id": 90, "options": { @@ -7374,7 +9889,7 @@ "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{service}} · Dropped", + "legendFormat": "{{service}} \u00b7 Dropped", "range": true, "refId": "A", "useBackend": false @@ -7450,7 +9965,7 @@ "h": 8, "w": 12, "x": 12, - "y": 325 + "y": 439 }, "id": 91, "options": { @@ -7478,7 +9993,7 @@ "hide": false, "includeNullMetadata": false, "instant": false, - "legendFormat": "{{service}} · Freq Announced Transactions Already in Pool", + "legendFormat": "{{service}} \u00b7 Freq Announced Transactions Already in Pool", "range": true, "refId": "A", "useBackend": false @@ -7495,7 +10010,7 @@ "hide": false, "includeNullMetadata": false, "instant": false, - "legendFormat": "{{service}} · Freq Received Transactions Already in Pool ", + "legendFormat": "{{service}} \u00b7 Freq Received Transactions Already in Pool ", "range": true, "refId": "B", "useBackend": false @@ -7570,7 +10085,7 @@ "h": 8, "w": 12, "x": 0, - "y": 333 + "y": 447 }, "id": 92, "options": { @@ -7594,7 +10109,7 @@ "editorMode": "builder", "expr": "reth_network_inflight_transaction_requests{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", "hide": false, - "legendFormat": "{{service}} · Inflight Transaction Requests", + "legendFormat": "{{service}} \u00b7 Inflight Transaction Requests", "range": true, "refId": "C" } @@ -7669,7 +10184,7 @@ "h": 8, "w": 12, "x": 12, - "y": 333 + "y": 447 }, "id": 93, "options": { @@ -7697,7 +10212,7 @@ "hide": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{service}} · Network Events", + "legendFormat": "{{service}} \u00b7 Network Events", "range": true, "refId": "B", "useBackend": false @@ -7714,7 +10229,7 @@ "hide": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{service}} · Transaction Events", + "legendFormat": "{{service}} \u00b7 Transaction Events", "range": true, "refId": "C", "useBackend": false @@ -7732,7 +10247,7 @@ "hide": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{service}} · Pending Transactions", + "legendFormat": "{{service}} \u00b7 Pending Transactions", "range": true, "refId": "D", "useBackend": false @@ -7749,7 +10264,7 @@ "hide": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{service}} · Pending Pool Imports", + "legendFormat": "{{service}} \u00b7 Pending Pool Imports", "range": true, "refId": "E", "useBackend": false @@ -7766,7 +10281,7 @@ "hide": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{service}} · Fetch Events", + "legendFormat": "{{service}} \u00b7 Fetch Events", "range": true, "refId": "F", "useBackend": false @@ -7783,7 +10298,7 @@ "hide": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{service}} · Commands", + "legendFormat": "{{service}} \u00b7 Commands", "range": true, "refId": "G", "useBackend": false @@ -7800,7 +10315,7 @@ "hide": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{service}} · Fetch Pending Hashes", + "legendFormat": "{{service}} \u00b7 Fetch Pending Hashes", "range": true, "refId": "A", "useBackend": false @@ -7817,7 +10332,7 @@ "hide": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{service}} · Total Transactions Manager Future", + "legendFormat": "{{service}} \u00b7 Total Transactions Manager Future", "range": true, "refId": "H", "useBackend": false @@ -7892,7 +10407,7 @@ "h": 8, "w": 12, "x": 0, - "y": 341 + "y": 455 }, "id": 94, "options": { @@ -7919,7 +10434,7 @@ "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{service}} · Hashes in Pending Fetch Cache", + "legendFormat": "{{service}} \u00b7 Hashes in Pending Fetch Cache", "range": true, "refId": "A", "useBackend": false @@ -7936,7 +10451,7 @@ "hide": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{service}} · Hashes in Inflight Requests", + "legendFormat": "{{service}} \u00b7 Hashes in Inflight Requests", "range": true, "refId": "B", "useBackend": false @@ -7950,7 +10465,7 @@ "expr": "sum by (service) (reth_network_hashes_inflight_transaction_requests{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}) + sum by (service) (reth_network_hashes_pending_fetch{chain=\"$chain\", role=~\"$role\", service=~\"$service\"})", "hide": false, "instant": false, - "legendFormat": "{{service}} · Total Hashes in Transaction Fetcher", + "legendFormat": "{{service}} \u00b7 Total Hashes in Transaction Fetcher", "range": true, "refId": "C" } @@ -8025,7 +10540,7 @@ "h": 8, "w": 12, "x": 12, - "y": 341 + "y": 455 }, "id": 95, "options": { @@ -8053,7 +10568,7 @@ "hide": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{service}} · Network Handle Messages", + "legendFormat": "{{service}} \u00b7 Network Handle Messages", "range": true, "refId": "A", "useBackend": false @@ -8070,7 +10585,7 @@ "hide": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{service}} · Swarm Events", + "legendFormat": "{{service}} \u00b7 Swarm Events", "range": true, "refId": "B", "useBackend": false @@ -8087,7 +10602,7 @@ "hide": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{service}} · Total Network Manager Future", + "legendFormat": "{{service}} \u00b7 Total Network Manager Future", "range": true, "refId": "C", "useBackend": false @@ -8163,7 +10678,7 @@ "h": 8, "w": 12, "x": 0, - "y": 349 + "y": 463 }, "id": 96, "options": { @@ -8191,7 +10706,7 @@ "hide": false, "includeNullMetadata": false, "instant": false, - "legendFormat": "{{service}} · Freq Announced Transactions Already Seen by Peer", + "legendFormat": "{{service}} \u00b7 Freq Announced Transactions Already Seen by Peer", "range": true, "refId": "A", "useBackend": false @@ -8208,7 +10723,7 @@ "hide": false, "includeNullMetadata": false, "instant": false, - "legendFormat": "{{service}} · Freq Received Transactions Already Seen by Peer", + "legendFormat": "{{service}} \u00b7 Freq Received Transactions Already Seen by Peer", "range": true, "refId": "B", "useBackend": false @@ -8284,7 +10799,7 @@ "h": 8, "w": 12, "x": 12, - "y": 349 + "y": 463 }, "id": 97, "options": { @@ -8312,7 +10827,7 @@ "hide": false, "includeNullMetadata": false, "instant": false, - "legendFormat": "{{service}} · Legacy", + "legendFormat": "{{service}} \u00b7 Legacy", "range": true, "refId": "A", "useBackend": false @@ -8329,7 +10844,7 @@ "hide": false, "includeNullMetadata": false, "instant": false, - "legendFormat": "{{service}} · EIP-2930", + "legendFormat": "{{service}} \u00b7 EIP-2930", "range": true, "refId": "B", "useBackend": false @@ -8346,7 +10861,7 @@ "hide": false, "includeNullMetadata": false, "instant": false, - "legendFormat": "{{service}} · EIP-1559", + "legendFormat": "{{service}} \u00b7 EIP-1559", "range": true, "refId": "C", "useBackend": false @@ -8363,7 +10878,7 @@ "hide": false, "includeNullMetadata": false, "instant": false, - "legendFormat": "{{service}} · EIP-4844", + "legendFormat": "{{service}} \u00b7 EIP-4844", "range": true, "refId": "D", "useBackend": false @@ -8380,7 +10895,7 @@ "hide": false, "includeNullMetadata": false, "instant": false, - "legendFormat": "{{service}} · EIP-7702", + "legendFormat": "{{service}} \u00b7 EIP-7702", "range": true, "refId": "E", "useBackend": false @@ -8397,7 +10912,7 @@ "hide": false, "includeNullMetadata": false, "instant": false, - "legendFormat": "{{service}} · Morph (0x7F/other)", + "legendFormat": "{{service}} \u00b7 Morph (0x7F/other)", "range": true, "refId": "F", "useBackend": false @@ -8473,7 +10988,7 @@ "h": 8, "w": 12, "x": 0, - "y": 357 + "y": 471 }, "id": 98, "options": { @@ -8501,7 +11016,7 @@ "hide": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{service}} · Find Idle Peer", + "legendFormat": "{{service}} \u00b7 Find Idle Peer", "range": true, "refId": "C", "useBackend": false @@ -8518,7 +11033,7 @@ "hide": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{service}} · Fill Request", + "legendFormat": "{{service}} \u00b7 Fill Request", "range": true, "refId": "B", "useBackend": false @@ -8594,7 +11109,7 @@ "h": 8, "w": 12, "x": 12, - "y": 357 + "y": 471 }, "id": 99, "options": { @@ -8622,7 +11137,7 @@ "hide": false, "includeNullMetadata": false, "instant": false, - "legendFormat": "{{service}} · Legacy", + "legendFormat": "{{service}} \u00b7 Legacy", "range": true, "refId": "A", "useBackend": false @@ -8639,7 +11154,7 @@ "hide": false, "includeNullMetadata": false, "instant": false, - "legendFormat": "{{service}} · Eip2930", + "legendFormat": "{{service}} \u00b7 Eip2930", "range": true, "refId": "B", "useBackend": false @@ -8656,7 +11171,7 @@ "hide": false, "includeNullMetadata": false, "instant": false, - "legendFormat": "{{service}} · Eip1559", + "legendFormat": "{{service}} \u00b7 Eip1559", "range": true, "refId": "C", "useBackend": false @@ -8673,7 +11188,7 @@ "hide": false, "includeNullMetadata": false, "instant": false, - "legendFormat": "{{service}} · Eip4844", + "legendFormat": "{{service}} \u00b7 Eip4844", "range": true, "refId": "D", "useBackend": false @@ -8690,7 +11205,7 @@ "hide": false, "includeNullMetadata": false, "instant": false, - "legendFormat": "{{service}} · Eip7702", + "legendFormat": "{{service}} \u00b7 Eip7702", "range": true, "refId": "E", "useBackend": false @@ -8766,7 +11281,7 @@ "h": 8, "w": 12, "x": 0, - "y": 365 + "y": 479 }, "id": 100, "options": { @@ -8788,7 +11303,7 @@ "expr": "reth_transaction_pool_pending_transactions_evicted{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", "fullMetaSearch": false, "includeNullMetadata": true, - "legendFormat": "{{service}} · PendingPool", + "legendFormat": "{{service}} \u00b7 PendingPool", "range": true, "refId": "A", "useBackend": false, @@ -8806,7 +11321,7 @@ "expr": "reth_transaction_pool_basefee_transactions_evicted{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", "hide": false, "instant": false, - "legendFormat": "{{service}} · BasefeePool", + "legendFormat": "{{service}} \u00b7 BasefeePool", "range": true, "refId": "B" }, @@ -8819,7 +11334,7 @@ "expr": "reth_transaction_pool_blob_transactions_evicted{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", "hide": false, "instant": false, - "legendFormat": "{{service}} · BlobPool", + "legendFormat": "{{service}} \u00b7 BlobPool", "range": true, "refId": "C" }, @@ -8832,7 +11347,7 @@ "expr": "reth_transaction_pool_queued_transactions_evicted{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", "hide": false, "instant": false, - "legendFormat": "{{service}} · QueuedPool", + "legendFormat": "{{service}} \u00b7 QueuedPool", "range": true, "refId": "D" } @@ -8850,7 +11365,7 @@ "h": 1, "w": 24, "x": 0, - "y": 141 + "y": 255 }, "id": 132, "panels": [ @@ -8917,7 +11432,7 @@ "overrides": [] }, "gridPos": { - "y": 293, + "y": 407, "x": 0, "w": 12, "h": 8 @@ -8943,7 +11458,7 @@ }, "editorMode": "builder", "expr": "reth_network_tracked_peers{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", - "legendFormat": "{{service}} · Tracked Peers", + "legendFormat": "{{service}} \u00b7 Tracked Peers", "range": true, "refId": "A" } @@ -9014,7 +11529,7 @@ "overrides": [] }, "gridPos": { - "y": 293, + "y": 407, "x": 12, "w": 12, "h": 8 @@ -9040,7 +11555,7 @@ }, "editorMode": "builder", "expr": "reth_network_outgoing_connections{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", - "legendFormat": "{{service}} · Outgoing Connections", + "legendFormat": "{{service}} \u00b7 Outgoing Connections", "range": true, "refId": "B" }, @@ -9052,7 +11567,7 @@ "editorMode": "builder", "expr": "reth_network_incoming_connections{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", "hide": false, - "legendFormat": "{{service}} · Incoming Connections", + "legendFormat": "{{service}} \u00b7 Incoming Connections", "range": true, "refId": "D" }, @@ -9064,7 +11579,7 @@ "editorMode": "builder", "expr": "reth_network_connected_peers{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", "hide": false, - "legendFormat": "{{service}} · Connected Peers", + "legendFormat": "{{service}} \u00b7 Connected Peers", "range": true, "refId": "E" } @@ -9137,7 +11652,7 @@ "overrides": [] }, "gridPos": { - "y": 301, + "y": 415, "x": 0, "w": 12, "h": 8 @@ -9166,7 +11681,7 @@ }, "editorMode": "builder", "expr": "rate(reth_p2pstream_disconnected_errors{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", - "legendFormat": "{{service}} · P2P Stream Disconnected", + "legendFormat": "{{service}} \u00b7 P2P Stream Disconnected", "range": true, "refId": "A" }, @@ -9178,7 +11693,7 @@ "editorMode": "builder", "expr": "rate(reth_network_pending_session_failures{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}[$__rate_interval])", "hide": false, - "legendFormat": "{{service}} · Failed Pending Sessions", + "legendFormat": "{{service}} \u00b7 Failed Pending Sessions", "range": true, "refId": "B" } @@ -9208,7 +11723,7 @@ "overrides": [] }, "gridPos": { - "y": 301, + "y": 415, "x": 12, "w": 12, "h": 8 @@ -9241,7 +11756,7 @@ }, "editorMode": "builder", "expr": "reth_network_useless_peer{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", - "legendFormat": "{{service}} · UselessPeer", + "legendFormat": "{{service}} \u00b7 UselessPeer", "range": true, "refId": "A" }, @@ -9253,7 +11768,7 @@ "editorMode": "builder", "expr": "reth_network_subprotocol_specific{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", "hide": false, - "legendFormat": "{{service}} · SubprotocolSpecific", + "legendFormat": "{{service}} \u00b7 SubprotocolSpecific", "range": true, "refId": "B" }, @@ -9265,7 +11780,7 @@ "editorMode": "builder", "expr": "reth_network_already_connected{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", "hide": false, - "legendFormat": "{{service}} · AlreadyConnected", + "legendFormat": "{{service}} \u00b7 AlreadyConnected", "range": true, "refId": "C" }, @@ -9277,7 +11792,7 @@ "editorMode": "builder", "expr": "reth_network_client_quitting{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", "hide": false, - "legendFormat": "{{service}} · ClientQuitting", + "legendFormat": "{{service}} \u00b7 ClientQuitting", "range": true, "refId": "D" }, @@ -9289,7 +11804,7 @@ "editorMode": "builder", "expr": "reth_network_unexpected_identity{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", "hide": false, - "legendFormat": "{{service}} · UnexpectedHandshakeIdentity", + "legendFormat": "{{service}} \u00b7 UnexpectedHandshakeIdentity", "range": true, "refId": "E" }, @@ -9301,7 +11816,7 @@ "editorMode": "builder", "expr": "reth_network_disconnect_requested{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", "hide": false, - "legendFormat": "{{service}} · DisconnectRequested", + "legendFormat": "{{service}} \u00b7 DisconnectRequested", "range": true, "refId": "F" }, @@ -9313,7 +11828,7 @@ "editorMode": "builder", "expr": "reth_network_null_node_identity{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", "hide": false, - "legendFormat": "{{service}} · NullNodeIdentity", + "legendFormat": "{{service}} \u00b7 NullNodeIdentity", "range": true, "refId": "G" }, @@ -9325,7 +11840,7 @@ "editorMode": "builder", "expr": "reth_network_tcp_subsystem_error{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", "hide": false, - "legendFormat": "{{service}} · TCPSubsystemError", + "legendFormat": "{{service}} \u00b7 TCPSubsystemError", "range": true, "refId": "H" }, @@ -9337,7 +11852,7 @@ "editorMode": "builder", "expr": "reth_network_incompatible{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", "hide": false, - "legendFormat": "{{service}} · IncompatibleP2PVersion", + "legendFormat": "{{service}} \u00b7 IncompatibleP2PVersion", "range": true, "refId": "I" }, @@ -9349,7 +11864,7 @@ "editorMode": "builder", "expr": "reth_network_protocol_breach{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", "hide": false, - "legendFormat": "{{service}} · ProtocolBreach", + "legendFormat": "{{service}} \u00b7 ProtocolBreach", "range": true, "refId": "J" }, @@ -9361,14 +11876,14 @@ "editorMode": "builder", "expr": "reth_network_too_many_peers{chain=\"$chain\", role=~\"$role\", service=~\"$service\"} > 0", "hide": false, - "legendFormat": "{{service}} · TooManyPeers", + "legendFormat": "{{service}} \u00b7 TooManyPeers", "range": true, "refId": "K" } ], "title": "P2P Peer Disconnect Reasons", "type": "piechart", - "description": "每种 disconnect reason 的计数。空 slice 表示该类断连没发生过(健康)。只要节点稳定运行,这个图应该为空。" + "description": "\u6bcf\u79cd disconnect reason \u7684\u8ba1\u6570\u3002\u7a7a slice \u8868\u793a\u8be5\u7c7b\u65ad\u8fde\u6ca1\u53d1\u751f\u8fc7(\u5065\u5eb7)\u3002\u53ea\u8981\u8282\u70b9\u7a33\u5b9a\u8fd0\u884c,\u8fd9\u4e2a\u56fe\u5e94\u8be5\u4e3a\u7a7a\u3002" }, { "datasource": { @@ -9433,7 +11948,7 @@ "overrides": [] }, "gridPos": { - "y": 309, + "y": 423, "x": 0, "w": 24, "h": 8 @@ -9459,7 +11974,7 @@ }, "editorMode": "builder", "expr": "reth_network_total_dial_successes{chain=\"$chain\", role=~\"$role\", service=~\"$service\"}", - "legendFormat": "{{service}} · Total Dial Successes", + "legendFormat": "{{service}} \u00b7 Total Dial Successes", "range": true, "refId": "A" } @@ -9477,7 +11992,7 @@ "h": 1, "w": 24, "x": 0, - "y": 142 + "y": 256 }, "id": 106, "panels": [ @@ -9545,7 +12060,7 @@ "h": 10, "w": 12, "x": 0, - "y": 390 + "y": 504 }, "id": 107, "options": { @@ -9572,7 +12087,7 @@ "expr": "avg by (service) (delta(reth_db_table_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\", table=\"PlainAccountState\"}[$interval]))", "instant": false, "interval": "$interval", - "legendFormat": "{{service}} · Account", + "legendFormat": "{{service}} \u00b7 Account", "range": true, "refId": "A" }, @@ -9586,7 +12101,7 @@ "hide": false, "instant": false, "interval": "$interval", - "legendFormat": "{{service}} · Storage", + "legendFormat": "{{service}} \u00b7 Storage", "range": true, "refId": "B" }, @@ -9600,7 +12115,7 @@ "hide": false, "instant": false, "interval": "$interval", - "legendFormat": "{{service}} · Bytecodes", + "legendFormat": "{{service}} \u00b7 Bytecodes", "range": true, "refId": "C" }, @@ -9614,7 +12129,7 @@ "hide": false, "instant": false, "interval": "$interval", - "legendFormat": "{{service}} · Total", + "legendFormat": "{{service}} \u00b7 Total", "range": true, "refId": "D" } @@ -9686,7 +12201,7 @@ "h": 10, "w": 12, "x": 12, - "y": 390 + "y": 504 }, "id": 108, "options": { @@ -9713,7 +12228,7 @@ "expr": "sum by (service) (reth_db_table_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\", table=\"PlainAccountState\"})", "instant": false, "interval": "10m", - "legendFormat": "{{service}} · Account", + "legendFormat": "{{service}} \u00b7 Account", "range": true, "refId": "A" }, @@ -9727,7 +12242,7 @@ "hide": false, "instant": false, "interval": "10m", - "legendFormat": "{{service}} · Storage", + "legendFormat": "{{service}} \u00b7 Storage", "range": true, "refId": "B" }, @@ -9741,7 +12256,7 @@ "hide": false, "instant": false, "interval": "10m", - "legendFormat": "{{service}} · Bytecodes", + "legendFormat": "{{service}} \u00b7 Bytecodes", "range": true, "refId": "C" }, @@ -9755,7 +12270,7 @@ "hide": false, "instant": false, "interval": "10m", - "legendFormat": "{{service}} · Total", + "legendFormat": "{{service}} \u00b7 Total", "range": true, "refId": "D" } @@ -9827,7 +12342,7 @@ "h": 10, "w": 12, "x": 0, - "y": 400 + "y": 514 }, "id": 109, "options": { @@ -9924,7 +12439,7 @@ "h": 10, "w": 12, "x": 12, - "y": 400 + "y": 514 }, "id": 110, "options": { @@ -10021,7 +12536,7 @@ "h": 10, "w": 12, "x": 0, - "y": 410 + "y": 524 }, "id": 111, "options": { @@ -10118,7 +12633,7 @@ "h": 10, "w": 12, "x": 12, - "y": 410 + "y": 524 }, "id": 112, "options": { @@ -10145,7 +12660,7 @@ "expr": "avg by (service) (delta(reth_db_table_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\", table=\"Headers\"}[$interval])) + avg by (service) (delta(reth_static_files_segment_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\", segment=\"headers\"}[$interval]))", "instant": false, "interval": "$interval", - "legendFormat": "{{service}} · Headers", + "legendFormat": "{{service}} \u00b7 Headers", "range": true, "refId": "A" }, @@ -10159,7 +12674,7 @@ "hide": false, "instant": false, "interval": "$interval", - "legendFormat": "{{service}} · Receipts", + "legendFormat": "{{service}} \u00b7 Receipts", "range": true, "refId": "B" }, @@ -10173,7 +12688,7 @@ "hide": false, "instant": false, "interval": "$interval", - "legendFormat": "{{service}} · Transactions", + "legendFormat": "{{service}} \u00b7 Transactions", "range": true, "refId": "C" } @@ -10266,7 +12781,7 @@ "h": 10, "w": 12, "x": 0, - "y": 420 + "y": 534 }, "id": 113, "options": { @@ -10293,7 +12808,7 @@ "expr": "sum by (service) (reth_db_table_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\", table=\"Headers\"}) + sum by (service) (reth_static_files_segment_size{chain=\"$chain\", role=~\"$role\", service=~\"$service\", segment=\"headers\"})", "instant": false, "interval": "10m", - "legendFormat": "{{service}} · Headers", + "legendFormat": "{{service}} \u00b7 Headers", "range": true, "refId": "A" }, @@ -10307,7 +12822,7 @@ "hide": false, "instant": false, "interval": "10m", - "legendFormat": "{{service}} · Receipts", + "legendFormat": "{{service}} \u00b7 Receipts", "range": true, "refId": "B" }, @@ -10321,7 +12836,7 @@ "hide": false, "instant": false, "interval": "10m", - "legendFormat": "{{service}} · Transactions", + "legendFormat": "{{service}} \u00b7 Transactions", "range": true, "refId": "C" } @@ -10409,7 +12924,7 @@ "h": 10, "w": 12, "x": 12, - "y": 420 + "y": 534 }, "id": 114, "options": { @@ -10506,7 +13021,7 @@ "h": 10, "w": 12, "x": 0, - "y": 430 + "y": 544 }, "id": 115, "options": { @@ -10603,7 +13118,7 @@ "h": 10, "w": 12, "x": 12, - "y": 430 + "y": 544 }, "id": 116, "options": { diff --git a/etc/prometheus/prometheus.yml b/etc/prometheus/prometheus.yml index a443a71c..125e0fac 100644 --- a/etc/prometheus/prometheus.yml +++ b/etc/prometheus/prometheus.yml @@ -11,4 +11,4 @@ scrape_configs: labels: service: morph-reth role: sequencer - chain: morph + chain: mainnet From 638530752a27966b2ea77fe4d7e4238eccbdd249 Mon Sep 17 00:00:00 2001 From: panos Date: Thu, 27 Aug 2026 00:19:49 +0800 Subject: [PATCH 12/12] fix(proofs): tolerate proof storage ahead on restart --- crates/proofs-exex/src/lib.rs | 76 ++++++++++++++++++++--------------- 1 file changed, 44 insertions(+), 32 deletions(-) diff --git a/crates/proofs-exex/src/lib.rs b/crates/proofs-exex/src/lib.rs index 1636cd23..55ce2209 100644 --- a/crates/proofs-exex/src/lib.rs +++ b/crates/proofs-exex/src/lib.rs @@ -1,7 +1,6 @@ //! Forward-only proof-history execution extension for Morph. //! -//! Adapted from Base commit `b2673bbd927cb34d7cfad4d448bfbd5bd30eae88` -//! under the MIT license. +//! Derived from an MIT-licensed upstream implementation. #![cfg_attr(not(test), warn(unused_crate_dependencies))] @@ -20,7 +19,7 @@ use morph_proofs::{ use reth_execution_types::Chain; use reth_exex::{ExExContext, ExExEvent, ExExNotification, ExExNotificationsStream}; use reth_node_api::{FullNodeComponents, NodePrimitives, NodeTypes}; -use reth_provider::{BlockHashReader, BlockNumReader, BlockReader, TransactionVariant}; +use reth_provider::{BlockNumReader, BlockReader, TransactionVariant}; pub use sync_target::{CachedBlockTrieData, SyncTarget, SyncTargetState}; use tokio::task; use tracing::{debug, error, info}; @@ -189,6 +188,13 @@ where "Storage behind tip, starting sync immediately" ); sync_target.update_state(SyncTargetState::SyncUpTo { to: best_block }); + } else if latest_stored > best_block { + info!( + target: "morph::proofs_exex", + latest_stored, + best_block, + "Proof storage ahead of canonical tip, retaining data while chain catches up" + ); } let prune_task = MorphProofStoragePrunerTask::new( @@ -222,11 +228,8 @@ where } }; - let (latest_block_number, latest_block_hash) = match self - .storage - .get_latest_block_number()? - { - Some(latest) => latest, + let latest_block_number = match self.storage.get_latest_block_number()? { + Some((n, _)) => n, None => { return Err(eyre::eyre!( "Proofs storage not initialized. Please run 'morph-reth proofs init --proofs-history.storage-path ' first." @@ -251,19 +254,10 @@ where } } - let canonical_hash = self - .ctx - .provider() - .block_hash(latest_block_number)? - .ok_or_else(|| eyre::eyre!( - "Proofs storage latest block {latest_block_number} is missing from the canonical chain" - ))?; - if canonical_hash != latest_block_hash { - return Err(eyre::eyre!( - "Proofs storage latest block hash mismatch at {latest_block_number}: stored {latest_block_hash}, canonical {canonical_hash}" - )); - } - + // Proof storage and Reth have independent commit boundaries, so the proof tip can be + // ahead after an ungraceful shutdown. Let forward sync retain those blocks and resume once + // the canonical chain passes the stored tip. Proof RPCs independently validate the + // canonical anchor before serving data. ProofHistoryMetrics::set_window(earliest_block_number, latest_block_number); Ok(()) @@ -1144,8 +1138,6 @@ mod tests { let exex = build_test_exex(ctx, proofs.clone()); let error = exex.ensure_initialized().expect_err("should return error"); - // Without this the test would also pass on the later canonical-hash check, - // which fires for the same fixture. assert!( error.to_string().contains("exceeds the safety threshold"), "expected the startup prune-threshold error, got: {error}" @@ -1153,7 +1145,7 @@ mod tests { } #[tokio::test] - async fn ensure_initialized_errors_when_latest_is_not_canonical() { + async fn ensure_initialized_allows_latest_hash_to_differ_from_canonical_chain() { let dir = tempdir_path(); let store = Arc::new(MdbxProofsStorage::new(dir.as_path()).expect("env")); let proofs: MorphProofsStorage> = Arc::clone(&store); @@ -1162,19 +1154,39 @@ mod tests { .await .expect("exex test context"); - // Anchor proof history at the right height but the wrong hash: this is the - // shape of a proofs DB restored next to a mismatched chain snapshot, and it - // must refuse to serve rather than emit proofs for a foreign chain. + // Startup accepts an independently committed proof tip. Proof RPCs still validate the + // canonical anchor before serving data. let genesis = handle.genesis.num_hash(); assert_ne!(genesis.hash, b256(0x00), "fixture must not collide"); init_storage_at(proofs.clone(), NumHash::new(genesis.number, b256(0x00))); let exex = build_test_exex(ctx, proofs.clone()); - let error = exex.ensure_initialized().expect_err("should return error"); - assert!( - error.to_string().contains("latest block hash mismatch"), - "expected a canonical-hash mismatch error, got: {error}" - ); + exex.ensure_initialized() + .expect("startup should not validate the proof tip against the canonical chain"); + } + + #[tokio::test] + async fn ensure_initialized_allows_proof_storage_ahead_of_canonical_chain() { + let dir = tempdir_path(); + let store = Arc::new(MdbxProofsStorage::new(dir.as_path()).expect("env")); + let proofs: MorphProofsStorage> = Arc::clone(&store); + + let (ctx, handle) = reth_exex_test_utils::test_exex_context() + .await + .expect("exex test context"); + + let genesis = handle.genesis.num_hash(); + init_storage_at(proofs.clone(), genesis); + proofs + .store_trie_updates( + BlockWithParent::new(genesis.hash, NumHash::new(1, b256(0x01))), + BlockStateDiff::default(), + ) + .expect("store proof tip ahead of canonical chain"); + + let exex = build_test_exex(ctx, proofs.clone()); + exex.ensure_initialized() + .expect("proof storage may be ahead after an ungraceful shutdown"); } #[tokio::test]