diff --git a/Cargo.toml b/Cargo.toml index 27e9b035..98481cf2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ members = ["crates/*"] resolver = "2" [workspace.package] -version = "0.17.2" +version = "0.18.0" edition = "2024" rust-version = "1.88" authors = ["init4"] @@ -34,34 +34,34 @@ debug = false incremental = false [workspace.dependencies] -signet-blobber = { version = "0.17.2", path = "crates/blobber" } -signet-block-processor = { version = "0.17.2", path = "crates/block-processor" } -signet-genesis = { version = "0.17.2", path = "crates/genesis" } -signet-host-reth = { version = "0.17.2", path = "crates/host-reth" } -signet-host-rpc = { version = "0.17.2", path = "crates/host-rpc" } -signet-node = { version = "0.17.2", path = "crates/node" } -signet-node-config = { version = "0.17.2", path = "crates/node-config" } -signet-node-tests = { version = "0.17.2", path = "crates/node-tests" } -signet-node-types = { version = "0.17.2", path = "crates/node-types" } -signet-rpc = { version = "0.17.2", path = "crates/rpc" } - -init4-bin-base = { version = "0.19.1", features = ["alloy"] } - -signet-bundle = "0.16.0" -signet-constants = "0.16.0" -signet-evm = "0.16.0" -signet-extract = "0.16.0" -signet-test-utils = "0.16.0" -signet-tx-cache = "0.16.0" -signet-types = "0.16.0" -signet-zenith = "0.16.0" -signet-journal = "0.16.0" -signet-storage = "0.7" -signet-cold = "0.7" -signet-hot = "0.7" -signet-hot-mdbx = "0.7" -signet-cold-mdbx = "0.7" -signet-storage-types = "0.7" +signet-blobber = { version = "0.18.0", path = "crates/blobber" } +signet-block-processor = { version = "0.18.0", path = "crates/block-processor" } +signet-genesis = { version = "0.18.0", path = "crates/genesis" } +signet-host-reth = { version = "0.18.0", path = "crates/host-reth" } +signet-host-rpc = { version = "0.18.0", path = "crates/host-rpc" } +signet-node = { version = "0.18.0", path = "crates/node" } +signet-node-config = { version = "0.18.0", path = "crates/node-config" } +signet-node-tests = { version = "0.18.0", path = "crates/node-tests" } +signet-node-types = { version = "0.18.0", path = "crates/node-types" } +signet-rpc = { version = "0.18.0", path = "crates/rpc" } + +init4-bin-base = { version = "0.21.0", features = ["alloy"] } + +signet-bundle = "0.18" +signet-constants = "0.18" +signet-evm = "0.18" +signet-extract = "0.18" +signet-test-utils = "0.18" +signet-tx-cache = "0.18" +signet-types = "0.18" +signet-zenith = "0.18" +signet-journal = "0.18" +signet-storage = "0.9" +signet-cold = "0.9" +signet-hot = "0.9" +signet-hot-mdbx = "0.9" +signet-cold-mdbx = "0.9" +signet-storage-types = "0.9" # ajj ajj = "0.7.0" @@ -116,3 +116,4 @@ url = "2.5.4" # Test Utils tempfile = "3.17.0" + diff --git a/crates/block-processor/src/v1/processor.rs b/crates/block-processor/src/v1/processor.rs index bbf374ce..30f89cc7 100644 --- a/crates/block-processor/src/v1/processor.rs +++ b/crates/block-processor/src/v1/processor.rs @@ -15,6 +15,7 @@ use signet_hot::{ model::{HotKv, HotKvRead, RevmRead}, }; use signet_storage_types::{DbSignetEvent, DbZenithHeader, ExecutedBlock, ExecutedBlockBuilder}; +use signet_types::primitives::SignetHeaderV1; use std::collections::VecDeque; use tracing::{error, instrument}; use trevm::revm::{ @@ -195,7 +196,7 @@ where block_extracts, to_alias, txns, - parent_header, + SignetHeaderV1::new_unchecked(parent_header.unseal()), self.constants.clone(), ); @@ -254,8 +255,11 @@ where let zenith_header = block_extracts.ru_header().map(DbZenithHeader::from); + // The storage API is signet-agnostic and stores headers as a plain + // `Sealed
`; unseal the `SignetHeaderV1` wrapper before + // handing it off. ExecutedBlockBuilder::new() - .header(header) + .header(header.into_inner()) .bundle(bundle) .transactions(transactions) .receipts(receipts) diff --git a/crates/host-rpc/src/notifier.rs b/crates/host-rpc/src/notifier.rs index d9d31496..31ca4399 100644 --- a/crates/host-rpc/src/notifier.rs +++ b/crates/host-rpc/src/notifier.rs @@ -3,14 +3,14 @@ use alloy::{ consensus::{BlockHeader, transaction::Recovered}, eips::{BlockId, BlockNumberOrTag}, network::BlockResponse, - primitives::{B256, Sealed}, + primitives::B256, providers::Provider, pubsub::SubscriptionStream, rpc::types::Header as RpcHeader, }; use futures_util::{StreamExt, TryStreamExt, stream}; use signet_node_types::{HostNotification, HostNotificationKind, HostNotifier, RevertRange}; -use signet_types::primitives::{RecoveredBlock, SealedBlock, TransactionSigned}; +use signet_types::primitives::{RecoveredBlock, SealedBlock, SignetHeaderV1, TransactionSigned}; use std::{collections::VecDeque, sync::Arc, time::Instant}; use tracing::{debug, info, warn}; @@ -189,7 +189,6 @@ where rpc_block: alloy::rpc::types::Block, rpc_receipts: Option>, ) -> RpcBlock { - let hash = rpc_block.header.hash; let block = rpc_block .map_transactions(|tx| { let recovered = tx.inner; @@ -198,7 +197,9 @@ where Recovered::new_unchecked(tx, signer) }) .into_consensus(); - let sealed_header = Sealed::new_unchecked(block.header, hash); + // SignetHeaderV1 reseals the header; the recomputed hash matches the + // RPC-supplied hash for any well-formed provider response. + let sealed_header = SignetHeaderV1::new_unchecked(block.header); let block: RecoveredBlock = SealedBlock::new(sealed_header, block.body.transactions); let receipts = rpc_receipts .unwrap_or_default() diff --git a/crates/node-config/Cargo.toml b/crates/node-config/Cargo.toml index 9a1a34cb..5a79905d 100644 --- a/crates/node-config/Cargo.toml +++ b/crates/node-config/Cargo.toml @@ -11,6 +11,9 @@ repository.workspace = true [dependencies] signet-blobber.workspace = true +signet-cold.workspace = true +signet-hot.workspace = true +signet-rpc.workspace = true signet-storage.workspace = true signet-types.workspace = true @@ -27,6 +30,6 @@ signet-genesis.workspace = true [features] -test_utils = [] -postgres = ["signet-storage/postgres", "init4-bin-base/cold-sql"] -sqlite = ["signet-storage/sqlite", "init4-bin-base/cold-sql"] +test_utils = ["signet-rpc/test-utils"] +postgres = ["signet-storage/postgres", "signet-rpc/postgres", "init4-bin-base/cold-sql"] +sqlite = ["signet-storage/sqlite", "signet-rpc/sqlite", "init4-bin-base/cold-sql"] diff --git a/crates/node-config/src/lib.rs b/crates/node-config/src/lib.rs index c3b6e8ac..2917b19a 100644 --- a/crates/node-config/src/lib.rs +++ b/crates/node-config/src/lib.rs @@ -11,6 +11,14 @@ #![deny(unused_must_use, rust_2018_idioms)] #![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] +// `signet_storage`, `signet_cold`, and `signet_hot` are only referenced +// from `build_storage`, which is gated to non-test_utils. Silence +// `unused_crate_dependencies` on test_utils builds where those paths are +// excluded. +use signet_cold as _; +use signet_hot as _; +use signet_storage as _; + mod core; pub use core::SignetNodeConfig; diff --git a/crates/node-config/src/storage.rs b/crates/node-config/src/storage.rs index 6270b565..fb85f403 100644 --- a/crates/node-config/src/storage.rs +++ b/crates/node-config/src/storage.rs @@ -1,12 +1,26 @@ use init4_bin_base::utils::from_env::FromEnv; +#[cfg(not(feature = "test_utils"))] +use signet_cold::ColdConnect; +#[cfg(not(feature = "test_utils"))] +use signet_hot::HotConnect; #[cfg(any(feature = "postgres", feature = "sqlite"))] use signet_storage::SqlConnector; -use signet_storage::{DatabaseEnv, MdbxConnector, UnifiedStorage, builder::StorageBuilder}; +#[cfg(not(feature = "test_utils"))] +use signet_storage::{DatabaseEnv, MdbxConnector, UnifiedStorage}; use std::borrow::Cow; #[cfg(any(feature = "postgres", feature = "sqlite"))] use std::time::Duration; +#[cfg(not(feature = "test_utils"))] use tokio_util::sync::CancellationToken; +// `signet-rpc` is no longer referenced from this crate. Keep the dep +// satisfied so `unused_crate_dependencies` does not fire. +#[cfg(feature = "test_utils")] +use eyre as _; +use signet_rpc as _; +#[cfg(feature = "test_utils")] +use tokio_util as _; + /// Configuration for signet unified storage. /// /// Reads hot and cold storage configuration from environment variables. @@ -22,23 +36,9 @@ use tokio_util::sync::CancellationToken; /// Exactly one of `SIGNET_COLD_PATH` or `SIGNET_COLD_SQL_URL` must be set. /// /// When using SQL cold storage, connection pool tuning is configured via -/// [`SqlConnector`]'s own environment variables (e.g. +/// the `SqlConnector`'s own environment variables (e.g. /// `SIGNET_COLD_SQL_MAX_CONNECTIONS`). See the `cold-sql` feature of /// `init4-bin-base` for the full list. -/// -/// [`SqlConnector`]: signet_storage::SqlConnector -/// -/// # Example -/// -/// ```rust,no_run -/// # use signet_node_config::StorageConfig; -/// # use tokio_util::sync::CancellationToken; -/// # async fn example(cfg: &StorageConfig) -> eyre::Result<()> { -/// let cancel = CancellationToken::new(); -/// let storage = cfg.build_storage(cancel).await?; -/// # Ok(()) -/// # } -/// ``` #[derive(Debug, Clone, FromEnv)] pub struct StorageConfig { /// Path to the hot MDBX database. @@ -160,11 +160,27 @@ impl StorageConfig { /// background task, and returns a [`UnifiedStorage`] ready for use. /// /// Exactly one of `cold_path` or `cold_sql` must be configured. + /// + /// Not available when the `test_utils` feature is enabled — tests + /// construct an in-memory `UnifiedStorage` directly instead. + /// + /// # Example + /// + /// ```rust,no_run + /// # use signet_node_config::StorageConfig; + /// # use tokio_util::sync::CancellationToken; + /// # async fn example(cfg: &StorageConfig) -> eyre::Result<()> { + /// let cancel = CancellationToken::new(); + /// let storage = cfg.build_storage(cancel).await?; + /// # Ok(()) + /// # } + /// ``` + #[cfg(not(feature = "test_utils"))] pub async fn build_storage( &self, cancel: CancellationToken, ) -> eyre::Result> { - let hot = MdbxConnector::new(self.hot_path.as_ref()); + let hot = HotConnect::connect(&MdbxConnector::new(self.hot_path.as_ref()))?; let has_mdbx = !self.cold_path.is_empty(); #[cfg(any(feature = "postgres", feature = "sqlite"))] @@ -173,22 +189,17 @@ impl StorageConfig { let has_sql = std::env::var("SIGNET_COLD_SQL_URL").is_ok_and(|v| !v.is_empty()); match (has_mdbx, has_sql) { - (true, false) => Ok(StorageBuilder::new() - .hot(hot) - .cold(MdbxConnector::new(self.cold_path.as_ref())) - .cancel_token(cancel) - .build() - .await?), + (true, false) => { + let cold = + ColdConnect::connect(&MdbxConnector::new(self.cold_path.as_ref())).await?; + Ok(UnifiedStorage::spawn_erased(hot, cold, cancel)) + } #[cfg(any(feature = "postgres", feature = "sqlite"))] (false, true) => { let connector = self.cold_sql.clone().expect("cold_sql must be Some when has_sql is true"); - Ok(StorageBuilder::new() - .hot(hot) - .cold(connector) - .cancel_token(cancel) - .build() - .await?) + let cold = connector.connect().await?; + Ok(UnifiedStorage::spawn_erased(hot, cold, cancel)) } #[cfg(not(any(feature = "postgres", feature = "sqlite")))] (false, true) => { diff --git a/crates/node-tests/Cargo.toml b/crates/node-tests/Cargo.toml index 9130f39e..9777d03a 100644 --- a/crates/node-tests/Cargo.toml +++ b/crates/node-tests/Cargo.toml @@ -15,7 +15,7 @@ signet-blobber = { workspace = true, features = ["test-utils"] } signet-node.workspace = true signet-node-config = { workspace = true, features = ["test_utils"] } signet-node-types.workspace = true -signet-rpc.workspace = true +signet-rpc = { workspace = true, features = ["test-utils"] } signet-cold = { workspace = true, features = ["in-memory"] } signet-constants.workspace = true diff --git a/crates/node-tests/src/context.rs b/crates/node-tests/src/context.rs index d2ad4434..5505e63d 100644 --- a/crates/node-tests/src/context.rs +++ b/crates/node-tests/src/context.rs @@ -15,7 +15,7 @@ use alloy::{ rpc::types::eth::{TransactionReceipt, TransactionRequest}, }; use signet_blobber::MemoryBlobSource; -use signet_cold::{ColdStorageReadHandle, mem::MemColdBackend}; +use signet_cold::{ColdStorage, mem::MemColdBackend}; use signet_hot::{ db::{HotDbRead, UnsafeDbWrite}, mem::MemKv, @@ -186,8 +186,11 @@ impl SignetTestContext { } // Create UnifiedStorage - let storage = - Arc::new(UnifiedStorage::spawn(hot, MemColdBackend::new(), cancel_token.clone())); + let storage = Arc::new(UnifiedStorage::spawn_erased( + hot, + MemColdBackend::new(), + cancel_token.clone(), + )); let alias_oracle: Arc>> = Arc::new(Mutex::new(HashSet::default())); @@ -288,7 +291,7 @@ impl SignetTestContext { } /// Get a cold storage read handle. - pub fn cold(&self) -> ColdStorageReadHandle { + pub fn cold(&self) -> ColdStorage { self.storage.cold_reader() } diff --git a/crates/node-tests/src/utils.rs b/crates/node-tests/src/utils.rs index 356ceadf..84d76bfe 100644 --- a/crates/node-tests/src/utils.rs +++ b/crates/node-tests/src/utils.rs @@ -5,7 +5,7 @@ use alloy::{ signers::{SignerSync, local::PrivateKeySigner}, uint, }; -use signet_types::primitives::{RecoveredBlock, Transaction, TransactionSigned}; +use signet_types::primitives::{RecoveredBlock, SignetHeaderV1, Transaction, TransactionSigned}; use signet_zenith::Zenith; use std::{panic, sync::Once}; use tracing_subscriber::EnvFilter; @@ -21,7 +21,7 @@ pub fn fake_block(number: u64) -> RecoveredBlock { excess_blob_gas: Some(0), ..Default::default() }; - RecoveredBlock::blank_with_header(header) + RecoveredBlock::blank_with_header(SignetHeaderV1::new_unchecked(header)) } /// Sign a transaction with a wallet. diff --git a/crates/node-tests/tests/db.rs b/crates/node-tests/tests/db.rs index 383c9d00..a26824ec 100644 --- a/crates/node-tests/tests/db.rs +++ b/crates/node-tests/tests/db.rs @@ -28,7 +28,8 @@ async fn test_genesis() { writer.commit().unwrap(); } - let storage = Arc::new(UnifiedStorage::spawn(hot, MemColdBackend::new(), cancel_token.clone())); + let storage = + Arc::new(UnifiedStorage::spawn_erased(hot, MemColdBackend::new(), cancel_token.clone())); // Create a dummy notifier (not used, we only check genesis loading) let (_sender, receiver) = mpsc::unbounded_channel(); diff --git a/crates/node/src/builder.rs b/crates/node/src/builder.rs index 3995070f..32198b81 100644 --- a/crates/node/src/builder.rs +++ b/crates/node/src/builder.rs @@ -26,7 +26,7 @@ pub struct NotAStorage; /// /// The builder requires the following components to be set before building: /// - A [`HostNotifier`], via [`Self::with_notifier`]. -/// - An [`Arc>`], via [`Self::with_storage`]. +/// - An `Arc>`, via [`Self::with_storage`]. /// - An [`AliasOracleFactory`], via [`Self::with_alias_oracle`]. /// - A [`CacheHandle`], via [`Self::with_blob_cacher`]. /// - A [`ServeConfig`], via [`Self::with_serve_config`]. diff --git a/crates/node/src/node.rs b/crates/node/src/node.rs index ea045fc3..8ec29788 100644 --- a/crates/node/src/node.rs +++ b/crates/node/src/node.rs @@ -151,7 +151,7 @@ where unwind_to = target, "storage layers inconsistent, reconciling" ); - self.storage.unwind_above(target)?; + self.storage.unwind_above(target).await?; } } @@ -282,7 +282,7 @@ where ); let executed = processor.process_block(block_extracts).await?; self.notify_new_block(&executed); - self.storage.append_blocks(vec![executed])?; + self.storage.append_blocks(vec![executed]).await?; processed = true; } Ok(processed) diff --git a/crates/rpc/Cargo.toml b/crates/rpc/Cargo.toml index 4a3994c1..a338a3ef 100644 --- a/crates/rpc/Cargo.toml +++ b/crates/rpc/Cargo.toml @@ -35,6 +35,13 @@ axum = "0.8.1" tower-http = { version = "0.6.2", features = ["cors"] } interprocess = { version = "2.2.2", features = ["tokio"] } +[features] +# Enables the in-memory cold backend used by integration tests. Must not be +# enabled in production builds. +test-utils = ["signet-cold/test-utils"] +postgres = ["signet-storage/postgres"] +sqlite = ["signet-storage/sqlite"] + [dev-dependencies] tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } tokio-util.workspace = true @@ -49,3 +56,7 @@ axum = "0.8" tower = { version = "0.5", features = ["util"] } http = "1" trevm.workspace = true + +[[test]] +name = "eth_rpc" +required-features = ["test-utils"] diff --git a/crates/rpc/src/config/ctx.rs b/crates/rpc/src/config/ctx.rs index 042a9e0f..c6fff06a 100644 --- a/crates/rpc/src/config/ctx.rs +++ b/crates/rpc/src/config/ctx.rs @@ -68,7 +68,7 @@ use alloy::{ eips::{BlockId, BlockNumberOrTag}, genesis::ChainConfig, }; -use signet_cold::ColdStorageReadHandle; +use signet_cold::ColdStorage; use signet_evm::EthereumHardfork; use signet_hot::{ HotKv, @@ -191,7 +191,7 @@ impl StorageRpcCtx { } /// Get a cold storage read handle. - pub fn cold(&self) -> ColdStorageReadHandle { + pub fn cold(&self) -> ColdStorage { self.inner.storage.cold_reader() } diff --git a/crates/rpc/src/config/gas_oracle.rs b/crates/rpc/src/config/gas_oracle.rs index a77d838c..a30a4d2a 100644 --- a/crates/rpc/src/config/gas_oracle.rs +++ b/crates/rpc/src/config/gas_oracle.rs @@ -8,7 +8,7 @@ use crate::config::{GasOracleCache, StorageRpcConfig}; use alloy::{consensus::Transaction, primitives::U256}; -use signet_cold::{ColdStorageError, ColdStorageReadHandle, HeaderSpecifier}; +use signet_cold::{ColdStorageError, HeaderSpecifier}; /// Suggest a tip cap based on recent transaction tips. /// @@ -23,7 +23,7 @@ use signet_cold::{ColdStorageError, ColdStorageReadHandle, HeaderSpecifier}; /// Uses the provided `cache` to avoid redundant cold storage reads /// when the tip has already been computed for the current block. pub(crate) async fn suggest_tip_cap( - cold: &ColdStorageReadHandle, + cold: &signet_cold::ColdStorage, latest: u64, config: &StorageRpcConfig, cache: &GasOracleCache, diff --git a/crates/rpc/tests/eth_rpc.rs b/crates/rpc/tests/eth_rpc.rs index c4bff04a..f35f6556 100644 --- a/crates/rpc/tests/eth_rpc.rs +++ b/crates/rpc/tests/eth_rpc.rs @@ -14,7 +14,7 @@ use alloy::{ use axum::body::Body; use http::Request; use serde_json::{Value, json}; -use signet_cold::{BlockData, ColdStorageHandle, ColdStorageTask, mem::MemColdBackend}; +use signet_cold::{BlockData, ColdStorage, mem::MemColdBackend}; use signet_constants::SignetSystemConstants; use signet_hot::{HotKv, db::UnsafeDbWrite, mem::MemKv}; use signet_rpc::{ChainNotifier, StorageRpcConfig, StorageRpcCtx}; @@ -32,7 +32,7 @@ use trevm::revm::bytecode::Bytecode; /// Everything needed to make RPC calls against the storage-backed router. struct TestHarness { app: axum::Router, - cold: ColdStorageHandle, + cold: ColdStorage, hot: MemKv, chain: ChainNotifier, #[allow(dead_code)] @@ -45,7 +45,7 @@ impl TestHarness { async fn new(latest: u64) -> Self { let cancel = CancellationToken::new(); let hot = MemKv::new(); - let cold = ColdStorageTask::spawn(MemColdBackend::new(), cancel.clone()); + let cold = ColdStorage::new_erased(MemColdBackend::new(), cancel.clone()); let storage = UnifiedStorage::new(hot.clone(), cold.clone()); let constants = SignetSystemConstants::test(); let chain = ChainNotifier::new(16);