chain add directly save special block without execute (#4013)

* db_exporter add verify blocks without save * chain add directly save special block without execute * db_exporter add output block execute (writeset and blockinfo) * add MAIN_DIRECT_SAVE_BLOCK_HASH_MAP * update verify block * fix execute_block_without_save * fix db_export verify-blocks * fix cargo fmt * fix compiler error * db exporter add apply block output * add MAIN_DIRECT_SAVE_BLOCK_HASH_MAP for barnard block * add save_startup_info * add MAIN_DIRECT_SAVE_BLOCK_HASH_MAP for barnard block * fix fmt
starcoinorg · Mar 5, 2024 · 50d34f8 · 50d34f8
1 parent e7ae35f
commit 50d34f8
Show file tree

Hide file tree

Showing 9 changed files with 1,157 additions and 7 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/chain/Cargo.toml b/chain/Cargo.toml
@@ -23,6 +23,8 @@ starcoin-types = { package = "starcoin-types", workspace = true }
 starcoin-vm-types = { workspace = true }
 starcoin-storage = { workspace = true }
 thiserror = { workspace = true }
+once_cell = { workspace = true }
+serde_json = { features = ["arbitrary_precision"], workspace = true }
 
 [dev-dependencies]
 proptest = { workspace = true }

diff --git a/chain/api/src/chain.rs b/chain/api/src/chain.rs
@@ -81,6 +81,7 @@ pub trait ChainReader {
     fn verify(&self, block: Block) -> Result<VerifiedBlock>;
     /// Execute block and verify it execute state, and save result base current chain, but do not change current chain.
     fn execute(&self, block: VerifiedBlock) -> Result<ExecutedBlock>;
+    fn execute_without_save(&self, verified_block: VerifiedBlock) -> Result<ExecutedBlock>;
     /// Get chain transaction infos
     fn get_transaction_infos(
         &self,

diff --git a/chain/src/chain.rs b/chain/src/chain.rs
diff --git a/chain/src/verifier/mod.rs b/chain/src/verifier/mod.rs
@@ -11,7 +11,7 @@ use starcoin_logger::prelude::debug;
 use starcoin_types::block::{Block, BlockHeader, ALLOWED_FUTURE_BLOCKTIME};
 use std::{collections::HashSet, str::FromStr};
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub enum Verifier {
     Basic,
     Consensus,

diff --git a/cmd/db-exporter/src/main.rs b/cmd/db-exporter/src/main.rs
@@ -235,6 +235,10 @@ enum Cmd {
     VerifyHeader(VerifyHeaderOptions),
     GenTurboSTMTransactions(GenTurboSTMTransactionsOptions),
     ApplyTurboSTMBlock(ApplyTurboSTMBlockOptions),
+    VerifyBlock(VerifyBlockOptions),
+    BlockOutput(BlockOutputOptions),
+    ApplyBlockOutput(ApplyBlockOutputOptions),
+    SaveStartupInfo(SaveStartupInfoOptions),
 }
 
 #[derive(Debug, Clone, Parser)]
@@ -444,6 +448,65 @@ pub struct ApplyTurboSTMBlockOptions {
     pub input_path: PathBuf,
 }
 
+#[derive(Debug, Parser)]
+#[clap(name = "verify-block-range", about = "verify block range")]
+pub struct VerifyBlockOptions {
+    #[clap(long, short = 'n')]
+    /// Chain Network
+    pub net: BuiltinNetworkID,
+    #[clap(long, short = 'i', parse(from_os_str))]
+    /// starcoin node db path. like ~/.starcoin/main
+    pub from_path: PathBuf,
+    #[clap(possible_values = Verifier::variants(), ignore_case = true)]
+    /// Verify type:  Basic, Consensus, Full, None, eg.
+    pub verifier: Option<Verifier>,
+    #[clap(long, short = 's')]
+    pub start: BlockNumber,
+    #[clap(long, short = 'e')]
+    pub end: Option<BlockNumber>,
+}
+
+#[derive(Debug, Parser)]
+#[clap(name = "block-output", about = "block output options")]
+pub struct BlockOutputOptions {
+    #[clap(long, short = 'n')]
+    /// Chain Network
+    pub net: BuiltinNetworkID,
+    #[clap(long, short = 'i', parse(from_os_str))]
+    /// starcoin node db path. like ~/.starcoin/main
+    pub from_path: PathBuf,
+    #[clap(long, short = 's')]
+    pub num: BlockNumber,
+}
+
+#[derive(Debug, Parser)]
+#[clap(name = "apply-block-output", about = "apply block output")]
+pub struct ApplyBlockOutputOptions {
+    #[clap(long, short = 'n')]
+    /// Chain Network
+    pub net: BuiltinNetworkID,
+    #[clap(long, short = 'o', parse(from_os_str))]
+    /// starcoin node db path. like ~/.starcoin/main
+    pub to_path: PathBuf,
+    #[clap(long, short = 'i', parse(from_os_str))]
+    /// input file, like accounts.csv
+    pub input_path: PathBuf,
+}
+
+#[derive(Debug, Parser)]
+#[clap(name = "save_startup_info", about = "save startup info")]
+pub struct SaveStartupInfoOptions {
+    #[clap(long, short = 'n')]
+    /// Chain Network
+    pub net: BuiltinNetworkID,
+    #[clap(long, short = 'o', parse(from_os_str))]
+    /// starcoin node db path. like ~/.starcoin/main
+    pub to_path: PathBuf,
+    /// startupinfo BlockNumber back off size
+    #[clap(long, short = 'b')]
+    pub hash_value: HashValue,
+}
+
 #[tokio::main(flavor = "multi_thread")]
 async fn main() -> anyhow::Result<()> {
     let opt = Opt::parse();
@@ -607,6 +670,29 @@ async fn main() -> anyhow::Result<()> {
                 apply_turbo_stm_block(option.to_path, option.turbo_stm_to_path, option.input_path);
             return result;
         }
+        Cmd::VerifyBlock(option) => {
+            let verifier = option.verifier.unwrap_or(Verifier::Basic);
+            let result = verify_block(
+                option.from_path,
+                option.net,
+                option.start,
+                option.end,
+                verifier,
+            );
+            return result;
+        }
+        Cmd::BlockOutput(option) => {
+            let result = block_output(option.from_path, option.net, option.num);
+            return result;
+        }
+        Cmd::ApplyBlockOutput(option) => {
+            let result = apply_block_output(option.to_path, option.input_path, option.net);
+            return result;
+        }
+        Cmd::SaveStartupInfo(option) => {
+            let result = save_startup_info(option.to_path, option.net, option.hash_value);
+            return result;
+        }
     }
     Ok(())
 }
@@ -2075,3 +2161,223 @@ pub fn apply_turbo_stm_block(
     println!("stm apply block use time: {:?}", use_time.as_secs());
     Ok(())
 }
+
+pub fn verify_block(
+    from_dir: PathBuf,
+    network: BuiltinNetworkID,
+    start: BlockNumber,
+    end: Option<BlockNumber>,
+    verifier: Verifier,
+) -> anyhow::Result<()> {
+    ::starcoin_logger::init();
+    let net = ChainNetwork::new_builtin(network);
+    let db_storage = DBStorage::open_with_cfs(
+        from_dir.join("starcoindb/db/starcoindb"),
+        StorageVersion::current_version()
+            .get_column_family_names()
+            .to_vec(),
+        true,
+        Default::default(),
+        None,
+    )?;
+    let storage = Arc::new(Storage::new(StorageInstance::new_cache_and_db_instance(
+        CacheStorage::new(None),
+        db_storage,
+    ))?);
+    let (chain_info, _) =
+        Genesis::init_and_check_storage(&net, storage.clone(), from_dir.as_ref())?;
+    let chain = BlockChain::new(
+        net.time_service(),
+        chain_info.head().id(),
+        storage.clone(),
+        None,
+    )
+    .expect("create block chain should success.");
+    let start_num = start;
+    let end_num = end.unwrap_or_else(|| chain.status().head().number());
+    let start_time = SystemTime::now();
+    let thread_cnt = num_cpus::get() / 2;
+    let avg = (end_num - start_num + 1) / (thread_cnt as u64);
+    let mut handles = vec![];
+    for i in 0..thread_cnt {
+        let st = start_num + i as u64 * avg;
+        let mut end = start_num + (i as u64 + 1) * avg - 1;
+        if end > end_num {
+            end = end_num;
+        }
+        let chain = BlockChain::new(
+            net.time_service(),
+            chain_info.head().id(),
+            storage.clone(),
+            None,
+        )
+        .expect("create block chain should success.");
+        let verifier2 = verifier.clone();
+        let handle = thread::spawn(move || {
+            verify_block_range(chain, st, end, verifier2).expect("verify_block_range panic")
+        });
+        handles.push(handle);
+    }
+
+    for handle in handles {
+        handle.join().unwrap();
+    }
+    let use_time = SystemTime::now().duration_since(start_time)?;
+    println!("verify block use time: {:?}", use_time.as_secs());
+    Ok(())
+}
+
+fn verify_block_range(
+    chain: BlockChain,
+    start_num: u64,
+    end_num: u64,
+    verifier: Verifier,
+) -> Result<()> {
+    let name = format!("file-{}-{}", start_num, end_num);
+    let mut file = File::create(name)?;
+    writeln!(file, "block [{}..{}]", start_num, end_num)?;
+    let bar = ProgressBar::new(end_num - start_num + 1);
+    bar.set_style(
+        ProgressStyle::default_bar()
+            .template("[{elapsed_precise}] {bar:100.cyan/blue} {percent}% {msg}"),
+    );
+    let mut cnt = 0;
+    for block_number in start_num..=end_num {
+        let block = chain
+            .get_block_by_number(block_number)?
+            .ok_or_else(|| format_err!("{} get block error", block_number))?;
+        let mut cur_chain = chain.fork(block.header().parent_hash())?;
+        let res = match verifier {
+            Verifier::Basic => cur_chain.verify_without_save::<BasicVerifier>(block),
+            Verifier::Consensus => cur_chain.verify_without_save::<ConsensusVerifier>(block),
+            Verifier::Full => cur_chain.verify_without_save::<FullVerifier>(block),
+            Verifier::None => cur_chain.verify_without_save::<NoneVerifier>(block),
+        };
+        if res.is_err() {
+            writeln!(file, "block {} verify error", block_number)?;
+        }
+        bar.set_message(format!("verify block {}", block_number));
+        bar.inc(1);
+        cnt += 1;
+        if cnt % 1000 == 0 {
+            writeln!(file, "block {} verify", start_num + cnt)?;
+        }
+    }
+    bar.finish();
+    file.flush()?;
+    Ok(())
+}
+
+pub fn block_output(
+    from_dir: PathBuf,
+    network: BuiltinNetworkID,
+    block_number: BlockNumber,
+) -> anyhow::Result<()> {
+    ::starcoin_logger::init();
+    let net = ChainNetwork::new_builtin(network);
+    let db_storage = DBStorage::open_with_cfs(
+        from_dir.join("starcoindb/db/starcoindb"),
+        StorageVersion::current_version()
+            .get_column_family_names()
+            .to_vec(),
+        true,
+        Default::default(),
+        None,
+    )?;
+    let storage = Arc::new(Storage::new(StorageInstance::new_cache_and_db_instance(
+        CacheStorage::new(None),
+        db_storage,
+    ))?);
+    let (chain_info, _) =
+        Genesis::init_and_check_storage(&net, storage.clone(), from_dir.as_ref())?;
+    let chain = BlockChain::new(
+        net.time_service(),
+        chain_info.head().id(),
+        storage.clone(),
+        None,
+    )
+    .expect("create block chain should success.");
+    let block = chain
+        .get_block_by_number(block_number)?
+        .ok_or_else(|| format_err!("{} get block error", block_number))?;
+    BlockChain::set_output_block();
+    let mut chain = BlockChain::new(
+        net.time_service(),
+        block.header.parent_hash(),
+        storage,
+        None,
+    )
+    .expect("create block chain should success.");
+    chain.verify_without_save::<BasicVerifier>(block)?;
+    Ok(())
+}
+
+pub fn apply_block_output(
+    to_dir: PathBuf,
+    input_path: PathBuf,
+    network: BuiltinNetworkID,
+) -> anyhow::Result<()> {
+    ::starcoin_logger::init();
+    let net = ChainNetwork::new_builtin(network);
+    let db_storage = DBStorage::new(to_dir.join("starcoindb/db"), RocksdbConfig::default(), None)?;
+    let storage = Arc::new(Storage::new(StorageInstance::new_cache_and_db_instance(
+        CacheStorage::new(None),
+        db_storage,
+    ))?);
+    let (_chain_info, _) = Genesis::init_and_check_storage(&net, storage.clone(), to_dir.as_ref())?;
+    let start_time = SystemTime::now();
+    let file_name = input_path.display().to_string();
+    let reader = BufReader::new(File::open(input_path)?);
+    let mut blocks = vec![];
+    for record in reader.lines() {
+        let record = record?;
+        let block: Block = serde_json::from_str(record.as_str())?;
+        blocks.push(block);
+    }
+    if blocks.is_empty() {
+        println!("file {} has apply", file_name);
+        return Ok(());
+    }
+
+    let use_time = SystemTime::now().duration_since(start_time)?;
+    println!("load blocks from file use time: {:?}", use_time.as_millis());
+    let bar = ProgressBar::new(blocks.len() as u64);
+    bar.set_style(
+        ProgressStyle::default_bar()
+            .template("[{elapsed_precise}] {bar:100.cyan/blue} {percent}% {msg}"),
+    );
+    BlockChain::set_output_block();
+    for block in blocks {
+        let block_number = block.header().number();
+        let mut chain = BlockChain::new(
+            net.time_service(),
+            block.header().parent_hash(),
+            storage.clone(),
+            None,
+        )
+        .expect("create block chain should success.");
+        chain.verify_without_save::<BasicVerifier>(block)?;
+        bar.set_message(format!("apply block {}", block_number));
+        bar.inc(1);
+    }
+    bar.finish();
+    Ok(())
+}
+
+fn save_startup_info(
+    to_dir: PathBuf,
+    network: BuiltinNetworkID,
+    hash_value: HashValue,
+) -> anyhow::Result<()> {
+    ::starcoin_logger::init();
+    let net = ChainNetwork::new_builtin(network);
+    let db_storage = DBStorage::new(to_dir.join("starcoindb/db"), RocksdbConfig::default(), None)?;
+    let storage = Arc::new(Storage::new(StorageInstance::new_cache_and_db_instance(
+        CacheStorage::new(None),
+        db_storage,
+    ))?);
+    let (_chain_info, _) = Genesis::init_and_check_storage(&net, storage.clone(), to_dir.as_ref())?;
+    let startup_info = StartupInfo::new(hash_value);
+    storage.save_startup_info(startup_info)?;
+    Ok(())
+}
diff --git a/cmd/db-exporter/src/verify_module.rs b/cmd/db-exporter/src/verify_module.rs
@@ -41,6 +41,10 @@ impl BatchCmdExec<VerifyModulesType, Block, VerifyModuleError> for Block {
                             Ok(compiled_module) => {
                                 match move_bytecode_verifier::verify_module(&compiled_module) {
                                     Err(e) => {
+                                        println!(
+                                            "verify module block height {}",
+                                            block.header().number()
+                                        );
                                         errors.push(VerifyModuleError {
                                             block_number: block.header().number(),
                                             transaction_hash: txn.id(),

diff --git a/executor/Cargo.toml b/executor/Cargo.toml
@@ -8,6 +8,7 @@ starcoin-types = { workspace = true }
 starcoin-vm-types = { workspace = true }
 starcoin-vm-runtime = { workspace = true }
 starcoin-statedb = { workspace = true }
+serde = { default-features = false, workspace = true }
 
 [dev-dependencies]
 bcs-ext = { package = "bcs-ext", workspace = true }