fix jh_hash function optimize bug in gcc11.4 (#3978)

* remove cryptonight cpp compiler flag Ofast * add block header nonce unit test * add block header nonce unit test * set opt-level=2 for cryptonight-rs package * fmt code * ubuntu 22 cryptonight use O2 * add hash_extra_jh_r rust function and unit test * remove Ofast * fix jh_hash function optimize bug in gcc11.4 * remove unused code * fix jh_hash optimize bug * fix ffi code * fix typo * add barnard block5061847 ubuntu20 * add barnard block5061847 ubuntu20 test * dev_setup: Update sccache version --------- Co-authored-by: simonjiao <simonjiaoh@gmail.com> Co-authored-by: sanlee42 <hit.cs.lijun@gmail.com>
starcoinorg · Nov 20, 2023 · 35c9a40 · 35c9a40
1 parent 2436a7a
commit 35c9a40
Show file tree

Hide file tree

Showing 10 changed files with 75 additions and 19 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -550,3 +550,6 @@ starcoin-vm-runtime.debug = 1
 
 [profile.release.package."*"]
 debug = false
+
+#[profile.release.package.cryptonight-rs]
+#opt-level = 2
diff --git a/consensus/Cargo.toml b/consensus/Cargo.toml
@@ -24,6 +24,8 @@ thiserror = { workspace = true }
 proptest = { workspace = true }
 proptest-derive = { workspace = true }
 stest = { workspace = true }
+serde = { features = ["derive"], workspace = true }
+serde_json = { features = ["arbitrary_precision"], workspace = true }
 
 [features]
 default = []

diff --git a/consensus/cryptonight-rs/ext/c_jh.c b/consensus/cryptonight-rs/ext/c_jh.c
@@ -213,16 +213,17 @@ static void E8(hashState *state)
 /*The compression function F8 */
 static void F8(hashState *state)
 {
-      uint64  i;
+    uint64_t* x = (uint64_t*)state->x;
+    const uint64_t* buf = (uint64*)state->buffer;
 
-      /*xor the 512-bit message with the fist half of the 1024-bit hash state*/
-      for (i = 0; i < 8; i++)  state->x[i >> 1][i & 1] ^= ((uint64*)state->buffer)[i];
+    /*xor the 512-bit message with the fist half of the 1024-bit hash state*/
+    for (int i = 0; i < 8; ++i) x[i] ^= buf[i];
 
       /*the bijective function E8 */
       E8(state);
 
       /*xor the 512-bit message with the second half of the 1024-bit hash state*/
-      for (i = 0; i < 8; i++)  state->x[(8+i) >> 1][(8+i) & 1] ^= ((uint64*)state->buffer)[i];
+    for (int i = 0; i < 8; ++i) x[i + 8] ^= buf[i];
 }
 
 /*before hashing a message, initialize the hash state as H0 */
@@ -240,6 +241,7 @@ static HashReturn Init(hashState *state, int hashbitlen)
             case 224: memcpy(state->x,JH224_H0,128); break;
             case 256: memcpy(state->x,JH256_H0,128); break;
             case 384: memcpy(state->x,JH384_H0,128); break;
+          default:
             case 512: memcpy(state->x,JH512_H0,128); break;
       }
 

diff --git a/consensus/cryptonight-rs/ext/hash-ops.h b/consensus/cryptonight-rs/ext/hash-ops.h
@@ -49,5 +49,11 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int
 
 void hash_extra_blake(const void *data, size_t length, char *hash);
 void hash_extra_groestl(const void *data, size_t length, char *hash);
+#ifdef __cplusplus
+extern "C" {
+#endif
 void hash_extra_jh(const void *data, size_t length, char *hash);
+#ifdef __cplusplus
+}
+#endif
 void hash_extra_skein(const void *data, size_t length, char *hash);
diff --git a/consensus/cryptonight-rs/src/lib.rs b/consensus/cryptonight-rs/src/lib.rs
@@ -4,21 +4,26 @@ use libc::{c_char, c_int, c_void, size_t};
 extern "C" {
     fn cryptonight_hash(
         data: *const c_char,
-        hash: *const c_char,
+        hash: *mut c_char,
         length: size_t,
         variant: c_int,
         height: u64,
     ) -> c_void;
 }
 
+#[link(name = "cryptonight", kind = "static")]
+extern "C" {
+    fn hash_extra_jh(data: *const c_char, length: size_t, hash: *mut c_char) -> c_void;
+}
 const VARIANT: i32 = 4;
 const HEIGHT: u64 = 0;
 
+// https://github.com/paritytech/rust-snappy/blob/master/snappy-sys/src/lib.rs#L19
 #[allow(clippy::unsound_collection_transmute)]
 pub fn cryptonight_r(data: &[u8], size: usize) -> Vec<u8> {
-    let hash: Vec<i8> = vec![0i8; 32];
+    let mut hash: Vec<i8> = vec![0i8; 32];
     let data_ptr: *const c_char = data.as_ptr() as *const c_char;
-    let hash_ptr: *const c_char = hash.as_ptr() as *const c_char;
+    let hash_ptr: *mut c_char = hash.as_mut_ptr() as *mut c_char;
     let mut hash = unsafe {
         cryptonight_hash(data_ptr, hash_ptr, size, VARIANT, HEIGHT);
         std::mem::transmute::<Vec<i8>, Vec<u8>>(hash)
@@ -27,5 +32,18 @@ pub fn cryptonight_r(data: &[u8], size: usize) -> Vec<u8> {
     hash
 }
 
+#[allow(clippy::unsound_collection_transmute)]
+pub fn hash_extra_jh_r(data: &[u8], size: usize) -> Vec<u8> {
+    let mut hash: Vec<i8> = vec![0i8; 32];
+    let data_ptr: *const c_char = data.as_ptr() as *const c_char;
+    let hash_ptr: *mut c_char = hash.as_mut_ptr() as *mut c_char;
+    let mut hash = unsafe {
+        hash_extra_jh(data_ptr, size, hash_ptr);
+        std::mem::transmute::<Vec<i8>, Vec<u8>>(hash)
+    };
+    hash.reverse();
+    hash
+}
+
 #[cfg(test)]
 mod tests;
diff --git a/consensus/cryptonight-rs/src/tests.rs b/consensus/cryptonight-rs/src/tests.rs
@@ -32,10 +32,21 @@ fn test_amd_ryzen() {
     assert_eq!(hash, data.output);
 }
 
+// this test is same test_amd_ryzen, trace the inner produce can get this data
+#[test]
+fn test_main_block2_jh_on_ubuntu22() {
+    let data = TestCase {
+        input: "d6a11a431fd56dc29c5927328e83eef11590362b70aef492f12c449bde657b4e3ac4e15816314613ca7339d152294b185ed86709acc1266259580d4aba729d37c733a89eb15ea91b28bbe49bd151cb1b947bb239c4ff10cfe24aa05f31a8cdb8e71345ea45cffeff6bbf9d2c7fe58f13f190b2921a531fb65841221d3e3e8483830a9b4a9c2e2843e74db1792c739d3ee27a973b61c838f300a54951cf8ac5fe99c2ced05d8b4cffa20a24f84dba4a4528b2b2cf035ae5b06b49f12364fcb5776db3692bfb6f029d".from_hex().unwrap(),
+        output: "0000008e67af69c3c670dab325e8fc3a9dc045c6e339aa35f43b415604513742".from_hex().unwrap(),
+    };
+    let hash = hash_extra_jh_r(&data.input[..], data.input.len());
+    assert_eq!(hash, data.output);
+}
+
 // add a test for ubuntu22
 // add -Ofast in build.rs, this test will failed on unbuntu22
 // the data is gen from verify_header_test_barnard_block3_ubuntu22
-// The test with 'ofast' flag passes using clang compiler under ubuntu22, but fails with using gcc
+// The test with 'Ofast' flag passes using clang compiler under ubuntu22, but fails with using gcc
 #[test]
 fn test_barnard_block3_on_ubuntu22() {
     let data = TestCase {

diff --git a/consensus/src/consensus_test.rs b/consensus/src/consensus_test.rs
@@ -5,12 +5,14 @@
 
 use crate::consensus::Consensus;
 use crate::difficulty::{get_next_target_helper, BlockDiffInfo};
-use crate::{difficult_to_target, target_to_difficulty, G_CRYPTONIGHT};
+use crate::{difficult_to_target, target_to_difficulty, G_ARGON, G_CRYPTONIGHT};
 use starcoin_crypto::hash::PlainCryptoHash;
 use starcoin_crypto::HashValue;
 use starcoin_time_service::{duration_since_epoch, MockTimeService, TimeService, TimeServiceType};
 use starcoin_types::account_address::AccountAddress;
-use starcoin_types::block::{BlockHeader, BlockHeaderBuilder, BlockHeaderExtra, RawBlockHeader};
+use starcoin_types::block::{
+    Block, BlockHeader, BlockHeaderBuilder, BlockHeaderExtra, RawBlockHeader,
+};
 use starcoin_types::U256;
 use starcoin_vm_types::genesis_config::ChainId;
 use std::collections::VecDeque;
@@ -94,6 +96,19 @@ fn verify_header_test_barnard_block3_ubuntu22() {
         .verify_header_difficulty(header.difficulty(), &header)
         .unwrap()
 }
+// see https://stcscan.io/barnard/address/0x01/resources
+// 0x00000000000000000000000000000001::Config::Config<0x00000000000000000000000000000001::ConsensusConfig::ConsensusConfig> "strategy":1
+// see https://stcscan.io/main/address/0x01/resources
+// 0x00000000000000000000000000000001::Config::Config<0x00000000000000000000000000000001::ConsensusConfig::ConsensusConfig> "strategy":3
+#[stest::test]
+fn verify_header_test_barnard_block5061847_ubuntu20() {
+    let str = r#"{"header":{"parent_hash":"0x45efb7914f0fe6bcd6baf4ed7a67da076953cf18128a25227e575a689236942f","timestamp":1654174610768,"number":5061847,"author":"0x7eec55ea1bafa8c4919101135b90b17b","author_auth_key":null,"txn_accumulator_root":"0x8c35c3d34ee16ca395c2a17668b61894b347c037d49d30477e7d2506dc69e936","block_accumulator_root":"0x0e602da30ea3f2f91b505f8a788dfc1c2b73619f328661a80ec6d59197527936","state_root":"0x42c63a367951160b01ce4e2c1e4559f08e786cf1e56a5e423da6db459f9507f7","gas_used":0,"difficulty":"0x0bee","body_hash":"0xc01e0329de6d899348a8ef4bd51db56175b3fa0988e57c3dcec8eaf13a164d97","chain_id":{"id":251},"nonce":2771673659,"extra":"0x00000000"},"body":{"transactions":[],"uncles":null}}"#;
+    let block: Block = serde_json::from_str(str).unwrap();
+    let header = block.header;
+    G_ARGON
+        .verify_header_difficulty(header.difficulty(), &header)
+        .unwrap()
+}
 
 fn simulate_blocks(time_plan: u64, init_difficulty: U256) -> u64 {
     fn liner_hash_pow(difficulty: U256, current: u64) -> u64 {

diff --git a/scripts/dev_setup.sh b/scripts/dev_setup.sh
@@ -16,9 +16,9 @@ set -eo pipefail
 
 SHELLCHECK_VERSION=0.7.1
 HADOLINT_VERSION=1.17.4
-SCCACHE_VERSION=0.2.16-alpha.0
+SCCACHE_VERSION=0.3.0
 #If installing sccache from a git repp set url@revision.
-SCCACHE_GIT='https://github.com/diem/sccache.git@ef50d87a58260c30767520045e242ccdbdb965af'
+#SCCACHE_GIT='https://github.com/diem/sccache.git@ef50d87a58260c30767520045e242ccdbdb965af'
 GRCOV_VERSION=0.8.2
 GUPPY_GIT='https://github.com/facebookincubator/cargo-guppy@39ec940f36b0a0df96a330243d127cbe2db9f919'
 KUBECTL_VERSION=1.18.6

diff --git a/scripts/verify_header.sh b/scripts/verify_header.sh
@@ -31,15 +31,13 @@ function download() {
 }
 
 function usage() {
-  echo -e "usage: verify_header.sh net to_dir"
+  echo -e "usage: verify_header.sh net"
   echo -e "net is main, barnard, proxima, halley"
-  echo -e "to_dir like ~/.starcoin/main，~/.starcoin/barnard"
 }
 
-function import_block() {
+function verify_header() {
   net=$1
   block_list=$2
-  to_dir=$3
 
   download "$net" "$block_list"
 
@@ -62,15 +60,14 @@ function import_block() {
   echo -e "$net verify-header succ"
 }
 
-if [ $# != 2 ]; then
+if [ $# != 1 ]; then
   usage
   exit 1
 fi
 net=$1
-to_dir=$2
 case $net in
 "main" | "barnard" | "proxima" |"halley")
-  import_block "$net" block_list.csv "$to_dir"
+  verify_header "$net" block_list.csv
   ;;
 *)
   echo "$net not supported"