From 4d50f38fe05374342b4b55953d188fd50d422663 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Thu, 13 Jul 2017 17:10:26 -0700 Subject: [PATCH 1/5] Support multi-block SHA256 transforms Extracted from a patch by Wladimir van der Laan. --- src/crypto/sha256.cpp | 177 +++++++++++++++++++++--------------------- 1 file changed, 90 insertions(+), 87 deletions(-) diff --git a/src/crypto/sha256.cpp b/src/crypto/sha256.cpp index 5b9f00a0a..45ee93d47 100644 --- a/src/crypto/sha256.cpp +++ b/src/crypto/sha256.cpp @@ -43,88 +43,91 @@ void inline Initialize(uint32_t* s) s[7] = 0x5be0cd19ul; } -/** Perform one SHA-256 transformation, processing a 64-byte chunk. */ -void Transform(uint32_t* s, const unsigned char* chunk) +/** Perform a number of SHA-256 transformations, processing 64-byte chunks. */ +void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks) { - uint32_t a = s[0], b = s[1], c = s[2], d = s[3], e = s[4], f = s[5], g = s[6], h = s[7]; - uint32_t w0, w1, w2, w3, w4, w5, w6, w7, w8, w9, w10, w11, w12, w13, w14, w15; - - Round(a, b, c, d, e, f, g, h, 0x428a2f98, w0 = ReadBE32(chunk + 0)); - Round(h, a, b, c, d, e, f, g, 0x71374491, w1 = ReadBE32(chunk + 4)); - Round(g, h, a, b, c, d, e, f, 0xb5c0fbcf, w2 = ReadBE32(chunk + 8)); - Round(f, g, h, a, b, c, d, e, 0xe9b5dba5, w3 = ReadBE32(chunk + 12)); - Round(e, f, g, h, a, b, c, d, 0x3956c25b, w4 = ReadBE32(chunk + 16)); - Round(d, e, f, g, h, a, b, c, 0x59f111f1, w5 = ReadBE32(chunk + 20)); - Round(c, d, e, f, g, h, a, b, 0x923f82a4, w6 = ReadBE32(chunk + 24)); - Round(b, c, d, e, f, g, h, a, 0xab1c5ed5, w7 = ReadBE32(chunk + 28)); - Round(a, b, c, d, e, f, g, h, 0xd807aa98, w8 = ReadBE32(chunk + 32)); - Round(h, a, b, c, d, e, f, g, 0x12835b01, w9 = ReadBE32(chunk + 36)); - Round(g, h, a, b, c, d, e, f, 0x243185be, w10 = ReadBE32(chunk + 40)); - Round(f, g, h, a, b, c, d, e, 0x550c7dc3, w11 = ReadBE32(chunk + 44)); - Round(e, f, g, h, a, b, c, d, 0x72be5d74, w12 = ReadBE32(chunk + 48)); - Round(d, e, f, g, h, a, b, c, 0x80deb1fe, w13 = ReadBE32(chunk + 52)); - Round(c, d, e, f, g, h, a, b, 0x9bdc06a7, w14 = ReadBE32(chunk + 56)); - Round(b, c, d, e, f, g, h, a, 0xc19bf174, w15 = ReadBE32(chunk + 60)); - - Round(a, b, c, d, e, f, g, h, 0xe49b69c1, w0 += sigma1(w14) + w9 + sigma0(w1)); - Round(h, a, b, c, d, e, f, g, 0xefbe4786, w1 += sigma1(w15) + w10 + sigma0(w2)); - Round(g, h, a, b, c, d, e, f, 0x0fc19dc6, w2 += sigma1(w0) + w11 + sigma0(w3)); - Round(f, g, h, a, b, c, d, e, 0x240ca1cc, w3 += sigma1(w1) + w12 + sigma0(w4)); - Round(e, f, g, h, a, b, c, d, 0x2de92c6f, w4 += sigma1(w2) + w13 + sigma0(w5)); - Round(d, e, f, g, h, a, b, c, 0x4a7484aa, w5 += sigma1(w3) + w14 + sigma0(w6)); - Round(c, d, e, f, g, h, a, b, 0x5cb0a9dc, w6 += sigma1(w4) + w15 + sigma0(w7)); - Round(b, c, d, e, f, g, h, a, 0x76f988da, w7 += sigma1(w5) + w0 + sigma0(w8)); - Round(a, b, c, d, e, f, g, h, 0x983e5152, w8 += sigma1(w6) + w1 + sigma0(w9)); - Round(h, a, b, c, d, e, f, g, 0xa831c66d, w9 += sigma1(w7) + w2 + sigma0(w10)); - Round(g, h, a, b, c, d, e, f, 0xb00327c8, w10 += sigma1(w8) + w3 + sigma0(w11)); - Round(f, g, h, a, b, c, d, e, 0xbf597fc7, w11 += sigma1(w9) + w4 + sigma0(w12)); - Round(e, f, g, h, a, b, c, d, 0xc6e00bf3, w12 += sigma1(w10) + w5 + sigma0(w13)); - Round(d, e, f, g, h, a, b, c, 0xd5a79147, w13 += sigma1(w11) + w6 + sigma0(w14)); - Round(c, d, e, f, g, h, a, b, 0x06ca6351, w14 += sigma1(w12) + w7 + sigma0(w15)); - Round(b, c, d, e, f, g, h, a, 0x14292967, w15 += sigma1(w13) + w8 + sigma0(w0)); - - Round(a, b, c, d, e, f, g, h, 0x27b70a85, w0 += sigma1(w14) + w9 + sigma0(w1)); - Round(h, a, b, c, d, e, f, g, 0x2e1b2138, w1 += sigma1(w15) + w10 + sigma0(w2)); - Round(g, h, a, b, c, d, e, f, 0x4d2c6dfc, w2 += sigma1(w0) + w11 + sigma0(w3)); - Round(f, g, h, a, b, c, d, e, 0x53380d13, w3 += sigma1(w1) + w12 + sigma0(w4)); - Round(e, f, g, h, a, b, c, d, 0x650a7354, w4 += sigma1(w2) + w13 + sigma0(w5)); - Round(d, e, f, g, h, a, b, c, 0x766a0abb, w5 += sigma1(w3) + w14 + sigma0(w6)); - Round(c, d, e, f, g, h, a, b, 0x81c2c92e, w6 += sigma1(w4) + w15 + sigma0(w7)); - Round(b, c, d, e, f, g, h, a, 0x92722c85, w7 += sigma1(w5) + w0 + sigma0(w8)); - Round(a, b, c, d, e, f, g, h, 0xa2bfe8a1, w8 += sigma1(w6) + w1 + sigma0(w9)); - Round(h, a, b, c, d, e, f, g, 0xa81a664b, w9 += sigma1(w7) + w2 + sigma0(w10)); - Round(g, h, a, b, c, d, e, f, 0xc24b8b70, w10 += sigma1(w8) + w3 + sigma0(w11)); - Round(f, g, h, a, b, c, d, e, 0xc76c51a3, w11 += sigma1(w9) + w4 + sigma0(w12)); - Round(e, f, g, h, a, b, c, d, 0xd192e819, w12 += sigma1(w10) + w5 + sigma0(w13)); - Round(d, e, f, g, h, a, b, c, 0xd6990624, w13 += sigma1(w11) + w6 + sigma0(w14)); - Round(c, d, e, f, g, h, a, b, 0xf40e3585, w14 += sigma1(w12) + w7 + sigma0(w15)); - Round(b, c, d, e, f, g, h, a, 0x106aa070, w15 += sigma1(w13) + w8 + sigma0(w0)); - - Round(a, b, c, d, e, f, g, h, 0x19a4c116, w0 += sigma1(w14) + w9 + sigma0(w1)); - Round(h, a, b, c, d, e, f, g, 0x1e376c08, w1 += sigma1(w15) + w10 + sigma0(w2)); - Round(g, h, a, b, c, d, e, f, 0x2748774c, w2 += sigma1(w0) + w11 + sigma0(w3)); - Round(f, g, h, a, b, c, d, e, 0x34b0bcb5, w3 += sigma1(w1) + w12 + sigma0(w4)); - Round(e, f, g, h, a, b, c, d, 0x391c0cb3, w4 += sigma1(w2) + w13 + sigma0(w5)); - Round(d, e, f, g, h, a, b, c, 0x4ed8aa4a, w5 += sigma1(w3) + w14 + sigma0(w6)); - Round(c, d, e, f, g, h, a, b, 0x5b9cca4f, w6 += sigma1(w4) + w15 + sigma0(w7)); - Round(b, c, d, e, f, g, h, a, 0x682e6ff3, w7 += sigma1(w5) + w0 + sigma0(w8)); - Round(a, b, c, d, e, f, g, h, 0x748f82ee, w8 += sigma1(w6) + w1 + sigma0(w9)); - Round(h, a, b, c, d, e, f, g, 0x78a5636f, w9 += sigma1(w7) + w2 + sigma0(w10)); - Round(g, h, a, b, c, d, e, f, 0x84c87814, w10 += sigma1(w8) + w3 + sigma0(w11)); - Round(f, g, h, a, b, c, d, e, 0x8cc70208, w11 += sigma1(w9) + w4 + sigma0(w12)); - Round(e, f, g, h, a, b, c, d, 0x90befffa, w12 += sigma1(w10) + w5 + sigma0(w13)); - Round(d, e, f, g, h, a, b, c, 0xa4506ceb, w13 += sigma1(w11) + w6 + sigma0(w14)); - Round(c, d, e, f, g, h, a, b, 0xbef9a3f7, w14 + sigma1(w12) + w7 + sigma0(w15)); - Round(b, c, d, e, f, g, h, a, 0xc67178f2, w15 + sigma1(w13) + w8 + sigma0(w0)); - - s[0] += a; - s[1] += b; - s[2] += c; - s[3] += d; - s[4] += e; - s[5] += f; - s[6] += g; - s[7] += h; + while (blocks--) { + uint32_t a = s[0], b = s[1], c = s[2], d = s[3], e = s[4], f = s[5], g = s[6], h = s[7]; + uint32_t w0, w1, w2, w3, w4, w5, w6, w7, w8, w9, w10, w11, w12, w13, w14, w15; + + Round(a, b, c, d, e, f, g, h, 0x428a2f98, w0 = ReadBE32(chunk + 0)); + Round(h, a, b, c, d, e, f, g, 0x71374491, w1 = ReadBE32(chunk + 4)); + Round(g, h, a, b, c, d, e, f, 0xb5c0fbcf, w2 = ReadBE32(chunk + 8)); + Round(f, g, h, a, b, c, d, e, 0xe9b5dba5, w3 = ReadBE32(chunk + 12)); + Round(e, f, g, h, a, b, c, d, 0x3956c25b, w4 = ReadBE32(chunk + 16)); + Round(d, e, f, g, h, a, b, c, 0x59f111f1, w5 = ReadBE32(chunk + 20)); + Round(c, d, e, f, g, h, a, b, 0x923f82a4, w6 = ReadBE32(chunk + 24)); + Round(b, c, d, e, f, g, h, a, 0xab1c5ed5, w7 = ReadBE32(chunk + 28)); + Round(a, b, c, d, e, f, g, h, 0xd807aa98, w8 = ReadBE32(chunk + 32)); + Round(h, a, b, c, d, e, f, g, 0x12835b01, w9 = ReadBE32(chunk + 36)); + Round(g, h, a, b, c, d, e, f, 0x243185be, w10 = ReadBE32(chunk + 40)); + Round(f, g, h, a, b, c, d, e, 0x550c7dc3, w11 = ReadBE32(chunk + 44)); + Round(e, f, g, h, a, b, c, d, 0x72be5d74, w12 = ReadBE32(chunk + 48)); + Round(d, e, f, g, h, a, b, c, 0x80deb1fe, w13 = ReadBE32(chunk + 52)); + Round(c, d, e, f, g, h, a, b, 0x9bdc06a7, w14 = ReadBE32(chunk + 56)); + Round(b, c, d, e, f, g, h, a, 0xc19bf174, w15 = ReadBE32(chunk + 60)); + + Round(a, b, c, d, e, f, g, h, 0xe49b69c1, w0 += sigma1(w14) + w9 + sigma0(w1)); + Round(h, a, b, c, d, e, f, g, 0xefbe4786, w1 += sigma1(w15) + w10 + sigma0(w2)); + Round(g, h, a, b, c, d, e, f, 0x0fc19dc6, w2 += sigma1(w0) + w11 + sigma0(w3)); + Round(f, g, h, a, b, c, d, e, 0x240ca1cc, w3 += sigma1(w1) + w12 + sigma0(w4)); + Round(e, f, g, h, a, b, c, d, 0x2de92c6f, w4 += sigma1(w2) + w13 + sigma0(w5)); + Round(d, e, f, g, h, a, b, c, 0x4a7484aa, w5 += sigma1(w3) + w14 + sigma0(w6)); + Round(c, d, e, f, g, h, a, b, 0x5cb0a9dc, w6 += sigma1(w4) + w15 + sigma0(w7)); + Round(b, c, d, e, f, g, h, a, 0x76f988da, w7 += sigma1(w5) + w0 + sigma0(w8)); + Round(a, b, c, d, e, f, g, h, 0x983e5152, w8 += sigma1(w6) + w1 + sigma0(w9)); + Round(h, a, b, c, d, e, f, g, 0xa831c66d, w9 += sigma1(w7) + w2 + sigma0(w10)); + Round(g, h, a, b, c, d, e, f, 0xb00327c8, w10 += sigma1(w8) + w3 + sigma0(w11)); + Round(f, g, h, a, b, c, d, e, 0xbf597fc7, w11 += sigma1(w9) + w4 + sigma0(w12)); + Round(e, f, g, h, a, b, c, d, 0xc6e00bf3, w12 += sigma1(w10) + w5 + sigma0(w13)); + Round(d, e, f, g, h, a, b, c, 0xd5a79147, w13 += sigma1(w11) + w6 + sigma0(w14)); + Round(c, d, e, f, g, h, a, b, 0x06ca6351, w14 += sigma1(w12) + w7 + sigma0(w15)); + Round(b, c, d, e, f, g, h, a, 0x14292967, w15 += sigma1(w13) + w8 + sigma0(w0)); + + Round(a, b, c, d, e, f, g, h, 0x27b70a85, w0 += sigma1(w14) + w9 + sigma0(w1)); + Round(h, a, b, c, d, e, f, g, 0x2e1b2138, w1 += sigma1(w15) + w10 + sigma0(w2)); + Round(g, h, a, b, c, d, e, f, 0x4d2c6dfc, w2 += sigma1(w0) + w11 + sigma0(w3)); + Round(f, g, h, a, b, c, d, e, 0x53380d13, w3 += sigma1(w1) + w12 + sigma0(w4)); + Round(e, f, g, h, a, b, c, d, 0x650a7354, w4 += sigma1(w2) + w13 + sigma0(w5)); + Round(d, e, f, g, h, a, b, c, 0x766a0abb, w5 += sigma1(w3) + w14 + sigma0(w6)); + Round(c, d, e, f, g, h, a, b, 0x81c2c92e, w6 += sigma1(w4) + w15 + sigma0(w7)); + Round(b, c, d, e, f, g, h, a, 0x92722c85, w7 += sigma1(w5) + w0 + sigma0(w8)); + Round(a, b, c, d, e, f, g, h, 0xa2bfe8a1, w8 += sigma1(w6) + w1 + sigma0(w9)); + Round(h, a, b, c, d, e, f, g, 0xa81a664b, w9 += sigma1(w7) + w2 + sigma0(w10)); + Round(g, h, a, b, c, d, e, f, 0xc24b8b70, w10 += sigma1(w8) + w3 + sigma0(w11)); + Round(f, g, h, a, b, c, d, e, 0xc76c51a3, w11 += sigma1(w9) + w4 + sigma0(w12)); + Round(e, f, g, h, a, b, c, d, 0xd192e819, w12 += sigma1(w10) + w5 + sigma0(w13)); + Round(d, e, f, g, h, a, b, c, 0xd6990624, w13 += sigma1(w11) + w6 + sigma0(w14)); + Round(c, d, e, f, g, h, a, b, 0xf40e3585, w14 += sigma1(w12) + w7 + sigma0(w15)); + Round(b, c, d, e, f, g, h, a, 0x106aa070, w15 += sigma1(w13) + w8 + sigma0(w0)); + + Round(a, b, c, d, e, f, g, h, 0x19a4c116, w0 += sigma1(w14) + w9 + sigma0(w1)); + Round(h, a, b, c, d, e, f, g, 0x1e376c08, w1 += sigma1(w15) + w10 + sigma0(w2)); + Round(g, h, a, b, c, d, e, f, 0x2748774c, w2 += sigma1(w0) + w11 + sigma0(w3)); + Round(f, g, h, a, b, c, d, e, 0x34b0bcb5, w3 += sigma1(w1) + w12 + sigma0(w4)); + Round(e, f, g, h, a, b, c, d, 0x391c0cb3, w4 += sigma1(w2) + w13 + sigma0(w5)); + Round(d, e, f, g, h, a, b, c, 0x4ed8aa4a, w5 += sigma1(w3) + w14 + sigma0(w6)); + Round(c, d, e, f, g, h, a, b, 0x5b9cca4f, w6 += sigma1(w4) + w15 + sigma0(w7)); + Round(b, c, d, e, f, g, h, a, 0x682e6ff3, w7 += sigma1(w5) + w0 + sigma0(w8)); + Round(a, b, c, d, e, f, g, h, 0x748f82ee, w8 += sigma1(w6) + w1 + sigma0(w9)); + Round(h, a, b, c, d, e, f, g, 0x78a5636f, w9 += sigma1(w7) + w2 + sigma0(w10)); + Round(g, h, a, b, c, d, e, f, 0x84c87814, w10 += sigma1(w8) + w3 + sigma0(w11)); + Round(f, g, h, a, b, c, d, e, 0x8cc70208, w11 += sigma1(w9) + w4 + sigma0(w12)); + Round(e, f, g, h, a, b, c, d, 0x90befffa, w12 += sigma1(w10) + w5 + sigma0(w13)); + Round(d, e, f, g, h, a, b, c, 0xa4506ceb, w13 += sigma1(w11) + w6 + sigma0(w14)); + Round(c, d, e, f, g, h, a, b, 0xbef9a3f7, w14 + sigma1(w12) + w7 + sigma0(w15)); + Round(b, c, d, e, f, g, h, a, 0xc67178f2, w15 + sigma1(w13) + w8 + sigma0(w0)); + + s[0] += a; + s[1] += b; + s[2] += c; + s[3] += d; + s[4] += e; + s[5] += f; + s[6] += g; + s[7] += h; + chunk += 64; + } } } // namespace sha256 @@ -147,14 +150,14 @@ CSHA256& CSHA256::Write(const unsigned char* data, size_t len) memcpy(buf + bufsize, data, 64 - bufsize); bytes += 64 - bufsize; data += 64 - bufsize; - sha256::Transform(s, buf); + sha256::Transform(s, buf, 1); bufsize = 0; } - while (end >= data + 64) { - // Process full chunks directly from the source. - sha256::Transform(s, data); - bytes += 64; - data += 64; + if (end - data >= 64) { + size_t blocks = (end - data) / 64; + sha256::Transform(s, data, blocks); + data += 64 * blocks; + bytes += 64 * blocks; } if (end > data) { // Fill the buffer with what remains. From 2991c91d88f3d17ac6b38bddfa6f1b6b8d835c0f Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Thu, 13 Jul 2017 23:26:04 -0700 Subject: [PATCH 2/5] Add SHA256 dispatcher --- src/bench/bench_bitcoin.cpp | 2 ++ src/crypto/sha256.cpp | 13 +++++++++++-- src/crypto/sha256.h | 6 ++++++ src/init.cpp | 2 ++ src/test/test_bitcoin.cpp | 2 ++ 5 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/bench/bench_bitcoin.cpp b/src/bench/bench_bitcoin.cpp index 226861aa7..37fd77243 100644 --- a/src/bench/bench_bitcoin.cpp +++ b/src/bench/bench_bitcoin.cpp @@ -4,6 +4,7 @@ #include "bench.h" +#include "crypto/sha256.h" #include "key.h" #include "validation.h" #include "util.h" @@ -12,6 +13,7 @@ int main(int argc, char** argv) { + SHA256AutoDetect(); RandomInit(); ECC_Start(); SetupEnvironment(); diff --git a/src/crypto/sha256.cpp b/src/crypto/sha256.cpp index 45ee93d47..3649dc4be 100644 --- a/src/crypto/sha256.cpp +++ b/src/crypto/sha256.cpp @@ -8,6 +8,8 @@ #include +#include + // Internal implementation code. namespace { @@ -131,8 +133,15 @@ void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks) } } // namespace sha256 + +void (*Transform)(uint32_t*, const unsigned char*, size_t) = sha256::Transform; + } // namespace +std::string SHA256AutoDetect() +{ + return "standard"; +} ////// SHA-256 @@ -150,12 +159,12 @@ CSHA256& CSHA256::Write(const unsigned char* data, size_t len) memcpy(buf + bufsize, data, 64 - bufsize); bytes += 64 - bufsize; data += 64 - bufsize; - sha256::Transform(s, buf, 1); + Transform(s, buf, 1); bufsize = 0; } if (end - data >= 64) { size_t blocks = (end - data) / 64; - sha256::Transform(s, data, blocks); + Transform(s, data, blocks); data += 64 * blocks; bytes += 64 * blocks; } diff --git a/src/crypto/sha256.h b/src/crypto/sha256.h index 127e62a22..aa4f3972c 100644 --- a/src/crypto/sha256.h +++ b/src/crypto/sha256.h @@ -7,6 +7,7 @@ #include #include +#include /** A hasher class for SHA-256. */ class CSHA256 @@ -25,4 +26,9 @@ public: CSHA256& Reset(); }; +/** Autodetect the best available SHA256 implementation. + * Returns the name of the implementation. + */ +std::string SHA256AutoDetect(); + #endif // BITCOIN_CRYPTO_SHA256_H diff --git a/src/init.cpp b/src/init.cpp index 1e8564201..b16653624 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1161,6 +1161,8 @@ bool AppInitSanityChecks() // ********************************************************* Step 4: sanity checks // Initialize elliptic curve code + std::string sha256_algo = SHA256AutoDetect(); + LogPrintf("Using the '%s' SHA256 implementation\n", sha256_algo); RandomInit(); ECC_Start(); globalVerifyHandle.reset(new ECCVerifyHandle()); diff --git a/src/test/test_bitcoin.cpp b/src/test/test_bitcoin.cpp index 3ba81ed17..0dd51a490 100644 --- a/src/test/test_bitcoin.cpp +++ b/src/test/test_bitcoin.cpp @@ -7,6 +7,7 @@ #include "chainparams.h" #include "consensus/consensus.h" #include "consensus/validation.h" +#include "crypto/sha256.h" #include "fs.h" #include "key.h" #include "validation.h" @@ -33,6 +34,7 @@ extern void noui_connect(); BasicTestingSetup::BasicTestingSetup(const std::string& chainName) { + SHA256AutoDetect(); RandomInit(); ECC_Start(); SetupEnvironment(); From c1ccb15b0e847eb95623f9d25dc522aa02dbdbe8 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Thu, 13 Jul 2017 23:33:11 -0700 Subject: [PATCH 3/5] Add SSE4 based SHA256 --- src/Makefile.am | 1 + src/crypto/sha256.cpp | 18 +- src/crypto/sha256_sse4.cpp | 1506 ++++++++++++++++++++++++++++++++++++ 3 files changed, 1523 insertions(+), 2 deletions(-) create mode 100644 src/crypto/sha256_sse4.cpp diff --git a/src/Makefile.am b/src/Makefile.am index 06b09404a..35bfeb466 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -263,6 +263,7 @@ crypto_libbitcoin_crypto_a_SOURCES = \ crypto/sha1.cpp \ crypto/sha1.h \ crypto/sha256.cpp \ + crypto/sha256_sse4.cpp \ crypto/sha256.h \ crypto/sha512.cpp \ crypto/sha512.h diff --git a/src/crypto/sha256.cpp b/src/crypto/sha256.cpp index 3649dc4be..4ecf50903 100644 --- a/src/crypto/sha256.cpp +++ b/src/crypto/sha256.cpp @@ -3,13 +3,19 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "crypto/sha256.h" - #include "crypto/common.h" #include - #include +#if defined(__x86_64__) || defined(__amd64__) +#include +namespace sha256_sse4 +{ +void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks); +} +#endif + // Internal implementation code. namespace { @@ -140,6 +146,14 @@ void (*Transform)(uint32_t*, const unsigned char*, size_t) = sha256::Transform; std::string SHA256AutoDetect() { +#if defined(__x86_64__) || defined(__amd64__) + uint32_t eax, ebx, ecx, edx; + if (__get_cpuid(1, &eax, &ebx, &ecx, &edx) && (ecx >> 19) & 1) { + Transform = sha256_sse4::Transform; + return "sse4"; + } +#endif + return "standard"; } diff --git a/src/crypto/sha256_sse4.cpp b/src/crypto/sha256_sse4.cpp new file mode 100644 index 000000000..89f529a3a --- /dev/null +++ b/src/crypto/sha256_sse4.cpp @@ -0,0 +1,1506 @@ +// Copyright (c) 2017 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// +// This is a translation to GCC extended asm syntax from YASM code by Intel +// (available at the bottom of this file). + +#include +#include + +#if defined(__x86_64__) || defined(__amd64__) + +namespace sha256_sse4 +{ +void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks) +{ + static const uint32_t K256 alignas(16) [] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, + 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, + 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, + 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, + 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, + }; + static const uint32_t FLIP_MASK alignas(16) [] = {0x00010203, 0x04050607, 0x08090a0b, 0x0c0d0e0f}; + static const uint32_t SHUF_00BA alignas(16) [] = {0x03020100, 0x0b0a0908, 0xffffffff, 0xffffffff}; + static const uint32_t SHUF_DC00 alignas(16) [] = {0xffffffff, 0xffffffff, 0x03020100, 0x0b0a0908}; + uint32_t a, b, c, d, f, g, h, y0, y1, y2; + uint64_t tbl; + uint64_t inp_end, inp; + uint32_t xfer alignas(16) [4]; + + __asm__ __volatile__( + "shl $0x6,%2;" + "je Ldone_hash_%=;" + "add %1,%2;" + "mov %2,%14;" + "mov (%0),%3;" + "mov 0x4(%0),%4;" + "mov 0x8(%0),%5;" + "mov 0xc(%0),%6;" + "mov 0x10(%0),%k2;" + "mov 0x14(%0),%7;" + "mov 0x18(%0),%8;" + "mov 0x1c(%0),%9;" + "movdqa %18,%%xmm12;" + "movdqa %19,%%xmm10;" + "movdqa %20,%%xmm11;" + + "Lloop0_%=:" + "lea %17,%13;" + "movdqu (%1),%%xmm4;" + "pshufb %%xmm12,%%xmm4;" + "movdqu 0x10(%1),%%xmm5;" + "pshufb %%xmm12,%%xmm5;" + "movdqu 0x20(%1),%%xmm6;" + "pshufb %%xmm12,%%xmm6;" + "movdqu 0x30(%1),%%xmm7;" + "pshufb %%xmm12,%%xmm7;" + "mov %1,%15;" + "mov $3,%1;" + + "Lloop1_%=:" + "movdqa 0x0(%13),%%xmm9;" + "paddd %%xmm4,%%xmm9;" + "movdqa %%xmm9,%16;" + "movdqa %%xmm7,%%xmm0;" + "mov %k2,%10;" + "ror $0xe,%10;" + "mov %3,%11;" + "palignr $0x4,%%xmm6,%%xmm0;" + "ror $0x9,%11;" + "xor %k2,%10;" + "mov %7,%12;" + "ror $0x5,%10;" + "movdqa %%xmm5,%%xmm1;" + "xor %3,%11;" + "xor %8,%12;" + "paddd %%xmm4,%%xmm0;" + "xor %k2,%10;" + "and %k2,%12;" + "ror $0xb,%11;" + "palignr $0x4,%%xmm4,%%xmm1;" + "xor %3,%11;" + "ror $0x6,%10;" + "xor %8,%12;" + "movdqa %%xmm1,%%xmm2;" + "ror $0x2,%11;" + "add %10,%12;" + "add %16,%12;" + "movdqa %%xmm1,%%xmm3;" + "mov %3,%10;" + "add %12,%9;" + "mov %3,%12;" + "pslld $0x19,%%xmm1;" + "or %5,%10;" + "add %9,%6;" + "and %5,%12;" + "psrld $0x7,%%xmm2;" + "and %4,%10;" + "add %11,%9;" + "por %%xmm2,%%xmm1;" + "or %12,%10;" + "add %10,%9;" + "movdqa %%xmm3,%%xmm2;" + "mov %6,%10;" + "mov %9,%11;" + "movdqa %%xmm3,%%xmm8;" + "ror $0xe,%10;" + "xor %6,%10;" + "mov %k2,%12;" + "ror $0x9,%11;" + "pslld $0xe,%%xmm3;" + "xor %9,%11;" + "ror $0x5,%10;" + "xor %7,%12;" + "psrld $0x12,%%xmm2;" + "ror $0xb,%11;" + "xor %6,%10;" + "and %6,%12;" + "ror $0x6,%10;" + "pxor %%xmm3,%%xmm1;" + "xor %9,%11;" + "xor %7,%12;" + "psrld $0x3,%%xmm8;" + "add %10,%12;" + "add 4+%16,%12;" + "ror $0x2,%11;" + "pxor %%xmm2,%%xmm1;" + "mov %9,%10;" + "add %12,%8;" + "mov %9,%12;" + "pxor %%xmm8,%%xmm1;" + "or %4,%10;" + "add %8,%5;" + "and %4,%12;" + "pshufd $0xfa,%%xmm7,%%xmm2;" + "and %3,%10;" + "add %11,%8;" + "paddd %%xmm1,%%xmm0;" + "or %12,%10;" + "add %10,%8;" + "movdqa %%xmm2,%%xmm3;" + "mov %5,%10;" + "mov %8,%11;" + "ror $0xe,%10;" + "movdqa %%xmm2,%%xmm8;" + "xor %5,%10;" + "ror $0x9,%11;" + "mov %6,%12;" + "xor %8,%11;" + "ror $0x5,%10;" + "psrlq $0x11,%%xmm2;" + "xor %k2,%12;" + "psrlq $0x13,%%xmm3;" + "xor %5,%10;" + "and %5,%12;" + "psrld $0xa,%%xmm8;" + "ror $0xb,%11;" + "xor %8,%11;" + "xor %k2,%12;" + "ror $0x6,%10;" + "pxor %%xmm3,%%xmm2;" + "add %10,%12;" + "ror $0x2,%11;" + "add 8+%16,%12;" + "pxor %%xmm2,%%xmm8;" + "mov %8,%10;" + "add %12,%7;" + "mov %8,%12;" + "pshufb %%xmm10,%%xmm8;" + "or %3,%10;" + "add %7,%4;" + "and %3,%12;" + "paddd %%xmm8,%%xmm0;" + "and %9,%10;" + "add %11,%7;" + "pshufd $0x50,%%xmm0,%%xmm2;" + "or %12,%10;" + "add %10,%7;" + "movdqa %%xmm2,%%xmm3;" + "mov %4,%10;" + "ror $0xe,%10;" + "mov %7,%11;" + "movdqa %%xmm2,%%xmm4;" + "ror $0x9,%11;" + "xor %4,%10;" + "mov %5,%12;" + "ror $0x5,%10;" + "psrlq $0x11,%%xmm2;" + "xor %7,%11;" + "xor %6,%12;" + "psrlq $0x13,%%xmm3;" + "xor %4,%10;" + "and %4,%12;" + "ror $0xb,%11;" + "psrld $0xa,%%xmm4;" + "xor %7,%11;" + "ror $0x6,%10;" + "xor %6,%12;" + "pxor %%xmm3,%%xmm2;" + "ror $0x2,%11;" + "add %10,%12;" + "add 12+%16,%12;" + "pxor %%xmm2,%%xmm4;" + "mov %7,%10;" + "add %12,%k2;" + "mov %7,%12;" + "pshufb %%xmm11,%%xmm4;" + "or %9,%10;" + "add %k2,%3;" + "and %9,%12;" + "paddd %%xmm0,%%xmm4;" + "and %8,%10;" + "add %11,%k2;" + "or %12,%10;" + "add %10,%k2;" + "movdqa 0x10(%13),%%xmm9;" + "paddd %%xmm5,%%xmm9;" + "movdqa %%xmm9,%16;" + "movdqa %%xmm4,%%xmm0;" + "mov %3,%10;" + "ror $0xe,%10;" + "mov %k2,%11;" + "palignr $0x4,%%xmm7,%%xmm0;" + "ror $0x9,%11;" + "xor %3,%10;" + "mov %4,%12;" + "ror $0x5,%10;" + "movdqa %%xmm6,%%xmm1;" + "xor %k2,%11;" + "xor %5,%12;" + "paddd %%xmm5,%%xmm0;" + "xor %3,%10;" + "and %3,%12;" + "ror $0xb,%11;" + "palignr $0x4,%%xmm5,%%xmm1;" + "xor %k2,%11;" + "ror $0x6,%10;" + "xor %5,%12;" + "movdqa %%xmm1,%%xmm2;" + "ror $0x2,%11;" + "add %10,%12;" + "add %16,%12;" + "movdqa %%xmm1,%%xmm3;" + "mov %k2,%10;" + "add %12,%6;" + "mov %k2,%12;" + "pslld $0x19,%%xmm1;" + "or %8,%10;" + "add %6,%9;" + "and %8,%12;" + "psrld $0x7,%%xmm2;" + "and %7,%10;" + "add %11,%6;" + "por %%xmm2,%%xmm1;" + "or %12,%10;" + "add %10,%6;" + "movdqa %%xmm3,%%xmm2;" + "mov %9,%10;" + "mov %6,%11;" + "movdqa %%xmm3,%%xmm8;" + "ror $0xe,%10;" + "xor %9,%10;" + "mov %3,%12;" + "ror $0x9,%11;" + "pslld $0xe,%%xmm3;" + "xor %6,%11;" + "ror $0x5,%10;" + "xor %4,%12;" + "psrld $0x12,%%xmm2;" + "ror $0xb,%11;" + "xor %9,%10;" + "and %9,%12;" + "ror $0x6,%10;" + "pxor %%xmm3,%%xmm1;" + "xor %6,%11;" + "xor %4,%12;" + "psrld $0x3,%%xmm8;" + "add %10,%12;" + "add 4+%16,%12;" + "ror $0x2,%11;" + "pxor %%xmm2,%%xmm1;" + "mov %6,%10;" + "add %12,%5;" + "mov %6,%12;" + "pxor %%xmm8,%%xmm1;" + "or %7,%10;" + "add %5,%8;" + "and %7,%12;" + "pshufd $0xfa,%%xmm4,%%xmm2;" + "and %k2,%10;" + "add %11,%5;" + "paddd %%xmm1,%%xmm0;" + "or %12,%10;" + "add %10,%5;" + "movdqa %%xmm2,%%xmm3;" + "mov %8,%10;" + "mov %5,%11;" + "ror $0xe,%10;" + "movdqa %%xmm2,%%xmm8;" + "xor %8,%10;" + "ror $0x9,%11;" + "mov %9,%12;" + "xor %5,%11;" + "ror $0x5,%10;" + "psrlq $0x11,%%xmm2;" + "xor %3,%12;" + "psrlq $0x13,%%xmm3;" + "xor %8,%10;" + "and %8,%12;" + "psrld $0xa,%%xmm8;" + "ror $0xb,%11;" + "xor %5,%11;" + "xor %3,%12;" + "ror $0x6,%10;" + "pxor %%xmm3,%%xmm2;" + "add %10,%12;" + "ror $0x2,%11;" + "add 8+%16,%12;" + "pxor %%xmm2,%%xmm8;" + "mov %5,%10;" + "add %12,%4;" + "mov %5,%12;" + "pshufb %%xmm10,%%xmm8;" + "or %k2,%10;" + "add %4,%7;" + "and %k2,%12;" + "paddd %%xmm8,%%xmm0;" + "and %6,%10;" + "add %11,%4;" + "pshufd $0x50,%%xmm0,%%xmm2;" + "or %12,%10;" + "add %10,%4;" + "movdqa %%xmm2,%%xmm3;" + "mov %7,%10;" + "ror $0xe,%10;" + "mov %4,%11;" + "movdqa %%xmm2,%%xmm5;" + "ror $0x9,%11;" + "xor %7,%10;" + "mov %8,%12;" + "ror $0x5,%10;" + "psrlq $0x11,%%xmm2;" + "xor %4,%11;" + "xor %9,%12;" + "psrlq $0x13,%%xmm3;" + "xor %7,%10;" + "and %7,%12;" + "ror $0xb,%11;" + "psrld $0xa,%%xmm5;" + "xor %4,%11;" + "ror $0x6,%10;" + "xor %9,%12;" + "pxor %%xmm3,%%xmm2;" + "ror $0x2,%11;" + "add %10,%12;" + "add 12+%16,%12;" + "pxor %%xmm2,%%xmm5;" + "mov %4,%10;" + "add %12,%3;" + "mov %4,%12;" + "pshufb %%xmm11,%%xmm5;" + "or %6,%10;" + "add %3,%k2;" + "and %6,%12;" + "paddd %%xmm0,%%xmm5;" + "and %5,%10;" + "add %11,%3;" + "or %12,%10;" + "add %10,%3;" + "movdqa 0x20(%13),%%xmm9;" + "paddd %%xmm6,%%xmm9;" + "movdqa %%xmm9,%16;" + "movdqa %%xmm5,%%xmm0;" + "mov %k2,%10;" + "ror $0xe,%10;" + "mov %3,%11;" + "palignr $0x4,%%xmm4,%%xmm0;" + "ror $0x9,%11;" + "xor %k2,%10;" + "mov %7,%12;" + "ror $0x5,%10;" + "movdqa %%xmm7,%%xmm1;" + "xor %3,%11;" + "xor %8,%12;" + "paddd %%xmm6,%%xmm0;" + "xor %k2,%10;" + "and %k2,%12;" + "ror $0xb,%11;" + "palignr $0x4,%%xmm6,%%xmm1;" + "xor %3,%11;" + "ror $0x6,%10;" + "xor %8,%12;" + "movdqa %%xmm1,%%xmm2;" + "ror $0x2,%11;" + "add %10,%12;" + "add %16,%12;" + "movdqa %%xmm1,%%xmm3;" + "mov %3,%10;" + "add %12,%9;" + "mov %3,%12;" + "pslld $0x19,%%xmm1;" + "or %5,%10;" + "add %9,%6;" + "and %5,%12;" + "psrld $0x7,%%xmm2;" + "and %4,%10;" + "add %11,%9;" + "por %%xmm2,%%xmm1;" + "or %12,%10;" + "add %10,%9;" + "movdqa %%xmm3,%%xmm2;" + "mov %6,%10;" + "mov %9,%11;" + "movdqa %%xmm3,%%xmm8;" + "ror $0xe,%10;" + "xor %6,%10;" + "mov %k2,%12;" + "ror $0x9,%11;" + "pslld $0xe,%%xmm3;" + "xor %9,%11;" + "ror $0x5,%10;" + "xor %7,%12;" + "psrld $0x12,%%xmm2;" + "ror $0xb,%11;" + "xor %6,%10;" + "and %6,%12;" + "ror $0x6,%10;" + "pxor %%xmm3,%%xmm1;" + "xor %9,%11;" + "xor %7,%12;" + "psrld $0x3,%%xmm8;" + "add %10,%12;" + "add 4+%16,%12;" + "ror $0x2,%11;" + "pxor %%xmm2,%%xmm1;" + "mov %9,%10;" + "add %12,%8;" + "mov %9,%12;" + "pxor %%xmm8,%%xmm1;" + "or %4,%10;" + "add %8,%5;" + "and %4,%12;" + "pshufd $0xfa,%%xmm5,%%xmm2;" + "and %3,%10;" + "add %11,%8;" + "paddd %%xmm1,%%xmm0;" + "or %12,%10;" + "add %10,%8;" + "movdqa %%xmm2,%%xmm3;" + "mov %5,%10;" + "mov %8,%11;" + "ror $0xe,%10;" + "movdqa %%xmm2,%%xmm8;" + "xor %5,%10;" + "ror $0x9,%11;" + "mov %6,%12;" + "xor %8,%11;" + "ror $0x5,%10;" + "psrlq $0x11,%%xmm2;" + "xor %k2,%12;" + "psrlq $0x13,%%xmm3;" + "xor %5,%10;" + "and %5,%12;" + "psrld $0xa,%%xmm8;" + "ror $0xb,%11;" + "xor %8,%11;" + "xor %k2,%12;" + "ror $0x6,%10;" + "pxor %%xmm3,%%xmm2;" + "add %10,%12;" + "ror $0x2,%11;" + "add 8+%16,%12;" + "pxor %%xmm2,%%xmm8;" + "mov %8,%10;" + "add %12,%7;" + "mov %8,%12;" + "pshufb %%xmm10,%%xmm8;" + "or %3,%10;" + "add %7,%4;" + "and %3,%12;" + "paddd %%xmm8,%%xmm0;" + "and %9,%10;" + "add %11,%7;" + "pshufd $0x50,%%xmm0,%%xmm2;" + "or %12,%10;" + "add %10,%7;" + "movdqa %%xmm2,%%xmm3;" + "mov %4,%10;" + "ror $0xe,%10;" + "mov %7,%11;" + "movdqa %%xmm2,%%xmm6;" + "ror $0x9,%11;" + "xor %4,%10;" + "mov %5,%12;" + "ror $0x5,%10;" + "psrlq $0x11,%%xmm2;" + "xor %7,%11;" + "xor %6,%12;" + "psrlq $0x13,%%xmm3;" + "xor %4,%10;" + "and %4,%12;" + "ror $0xb,%11;" + "psrld $0xa,%%xmm6;" + "xor %7,%11;" + "ror $0x6,%10;" + "xor %6,%12;" + "pxor %%xmm3,%%xmm2;" + "ror $0x2,%11;" + "add %10,%12;" + "add 12+%16,%12;" + "pxor %%xmm2,%%xmm6;" + "mov %7,%10;" + "add %12,%k2;" + "mov %7,%12;" + "pshufb %%xmm11,%%xmm6;" + "or %9,%10;" + "add %k2,%3;" + "and %9,%12;" + "paddd %%xmm0,%%xmm6;" + "and %8,%10;" + "add %11,%k2;" + "or %12,%10;" + "add %10,%k2;" + "movdqa 0x30(%13),%%xmm9;" + "paddd %%xmm7,%%xmm9;" + "movdqa %%xmm9,%16;" + "add $0x40,%13;" + "movdqa %%xmm6,%%xmm0;" + "mov %3,%10;" + "ror $0xe,%10;" + "mov %k2,%11;" + "palignr $0x4,%%xmm5,%%xmm0;" + "ror $0x9,%11;" + "xor %3,%10;" + "mov %4,%12;" + "ror $0x5,%10;" + "movdqa %%xmm4,%%xmm1;" + "xor %k2,%11;" + "xor %5,%12;" + "paddd %%xmm7,%%xmm0;" + "xor %3,%10;" + "and %3,%12;" + "ror $0xb,%11;" + "palignr $0x4,%%xmm7,%%xmm1;" + "xor %k2,%11;" + "ror $0x6,%10;" + "xor %5,%12;" + "movdqa %%xmm1,%%xmm2;" + "ror $0x2,%11;" + "add %10,%12;" + "add %16,%12;" + "movdqa %%xmm1,%%xmm3;" + "mov %k2,%10;" + "add %12,%6;" + "mov %k2,%12;" + "pslld $0x19,%%xmm1;" + "or %8,%10;" + "add %6,%9;" + "and %8,%12;" + "psrld $0x7,%%xmm2;" + "and %7,%10;" + "add %11,%6;" + "por %%xmm2,%%xmm1;" + "or %12,%10;" + "add %10,%6;" + "movdqa %%xmm3,%%xmm2;" + "mov %9,%10;" + "mov %6,%11;" + "movdqa %%xmm3,%%xmm8;" + "ror $0xe,%10;" + "xor %9,%10;" + "mov %3,%12;" + "ror $0x9,%11;" + "pslld $0xe,%%xmm3;" + "xor %6,%11;" + "ror $0x5,%10;" + "xor %4,%12;" + "psrld $0x12,%%xmm2;" + "ror $0xb,%11;" + "xor %9,%10;" + "and %9,%12;" + "ror $0x6,%10;" + "pxor %%xmm3,%%xmm1;" + "xor %6,%11;" + "xor %4,%12;" + "psrld $0x3,%%xmm8;" + "add %10,%12;" + "add 4+%16,%12;" + "ror $0x2,%11;" + "pxor %%xmm2,%%xmm1;" + "mov %6,%10;" + "add %12,%5;" + "mov %6,%12;" + "pxor %%xmm8,%%xmm1;" + "or %7,%10;" + "add %5,%8;" + "and %7,%12;" + "pshufd $0xfa,%%xmm6,%%xmm2;" + "and %k2,%10;" + "add %11,%5;" + "paddd %%xmm1,%%xmm0;" + "or %12,%10;" + "add %10,%5;" + "movdqa %%xmm2,%%xmm3;" + "mov %8,%10;" + "mov %5,%11;" + "ror $0xe,%10;" + "movdqa %%xmm2,%%xmm8;" + "xor %8,%10;" + "ror $0x9,%11;" + "mov %9,%12;" + "xor %5,%11;" + "ror $0x5,%10;" + "psrlq $0x11,%%xmm2;" + "xor %3,%12;" + "psrlq $0x13,%%xmm3;" + "xor %8,%10;" + "and %8,%12;" + "psrld $0xa,%%xmm8;" + "ror $0xb,%11;" + "xor %5,%11;" + "xor %3,%12;" + "ror $0x6,%10;" + "pxor %%xmm3,%%xmm2;" + "add %10,%12;" + "ror $0x2,%11;" + "add 8+%16,%12;" + "pxor %%xmm2,%%xmm8;" + "mov %5,%10;" + "add %12,%4;" + "mov %5,%12;" + "pshufb %%xmm10,%%xmm8;" + "or %k2,%10;" + "add %4,%7;" + "and %k2,%12;" + "paddd %%xmm8,%%xmm0;" + "and %6,%10;" + "add %11,%4;" + "pshufd $0x50,%%xmm0,%%xmm2;" + "or %12,%10;" + "add %10,%4;" + "movdqa %%xmm2,%%xmm3;" + "mov %7,%10;" + "ror $0xe,%10;" + "mov %4,%11;" + "movdqa %%xmm2,%%xmm7;" + "ror $0x9,%11;" + "xor %7,%10;" + "mov %8,%12;" + "ror $0x5,%10;" + "psrlq $0x11,%%xmm2;" + "xor %4,%11;" + "xor %9,%12;" + "psrlq $0x13,%%xmm3;" + "xor %7,%10;" + "and %7,%12;" + "ror $0xb,%11;" + "psrld $0xa,%%xmm7;" + "xor %4,%11;" + "ror $0x6,%10;" + "xor %9,%12;" + "pxor %%xmm3,%%xmm2;" + "ror $0x2,%11;" + "add %10,%12;" + "add 12+%16,%12;" + "pxor %%xmm2,%%xmm7;" + "mov %4,%10;" + "add %12,%3;" + "mov %4,%12;" + "pshufb %%xmm11,%%xmm7;" + "or %6,%10;" + "add %3,%k2;" + "and %6,%12;" + "paddd %%xmm0,%%xmm7;" + "and %5,%10;" + "add %11,%3;" + "or %12,%10;" + "add %10,%3;" + "sub $0x1,%1;" + "jne Lloop1_%=;" + "mov $0x2,%1;" + + "Lloop2_%=:" + "paddd 0x0(%13),%%xmm4;" + "movdqa %%xmm4,%16;" + "mov %k2,%10;" + "ror $0xe,%10;" + "mov %3,%11;" + "xor %k2,%10;" + "ror $0x9,%11;" + "mov %7,%12;" + "xor %3,%11;" + "ror $0x5,%10;" + "xor %8,%12;" + "xor %k2,%10;" + "ror $0xb,%11;" + "and %k2,%12;" + "xor %3,%11;" + "ror $0x6,%10;" + "xor %8,%12;" + "add %10,%12;" + "ror $0x2,%11;" + "add %16,%12;" + "mov %3,%10;" + "add %12,%9;" + "mov %3,%12;" + "or %5,%10;" + "add %9,%6;" + "and %5,%12;" + "and %4,%10;" + "add %11,%9;" + "or %12,%10;" + "add %10,%9;" + "mov %6,%10;" + "ror $0xe,%10;" + "mov %9,%11;" + "xor %6,%10;" + "ror $0x9,%11;" + "mov %k2,%12;" + "xor %9,%11;" + "ror $0x5,%10;" + "xor %7,%12;" + "xor %6,%10;" + "ror $0xb,%11;" + "and %6,%12;" + "xor %9,%11;" + "ror $0x6,%10;" + "xor %7,%12;" + "add %10,%12;" + "ror $0x2,%11;" + "add 4+%16,%12;" + "mov %9,%10;" + "add %12,%8;" + "mov %9,%12;" + "or %4,%10;" + "add %8,%5;" + "and %4,%12;" + "and %3,%10;" + "add %11,%8;" + "or %12,%10;" + "add %10,%8;" + "mov %5,%10;" + "ror $0xe,%10;" + "mov %8,%11;" + "xor %5,%10;" + "ror $0x9,%11;" + "mov %6,%12;" + "xor %8,%11;" + "ror $0x5,%10;" + "xor %k2,%12;" + "xor %5,%10;" + "ror $0xb,%11;" + "and %5,%12;" + "xor %8,%11;" + "ror $0x6,%10;" + "xor %k2,%12;" + "add %10,%12;" + "ror $0x2,%11;" + "add 8+%16,%12;" + "mov %8,%10;" + "add %12,%7;" + "mov %8,%12;" + "or %3,%10;" + "add %7,%4;" + "and %3,%12;" + "and %9,%10;" + "add %11,%7;" + "or %12,%10;" + "add %10,%7;" + "mov %4,%10;" + "ror $0xe,%10;" + "mov %7,%11;" + "xor %4,%10;" + "ror $0x9,%11;" + "mov %5,%12;" + "xor %7,%11;" + "ror $0x5,%10;" + "xor %6,%12;" + "xor %4,%10;" + "ror $0xb,%11;" + "and %4,%12;" + "xor %7,%11;" + "ror $0x6,%10;" + "xor %6,%12;" + "add %10,%12;" + "ror $0x2,%11;" + "add 12+%16,%12;" + "mov %7,%10;" + "add %12,%k2;" + "mov %7,%12;" + "or %9,%10;" + "add %k2,%3;" + "and %9,%12;" + "and %8,%10;" + "add %11,%k2;" + "or %12,%10;" + "add %10,%k2;" + "paddd 0x10(%13),%%xmm5;" + "movdqa %%xmm5,%16;" + "add $0x20,%13;" + "mov %3,%10;" + "ror $0xe,%10;" + "mov %k2,%11;" + "xor %3,%10;" + "ror $0x9,%11;" + "mov %4,%12;" + "xor %k2,%11;" + "ror $0x5,%10;" + "xor %5,%12;" + "xor %3,%10;" + "ror $0xb,%11;" + "and %3,%12;" + "xor %k2,%11;" + "ror $0x6,%10;" + "xor %5,%12;" + "add %10,%12;" + "ror $0x2,%11;" + "add %16,%12;" + "mov %k2,%10;" + "add %12,%6;" + "mov %k2,%12;" + "or %8,%10;" + "add %6,%9;" + "and %8,%12;" + "and %7,%10;" + "add %11,%6;" + "or %12,%10;" + "add %10,%6;" + "mov %9,%10;" + "ror $0xe,%10;" + "mov %6,%11;" + "xor %9,%10;" + "ror $0x9,%11;" + "mov %3,%12;" + "xor %6,%11;" + "ror $0x5,%10;" + "xor %4,%12;" + "xor %9,%10;" + "ror $0xb,%11;" + "and %9,%12;" + "xor %6,%11;" + "ror $0x6,%10;" + "xor %4,%12;" + "add %10,%12;" + "ror $0x2,%11;" + "add 4+%16,%12;" + "mov %6,%10;" + "add %12,%5;" + "mov %6,%12;" + "or %7,%10;" + "add %5,%8;" + "and %7,%12;" + "and %k2,%10;" + "add %11,%5;" + "or %12,%10;" + "add %10,%5;" + "mov %8,%10;" + "ror $0xe,%10;" + "mov %5,%11;" + "xor %8,%10;" + "ror $0x9,%11;" + "mov %9,%12;" + "xor %5,%11;" + "ror $0x5,%10;" + "xor %3,%12;" + "xor %8,%10;" + "ror $0xb,%11;" + "and %8,%12;" + "xor %5,%11;" + "ror $0x6,%10;" + "xor %3,%12;" + "add %10,%12;" + "ror $0x2,%11;" + "add 8+%16,%12;" + "mov %5,%10;" + "add %12,%4;" + "mov %5,%12;" + "or %k2,%10;" + "add %4,%7;" + "and %k2,%12;" + "and %6,%10;" + "add %11,%4;" + "or %12,%10;" + "add %10,%4;" + "mov %7,%10;" + "ror $0xe,%10;" + "mov %4,%11;" + "xor %7,%10;" + "ror $0x9,%11;" + "mov %8,%12;" + "xor %4,%11;" + "ror $0x5,%10;" + "xor %9,%12;" + "xor %7,%10;" + "ror $0xb,%11;" + "and %7,%12;" + "xor %4,%11;" + "ror $0x6,%10;" + "xor %9,%12;" + "add %10,%12;" + "ror $0x2,%11;" + "add 12+%16,%12;" + "mov %4,%10;" + "add %12,%3;" + "mov %4,%12;" + "or %6,%10;" + "add %3,%k2;" + "and %6,%12;" + "and %5,%10;" + "add %11,%3;" + "or %12,%10;" + "add %10,%3;" + "movdqa %%xmm6,%%xmm4;" + "movdqa %%xmm7,%%xmm5;" + "sub $0x1,%1;" + "jne Lloop2_%=;" + "add (%0),%3;" + "mov %3,(%0);" + "add 0x4(%0),%4;" + "mov %4,0x4(%0);" + "add 0x8(%0),%5;" + "mov %5,0x8(%0);" + "add 0xc(%0),%6;" + "mov %6,0xc(%0);" + "add 0x10(%0),%k2;" + "mov %k2,0x10(%0);" + "add 0x14(%0),%7;" + "mov %7,0x14(%0);" + "add 0x18(%0),%8;" + "mov %8,0x18(%0);" + "add 0x1c(%0),%9;" + "mov %9,0x1c(%0);" + "mov %15,%1;" + "add $0x40,%1;" + "cmp %14,%1;" + "jne Lloop0_%=;" + + "Ldone_hash_%=:" + + : "+r"(s), "+r"(chunk), "+r"(blocks), "=r"(a), "=r"(b), "=r"(c), "=r"(d), /* e = chunk */ "=r"(f), "=r"(g), "=r"(h), "=r"(y0), "=r"(y1), "=r"(y2), "=r"(tbl), "+m"(inp_end), "+m"(inp), "+m"(xfer) + : "m"(K256), "m"(FLIP_MASK), "m"(SHUF_00BA), "m"(SHUF_DC00) + : "cc", "memory", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12" + ); +} +} + +/* +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; Copyright (c) 2012, Intel Corporation +; +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are +; met: +; +; * Redistributions of source code must retain the above copyright +; notice, this list of conditions and the following disclaimer. +; +; * Redistributions in binary form must reproduce the above copyright +; notice, this list of conditions and the following disclaimer in the +; documentation and/or other materials provided with the +; distribution. +; +; * Neither the name of the Intel Corporation nor the names of its +; contributors may be used to endorse or promote products derived from +; this software without specific prior written permission. +; +; +; THIS SOFTWARE IS PROVIDED BY INTEL CORPORATION "AS IS" AND ANY +; EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +; PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL INTEL CORPORATION OR +; CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +; EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +; PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +; PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +; LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +; NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; Example YASM command lines: +; Windows: yasm -Xvc -f x64 -rnasm -pnasm -o sha256_sse4.obj -g cv8 sha256_sse4.asm +; Linux: yasm -f x64 -f elf64 -X gnu -g dwarf2 -D LINUX -o sha256_sse4.o sha256_sse4.asm +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; +; This code is described in an Intel White-Paper: +; "Fast SHA-256 Implementations on Intel Architecture Processors" +; +; To find it, surf to http://www.intel.com/p/en_US/embedded +; and search for that title. +; The paper is expected to be released roughly at the end of April, 2012 +; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; This code schedules 1 blocks at a time, with 4 lanes per block +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%define MOVDQ movdqu ;; assume buffers not aligned + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Define Macros + +; addm [mem], reg +; Add reg to mem using reg-mem add and store +%macro addm 2 + add %2, %1 + mov %1, %2 +%endm + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +; COPY_XMM_AND_BSWAP xmm, [mem], byte_flip_mask +; Load xmm with mem and byte swap each dword +%macro COPY_XMM_AND_BSWAP 3 + MOVDQ %1, %2 + pshufb %1, %3 +%endmacro + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%define X0 xmm4 +%define X1 xmm5 +%define X2 xmm6 +%define X3 xmm7 + +%define XTMP0 xmm0 +%define XTMP1 xmm1 +%define XTMP2 xmm2 +%define XTMP3 xmm3 +%define XTMP4 xmm8 +%define XFER xmm9 + +%define SHUF_00BA xmm10 ; shuffle xBxA -> 00BA +%define SHUF_DC00 xmm11 ; shuffle xDxC -> DC00 +%define BYTE_FLIP_MASK xmm12 + +%ifdef LINUX +%define NUM_BLKS rdx ; 3rd arg +%define CTX rsi ; 2nd arg +%define INP rdi ; 1st arg + +%define SRND rdi ; clobbers INP +%define c ecx +%define d r8d +%define e edx +%else +%define NUM_BLKS r8 ; 3rd arg +%define CTX rdx ; 2nd arg +%define INP rcx ; 1st arg + +%define SRND rcx ; clobbers INP +%define c edi +%define d esi +%define e r8d + +%endif +%define TBL rbp +%define a eax +%define b ebx + +%define f r9d +%define g r10d +%define h r11d + +%define y0 r13d +%define y1 r14d +%define y2 r15d + + + +_INP_END_SIZE equ 8 +_INP_SIZE equ 8 +_XFER_SIZE equ 8 +%ifdef LINUX +_XMM_SAVE_SIZE equ 0 +%else +_XMM_SAVE_SIZE equ 7*16 +%endif +; STACK_SIZE plus pushes must be an odd multiple of 8 +_ALIGN_SIZE equ 8 + +_INP_END equ 0 +_INP equ _INP_END + _INP_END_SIZE +_XFER equ _INP + _INP_SIZE +_XMM_SAVE equ _XFER + _XFER_SIZE + _ALIGN_SIZE +STACK_SIZE equ _XMM_SAVE + _XMM_SAVE_SIZE + +; rotate_Xs +; Rotate values of symbols X0...X3 +%macro rotate_Xs 0 +%xdefine X_ X0 +%xdefine X0 X1 +%xdefine X1 X2 +%xdefine X2 X3 +%xdefine X3 X_ +%endm + +; ROTATE_ARGS +; Rotate values of symbols a...h +%macro ROTATE_ARGS 0 +%xdefine TMP_ h +%xdefine h g +%xdefine g f +%xdefine f e +%xdefine e d +%xdefine d c +%xdefine c b +%xdefine b a +%xdefine a TMP_ +%endm + +%macro FOUR_ROUNDS_AND_SCHED 0 + ;; compute s0 four at a time and s1 two at a time + ;; compute W[-16] + W[-7] 4 at a time + movdqa XTMP0, X3 + mov y0, e ; y0 = e + ror y0, (25-11) ; y0 = e >> (25-11) + mov y1, a ; y1 = a + palignr XTMP0, X2, 4 ; XTMP0 = W[-7] + ror y1, (22-13) ; y1 = a >> (22-13) + xor y0, e ; y0 = e ^ (e >> (25-11)) + mov y2, f ; y2 = f + ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6)) + movdqa XTMP1, X1 + xor y1, a ; y1 = a ^ (a >> (22-13) + xor y2, g ; y2 = f^g + paddd XTMP0, X0 ; XTMP0 = W[-7] + W[-16] + xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) + and y2, e ; y2 = (f^g)&e + ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2)) + ;; compute s0 + palignr XTMP1, X0, 4 ; XTMP1 = W[-15] + xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) + ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) + xor y2, g ; y2 = CH = ((f^g)&e)^g + movdqa XTMP2, XTMP1 ; XTMP2 = W[-15] + ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) + add y2, y0 ; y2 = S1 + CH + add y2, [rsp + _XFER + 0*4] ; y2 = k + w + S1 + CH + movdqa XTMP3, XTMP1 ; XTMP3 = W[-15] + mov y0, a ; y0 = a + add h, y2 ; h = h + S1 + CH + k + w + mov y2, a ; y2 = a + pslld XTMP1, (32-7) + or y0, c ; y0 = a|c + add d, h ; d = d + h + S1 + CH + k + w + and y2, c ; y2 = a&c + psrld XTMP2, 7 + and y0, b ; y0 = (a|c)&b + add h, y1 ; h = h + S1 + CH + k + w + S0 + por XTMP1, XTMP2 ; XTMP1 = W[-15] ror 7 + or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c) + add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ + +ROTATE_ARGS + movdqa XTMP2, XTMP3 ; XTMP2 = W[-15] + mov y0, e ; y0 = e + mov y1, a ; y1 = a + movdqa XTMP4, XTMP3 ; XTMP4 = W[-15] + ror y0, (25-11) ; y0 = e >> (25-11) + xor y0, e ; y0 = e ^ (e >> (25-11)) + mov y2, f ; y2 = f + ror y1, (22-13) ; y1 = a >> (22-13) + pslld XTMP3, (32-18) + xor y1, a ; y1 = a ^ (a >> (22-13) + ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6)) + xor y2, g ; y2 = f^g + psrld XTMP2, 18 + ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2)) + xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) + and y2, e ; y2 = (f^g)&e + ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) + pxor XTMP1, XTMP3 + xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) + xor y2, g ; y2 = CH = ((f^g)&e)^g + psrld XTMP4, 3 ; XTMP4 = W[-15] >> 3 + add y2, y0 ; y2 = S1 + CH + add y2, [rsp + _XFER + 1*4] ; y2 = k + w + S1 + CH + ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) + pxor XTMP1, XTMP2 ; XTMP1 = W[-15] ror 7 ^ W[-15] ror 18 + mov y0, a ; y0 = a + add h, y2 ; h = h + S1 + CH + k + w + mov y2, a ; y2 = a + pxor XTMP1, XTMP4 ; XTMP1 = s0 + or y0, c ; y0 = a|c + add d, h ; d = d + h + S1 + CH + k + w + and y2, c ; y2 = a&c + ;; compute low s1 + pshufd XTMP2, X3, 11111010b ; XTMP2 = W[-2] {BBAA} + and y0, b ; y0 = (a|c)&b + add h, y1 ; h = h + S1 + CH + k + w + S0 + paddd XTMP0, XTMP1 ; XTMP0 = W[-16] + W[-7] + s0 + or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c) + add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ + +ROTATE_ARGS + movdqa XTMP3, XTMP2 ; XTMP3 = W[-2] {BBAA} + mov y0, e ; y0 = e + mov y1, a ; y1 = a + ror y0, (25-11) ; y0 = e >> (25-11) + movdqa XTMP4, XTMP2 ; XTMP4 = W[-2] {BBAA} + xor y0, e ; y0 = e ^ (e >> (25-11)) + ror y1, (22-13) ; y1 = a >> (22-13) + mov y2, f ; y2 = f + xor y1, a ; y1 = a ^ (a >> (22-13) + ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6)) + psrlq XTMP2, 17 ; XTMP2 = W[-2] ror 17 {xBxA} + xor y2, g ; y2 = f^g + psrlq XTMP3, 19 ; XTMP3 = W[-2] ror 19 {xBxA} + xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) + and y2, e ; y2 = (f^g)&e + psrld XTMP4, 10 ; XTMP4 = W[-2] >> 10 {BBAA} + ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2)) + xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) + xor y2, g ; y2 = CH = ((f^g)&e)^g + ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) + pxor XTMP2, XTMP3 + add y2, y0 ; y2 = S1 + CH + ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) + add y2, [rsp + _XFER + 2*4] ; y2 = k + w + S1 + CH + pxor XTMP4, XTMP2 ; XTMP4 = s1 {xBxA} + mov y0, a ; y0 = a + add h, y2 ; h = h + S1 + CH + k + w + mov y2, a ; y2 = a + pshufb XTMP4, SHUF_00BA ; XTMP4 = s1 {00BA} + or y0, c ; y0 = a|c + add d, h ; d = d + h + S1 + CH + k + w + and y2, c ; y2 = a&c + paddd XTMP0, XTMP4 ; XTMP0 = {..., ..., W[1], W[0]} + and y0, b ; y0 = (a|c)&b + add h, y1 ; h = h + S1 + CH + k + w + S0 + ;; compute high s1 + pshufd XTMP2, XTMP0, 01010000b ; XTMP2 = W[-2] {DDCC} + or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c) + add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ + +ROTATE_ARGS + movdqa XTMP3, XTMP2 ; XTMP3 = W[-2] {DDCC} + mov y0, e ; y0 = e + ror y0, (25-11) ; y0 = e >> (25-11) + mov y1, a ; y1 = a + movdqa X0, XTMP2 ; X0 = W[-2] {DDCC} + ror y1, (22-13) ; y1 = a >> (22-13) + xor y0, e ; y0 = e ^ (e >> (25-11)) + mov y2, f ; y2 = f + ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6)) + psrlq XTMP2, 17 ; XTMP2 = W[-2] ror 17 {xDxC} + xor y1, a ; y1 = a ^ (a >> (22-13) + xor y2, g ; y2 = f^g + psrlq XTMP3, 19 ; XTMP3 = W[-2] ror 19 {xDxC} + xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) + and y2, e ; y2 = (f^g)&e + ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2)) + psrld X0, 10 ; X0 = W[-2] >> 10 {DDCC} + xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) + ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) + xor y2, g ; y2 = CH = ((f^g)&e)^g + pxor XTMP2, XTMP3 + ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) + add y2, y0 ; y2 = S1 + CH + add y2, [rsp + _XFER + 3*4] ; y2 = k + w + S1 + CH + pxor X0, XTMP2 ; X0 = s1 {xDxC} + mov y0, a ; y0 = a + add h, y2 ; h = h + S1 + CH + k + w + mov y2, a ; y2 = a + pshufb X0, SHUF_DC00 ; X0 = s1 {DC00} + or y0, c ; y0 = a|c + add d, h ; d = d + h + S1 + CH + k + w + and y2, c ; y2 = a&c + paddd X0, XTMP0 ; X0 = {W[3], W[2], W[1], W[0]} + and y0, b ; y0 = (a|c)&b + add h, y1 ; h = h + S1 + CH + k + w + S0 + or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c) + add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ + +ROTATE_ARGS +rotate_Xs +%endm + +;; input is [rsp + _XFER + %1 * 4] +%macro DO_ROUND 1 + mov y0, e ; y0 = e + ror y0, (25-11) ; y0 = e >> (25-11) + mov y1, a ; y1 = a + xor y0, e ; y0 = e ^ (e >> (25-11)) + ror y1, (22-13) ; y1 = a >> (22-13) + mov y2, f ; y2 = f + xor y1, a ; y1 = a ^ (a >> (22-13) + ror y0, (11-6) ; y0 = (e >> (11-6)) ^ (e >> (25-6)) + xor y2, g ; y2 = f^g + xor y0, e ; y0 = e ^ (e >> (11-6)) ^ (e >> (25-6)) + ror y1, (13-2) ; y1 = (a >> (13-2)) ^ (a >> (22-2)) + and y2, e ; y2 = (f^g)&e + xor y1, a ; y1 = a ^ (a >> (13-2)) ^ (a >> (22-2)) + ror y0, 6 ; y0 = S1 = (e>>6) & (e>>11) ^ (e>>25) + xor y2, g ; y2 = CH = ((f^g)&e)^g + add y2, y0 ; y2 = S1 + CH + ror y1, 2 ; y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22) + add y2, [rsp + _XFER + %1 * 4] ; y2 = k + w + S1 + CH + mov y0, a ; y0 = a + add h, y2 ; h = h + S1 + CH + k + w + mov y2, a ; y2 = a + or y0, c ; y0 = a|c + add d, h ; d = d + h + S1 + CH + k + w + and y2, c ; y2 = a&c + and y0, b ; y0 = (a|c)&b + add h, y1 ; h = h + S1 + CH + k + w + S0 + or y0, y2 ; y0 = MAJ = (a|c)&b)|(a&c) + add h, y0 ; h = h + S1 + CH + k + w + S0 + MAJ + ROTATE_ARGS +%endm + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; void sha256_sse4(void *input_data, UINT32 digest[8], UINT64 num_blks) +;; arg 1 : pointer to input data +;; arg 2 : pointer to digest +;; arg 3 : Num blocks +section .text +global sha256_sse4 +align 32 +sha256_sse4: + push rbx +%ifndef LINUX + push rsi + push rdi +%endif + push rbp + push r13 + push r14 + push r15 + + sub rsp,STACK_SIZE +%ifndef LINUX + movdqa [rsp + _XMM_SAVE + 0*16],xmm6 + movdqa [rsp + _XMM_SAVE + 1*16],xmm7 + movdqa [rsp + _XMM_SAVE + 2*16],xmm8 + movdqa [rsp + _XMM_SAVE + 3*16],xmm9 + movdqa [rsp + _XMM_SAVE + 4*16],xmm10 + movdqa [rsp + _XMM_SAVE + 5*16],xmm11 + movdqa [rsp + _XMM_SAVE + 6*16],xmm12 +%endif + + shl NUM_BLKS, 6 ; convert to bytes + jz done_hash + add NUM_BLKS, INP ; pointer to end of data + mov [rsp + _INP_END], NUM_BLKS + + ;; load initial digest + mov a,[4*0 + CTX] + mov b,[4*1 + CTX] + mov c,[4*2 + CTX] + mov d,[4*3 + CTX] + mov e,[4*4 + CTX] + mov f,[4*5 + CTX] + mov g,[4*6 + CTX] + mov h,[4*7 + CTX] + + movdqa BYTE_FLIP_MASK, [PSHUFFLE_BYTE_FLIP_MASK wrt rip] + movdqa SHUF_00BA, [_SHUF_00BA wrt rip] + movdqa SHUF_DC00, [_SHUF_DC00 wrt rip] + +loop0: + lea TBL,[K256 wrt rip] + + ;; byte swap first 16 dwords + COPY_XMM_AND_BSWAP X0, [INP + 0*16], BYTE_FLIP_MASK + COPY_XMM_AND_BSWAP X1, [INP + 1*16], BYTE_FLIP_MASK + COPY_XMM_AND_BSWAP X2, [INP + 2*16], BYTE_FLIP_MASK + COPY_XMM_AND_BSWAP X3, [INP + 3*16], BYTE_FLIP_MASK + + mov [rsp + _INP], INP + + ;; schedule 48 input dwords, by doing 3 rounds of 16 each + mov SRND, 3 +align 16 +loop1: + movdqa XFER, [TBL + 0*16] + paddd XFER, X0 + movdqa [rsp + _XFER], XFER + FOUR_ROUNDS_AND_SCHED + + movdqa XFER, [TBL + 1*16] + paddd XFER, X0 + movdqa [rsp + _XFER], XFER + FOUR_ROUNDS_AND_SCHED + + movdqa XFER, [TBL + 2*16] + paddd XFER, X0 + movdqa [rsp + _XFER], XFER + FOUR_ROUNDS_AND_SCHED + + movdqa XFER, [TBL + 3*16] + paddd XFER, X0 + movdqa [rsp + _XFER], XFER + add TBL, 4*16 + FOUR_ROUNDS_AND_SCHED + + sub SRND, 1 + jne loop1 + + mov SRND, 2 +loop2: + paddd X0, [TBL + 0*16] + movdqa [rsp + _XFER], X0 + DO_ROUND 0 + DO_ROUND 1 + DO_ROUND 2 + DO_ROUND 3 + paddd X1, [TBL + 1*16] + movdqa [rsp + _XFER], X1 + add TBL, 2*16 + DO_ROUND 0 + DO_ROUND 1 + DO_ROUND 2 + DO_ROUND 3 + + movdqa X0, X2 + movdqa X1, X3 + + sub SRND, 1 + jne loop2 + + addm [4*0 + CTX],a + addm [4*1 + CTX],b + addm [4*2 + CTX],c + addm [4*3 + CTX],d + addm [4*4 + CTX],e + addm [4*5 + CTX],f + addm [4*6 + CTX],g + addm [4*7 + CTX],h + + mov INP, [rsp + _INP] + add INP, 64 + cmp INP, [rsp + _INP_END] + jne loop0 + +done_hash: +%ifndef LINUX + movdqa xmm6,[rsp + _XMM_SAVE + 0*16] + movdqa xmm7,[rsp + _XMM_SAVE + 1*16] + movdqa xmm8,[rsp + _XMM_SAVE + 2*16] + movdqa xmm9,[rsp + _XMM_SAVE + 3*16] + movdqa xmm10,[rsp + _XMM_SAVE + 4*16] + movdqa xmm11,[rsp + _XMM_SAVE + 5*16] + movdqa xmm12,[rsp + _XMM_SAVE + 6*16] +%endif + + add rsp, STACK_SIZE + + pop r15 + pop r14 + pop r13 + pop rbp +%ifndef LINUX + pop rdi + pop rsi +%endif + pop rbx + + ret + + +section .data +align 64 +K256: + dd 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 + dd 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 + dd 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 + dd 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 + dd 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc + dd 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da + dd 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 + dd 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 + dd 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 + dd 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 + dd 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 + dd 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 + dd 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 + dd 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 + dd 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 + dd 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 + +PSHUFFLE_BYTE_FLIP_MASK: ddq 0x0c0d0e0f08090a0b0405060700010203 + +; shuffle xBxA -> 00BA +_SHUF_00BA: ddq 0xFFFFFFFFFFFFFFFF0b0a090803020100 + +; shuffle xDxC -> DC00 +_SHUF_DC00: ddq 0x0b0a090803020100FFFFFFFFFFFFFFFF +*/ + +#endif From fa9be909c945c3689591590ac19504aa53154c2f Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sun, 16 Jul 2017 11:30:27 -0700 Subject: [PATCH 4/5] Add selftest for SHA256 transform --- src/crypto/sha256.cpp | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/src/crypto/sha256.cpp b/src/crypto/sha256.cpp index 4ecf50903..25f4eabd4 100644 --- a/src/crypto/sha256.cpp +++ b/src/crypto/sha256.cpp @@ -5,6 +5,7 @@ #include "crypto/sha256.h" #include "crypto/common.h" +#include #include #include @@ -140,7 +141,36 @@ void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks) } // namespace sha256 -void (*Transform)(uint32_t*, const unsigned char*, size_t) = sha256::Transform; +typedef void (*TransformType)(uint32_t*, const unsigned char*, size_t); + +bool SelfTest(TransformType tr) { + static const unsigned char in1[65] = {0, 0x80}; + static const unsigned char in2[129] = { + 0, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0 + }; + static const uint32_t init[8] = {0x6a09e667ul, 0xbb67ae85ul, 0x3c6ef372ul, 0xa54ff53aul, 0x510e527ful, 0x9b05688cul, 0x1f83d9abul, 0x5be0cd19ul}; + static const uint32_t out1[8] = {0xe3b0c442ul, 0x98fc1c14ul, 0x9afbf4c8ul, 0x996fb924ul, 0x27ae41e4ul, 0x649b934cul, 0xa495991bul, 0x7852b855ul}; + static const uint32_t out2[8] = {0xce4153b0ul, 0x147c2a86ul, 0x3ed4298eul, 0xe0676bc8ul, 0x79fc77a1ul, 0x2abe1f49ul, 0xb2b055dful, 0x1069523eul}; + uint32_t buf[8]; + memcpy(buf, init, sizeof(buf)); + // Process nothing, and check we remain in the initial state. + tr(buf, nullptr, 0); + if (memcmp(buf, init, sizeof(buf))) return false; + // Process the padded empty string (unaligned) + tr(buf, in1 + 1, 1); + if (memcmp(buf, out1, sizeof(buf))) return false; + // Process 64 spaces (unaligned) + memcpy(buf, init, sizeof(buf)); + tr(buf, in2 + 1, 2); + if (memcmp(buf, out2, sizeof(buf))) return false; + return true; +} + +TransformType Transform = sha256::Transform; } // namespace @@ -150,10 +180,12 @@ std::string SHA256AutoDetect() uint32_t eax, ebx, ecx, edx; if (__get_cpuid(1, &eax, &ebx, &ecx, &edx) && (ecx >> 19) & 1) { Transform = sha256_sse4::Transform; + assert(SelfTest(Transform)); return "sse4"; } #endif + assert(SelfTest(Transform)); return "standard"; } From 6b8d872e5e2dd68a5229ec55f5261dae34ff9bdb Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Tue, 18 Jul 2017 01:06:11 -0700 Subject: [PATCH 5/5] Protect SSE4 code behind a compile-time flag --- configure.ac | 11 +++++++++++ src/Makefile.am | 5 ++++- src/crypto/sha256.cpp | 4 +++- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 835dcdbf5..aea5d7160 100644 --- a/configure.ac +++ b/configure.ac @@ -177,6 +177,16 @@ AC_ARG_ENABLE([glibc-back-compat], [use_glibc_compat=$enableval], [use_glibc_compat=no]) +AC_ARG_ENABLE([experimental-asm], + [AS_HELP_STRING([--enable-experimental-asm], + [Enable experimental assembly routines (default is no)])], + [experimental_asm=$enableval], + [experimental_asm=no]) + +if test "x$experimental_asm" = xyes; then + AC_DEFINE(EXPERIMENTAL_ASM, 1, [Define this symbol to build in experimental assembly routines]) +fi + AC_ARG_WITH([system-univalue], [AS_HELP_STRING([--with-system-univalue], [Build with system UniValue (default is no)])], @@ -1162,6 +1172,7 @@ AM_CONDITIONAL([USE_LCOV],[test x$use_lcov = xyes]) AM_CONDITIONAL([GLIBC_BACK_COMPAT],[test x$use_glibc_compat = xyes]) AM_CONDITIONAL([HARDEN],[test x$use_hardening = xyes]) AM_CONDITIONAL([ENABLE_HWCRC32],[test x$enable_hwcrc32 = xyes]) +AM_CONDITIONAL([EXPERIMENTAL_ASM],[test x$experimental_asm = xyes]) AC_DEFINE(CLIENT_VERSION_MAJOR, _CLIENT_VERSION_MAJOR, [Major version]) AC_DEFINE(CLIENT_VERSION_MINOR, _CLIENT_VERSION_MINOR, [Minor version]) diff --git a/src/Makefile.am b/src/Makefile.am index 35bfeb466..f7abab482 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -263,11 +263,14 @@ crypto_libbitcoin_crypto_a_SOURCES = \ crypto/sha1.cpp \ crypto/sha1.h \ crypto/sha256.cpp \ - crypto/sha256_sse4.cpp \ crypto/sha256.h \ crypto/sha512.cpp \ crypto/sha512.h +if EXPERIMENTAL_ASM +crypto_libbitcoin_crypto_a_SOURCES += crypto/sha256_sse4.cpp +endif + # consensus: shared between all executables that validate any consensus rules. libbitcoin_consensus_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_consensus_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) diff --git a/src/crypto/sha256.cpp b/src/crypto/sha256.cpp index 25f4eabd4..15d6db90c 100644 --- a/src/crypto/sha256.cpp +++ b/src/crypto/sha256.cpp @@ -10,12 +10,14 @@ #include #if defined(__x86_64__) || defined(__amd64__) +#if defined(EXPERIMENTAL_ASM) #include namespace sha256_sse4 { void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks); } #endif +#endif // Internal implementation code. namespace @@ -176,7 +178,7 @@ TransformType Transform = sha256::Transform; std::string SHA256AutoDetect() { -#if defined(__x86_64__) || defined(__amd64__) +#if defined(EXPERIMENTAL_ASM) && (defined(__x86_64__) || defined(__amd64__)) uint32_t eax, ebx, ecx, edx; if (__get_cpuid(1, &eax, &ebx, &ecx, &edx) && (ecx >> 19) & 1) { Transform = sha256_sse4::Transform;