From f1fcd76ba7514003135f3d76ef2308b86b754507 Mon Sep 17 00:00:00 2001 From: Jeff Garzik Date: Mon, 6 Dec 2010 20:14:58 -0500 Subject: [PATCH] sha256_cryptopp: Add crypto++ 32-bit assembly implementation --- cpu-miner.c | 19 +- miner.h | 7 + sha256_cryptopp.c | 494 +++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 518 insertions(+), 2 deletions(-) diff --git a/cpu-miner.c b/cpu-miner.c index 70974c32..db1fd575 100644 --- a/cpu-miner.c +++ b/cpu-miner.c @@ -39,7 +39,8 @@ enum sha256_algos { ALGO_C, /* plain C */ ALGO_4WAY, /* parallel SSE2 */ ALGO_VIA, /* VIA padlock */ - ALGO_CRYPTOPP, /* Crypto++ */ + ALGO_CRYPTOPP, /* Crypto++ (C) */ + ALGO_CRYPTOPP_ASM32, /* Crypto++ 32-bit assembly */ }; static const char *algo_names[] = { @@ -51,6 +52,9 @@ static const char *algo_names[] = { [ALGO_VIA] = "via", #endif [ALGO_CRYPTOPP] = "cryptopp", +#ifdef WANT_CRYPTOPP_ASM32 + [ALGO_CRYPTOPP_ASM32] = "cryptopp_asm32", +#endif }; bool opt_debug = false; @@ -82,6 +86,9 @@ static struct option_help options_help[] = { "\n\tvia\t\tVIA padlock implementation" #endif "\n\tcryptopp\tCrypto++ library implementation (EXPERIMENTAL)" +#ifdef WANT_CRYPTOPP_ASM32 + "\n\tcryptopp_asm32\tCrypto++ library implementation (EXPERIMENTAL)" +#endif }, { "debug", @@ -294,6 +301,16 @@ static void *miner_thread(void *thr_id_int) work.hash1, work.hash, &hashes_done); break; +#ifdef WANT_CRYPTOPP_ASM32 + case ALGO_CRYPTOPP_ASM32: + rc = scanhash_asm32(work.midstate, work.data + 64, + work.hash1, work.hash, &hashes_done); + break; +#endif + + default: + /* should never happen */ + return NULL; } hashmeter(thr_id, &tv_start, hashes_done); diff --git a/miner.h b/miner.h index 09157fa6..b77bf0d4 100644 --- a/miner.h +++ b/miner.h @@ -14,6 +14,10 @@ #define WANT_VIA_PADLOCK 1 #endif +#if defined(__i386__) +#define WANT_CRYPTOPP_ASM32 +#endif + #ifndef ARRAY_SIZE #define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0])) #endif @@ -40,6 +44,9 @@ extern bool scanhash_c(const unsigned char *midstate, unsigned char *data, extern bool scanhash_cryptopp(const unsigned char *midstate,unsigned char *data, unsigned char *hash1, unsigned char *hash, unsigned long *hashes_done); +extern bool scanhash_asm32(const unsigned char *midstate,unsigned char *data, + unsigned char *hash1, unsigned char *hash, + unsigned long *hashes_done); extern int timeval_subtract (struct timeval *result, struct timeval *x, struct timeval *y); diff --git a/sha256_cryptopp.c b/sha256_cryptopp.c index 40772f41..3c200434 100644 --- a/sha256_cryptopp.c +++ b/sha256_cryptopp.c @@ -59,7 +59,7 @@ static const word32 SHA256_K[64] = { static void SHA256_Transform(word32 *state, const word32 *data) { - word32 W[16]; + word32 W[16] = { }; word32 T[8]; unsigned int j; @@ -131,3 +131,495 @@ bool scanhash_cryptopp(const unsigned char *midstate, unsigned char *data, } } +#if defined(WANT_CRYPTOPP_ASM32) + +#define CRYPTOPP_FASTCALL +#define CRYPTOPP_BOOL_X86 1 +#define CRYPTOPP_BOOL_X64 0 +#define CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE 0 + +#ifdef CRYPTOPP_GENERATE_X64_MASM + #define AS1(x) x*newline* + #define AS2(x, y) x, y*newline* + #define AS3(x, y, z) x, y, z*newline* + #define ASS(x, y, a, b, c, d) x, y, a*64+b*16+c*4+d*newline* + #define ASL(x) label##x:*newline* + #define ASJ(x, y, z) x label##y*newline* + #define ASC(x, y) x label##y*newline* + #define AS_HEX(y) 0##y##h +#elif defined(_MSC_VER) || defined(__BORLANDC__) + #define CRYPTOPP_MS_STYLE_INLINE_ASSEMBLY + #define AS1(x) __asm {x} + #define AS2(x, y) __asm {x, y} + #define AS3(x, y, z) __asm {x, y, z} + #define ASS(x, y, a, b, c, d) __asm {x, y, (a)*64+(b)*16+(c)*4+(d)} + #define ASL(x) __asm {label##x:} + #define ASJ(x, y, z) __asm {x label##y} + #define ASC(x, y) __asm {x label##y} + #define CRYPTOPP_NAKED __declspec(naked) + #define AS_HEX(y) 0x##y +#else + #define CRYPTOPP_GNU_STYLE_INLINE_ASSEMBLY + // define these in two steps to allow arguments to be expanded + #define GNU_AS1(x) #x ";" + #define GNU_AS2(x, y) #x ", " #y ";" + #define GNU_AS3(x, y, z) #x ", " #y ", " #z ";" + #define GNU_ASL(x) "\n" #x ":" + #define GNU_ASJ(x, y, z) #x " " #y #z ";" + #define AS1(x) GNU_AS1(x) + #define AS2(x, y) GNU_AS2(x, y) + #define AS3(x, y, z) GNU_AS3(x, y, z) + #define ASS(x, y, a, b, c, d) #x ", " #y ", " #a "*64+" #b "*16+" #c "*4+" #d ";" + #define ASL(x) GNU_ASL(x) + #define ASJ(x, y, z) GNU_ASJ(x, y, z) + #define ASC(x, y) #x " " #y ";" + #define CRYPTOPP_NAKED + #define AS_HEX(y) 0x##y +#endif + +#define IF0(y) +#define IF1(y) y + +#ifdef CRYPTOPP_GENERATE_X64_MASM +#define ASM_MOD(x, y) ((x) MOD (y)) +#define XMMWORD_PTR XMMWORD PTR +#else +// GNU assembler doesn't seem to have mod operator +#define ASM_MOD(x, y) ((x)-((x)/(y))*(y)) +// GAS 2.15 doesn't support XMMWORD PTR. it seems necessary only for MASM +#define XMMWORD_PTR +#endif + +#if CRYPTOPP_BOOL_X86 + #define AS_REG_1 ecx + #define AS_REG_2 edx + #define AS_REG_3 esi + #define AS_REG_4 edi + #define AS_REG_5 eax + #define AS_REG_6 ebx + #define AS_REG_7 ebp + #define AS_REG_1d ecx + #define AS_REG_2d edx + #define AS_REG_3d esi + #define AS_REG_4d edi + #define AS_REG_5d eax + #define AS_REG_6d ebx + #define AS_REG_7d ebp + #define WORD_SZ 4 + #define WORD_REG(x) e##x + #define WORD_PTR DWORD PTR + #define AS_PUSH_IF86(x) AS1(push e##x) + #define AS_POP_IF86(x) AS1(pop e##x) + #define AS_JCXZ jecxz +#elif CRYPTOPP_BOOL_X64 + #ifdef CRYPTOPP_GENERATE_X64_MASM + #define AS_REG_1 rcx + #define AS_REG_2 rdx + #define AS_REG_3 r8 + #define AS_REG_4 r9 + #define AS_REG_5 rax + #define AS_REG_6 r10 + #define AS_REG_7 r11 + #define AS_REG_1d ecx + #define AS_REG_2d edx + #define AS_REG_3d r8d + #define AS_REG_4d r9d + #define AS_REG_5d eax + #define AS_REG_6d r10d + #define AS_REG_7d r11d + #else + #define AS_REG_1 rdi + #define AS_REG_2 rsi + #define AS_REG_3 rdx + #define AS_REG_4 rcx + #define AS_REG_5 r8 + #define AS_REG_6 r9 + #define AS_REG_7 r10 + #define AS_REG_1d edi + #define AS_REG_2d esi + #define AS_REG_3d edx + #define AS_REG_4d ecx + #define AS_REG_5d r8d + #define AS_REG_6d r9d + #define AS_REG_7d r10d + #endif + #define WORD_SZ 8 + #define WORD_REG(x) r##x + #define WORD_PTR QWORD PTR + #define AS_PUSH_IF86(x) + #define AS_POP_IF86(x) + #define AS_JCXZ jrcxz +#endif + +static void CRYPTOPP_FASTCALL X86_SHA256_HashBlocks(word32 *state, const word32 *data, size_t len +#if defined(_MSC_VER) && (_MSC_VER == 1200) + , ... // VC60 workaround: prevent VC 6 from inlining this function +#endif + ) +{ +#if defined(_MSC_VER) && (_MSC_VER == 1200) + AS2(mov ecx, [state]) + AS2(mov edx, [data]) +#endif + + #define LOCALS_SIZE 8*4 + 16*4 + 4*WORD_SZ + #define H(i) [BASE+ASM_MOD(1024+7-(i),8)*4] + #define G(i) H(i+1) + #define F(i) H(i+2) + #define E(i) H(i+3) + #define D(i) H(i+4) + #define C(i) H(i+5) + #define B(i) H(i+6) + #define A(i) H(i+7) + #define Wt(i) BASE+8*4+ASM_MOD(1024+15-(i),16)*4 + #define Wt_2(i) Wt((i)-2) + #define Wt_15(i) Wt((i)-15) + #define Wt_7(i) Wt((i)-7) + #define K_END [BASE+8*4+16*4+0*WORD_SZ] + #define STATE_SAVE [BASE+8*4+16*4+1*WORD_SZ] + #define DATA_SAVE [BASE+8*4+16*4+2*WORD_SZ] + #define DATA_END [BASE+8*4+16*4+3*WORD_SZ] + #define Kt(i) WORD_REG(si)+(i)*4 +#if CRYPTOPP_BOOL_X86 + #define BASE esp+4 +#elif defined(__GNUC__) + #define BASE r8 +#else + #define BASE rsp +#endif + +#define RA0(i, edx, edi) \ + AS2( add edx, [Kt(i)] )\ + AS2( add edx, [Wt(i)] )\ + AS2( add edx, H(i) )\ + +#define RA1(i, edx, edi) + +#define RB0(i, edx, edi) + +#define RB1(i, edx, edi) \ + AS2( mov AS_REG_7d, [Wt_2(i)] )\ + AS2( mov edi, [Wt_15(i)])\ + AS2( mov ebx, AS_REG_7d )\ + AS2( shr AS_REG_7d, 10 )\ + AS2( ror ebx, 17 )\ + AS2( xor AS_REG_7d, ebx )\ + AS2( ror ebx, 2 )\ + AS2( xor ebx, AS_REG_7d )/* s1(W_t-2) */\ + AS2( add ebx, [Wt_7(i)])\ + AS2( mov AS_REG_7d, edi )\ + AS2( shr AS_REG_7d, 3 )\ + AS2( ror edi, 7 )\ + AS2( add ebx, [Wt(i)])/* s1(W_t-2) + W_t-7 + W_t-16 */\ + AS2( xor AS_REG_7d, edi )\ + AS2( add edx, [Kt(i)])\ + AS2( ror edi, 11 )\ + AS2( add edx, H(i) )\ + AS2( xor AS_REG_7d, edi )/* s0(W_t-15) */\ + AS2( add AS_REG_7d, ebx )/* W_t = s1(W_t-2) + W_t-7 + s0(W_t-15) W_t-16*/\ + AS2( mov [Wt(i)], AS_REG_7d)\ + AS2( add edx, AS_REG_7d )\ + +#define ROUND(i, r, eax, ecx, edi, edx)\ + /* in: edi = E */\ + /* unused: eax, ecx, temp: ebx, AS_REG_7d, out: edx = T1 */\ + AS2( mov edx, F(i) )\ + AS2( xor edx, G(i) )\ + AS2( and edx, edi )\ + AS2( xor edx, G(i) )/* Ch(E,F,G) = (G^(E&(F^G))) */\ + AS2( mov AS_REG_7d, edi )\ + AS2( ror edi, 6 )\ + AS2( ror AS_REG_7d, 25 )\ + RA##r(i, edx, edi )/* H + Wt + Kt + Ch(E,F,G) */\ + AS2( xor AS_REG_7d, edi )\ + AS2( ror edi, 5 )\ + AS2( xor AS_REG_7d, edi )/* S1(E) */\ + AS2( add edx, AS_REG_7d )/* T1 = S1(E) + Ch(E,F,G) + H + Wt + Kt */\ + RB##r(i, edx, edi )/* H + Wt + Kt + Ch(E,F,G) */\ + /* in: ecx = A, eax = B^C, edx = T1 */\ + /* unused: edx, temp: ebx, AS_REG_7d, out: eax = A, ecx = B^C, edx = E */\ + AS2( mov ebx, ecx )\ + AS2( xor ecx, B(i) )/* A^B */\ + AS2( and eax, ecx )\ + AS2( xor eax, B(i) )/* Maj(A,B,C) = B^((A^B)&(B^C) */\ + AS2( mov AS_REG_7d, ebx )\ + AS2( ror ebx, 2 )\ + AS2( add eax, edx )/* T1 + Maj(A,B,C) */\ + AS2( add edx, D(i) )\ + AS2( mov D(i), edx )\ + AS2( ror AS_REG_7d, 22 )\ + AS2( xor AS_REG_7d, ebx )\ + AS2( ror ebx, 11 )\ + AS2( xor AS_REG_7d, ebx )\ + AS2( add eax, AS_REG_7d )/* T1 + S0(A) + Maj(A,B,C) */\ + AS2( mov H(i), eax )\ + +#define SWAP_COPY(i) \ + AS2( mov WORD_REG(bx), [WORD_REG(dx)+i*WORD_SZ])\ + AS1( bswap WORD_REG(bx))\ + AS2( mov [Wt(i*(1+CRYPTOPP_BOOL_X64)+CRYPTOPP_BOOL_X64)], WORD_REG(bx)) + +#if defined(__GNUC__) + #if CRYPTOPP_BOOL_X64 + FixedSizeAlignedSecBlock workspace; + #endif + __asm__ __volatile__ + ( + #if CRYPTOPP_BOOL_X64 + "lea %4, %%r8;" + #endif + ".intel_syntax noprefix;" +#elif defined(CRYPTOPP_GENERATE_X64_MASM) + ALIGN 8 + X86_SHA256_HashBlocks PROC FRAME + rex_push_reg rsi + push_reg rdi + push_reg rbx + push_reg rbp + alloc_stack(LOCALS_SIZE+8) + .endprolog + mov rdi, r8 + lea rsi, [?SHA256_K@CryptoPP@@3QBIB + 48*4] +#endif + +#if CRYPTOPP_BOOL_X86 + #ifndef __GNUC__ + AS2( mov edi, [len]) + AS2( lea WORD_REG(si), [SHA256_K+48*4]) + #endif + #if !defined(_MSC_VER) || (_MSC_VER < 1400) + AS_PUSH_IF86(bx) + #endif + + AS_PUSH_IF86(bp) + AS2( mov ebx, esp) + AS2( and esp, -16) + AS2( sub WORD_REG(sp), LOCALS_SIZE) + AS_PUSH_IF86(bx) +#endif + AS2( mov STATE_SAVE, WORD_REG(cx)) + AS2( mov DATA_SAVE, WORD_REG(dx)) + AS2( lea WORD_REG(ax), [WORD_REG(di) + WORD_REG(dx)]) + AS2( mov DATA_END, WORD_REG(ax)) + AS2( mov K_END, WORD_REG(si)) + +#if CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE +#if CRYPTOPP_BOOL_X86 + AS2( test edi, 1) + ASJ( jnz, 2, f) + AS1( dec DWORD PTR K_END) +#endif + AS2( movdqa xmm0, XMMWORD_PTR [WORD_REG(cx)+0*16]) + AS2( movdqa xmm1, XMMWORD_PTR [WORD_REG(cx)+1*16]) +#endif + +#if CRYPTOPP_BOOL_X86 +#if CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE + ASJ( jmp, 0, f) +#endif + ASL(2) // non-SSE2 + AS2( mov esi, ecx) + AS2( lea edi, A(0)) + AS2( mov ecx, 8) + AS1( rep movsd) + AS2( mov esi, K_END) + ASJ( jmp, 3, f) +#endif + +#if CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE + ASL(0) + AS2( movdqa E(0), xmm1) + AS2( movdqa A(0), xmm0) +#endif +#if CRYPTOPP_BOOL_X86 + ASL(3) +#endif + AS2( sub WORD_REG(si), 48*4) + SWAP_COPY(0) SWAP_COPY(1) SWAP_COPY(2) SWAP_COPY(3) + SWAP_COPY(4) SWAP_COPY(5) SWAP_COPY(6) SWAP_COPY(7) +#if CRYPTOPP_BOOL_X86 + SWAP_COPY(8) SWAP_COPY(9) SWAP_COPY(10) SWAP_COPY(11) + SWAP_COPY(12) SWAP_COPY(13) SWAP_COPY(14) SWAP_COPY(15) +#endif + AS2( mov edi, E(0)) // E + AS2( mov eax, B(0)) // B + AS2( xor eax, C(0)) // B^C + AS2( mov ecx, A(0)) // A + + ROUND(0, 0, eax, ecx, edi, edx) + ROUND(1, 0, ecx, eax, edx, edi) + ROUND(2, 0, eax, ecx, edi, edx) + ROUND(3, 0, ecx, eax, edx, edi) + ROUND(4, 0, eax, ecx, edi, edx) + ROUND(5, 0, ecx, eax, edx, edi) + ROUND(6, 0, eax, ecx, edi, edx) + ROUND(7, 0, ecx, eax, edx, edi) + ROUND(8, 0, eax, ecx, edi, edx) + ROUND(9, 0, ecx, eax, edx, edi) + ROUND(10, 0, eax, ecx, edi, edx) + ROUND(11, 0, ecx, eax, edx, edi) + ROUND(12, 0, eax, ecx, edi, edx) + ROUND(13, 0, ecx, eax, edx, edi) + ROUND(14, 0, eax, ecx, edi, edx) + ROUND(15, 0, ecx, eax, edx, edi) + + ASL(1) + AS2(add WORD_REG(si), 4*16) + ROUND(0, 1, eax, ecx, edi, edx) + ROUND(1, 1, ecx, eax, edx, edi) + ROUND(2, 1, eax, ecx, edi, edx) + ROUND(3, 1, ecx, eax, edx, edi) + ROUND(4, 1, eax, ecx, edi, edx) + ROUND(5, 1, ecx, eax, edx, edi) + ROUND(6, 1, eax, ecx, edi, edx) + ROUND(7, 1, ecx, eax, edx, edi) + ROUND(8, 1, eax, ecx, edi, edx) + ROUND(9, 1, ecx, eax, edx, edi) + ROUND(10, 1, eax, ecx, edi, edx) + ROUND(11, 1, ecx, eax, edx, edi) + ROUND(12, 1, eax, ecx, edi, edx) + ROUND(13, 1, ecx, eax, edx, edi) + ROUND(14, 1, eax, ecx, edi, edx) + ROUND(15, 1, ecx, eax, edx, edi) + AS2( cmp WORD_REG(si), K_END) + ASJ( jb, 1, b) + + AS2( mov WORD_REG(dx), DATA_SAVE) + AS2( add WORD_REG(dx), 64) + AS2( mov AS_REG_7, STATE_SAVE) + AS2( mov DATA_SAVE, WORD_REG(dx)) + +#if CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE +#if CRYPTOPP_BOOL_X86 + AS2( test DWORD PTR K_END, 1) + ASJ( jz, 4, f) +#endif + AS2( movdqa xmm1, XMMWORD_PTR [AS_REG_7+1*16]) + AS2( movdqa xmm0, XMMWORD_PTR [AS_REG_7+0*16]) + AS2( paddd xmm1, E(0)) + AS2( paddd xmm0, A(0)) + AS2( movdqa [AS_REG_7+1*16], xmm1) + AS2( movdqa [AS_REG_7+0*16], xmm0) + AS2( cmp WORD_REG(dx), DATA_END) + ASJ( jb, 0, b) +#endif + +#if CRYPTOPP_BOOL_X86 +#if CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE + ASJ( jmp, 5, f) + ASL(4) // non-SSE2 +#endif + AS2( add [AS_REG_7+0*4], ecx) // A + AS2( add [AS_REG_7+4*4], edi) // E + AS2( mov eax, B(0)) + AS2( mov ebx, C(0)) + AS2( mov ecx, D(0)) + AS2( add [AS_REG_7+1*4], eax) + AS2( add [AS_REG_7+2*4], ebx) + AS2( add [AS_REG_7+3*4], ecx) + AS2( mov eax, F(0)) + AS2( mov ebx, G(0)) + AS2( mov ecx, H(0)) + AS2( add [AS_REG_7+5*4], eax) + AS2( add [AS_REG_7+6*4], ebx) + AS2( add [AS_REG_7+7*4], ecx) + AS2( mov ecx, AS_REG_7d) + AS2( cmp WORD_REG(dx), DATA_END) + ASJ( jb, 2, b) +#if CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE + ASL(5) +#endif +#endif + + AS_POP_IF86(sp) + AS_POP_IF86(bp) + #if !defined(_MSC_VER) || (_MSC_VER < 1400) + AS_POP_IF86(bx) + #endif + +#ifdef CRYPTOPP_GENERATE_X64_MASM + add rsp, LOCALS_SIZE+8 + pop rbp + pop rbx + pop rdi + pop rsi + ret + X86_SHA256_HashBlocks ENDP +#endif + +#ifdef __GNUC__ + ".att_syntax prefix;" + : + : "c" (state), "d" (data), "S" (SHA256_K+48), "D" (len) + #if CRYPTOPP_BOOL_X64 + , "m" (workspace[0]) + #endif + : "memory", "cc", "%eax" + #if CRYPTOPP_BOOL_X64 + , "%rbx", "%r8", "%r10" + #endif + ); +#endif +} + +static inline bool HasSSE2(void) { return false; } + +static void SHA256_Transform32(word32 *state, const word32 *data) +{ + word32 W[16]; + int i; + + for (i = 0; i < 16; i++) + W[i] = ((word32 *)(data))[i]; + + X86_SHA256_HashBlocks(state, W, 16 * 4); +} + +static void runhash32(void *state, const void *input, const void *init) +{ + memcpy(state, init, 32); + SHA256_Transform32(state, input); +} + +/* suspiciously similar to ScanHash* from bitcoin */ +bool scanhash_asm32(const unsigned char *midstate, unsigned char *data, + unsigned char *hash1, unsigned char *hash, + unsigned long *hashes_done) +{ + uint32_t *hash32 = (uint32_t *) hash; + uint32_t *nonce = (uint32_t *)(data + 12); + uint32_t n = 0; + unsigned long stat_ctr = 0; + + while (1) { + n++; + *nonce = n; + + runhash32(hash1, data, midstate); + runhash32(hash, hash1, sha256_init_state); + + stat_ctr++; + + if (hash32[7] == 0) { + char *hexstr; + + hexstr = bin2hex(hash, 32); + fprintf(stderr, + "DBG: found zeroes in hash:\n%s\n", + hexstr); + free(hexstr); + + *hashes_done = stat_ctr; + return true; + } + + if ((n & 0xffffff) == 0) { + if (opt_debug) + fprintf(stderr, "DBG: end of nonce range\n"); + *hashes_done = stat_ctr; + return false; + } + } +} + +#endif // #if defined(WANT_CRYPTOPP_ASM32)