Browse Source

use builtin __AVX__ and __AES__ macros and reduce code duplication

pull/1204/head
yangfl 7 years ago
parent
commit
dec848f072
  1. 2
      Makefile.homebrew
  2. 2
      Makefile.linux
  3. 2
      Makefile.mingw
  4. 2
      Makefile.osx
  5. 1
      build/CMakeLists.txt
  6. 14
      libi2pd/CPU.cpp
  7. 112
      libi2pd/Crypto.cpp
  8. 13
      libi2pd/Crypto.h
  9. 11
      libi2pd/Identity.cpp

2
Makefile.homebrew

@ -34,7 +34,7 @@ endif
# Seems like all recent Mac's have AES-NI, after firmware upgrade 2.2 # Seems like all recent Mac's have AES-NI, after firmware upgrade 2.2
# Found no good way to detect it from command line. TODO: Might be some osx sysinfo magic # Found no good way to detect it from command line. TODO: Might be some osx sysinfo magic
ifeq ($(USE_AESNI),yes) ifeq ($(USE_AESNI),yes)
CXXFLAGS += -maes -DAESNI CXXFLAGS += -maes
endif endif
ifeq ($(USE_AVX),1) ifeq ($(USE_AVX),1)
CXXFLAGS += -mavx CXXFLAGS += -mavx

2
Makefile.linux

@ -64,7 +64,7 @@ ifneq ($(shell $(GREP) -c aes /proc/cpuinfo),0)
ifeq ($(machine), aarch64) ifeq ($(machine), aarch64)
CXXFLAGS += -DARM64AES CXXFLAGS += -DARM64AES
else else
CPU_FLAGS += -maes -DAESNI CPU_FLAGS += -maes
endif endif
endif endif
endif endif

2
Makefile.mingw

@ -37,7 +37,7 @@ endif
# don't change following line to ifeq ($(USE_AESNI),yes) !!! # don't change following line to ifeq ($(USE_AESNI),yes) !!!
ifeq ($(USE_AESNI),1) ifeq ($(USE_AESNI),1)
CPU_FLAGS += -maes -DAESNI CPU_FLAGS += -maes
else else
CPU_FLAGS += -msse CPU_FLAGS += -msse
endif endif

2
Makefile.osx vendored

@ -21,7 +21,7 @@ ifeq ($(USE_UPNP),yes)
endif endif
ifeq ($(USE_AESNI),1) ifeq ($(USE_AESNI),1)
CXXFLAGS += -maes -DAESNI CXXFLAGS += -maes
else else
CXXFLAGS += -msse CXXFLAGS += -msse
endif endif

1
build/CMakeLists.txt

@ -234,7 +234,6 @@ endif ()
if (WITH_AESNI) if (WITH_AESNI)
set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -maes" ) set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -maes" )
add_definitions ( -DAESNI )
endif() endif()
if (WITH_AVX) if (WITH_AVX)

14
libi2pd/CPU.cpp

@ -21,23 +21,35 @@ namespace cpu
void Detect() void Detect()
{ {
#if defined(__AES__) || defined(__AVX__)
#if defined(__x86_64__) || defined(__i386__) #if defined(__x86_64__) || defined(__i386__)
int info[4]; int info[4];
__cpuid(0, info[0], info[1], info[2], info[3]); __cpuid(0, info[0], info[1], info[2], info[3]);
if (info[0] >= 0x00000001) { if (info[0] >= 0x00000001) {
__cpuid(0x00000001, info[0], info[1], info[2], info[3]); __cpuid(0x00000001, info[0], info[1], info[2], info[3]);
#ifdef __AES__
aesni = info[2] & bit_AES; // AESNI aesni = info[2] & bit_AES; // AESNI
#endif // __AES__
#ifdef __AVX__
avx = info[2] & bit_AVX; // AVX avx = info[2] & bit_AVX; // AVX
#endif // __AVX__
} }
#endif #endif // defined(__x86_64__) || defined(__i386__)
#ifdef __AES__
if(aesni) if(aesni)
{ {
LogPrint(eLogInfo, "AESNI enabled"); LogPrint(eLogInfo, "AESNI enabled");
} }
#endif // __AES__
#ifdef __AVX__
if(avx) if(avx)
{ {
LogPrint(eLogInfo, "AVX enabled"); LogPrint(eLogInfo, "AVX enabled");
} }
#endif // __AVX__
#endif // defined(__AES__) || defined(__AVX__)
} }
} }
} }

112
libi2pd/Crypto.cpp

@ -522,9 +522,9 @@ namespace crypto
{ {
uint64_t buf[256]; uint64_t buf[256];
uint64_t hash[12]; // 96 bytes uint64_t hash[12]; // 96 bytes
#ifdef __AVX__
if(i2p::cpu::avx) if(i2p::cpu::avx)
{ {
#ifdef AVX
__asm__ __asm__
( (
"vmovups %[key], %%ymm0 \n" "vmovups %[key], %%ymm0 \n"
@ -543,30 +543,9 @@ namespace crypto
[buf]"r"(buf), [hash]"r"(hash) [buf]"r"(buf), [hash]"r"(hash)
: "memory", "%xmm0" // TODO: change to %ymm0 later : "memory", "%xmm0" // TODO: change to %ymm0 later
); );
#else
// ikeypad
buf[0] = key.GetLL ()[0] ^ IPAD;
buf[1] = key.GetLL ()[1] ^ IPAD;
buf[2] = key.GetLL ()[2] ^ IPAD;
buf[3] = key.GetLL ()[3] ^ IPAD;
buf[4] = IPAD;
buf[5] = IPAD;
buf[6] = IPAD;
buf[7] = IPAD;
// okeypad
hash[0] = key.GetLL ()[0] ^ OPAD;
hash[1] = key.GetLL ()[1] ^ OPAD;
hash[2] = key.GetLL ()[2] ^ OPAD;
hash[3] = key.GetLL ()[3] ^ OPAD;
hash[4] = OPAD;
hash[5] = OPAD;
hash[6] = OPAD;
hash[7] = OPAD;
// fill last 16 bytes with zeros (first hash size assumed 32 bytes in I2P)
memset (hash + 10, 0, 16);
#endif
} }
else else
#endif
{ {
// ikeypad // ikeypad
buf[0] = key.GetLL ()[0] ^ IPAD; buf[0] = key.GetLL ()[0] ^ IPAD;
@ -600,7 +579,7 @@ namespace crypto
} }
// AES // AES
#ifdef AESNI #ifdef __AES__
#ifdef ARM64AES #ifdef ARM64AES
void init_aesenc(void){ void init_aesenc(void){
// TODO: Implementation // TODO: Implementation
@ -632,7 +611,7 @@ namespace crypto
"movaps %%xmm3, "#round1"(%[sched]) \n" "movaps %%xmm3, "#round1"(%[sched]) \n"
#endif #endif
#ifdef AESNI #ifdef __AES__
void ECBCryptoAESNI::ExpandKey (const AESKey& key) void ECBCryptoAESNI::ExpandKey (const AESKey& key)
{ {
__asm__ __asm__
@ -673,7 +652,7 @@ namespace crypto
#endif #endif
#if AESNI #ifdef __AES__
#define EncryptAES256(sched) \ #define EncryptAES256(sched) \
"pxor (%["#sched"]), %%xmm0 \n" \ "pxor (%["#sched"]), %%xmm0 \n" \
"aesenc 16(%["#sched"]), %%xmm0 \n" \ "aesenc 16(%["#sched"]), %%xmm0 \n" \
@ -694,9 +673,9 @@ namespace crypto
void ECBEncryption::Encrypt (const ChipherBlock * in, ChipherBlock * out) void ECBEncryption::Encrypt (const ChipherBlock * in, ChipherBlock * out)
{ {
#ifdef __AES__
if(i2p::cpu::aesni) if(i2p::cpu::aesni)
{ {
#ifdef AESNI
__asm__ __asm__
( (
"movups (%[in]), %%xmm0 \n" "movups (%[in]), %%xmm0 \n"
@ -704,17 +683,15 @@ namespace crypto
"movups %%xmm0, (%[out]) \n" "movups %%xmm0, (%[out]) \n"
: : [sched]"r"(GetKeySchedule ()), [in]"r"(in), [out]"r"(out) : "%xmm0", "memory" : : [sched]"r"(GetKeySchedule ()), [in]"r"(in), [out]"r"(out) : "%xmm0", "memory"
); );
#else
AES_encrypt (in->buf, out->buf, &m_Key);
#endif
} }
else else
#endif
{ {
AES_encrypt (in->buf, out->buf, &m_Key); AES_encrypt (in->buf, out->buf, &m_Key);
} }
} }
#ifdef AESNI #ifdef __AES__
#define DecryptAES256(sched) \ #define DecryptAES256(sched) \
"pxor 224(%["#sched"]), %%xmm0 \n" \ "pxor 224(%["#sched"]), %%xmm0 \n" \
"aesdec 208(%["#sched"]), %%xmm0 \n" \ "aesdec 208(%["#sched"]), %%xmm0 \n" \
@ -735,9 +712,9 @@ namespace crypto
void ECBDecryption::Decrypt (const ChipherBlock * in, ChipherBlock * out) void ECBDecryption::Decrypt (const ChipherBlock * in, ChipherBlock * out)
{ {
#ifdef __AES__
if(i2p::cpu::aesni) if(i2p::cpu::aesni)
{ {
#ifdef AESNI
__asm__ __asm__
( (
"movups (%[in]), %%xmm0 \n" "movups (%[in]), %%xmm0 \n"
@ -745,17 +722,15 @@ namespace crypto
"movups %%xmm0, (%[out]) \n" "movups %%xmm0, (%[out]) \n"
: : [sched]"r"(GetKeySchedule ()), [in]"r"(in), [out]"r"(out) : "%xmm0", "memory" : : [sched]"r"(GetKeySchedule ()), [in]"r"(in), [out]"r"(out) : "%xmm0", "memory"
); );
#else
AES_decrypt (in->buf, out->buf, &m_Key);
#endif
} }
else else
#endif
{ {
AES_decrypt (in->buf, out->buf, &m_Key); AES_decrypt (in->buf, out->buf, &m_Key);
} }
} }
#ifdef AESNI #ifdef __AES__
#define CallAESIMC(offset) \ #define CallAESIMC(offset) \
"movaps "#offset"(%[shed]), %%xmm0 \n" \ "movaps "#offset"(%[shed]), %%xmm0 \n" \
"aesimc %%xmm0, %%xmm0 \n" \ "aesimc %%xmm0, %%xmm0 \n" \
@ -764,15 +739,13 @@ namespace crypto
void ECBEncryption::SetKey (const AESKey& key) void ECBEncryption::SetKey (const AESKey& key)
{ {
#ifdef __AES__
if(i2p::cpu::aesni) if(i2p::cpu::aesni)
{ {
#ifdef AESNI
ExpandKey (key); ExpandKey (key);
#else
AES_set_encrypt_key (key, 256, &m_Key);
#endif
} }
else else
#endif
{ {
AES_set_encrypt_key (key, 256, &m_Key); AES_set_encrypt_key (key, 256, &m_Key);
} }
@ -780,9 +753,9 @@ namespace crypto
void ECBDecryption::SetKey (const AESKey& key) void ECBDecryption::SetKey (const AESKey& key)
{ {
#ifdef __AES__
if(i2p::cpu::aesni) if(i2p::cpu::aesni)
{ {
#ifdef AESNI
ExpandKey (key); // expand encryption key first ExpandKey (key); // expand encryption key first
// then invert it using aesimc // then invert it using aesimc
__asm__ __asm__
@ -802,11 +775,9 @@ namespace crypto
CallAESIMC(208) CallAESIMC(208)
: : [shed]"r"(GetKeySchedule ()) : "%xmm0", "memory" : : [shed]"r"(GetKeySchedule ()) : "%xmm0", "memory"
); );
#else
AES_set_decrypt_key (key, 256, &m_Key);
#endif
} }
else else
#endif
{ {
AES_set_decrypt_key (key, 256, &m_Key); AES_set_decrypt_key (key, 256, &m_Key);
} }
@ -815,9 +786,9 @@ namespace crypto
void CBCEncryption::Encrypt (int numBlocks, const ChipherBlock * in, ChipherBlock * out) void CBCEncryption::Encrypt (int numBlocks, const ChipherBlock * in, ChipherBlock * out)
{ {
#ifdef __AES__
if(i2p::cpu::aesni) if(i2p::cpu::aesni)
{ {
#ifdef AESNI
__asm__ __asm__
( (
"movups (%[iv]), %%xmm1 \n" "movups (%[iv]), %%xmm1 \n"
@ -837,16 +808,9 @@ namespace crypto
[in]"r"(in), [out]"r"(out), [num]"r"(numBlocks) [in]"r"(in), [out]"r"(out), [num]"r"(numBlocks)
: "%xmm0", "%xmm1", "cc", "memory" : "%xmm0", "%xmm1", "cc", "memory"
); );
#else
for (int i = 0; i < numBlocks; i++)
{
*m_LastBlock.GetChipherBlock () ^= in[i];
m_ECBEncryption.Encrypt (m_LastBlock.GetChipherBlock (), m_LastBlock.GetChipherBlock ());
out[i] = *m_LastBlock.GetChipherBlock ();
}
#endif
} }
else else
#endif
{ {
for (int i = 0; i < numBlocks; i++) for (int i = 0; i < numBlocks; i++)
{ {
@ -867,9 +831,9 @@ namespace crypto
void CBCEncryption::Encrypt (const uint8_t * in, uint8_t * out) void CBCEncryption::Encrypt (const uint8_t * in, uint8_t * out)
{ {
#ifdef __AES__
if(i2p::cpu::aesni) if(i2p::cpu::aesni)
{ {
#ifdef AESNI
__asm__ __asm__
( (
"movups (%[iv]), %%xmm1 \n" "movups (%[iv]), %%xmm1 \n"
@ -883,19 +847,17 @@ namespace crypto
[in]"r"(in), [out]"r"(out) [in]"r"(in), [out]"r"(out)
: "%xmm0", "%xmm1", "memory" : "%xmm0", "%xmm1", "memory"
); );
#else
Encrypt (1, (const ChipherBlock *)in, (ChipherBlock *)out);
#endif
} }
else else
#endif
Encrypt (1, (const ChipherBlock *)in, (ChipherBlock *)out); Encrypt (1, (const ChipherBlock *)in, (ChipherBlock *)out);
} }
void CBCDecryption::Decrypt (int numBlocks, const ChipherBlock * in, ChipherBlock * out) void CBCDecryption::Decrypt (int numBlocks, const ChipherBlock * in, ChipherBlock * out)
{ {
#ifdef __AES__
if(i2p::cpu::aesni) if(i2p::cpu::aesni)
{ {
#ifdef AESNI
__asm__ __asm__
( (
"movups (%[iv]), %%xmm1 \n" "movups (%[iv]), %%xmm1 \n"
@ -916,17 +878,9 @@ namespace crypto
[in]"r"(in), [out]"r"(out), [num]"r"(numBlocks) [in]"r"(in), [out]"r"(out), [num]"r"(numBlocks)
: "%xmm0", "%xmm1", "%xmm2", "cc", "memory" : "%xmm0", "%xmm1", "%xmm2", "cc", "memory"
); );
#else
for (int i = 0; i < numBlocks; i++)
{
ChipherBlock tmp = in[i];
m_ECBDecryption.Decrypt (in + i, out + i);
out[i] ^= *m_IV.GetChipherBlock ();
*m_IV.GetChipherBlock () = tmp;
}
#endif
} }
else else
#endif
{ {
for (int i = 0; i < numBlocks; i++) for (int i = 0; i < numBlocks; i++)
{ {
@ -947,9 +901,9 @@ namespace crypto
void CBCDecryption::Decrypt (const uint8_t * in, uint8_t * out) void CBCDecryption::Decrypt (const uint8_t * in, uint8_t * out)
{ {
#ifdef __AES__
if(i2p::cpu::aesni) if(i2p::cpu::aesni)
{ {
#ifdef AESNI
__asm__ __asm__
( (
"movups (%[iv]), %%xmm1 \n" "movups (%[iv]), %%xmm1 \n"
@ -963,19 +917,17 @@ namespace crypto
[in]"r"(in), [out]"r"(out) [in]"r"(in), [out]"r"(out)
: "%xmm0", "%xmm1", "memory" : "%xmm0", "%xmm1", "memory"
); );
#else
Decrypt (1, (const ChipherBlock *)in, (ChipherBlock *)out);
#endif
} }
else else
#endif
Decrypt (1, (const ChipherBlock *)in, (ChipherBlock *)out); Decrypt (1, (const ChipherBlock *)in, (ChipherBlock *)out);
} }
void TunnelEncryption::Encrypt (const uint8_t * in, uint8_t * out) void TunnelEncryption::Encrypt (const uint8_t * in, uint8_t * out)
{ {
#ifdef __AES__
if(i2p::cpu::aesni) if(i2p::cpu::aesni)
{ {
#ifdef AESNI
__asm__ __asm__
( (
// encrypt IV // encrypt IV
@ -1001,14 +953,9 @@ namespace crypto
[in]"r"(in), [out]"r"(out), [num]"r"(63) // 63 blocks = 1008 bytes [in]"r"(in), [out]"r"(out), [num]"r"(63) // 63 blocks = 1008 bytes
: "%xmm0", "%xmm1", "cc", "memory" : "%xmm0", "%xmm1", "cc", "memory"
); );
#else
m_IVEncryption.Encrypt ((const ChipherBlock *)in, (ChipherBlock *)out); // iv
m_LayerEncryption.SetIV (out);
m_LayerEncryption.Encrypt (in + 16, i2p::tunnel::TUNNEL_DATA_ENCRYPTED_SIZE, out + 16); // data
m_IVEncryption.Encrypt ((ChipherBlock *)out, (ChipherBlock *)out); // double iv
#endif
} }
else else
#endif
{ {
m_IVEncryption.Encrypt ((const ChipherBlock *)in, (ChipherBlock *)out); // iv m_IVEncryption.Encrypt ((const ChipherBlock *)in, (ChipherBlock *)out); // iv
m_LayerEncryption.SetIV (out); m_LayerEncryption.SetIV (out);
@ -1019,9 +966,9 @@ namespace crypto
void TunnelDecryption::Decrypt (const uint8_t * in, uint8_t * out) void TunnelDecryption::Decrypt (const uint8_t * in, uint8_t * out)
{ {
#ifdef __AES__
if(i2p::cpu::aesni) if(i2p::cpu::aesni)
{ {
#ifdef AESNI
__asm__ __asm__
( (
// decrypt IV // decrypt IV
@ -1048,14 +995,9 @@ namespace crypto
[in]"r"(in), [out]"r"(out), [num]"r"(63) // 63 blocks = 1008 bytes [in]"r"(in), [out]"r"(out), [num]"r"(63) // 63 blocks = 1008 bytes
: "%xmm0", "%xmm1", "%xmm2", "cc", "memory" : "%xmm0", "%xmm1", "%xmm2", "cc", "memory"
); );
#else
m_IVDecryption.Decrypt ((const ChipherBlock *)in, (ChipherBlock *)out); // iv
m_LayerDecryption.SetIV (out);
m_LayerDecryption.Decrypt (in + 16, i2p::tunnel::TUNNEL_DATA_ENCRYPTED_SIZE, out + 16); // data
m_IVDecryption.Decrypt ((ChipherBlock *)out, (ChipherBlock *)out); // double iv
#endif
} }
else else
#endif
{ {
m_IVDecryption.Decrypt ((const ChipherBlock *)in, (ChipherBlock *)out); // iv m_IVDecryption.Decrypt ((const ChipherBlock *)in, (ChipherBlock *)out); // iv
m_LayerDecryption.SetIV (out); m_LayerDecryption.SetIV (out);

13
libi2pd/Crypto.h

@ -69,9 +69,9 @@ namespace crypto
void operator^=(const ChipherBlock& other) // XOR void operator^=(const ChipherBlock& other) // XOR
{ {
#ifdef __AVX__
if (i2p::cpu::avx) if (i2p::cpu::avx)
{ {
#ifdef AVX
__asm__ __asm__
( (
"vmovups (%[buf]), %%xmm0 \n" "vmovups (%[buf]), %%xmm0 \n"
@ -82,12 +82,9 @@ namespace crypto
: [buf]"r"(buf), [other]"r"(other.buf) : [buf]"r"(buf), [other]"r"(other.buf)
: "%xmm0", "%xmm1", "memory" : "%xmm0", "%xmm1", "memory"
); );
#else
for (int i = 0; i < 16; i++)
buf[i] ^= other.buf[i];
#endif
} }
else else
#endif
{ {
// TODO: implement it better // TODO: implement it better
for (int i = 0; i < 16; i++) for (int i = 0; i < 16; i++)
@ -123,7 +120,7 @@ namespace crypto
}; };
#ifdef AESNI #ifdef __AES__
#ifdef ARM64AES #ifdef ARM64AES
void init_aesenc(void) __attribute__((constructor)); void init_aesenc(void) __attribute__((constructor));
#endif #endif
@ -143,7 +140,7 @@ namespace crypto
}; };
#endif #endif
#ifdef AESNI #ifdef __AES__
class ECBEncryption: public ECBCryptoAESNI class ECBEncryption: public ECBCryptoAESNI
#else #else
class ECBEncryption class ECBEncryption
@ -159,7 +156,7 @@ namespace crypto
AES_KEY m_Key; AES_KEY m_Key;
}; };
#ifdef AESNI #ifdef __AES__
class ECBDecryption: public ECBCryptoAESNI class ECBDecryption: public ECBCryptoAESNI
#else #else
class ECBDecryption class ECBDecryption

11
libi2pd/Identity.cpp

@ -719,7 +719,9 @@ namespace data
XORMetric operator^(const IdentHash& key1, const IdentHash& key2) XORMetric operator^(const IdentHash& key1, const IdentHash& key2)
{ {
XORMetric m; XORMetric m;
#if defined(__AVX__) // for AVX #ifdef __AVX__
if(i2p::cpu::avx)
{
__asm__ __asm__
( (
"vmovups %1, %%ymm0 \n" "vmovups %1, %%ymm0 \n"
@ -730,13 +732,16 @@ namespace data
: "m"(*key1), "m"(*key2) : "m"(*key1), "m"(*key2)
: "memory", "%xmm0", "%xmm1" // should be replaced by %ymm0/1 once supported by compiler : "memory", "%xmm0", "%xmm1" // should be replaced by %ymm0/1 once supported by compiler
); );
#else }
else
#endif
{
const uint64_t * hash1 = key1.GetLL (), * hash2 = key2.GetLL (); const uint64_t * hash1 = key1.GetLL (), * hash2 = key2.GetLL ();
m.metric_ll[0] = hash1[0] ^ hash2[0]; m.metric_ll[0] = hash1[0] ^ hash2[0];
m.metric_ll[1] = hash1[1] ^ hash2[1]; m.metric_ll[1] = hash1[1] ^ hash2[1];
m.metric_ll[2] = hash1[2] ^ hash2[2]; m.metric_ll[2] = hash1[2] ^ hash2[2];
m.metric_ll[3] = hash1[3] ^ hash2[3]; m.metric_ll[3] = hash1[3] ^ hash2[3];
#endif }
return m; return m;
} }

Loading…
Cancel
Save