PurpleI2P
/
i2pd
mirror of https://github.com/PurpleI2P/i2pd.git
1
0
Fork 0
Code Issues Releases Activity
Browse Source

initial code for runtime detection of aesni/avx

pull/1115/head
Jeff Becker 6 years ago
parent
b6e75e9c5a
commit
91e45d9a4a
No known key found for this signature in database
GPG Key ID: F357B3B42F6F9B05
2 changed files with 419 additions and 299 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 175
      libi2pd/Crypto.cpp
  2. 91
      libi2pd/Crypto.h

175
libi2pd/Crypto.cpp
Unescape View File

@ -479,10 +479,9 @@ namespace crypto @@ -479,10 +479,9 @@ namespace crypto
const uint64_t IPAD = 0x3636363636363636;
const uint64_t OPAD = 0x5C5C5C5C5C5C5C5C;
#if defined(__AVX__)
static const uint64_t ipads[] = { IPAD, IPAD, IPAD, IPAD };
static const uint64_t opads[] = { OPAD, OPAD, OPAD, OPAD };
#endif
void HMACMD5Digest (uint8_t * msg, size_t len, const MACKey& key, uint8_t * digest)
// key is 32 bytes
@ -491,7 +490,9 @@ namespace crypto @@ -491,7 +490,9 @@ namespace crypto
{
uint64_t buf[256];
uint64_t hash[12]; // 96 bytes
#if defined(__AVX__) // for AVX
if(i2p::cpu::avx)
{
#ifdef AVX
__asm__
(
"vmovups %[key], %%ymm0 \n"
@ -532,6 +533,30 @@ namespace crypto @@ -532,6 +533,30 @@ namespace crypto
// fill last 16 bytes with zeros (first hash size assumed 32 bytes in I2P)
memset (hash + 10, 0, 16);
#endif
}
else
{
// ikeypad
buf[0] = key.GetLL ()[0] ^ IPAD;
buf[1] = key.GetLL ()[1] ^ IPAD;
buf[2] = key.GetLL ()[2] ^ IPAD;
buf[3] = key.GetLL ()[3] ^ IPAD;
buf[4] = IPAD;
buf[5] = IPAD;
buf[6] = IPAD;
buf[7] = IPAD;
// okeypad
hash[0] = key.GetLL ()[0] ^ OPAD;
hash[1] = key.GetLL ()[1] ^ OPAD;
hash[2] = key.GetLL ()[2] ^ OPAD;
hash[3] = key.GetLL ()[3] ^ OPAD;
hash[4] = OPAD;
hash[5] = OPAD;
hash[6] = OPAD;
hash[7] = OPAD;
// fill last 16 bytes with zeros (first hash size assumed 32 bytes in I2P)
memset (hash + 10, 0, 16);
}
// concatenate with msg
memcpy (buf + 8, msg, len);
@ -543,8 +568,7 @@ namespace crypto @@ -543,8 +568,7 @@ namespace crypto
}
// AES
#ifdef AESNI
#ifdef AESNI
#define KeyExpansion256(round0,round1) \
"pshufd $0xff, %%xmm2, %%xmm2 \n" \
"movaps %%xmm1, %%xmm4 \n" \
@ -567,7 +591,9 @@ namespace crypto @@ -567,7 +591,9 @@ namespace crypto
"pxor %%xmm4, %%xmm3 \n" \
"pxor %%xmm2, %%xmm3 \n" \
"movaps %%xmm3, "#round1"(%[sched]) \n"
#endif
#ifdef AESNI
void ECBCryptoAESNI::ExpandKey (const AESKey& key)
{
__asm__
@ -605,7 +631,10 @@ namespace crypto @@ -605,7 +631,10 @@ namespace crypto
: "%xmm1", "%xmm2", "%xmm3", "%xmm4", "memory" // clogged
);
}
#endif
#if AESNI
#define EncryptAES256(sched) \
"pxor (%["#sched"]), %%xmm0 \n" \
"aesenc 16(%["#sched"]), %%xmm0 \n" \
@ -622,9 +651,13 @@ namespace crypto @@ -622,9 +651,13 @@ namespace crypto
"aesenc 192(%["#sched"]), %%xmm0 \n" \
"aesenc 208(%["#sched"]), %%xmm0 \n" \
"aesenclast 224(%["#sched"]), %%xmm0 \n"
#endif
void ECBEncryptionAESNI::Encrypt (const ChipherBlock * in, ChipherBlock * out)
void ECBEncryption::Encrypt (const ChipherBlock * in, ChipherBlock * out)
{
if(i2p::cpu::aesni)
{
#ifdef AESNI
__asm__
(
"movups (%[in]), %%xmm0 \n"
@ -632,8 +665,17 @@ namespace crypto @@ -632,8 +665,17 @@ namespace crypto
"movups %%xmm0, (%[out]) \n"
: : [sched]"r"(GetKeySchedule ()), [in]"r"(in), [out]"r"(out) : "%xmm0", "memory"
);
#else
AES_encrypt (in->buf, out->buf, &m_Key);
#endif
}
else
{
AES_encrypt (in->buf, out->buf, &m_Key);
}
}
#ifdef AESNI
#define DecryptAES256(sched) \
"pxor 224(%["#sched"]), %%xmm0 \n" \
"aesdec 208(%["#sched"]), %%xmm0 \n" \
@ -650,9 +692,13 @@ namespace crypto @@ -650,9 +692,13 @@ namespace crypto
"aesdec 32(%["#sched"]), %%xmm0 \n" \
"aesdec 16(%["#sched"]), %%xmm0 \n" \
"aesdeclast (%["#sched"]), %%xmm0 \n"
#endif
void ECBDecryptionAESNI::Decrypt (const ChipherBlock * in, ChipherBlock * out)
void ECBDecryption::Decrypt (const ChipherBlock * in, ChipherBlock * out)
{
if(i2p::cpu::aesni)
{
#ifdef AESNI
__asm__
(
"movups (%[in]), %%xmm0 \n"
@ -660,15 +706,28 @@ namespace crypto @@ -660,15 +706,28 @@ namespace crypto
"movups %%xmm0, (%[out]) \n"
: : [sched]"r"(GetKeySchedule ()), [in]"r"(in), [out]"r"(out) : "%xmm0", "memory"
);
#else
AES_decrypt (in->buf, out->buf, &m_Key);
#endif
}
else
{
AES_decrypt (in->buf, out->buf, &m_Key);
}
}
#ifdef AESNI
#define CallAESIMC(offset) \
"movaps "#offset"(%[shed]), %%xmm0 \n" \
"aesimc %%xmm0, %%xmm0 \n" \
"movaps %%xmm0, "#offset"(%[shed]) \n"
#endif
void ECBDecryptionAESNI::SetKey (const AESKey& key)
void ECBEncryption::SetKey (const AESKey& key)
{
if(i2p::cpu::aesni)
{
#ifdef AESNI
ExpandKey (key); // expand encryption key first
// then invert it using aesimc
__asm__
@ -688,13 +747,55 @@ namespace crypto @@ -688,13 +747,55 @@ namespace crypto
CallAESIMC(208)
: : [shed]"r"(GetKeySchedule ()) : "%xmm0", "memory"
);
#else
AES_set_encrypt_key (key, 256, &m_Key);
#endif
}
else
{
AES_set_encrypt_key (key, 256, &m_Key);
}
}
void ECBDecryption::SetKey (const AESKey& key)
{
if(i2p::cpu::aesni)
{
#ifdef AESNI
ExpandKey (key); // expand encryption key first
// then invert it using aesimc
__asm__
(
CallAESIMC(16)
CallAESIMC(32)
CallAESIMC(48)
CallAESIMC(64)
CallAESIMC(80)
CallAESIMC(96)
CallAESIMC(112)
CallAESIMC(128)
CallAESIMC(144)
CallAESIMC(160)
CallAESIMC(176)
CallAESIMC(192)
CallAESIMC(208)
: : [shed]"r"(GetKeySchedule ()) : "%xmm0", "memory"
);
#else
AES_set_decrypt_key (key, 256, &m_Key);
#endif
}
else
{
AES_set_decrypt_key (key, 256, &m_Key);
}
}
void CBCEncryption::Encrypt (int numBlocks, const ChipherBlock * in, ChipherBlock * out)
{
if(i2p::cpu::aesni)
{
#ifdef AESNI
__asm__
(
@ -724,6 +825,16 @@ namespace crypto @@ -724,6 +825,16 @@ namespace crypto
}
#endif
}
else
{
for (int i = 0; i < numBlocks; i++)
{
*m_LastBlock.GetChipherBlock () ^= in[i];
m_ECBEncryption.Encrypt (m_LastBlock.GetChipherBlock (), m_LastBlock.GetChipherBlock ());
out[i] = *m_LastBlock.GetChipherBlock ();
}
}
}
void CBCEncryption::Encrypt (const uint8_t * in, std::size_t len, uint8_t * out)
{
@ -735,6 +846,8 @@ namespace crypto @@ -735,6 +846,8 @@ namespace crypto
void CBCEncryption::Encrypt (const uint8_t * in, uint8_t * out)
{
if(i2p::cpu::aesni)
{
#ifdef AESNI
__asm__
(
@ -753,9 +866,14 @@ namespace crypto @@ -753,9 +866,14 @@ namespace crypto
Encrypt (1, (const ChipherBlock *)in, (ChipherBlock *)out);
#endif
}
else
Encrypt (1, (const ChipherBlock *)in, (ChipherBlock *)out);
}
void CBCDecryption::Decrypt (int numBlocks, const ChipherBlock * in, ChipherBlock * out)
{
if(i2p::cpu::aesni)
{
#ifdef AESNI
__asm__
(
@ -787,6 +905,17 @@ namespace crypto @@ -787,6 +905,17 @@ namespace crypto
}
#endif
}
else
{
for (int i = 0; i < numBlocks; i++)
{
ChipherBlock tmp = in[i];
m_ECBDecryption.Decrypt (in + i, out + i);
out[i] ^= *m_IV.GetChipherBlock ();
*m_IV.GetChipherBlock () = tmp;
}
}
}
void CBCDecryption::Decrypt (const uint8_t * in, std::size_t len, uint8_t * out)
{
@ -797,6 +926,8 @@ namespace crypto @@ -797,6 +926,8 @@ namespace crypto
void CBCDecryption::Decrypt (const uint8_t * in, uint8_t * out)
{
if(i2p::cpu::aesni)
{
#ifdef AESNI
__asm__
(
@ -815,9 +946,14 @@ namespace crypto @@ -815,9 +946,14 @@ namespace crypto
Decrypt (1, (const ChipherBlock *)in, (ChipherBlock *)out);
#endif
}
else
Decrypt (1, (const ChipherBlock *)in, (ChipherBlock *)out);
}
void TunnelEncryption::Encrypt (const uint8_t * in, uint8_t * out)
{
if(i2p::cpu::aesni)
{
#ifdef AESNI
__asm__
(
@ -840,7 +976,7 @@ namespace crypto @@ -840,7 +976,7 @@ namespace crypto
"dec %[num] \n"
"jnz 1b \n"
:
: [sched_iv]"r"(m_IVEncryption.GetKeySchedule ()), [sched_l]"r"(m_LayerEncryption.GetKeySchedule ()),
: [sched_iv]"r"(m_IVEncryption.GetKeySchedule ()), [sched_l]"r"(m_LayerEncryption.ECB().GetKeySchedule ()),
[in]"r"(in), [out]"r"(out), [num]"r"(63) // 63 blocks = 1008 bytes
: "%xmm0", "%xmm1", "cc", "memory"
);
@ -851,9 +987,19 @@ namespace crypto @@ -851,9 +987,19 @@ namespace crypto
m_IVEncryption.Encrypt ((ChipherBlock *)out, (ChipherBlock *)out); // double iv
#endif
}
else
{
m_IVEncryption.Encrypt ((const ChipherBlock *)in, (ChipherBlock *)out); // iv
m_LayerEncryption.SetIV (out);
m_LayerEncryption.Encrypt (in + 16, i2p::tunnel::TUNNEL_DATA_ENCRYPTED_SIZE, out + 16); // data
m_IVEncryption.Encrypt ((ChipherBlock *)out, (ChipherBlock *)out); // double iv
}
}
void TunnelDecryption::Decrypt (const uint8_t * in, uint8_t * out)
{
if(i2p::cpu::aesni)
{
#ifdef AESNI
__asm__
(
@ -877,7 +1023,7 @@ namespace crypto @@ -877,7 +1023,7 @@ namespace crypto
"dec %[num] \n"
"jnz 1b \n"
:
: [sched_iv]"r"(m_IVDecryption.GetKeySchedule ()), [sched_l]"r"(m_LayerDecryption.GetKeySchedule ()),
: [sched_iv]"r"(m_IVDecryption.GetKeySchedule ()), [sched_l]"r"(m_LayerDecryption.ECB().GetKeySchedule ()),
[in]"r"(in), [out]"r"(out), [num]"r"(63) // 63 blocks = 1008 bytes
: "%xmm0", "%xmm1", "%xmm2", "cc", "memory"
);
@ -888,6 +1034,14 @@ namespace crypto @@ -888,6 +1034,14 @@ namespace crypto
m_IVDecryption.Decrypt ((ChipherBlock *)out, (ChipherBlock *)out); // double iv
#endif
}
else
{
m_IVDecryption.Decrypt ((const ChipherBlock *)in, (ChipherBlock *)out); // iv
m_LayerDecryption.SetIV (out);
m_LayerDecryption.Decrypt (in + 16, i2p::tunnel::TUNNEL_DATA_ENCRYPTED_SIZE, out + 16); // data
m_IVDecryption.Decrypt ((ChipherBlock *)out, (ChipherBlock *)out); // double iv
}
}
/* std::vector <std::unique_ptr<std::mutex> > m_OpenSSLMutexes;
static void OpensslLockingCallback(int mode, int type, const char * file, int line)
@ -904,6 +1058,7 @@ namespace crypto @@ -904,6 +1058,7 @@ namespace crypto
void InitCrypto (bool precomputation)
{
i2p::cpu::Detect ();
SSL_library_init ();
/* auto numLocks = CRYPTO_num_locks();
for (int i = 0; i < numLocks; i++)

91
libi2pd/Crypto.h
Unescape View File

@ -16,6 +16,7 @@ @@ -16,6 +16,7 @@
#include "Base.h"
#include "Tag.h"
#include "CPU.h"
namespace i2p
{
@ -68,7 +69,9 @@ namespace crypto @@ -68,7 +69,9 @@ namespace crypto
void operator^=(const ChipherBlock& other) // XOR
{
#if defined(__AVX__) // AVX
if (i2p::cpu::avx)
{
#ifdef AVX
__asm__
(
"vmovups (%[buf]), %%xmm0 \n"
@ -79,23 +82,18 @@ namespace crypto @@ -79,23 +82,18 @@ namespace crypto
: [buf]"r"(buf), [other]"r"(other.buf)
: "%xmm0", "%xmm1", "memory"
);
#elif defined(__SSE__) // SSE
__asm__
(
"movups (%[buf]), %%xmm0 \n"
"movups (%[other]), %%xmm1 \n"
"pxor %%xmm1, %%xmm0 \n"
"movups %%xmm0, (%[buf]) \n"
:
: [buf]"r"(buf), [other]"r"(other.buf)
: "%xmm0", "%xmm1", "memory"
);
#else
// TODO: implement it better
for (int i = 0; i < 16; i++)
buf[i] ^= other.buf[i];
#endif
}
else
{
// TODO: implement it better
for (int i = 0; i < 16; i++)
buf[i] ^= other.buf[i];
}
}
};
typedef i2p::data::Tag<32> AESKey;
@ -140,67 +138,38 @@ namespace crypto @@ -140,67 +138,38 @@ namespace crypto
AESAlignedBuffer<240> m_KeySchedule; // 14 rounds for AES-256, 240 bytes
};
#endif
class ECBEncryptionAESNI: public ECBCryptoAESNI
{
public:
void SetKey (const AESKey& key) { ExpandKey (key); };
void Encrypt (const ChipherBlock * in, ChipherBlock * out);
};
class ECBDecryptionAESNI: public ECBCryptoAESNI
#ifdef AESNI
class ECBEncryption: public ECBCryptoAESNI
#else
class ECBEncryption
#endif
{
public:
void SetKey (const AESKey& key);
void Decrypt (const ChipherBlock * in, ChipherBlock * out);
};
typedef ECBEncryptionAESNI ECBEncryption;
typedef ECBDecryptionAESNI ECBDecryption;
#else // use openssl
class ECBEncryption
{
public:
void SetKey (const AESKey& key)
{
AES_set_encrypt_key (key, 256, &m_Key);
}
void Encrypt (const ChipherBlock * in, ChipherBlock * out)
{
AES_encrypt (in->buf, out->buf, &m_Key);
}
void Encrypt(const ChipherBlock * in, ChipherBlock * out);
private:
AES_KEY m_Key;
};
#ifdef AESNI
class ECBDecryption: public ECBCryptoAESNI
#else
class ECBDecryption
#endif
{
public:
void SetKey (const AESKey& key)
{
AES_set_decrypt_key (key, 256, &m_Key);
}
void Decrypt (const ChipherBlock * in, ChipherBlock * out)
{
AES_decrypt (in->buf, out->buf, &m_Key);
}
void SetKey (const AESKey& key);
void Decrypt (const ChipherBlock * in, ChipherBlock * out);
private:
AES_KEY m_Key;
};
#endif
class CBCEncryption
{
public:
@ -214,6 +183,8 @@ namespace crypto @@ -214,6 +183,8 @@ namespace crypto
void Encrypt (const uint8_t * in, std::size_t len, uint8_t * out);
void Encrypt (const uint8_t * in, uint8_t * out); // one block
ECBEncryption & ECB() { return m_ECBEncryption; }
private:
AESAlignedBuffer<16> m_LastBlock;
@ -234,6 +205,8 @@ namespace crypto @@ -234,6 +205,8 @@ namespace crypto
void Decrypt (const uint8_t * in, std::size_t len, uint8_t * out);
void Decrypt (const uint8_t * in, uint8_t * out); // one block
ECBDecryption & ECB() { return m_ECBDecryption; }
private:
AESAlignedBuffer<16> m_IV;
@ -255,11 +228,7 @@ namespace crypto @@ -255,11 +228,7 @@ namespace crypto
private:
ECBEncryption m_IVEncryption;
#ifdef AESNI
ECBEncryption m_LayerEncryption;
#else
CBCEncryption m_LayerEncryption;
#endif
};
class TunnelDecryption // with double IV encryption
@ -277,11 +246,7 @@ namespace crypto @@ -277,11 +246,7 @@ namespace crypto
private:
ECBDecryption m_IVDecryption;
#ifdef AESNI
ECBDecryption m_LayerDecryption;
#else
CBCDecryption m_LayerDecryption;
#endif
};
void InitCrypto (bool precomputation);

Write Preview
Loading…
Cancel
Save