PurpleI2P
/
i2pd
mirror of https://github.com/PurpleI2P/i2pd.git
1
0
Fork 0
Code Issues Releases Activity
Browse Source

Merge pull request #1115 from majestrate/aesni-avx-runtime-detect 

AESNI/AVX runtime detection [WIP]
pull/1119/head
orignal 7 years ago committed by GitHub
parent
dea6fbf285 486a4cfdd6
commit
f5349dcef9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 452 additions and 299 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 1
      build/CMakeLists.txt
  2. 34
      libi2pd/CPU.cpp
  3. 15
      libi2pd/CPU.h
  4. 577
      libi2pd/Crypto.cpp
  5. 123
      libi2pd/Crypto.h
  6. 1
      qt/i2pd_qt/i2pd_qt.pro

1
build/CMakeLists.txt
Unescape View File

@ -39,6 +39,7 @@ include_directories(${LIBI2PD_CLIENT_SRC_DIR}) @@ -39,6 +39,7 @@ include_directories(${LIBI2PD_CLIENT_SRC_DIR})
set (LIBI2PD_SRC
"${LIBI2PD_SRC_DIR}/BloomFilter.cpp"
"${LIBI2PD_SRC_DIR}/Config.cpp"
"${LIBI2PD_SRC_DIR}/CPU.cpp"
"${LIBI2PD_SRC_DIR}/Crypto.cpp"
"${LIBI2PD_SRC_DIR}/CryptoKey.cpp"
"${LIBI2PD_SRC_DIR}/Garlic.cpp"

34
libi2pd/CPU.cpp
Unescape View File

@ -0,0 +1,34 @@ @@ -0,0 +1,34 @@
#include "CPU.h"
#if defined(__x86_64__) || defined(__i386__)
#include <cpuid.h>
#endif
#include "Log.h"
namespace i2p
{
namespace cpu
{
bool aesni = false;
bool avx = false;
void Detect()
{
#if defined(__x86_64__) || defined(__i386__)
int info[4];
__cpuid(0, info[0], info[1], info[2], info[3]);
if (info[0] >= 0x00000001) {
__cpuid(0x00000001, info[0], info[1], info[2], info[3]);
aesni = info[2] & bit_AES; // AESNI
avx = info[2] & bit_AVX; // AVX
}
#endif
if(aesni)
{
LogPrint(eLogInfo, "AESNI enabled");
}
if(avx)
{
LogPrint(eLogInfo, "AVX enabled");
}
}
}
}

15
libi2pd/CPU.h
Unescape View File

@ -0,0 +1,15 @@ @@ -0,0 +1,15 @@
#ifndef LIBI2PD_CPU_H
#define LIBI2PD_CPU_H
namespace i2p
{
namespace cpu
{
extern bool aesni;
extern bool avx;
void Detect();
}
}
#endif

577
libi2pd/Crypto.cpp
Unescape View File

@ -479,10 +479,9 @@ namespace crypto @@ -479,10 +479,9 @@ namespace crypto
const uint64_t IPAD = 0x3636363636363636;
const uint64_t OPAD = 0x5C5C5C5C5C5C5C5C;
#if defined(__AVX__)
static const uint64_t ipads[] = { IPAD, IPAD, IPAD, IPAD };
static const uint64_t opads[] = { OPAD, OPAD, OPAD, OPAD };
#endif
void HMACMD5Digest (uint8_t * msg, size_t len, const MACKey& key, uint8_t * digest)
// key is 32 bytes
@ -491,47 +490,73 @@ namespace crypto @@ -491,47 +490,73 @@ namespace crypto
{
uint64_t buf[256];
uint64_t hash[12]; // 96 bytes
#if defined(__AVX__) // for AVX
__asm__
(
"vmovups %[key], %%ymm0 \n"
"vmovups %[ipad], %%ymm1 \n"
"vmovups %%ymm1, 32(%[buf]) \n"
"vxorps %%ymm0, %%ymm1, %%ymm1 \n"
"vmovups %%ymm1, (%[buf]) \n"
"vmovups %[opad], %%ymm1 \n"
"vmovups %%ymm1, 32(%[hash]) \n"
"vxorps %%ymm0, %%ymm1, %%ymm1 \n"
"vmovups %%ymm1, (%[hash]) \n"
"vzeroall \n" // end of AVX
"movups %%xmm0, 80(%[hash]) \n" // zero last 16 bytes
:
: [key]"m"(*(const uint8_t *)key), [ipad]"m"(*ipads), [opad]"m"(*opads),
[buf]"r"(buf), [hash]"r"(hash)
: "memory", "%xmm0" // TODO: change to %ymm0 later
);
if(i2p::cpu::avx)
{
#ifdef AVX
__asm__
(
"vmovups %[key], %%ymm0 \n"
"vmovups %[ipad], %%ymm1 \n"
"vmovups %%ymm1, 32(%[buf]) \n"
"vxorps %%ymm0, %%ymm1, %%ymm1 \n"
"vmovups %%ymm1, (%[buf]) \n"
"vmovups %[opad], %%ymm1 \n"
"vmovups %%ymm1, 32(%[hash]) \n"
"vxorps %%ymm0, %%ymm1, %%ymm1 \n"
"vmovups %%ymm1, (%[hash]) \n"
"vzeroall \n" // end of AVX
"movups %%xmm0, 80(%[hash]) \n" // zero last 16 bytes
:
: [key]"m"(*(const uint8_t *)key), [ipad]"m"(*ipads), [opad]"m"(*opads),
[buf]"r"(buf), [hash]"r"(hash)
: "memory", "%xmm0" // TODO: change to %ymm0 later
);
#else
// ikeypad
buf[0] = key.GetLL ()[0] ^ IPAD;
buf[1] = key.GetLL ()[1] ^ IPAD;
buf[2] = key.GetLL ()[2] ^ IPAD;
buf[3] = key.GetLL ()[3] ^ IPAD;
buf[4] = IPAD;
buf[5] = IPAD;
buf[6] = IPAD;
buf[7] = IPAD;
// okeypad
hash[0] = key.GetLL ()[0] ^ OPAD;
hash[1] = key.GetLL ()[1] ^ OPAD;
hash[2] = key.GetLL ()[2] ^ OPAD;
hash[3] = key.GetLL ()[3] ^ OPAD;
hash[4] = OPAD;
hash[5] = OPAD;
hash[6] = OPAD;
hash[7] = OPAD;
// fill last 16 bytes with zeros (first hash size assumed 32 bytes in I2P)
memset (hash + 10, 0, 16);
// ikeypad
buf[0] = key.GetLL ()[0] ^ IPAD;
buf[1] = key.GetLL ()[1] ^ IPAD;
buf[2] = key.GetLL ()[2] ^ IPAD;
buf[3] = key.GetLL ()[3] ^ IPAD;
buf[4] = IPAD;
buf[5] = IPAD;
buf[6] = IPAD;
buf[7] = IPAD;
// okeypad
hash[0] = key.GetLL ()[0] ^ OPAD;
hash[1] = key.GetLL ()[1] ^ OPAD;
hash[2] = key.GetLL ()[2] ^ OPAD;
hash[3] = key.GetLL ()[3] ^ OPAD;
hash[4] = OPAD;
hash[5] = OPAD;
hash[6] = OPAD;
hash[7] = OPAD;
// fill last 16 bytes with zeros (first hash size assumed 32 bytes in I2P)
memset (hash + 10, 0, 16);
#endif
}
else
{
// ikeypad
buf[0] = key.GetLL ()[0] ^ IPAD;
buf[1] = key.GetLL ()[1] ^ IPAD;
buf[2] = key.GetLL ()[2] ^ IPAD;
buf[3] = key.GetLL ()[3] ^ IPAD;
buf[4] = IPAD;
buf[5] = IPAD;
buf[6] = IPAD;
buf[7] = IPAD;
// okeypad
hash[0] = key.GetLL ()[0] ^ OPAD;
hash[1] = key.GetLL ()[1] ^ OPAD;
hash[2] = key.GetLL ()[2] ^ OPAD;
hash[3] = key.GetLL ()[3] ^ OPAD;
hash[4] = OPAD;
hash[5] = OPAD;
hash[6] = OPAD;
hash[7] = OPAD;
// fill last 16 bytes with zeros (first hash size assumed 32 bytes in I2P)
memset (hash + 10, 0, 16);
}
// concatenate with msg
memcpy (buf + 8, msg, len);
@ -543,8 +568,7 @@ namespace crypto @@ -543,8 +568,7 @@ namespace crypto
}
// AES
#ifdef AESNI
#ifdef AESNI
#define KeyExpansion256(round0,round1) \
"pshufd $0xff, %%xmm2, %%xmm2 \n" \
"movaps %%xmm1, %%xmm4 \n" \
@ -567,7 +591,9 @@ namespace crypto @@ -567,7 +591,9 @@ namespace crypto
"pxor %%xmm4, %%xmm3 \n" \
"pxor %%xmm2, %%xmm3 \n" \
"movaps %%xmm3, "#round1"(%[sched]) \n"
#endif
#ifdef AESNI
void ECBCryptoAESNI::ExpandKey (const AESKey& key)
{
__asm__
@ -604,8 +630,11 @@ namespace crypto @@ -604,8 +630,11 @@ namespace crypto
: [key]"r"((const uint8_t *)key), [sched]"r"(GetKeySchedule ()) // input
: "%xmm1", "%xmm2", "%xmm3", "%xmm4", "memory" // clogged
);
}
}
#endif
#if AESNI
#define EncryptAES256(sched) \
"pxor (%["#sched"]), %%xmm0 \n" \
"aesenc 16(%["#sched"]), %%xmm0 \n" \
@ -622,18 +651,31 @@ namespace crypto @@ -622,18 +651,31 @@ namespace crypto
"aesenc 192(%["#sched"]), %%xmm0 \n" \
"aesenc 208(%["#sched"]), %%xmm0 \n" \
"aesenclast 224(%["#sched"]), %%xmm0 \n"
void ECBEncryptionAESNI::Encrypt (const ChipherBlock * in, ChipherBlock * out)
#endif
void ECBEncryption::Encrypt (const ChipherBlock * in, ChipherBlock * out)
{
__asm__
(
"movups (%[in]), %%xmm0 \n"
EncryptAES256(sched)
"movups %%xmm0, (%[out]) \n"
: : [sched]"r"(GetKeySchedule ()), [in]"r"(in), [out]"r"(out) : "%xmm0", "memory"
);
if(i2p::cpu::aesni)
{
#ifdef AESNI
__asm__
(
"movups (%[in]), %%xmm0 \n"
EncryptAES256(sched)
"movups %%xmm0, (%[out]) \n"
: : [sched]"r"(GetKeySchedule ()), [in]"r"(in), [out]"r"(out) : "%xmm0", "memory"
);
#else
AES_encrypt (in->buf, out->buf, &m_Key);
#endif
}
else
{
AES_encrypt (in->buf, out->buf, &m_Key);
}
}
#ifdef AESNI
#define DecryptAES256(sched) \
"pxor 224(%["#sched"]), %%xmm0 \n" \
"aesdec 208(%["#sched"]), %%xmm0 \n" \
@ -650,79 +692,130 @@ namespace crypto @@ -650,79 +692,130 @@ namespace crypto
"aesdec 32(%["#sched"]), %%xmm0 \n" \
"aesdec 16(%["#sched"]), %%xmm0 \n" \
"aesdeclast (%["#sched"]), %%xmm0 \n"
void ECBDecryptionAESNI::Decrypt (const ChipherBlock * in, ChipherBlock * out)
#endif
void ECBDecryption::Decrypt (const ChipherBlock * in, ChipherBlock * out)
{
__asm__
(
"movups (%[in]), %%xmm0 \n"
DecryptAES256(sched)
"movups %%xmm0, (%[out]) \n"
: : [sched]"r"(GetKeySchedule ()), [in]"r"(in), [out]"r"(out) : "%xmm0", "memory"
);
if(i2p::cpu::aesni)
{
#ifdef AESNI
__asm__
(
"movups (%[in]), %%xmm0 \n"
DecryptAES256(sched)
"movups %%xmm0, (%[out]) \n"
: : [sched]"r"(GetKeySchedule ()), [in]"r"(in), [out]"r"(out) : "%xmm0", "memory"
);
#else
AES_decrypt (in->buf, out->buf, &m_Key);
#endif
}
else
{
AES_decrypt (in->buf, out->buf, &m_Key);
}
}
#ifdef AESNI
#define CallAESIMC(offset) \
"movaps "#offset"(%[shed]), %%xmm0 \n" \
"aesimc %%xmm0, %%xmm0 \n" \
"movaps %%xmm0, "#offset"(%[shed]) \n"
#endif
void ECBDecryptionAESNI::SetKey (const AESKey& key)
void ECBEncryption::SetKey (const AESKey& key)
{
ExpandKey (key); // expand encryption key first
// then invert it using aesimc
__asm__
(
CallAESIMC(16)
CallAESIMC(32)
CallAESIMC(48)
CallAESIMC(64)
CallAESIMC(80)
CallAESIMC(96)
CallAESIMC(112)
CallAESIMC(128)
CallAESIMC(144)
CallAESIMC(160)
CallAESIMC(176)
CallAESIMC(192)
CallAESIMC(208)
: : [shed]"r"(GetKeySchedule ()) : "%xmm0", "memory"
);
if(i2p::cpu::aesni)
{
#ifdef AESNI
ExpandKey (key);
#else
AES_set_encrypt_key (key, 256, &m_Key);
#endif
}
else
{
AES_set_encrypt_key (key, 256, &m_Key);
}
}
void ECBDecryption::SetKey (const AESKey& key)
{
if(i2p::cpu::aesni)
{
#ifdef AESNI
ExpandKey (key); // expand encryption key first
// then invert it using aesimc
__asm__
(
CallAESIMC(16)
CallAESIMC(32)
CallAESIMC(48)
CallAESIMC(64)
CallAESIMC(80)
CallAESIMC(96)
CallAESIMC(112)
CallAESIMC(128)
CallAESIMC(144)
CallAESIMC(160)
CallAESIMC(176)
CallAESIMC(192)
CallAESIMC(208)
: : [shed]"r"(GetKeySchedule ()) : "%xmm0", "memory"
);
#else
AES_set_decrypt_key (key, 256, &m_Key);
#endif
}
else
{
AES_set_decrypt_key (key, 256, &m_Key);
}
}
void CBCEncryption::Encrypt (int numBlocks, const ChipherBlock * in, ChipherBlock * out)
{
if(i2p::cpu::aesni)
{
#ifdef AESNI
__asm__
(
"movups (%[iv]), %%xmm1 \n"
"1: \n"
"movups (%[in]), %%xmm0 \n"
"pxor %%xmm1, %%xmm0 \n"
EncryptAES256(sched)
"movaps %%xmm0, %%xmm1 \n"
"movups %%xmm0, (%[out]) \n"
"add $16, %[in] \n"
"add $16, %[out] \n"
"dec %[num] \n"
"jnz 1b \n"
"movups %%xmm1, (%[iv]) \n"
:
: [iv]"r"((uint8_t *)m_LastBlock), [sched]"r"(m_ECBEncryption.GetKeySchedule ()),
[in]"r"(in), [out]"r"(out), [num]"r"(numBlocks)
: "%xmm0", "%xmm1", "cc", "memory"
);
__asm__
(
"movups (%[iv]), %%xmm1 \n"
"1: \n"
"movups (%[in]), %%xmm0 \n"
"pxor %%xmm1, %%xmm0 \n"
EncryptAES256(sched)
"movaps %%xmm0, %%xmm1 \n"
"movups %%xmm0, (%[out]) \n"
"add $16, %[in] \n"
"add $16, %[out] \n"
"dec %[num] \n"
"jnz 1b \n"
"movups %%xmm1, (%[iv]) \n"
:
: [iv]"r"((uint8_t *)m_LastBlock), [sched]"r"(m_ECBEncryption.GetKeySchedule ()),
[in]"r"(in), [out]"r"(out), [num]"r"(numBlocks)
: "%xmm0", "%xmm1", "cc", "memory"
);
#else
for (int i = 0; i < numBlocks; i++)
for (int i = 0; i < numBlocks; i++)
{
*m_LastBlock.GetChipherBlock () ^= in[i];
m_ECBEncryption.Encrypt (m_LastBlock.GetChipherBlock (), m_LastBlock.GetChipherBlock ());
out[i] = *m_LastBlock.GetChipherBlock ();
}
#endif
}
else
{
*m_LastBlock.GetChipherBlock () ^= in[i];
m_ECBEncryption.Encrypt (m_LastBlock.GetChipherBlock (), m_LastBlock.GetChipherBlock ());
out[i] = *m_LastBlock.GetChipherBlock ();
for (int i = 0; i < numBlocks; i++)
{
*m_LastBlock.GetChipherBlock () ^= in[i];
m_ECBEncryption.Encrypt (m_LastBlock.GetChipherBlock (), m_LastBlock.GetChipherBlock ());
out[i] = *m_LastBlock.GetChipherBlock ();
}
}
#endif
}
void CBCEncryption::Encrypt (const uint8_t * in, std::size_t len, uint8_t * out)
@ -735,57 +828,75 @@ namespace crypto @@ -735,57 +828,75 @@ namespace crypto
void CBCEncryption::Encrypt (const uint8_t * in, uint8_t * out)
{
if(i2p::cpu::aesni)
{
#ifdef AESNI
__asm__
(
"movups (%[iv]), %%xmm1 \n"
"movups (%[in]), %%xmm0 \n"
"pxor %%xmm1, %%xmm0 \n"
EncryptAES256(sched)
"movups %%xmm0, (%[out]) \n"
"movups %%xmm0, (%[iv]) \n"
:
: [iv]"r"((uint8_t *)m_LastBlock), [sched]"r"(m_ECBEncryption.GetKeySchedule ()),
[in]"r"(in), [out]"r"(out)
: "%xmm0", "%xmm1", "memory"
);
__asm__
(
"movups (%[iv]), %%xmm1 \n"
"movups (%[in]), %%xmm0 \n"
"pxor %%xmm1, %%xmm0 \n"
EncryptAES256(sched)
"movups %%xmm0, (%[out]) \n"
"movups %%xmm0, (%[iv]) \n"
:
: [iv]"r"((uint8_t *)m_LastBlock), [sched]"r"(m_ECBEncryption.GetKeySchedule ()),
[in]"r"(in), [out]"r"(out)
: "%xmm0", "%xmm1", "memory"
);
#else
Encrypt (1, (const ChipherBlock *)in, (ChipherBlock *)out);
Encrypt (1, (const ChipherBlock *)in, (ChipherBlock *)out);
#endif
}
else
Encrypt (1, (const ChipherBlock *)in, (ChipherBlock *)out);
}
void CBCDecryption::Decrypt (int numBlocks, const ChipherBlock * in, ChipherBlock * out)
{
if(i2p::cpu::aesni)
{
#ifdef AESNI
__asm__
(
"movups (%[iv]), %%xmm1 \n"
"1: \n"
"movups (%[in]), %%xmm0 \n"
"movaps %%xmm0, %%xmm2 \n"
DecryptAES256(sched)
"pxor %%xmm1, %%xmm0 \n"
"movups %%xmm0, (%[out]) \n"
"movaps %%xmm2, %%xmm1 \n"
"add $16, %[in] \n"
"add $16, %[out] \n"
"dec %[num] \n"
"jnz 1b \n"
"movups %%xmm1, (%[iv]) \n"
:
: [iv]"r"((uint8_t *)m_IV), [sched]"r"(m_ECBDecryption.GetKeySchedule ()),
[in]"r"(in), [out]"r"(out), [num]"r"(numBlocks)
: "%xmm0", "%xmm1", "%xmm2", "cc", "memory"
);
__asm__
(
"movups (%[iv]), %%xmm1 \n"
"1: \n"
"movups (%[in]), %%xmm0 \n"
"movaps %%xmm0, %%xmm2 \n"
DecryptAES256(sched)
"pxor %%xmm1, %%xmm0 \n"
"movups %%xmm0, (%[out]) \n"
"movaps %%xmm2, %%xmm1 \n"
"add $16, %[in] \n"
"add $16, %[out] \n"
"dec %[num] \n"
"jnz 1b \n"
"movups %%xmm1, (%[iv]) \n"
:
: [iv]"r"((uint8_t *)m_IV), [sched]"r"(m_ECBDecryption.GetKeySchedule ()),
[in]"r"(in), [out]"r"(out), [num]"r"(numBlocks)
: "%xmm0", "%xmm1", "%xmm2", "cc", "memory"
);
#else
for (int i = 0; i < numBlocks; i++)
for (int i = 0; i < numBlocks; i++)
{
ChipherBlock tmp = in[i];
m_ECBDecryption.Decrypt (in + i, out + i);
out[i] ^= *m_IV.GetChipherBlock ();
*m_IV.GetChipherBlock () = tmp;
}
#endif
}
else
{
ChipherBlock tmp = in[i];
m_ECBDecryption.Decrypt (in + i, out + i);
out[i] ^= *m_IV.GetChipherBlock ();
*m_IV.GetChipherBlock () = tmp;
for (int i = 0; i < numBlocks; i++)
{
ChipherBlock tmp = in[i];
m_ECBDecryption.Decrypt (in + i, out + i);
out[i] ^= *m_IV.GetChipherBlock ();
*m_IV.GetChipherBlock () = tmp;
}
}
#endif
}
void CBCDecryption::Decrypt (const uint8_t * in, std::size_t len, uint8_t * out)
@ -797,96 +908,121 @@ namespace crypto @@ -797,96 +908,121 @@ namespace crypto
void CBCDecryption::Decrypt (const uint8_t * in, uint8_t * out)
{
if(i2p::cpu::aesni)
{
#ifdef AESNI
__asm__
(
"movups (%[iv]), %%xmm1 \n"
"movups (%[in]), %%xmm0 \n"
"movups %%xmm0, (%[iv]) \n"
DecryptAES256(sched)
"pxor %%xmm1, %%xmm0 \n"
"movups %%xmm0, (%[out]) \n"
:
: [iv]"r"((uint8_t *)m_IV), [sched]"r"(m_ECBDecryption.GetKeySchedule ()),
[in]"r"(in), [out]"r"(out)
: "%xmm0", "%xmm1", "memory"
);
__asm__
(
"movups (%[iv]), %%xmm1 \n"
"movups (%[in]), %%xmm0 \n"
"movups %%xmm0, (%[iv]) \n"
DecryptAES256(sched)
"pxor %%xmm1, %%xmm0 \n"
"movups %%xmm0, (%[out]) \n"
:
: [iv]"r"((uint8_t *)m_IV), [sched]"r"(m_ECBDecryption.GetKeySchedule ()),
[in]"r"(in), [out]"r"(out)
: "%xmm0", "%xmm1", "memory"
);
#else
Decrypt (1, (const ChipherBlock *)in, (ChipherBlock *)out);
Decrypt (1, (const ChipherBlock *)in, (ChipherBlock *)out);
#endif
}
else
Decrypt (1, (const ChipherBlock *)in, (ChipherBlock *)out);
}
void TunnelEncryption::Encrypt (const uint8_t * in, uint8_t * out)
{
if(i2p::cpu::aesni)
{
#ifdef AESNI
__asm__
(
// encrypt IV
"movups (%[in]), %%xmm0 \n"
EncryptAES256(sched_iv)
"movaps %%xmm0, %%xmm1 \n"
// double IV encryption
EncryptAES256(sched_iv)
"movups %%xmm0, (%[out]) \n"
// encrypt data, IV is xmm1
"1: \n"
"add $16, %[in] \n"
"add $16, %[out] \n"
"movups (%[in]), %%xmm0 \n"
"pxor %%xmm1, %%xmm0 \n"
EncryptAES256(sched_l)
"movaps %%xmm0, %%xmm1 \n"
"movups %%xmm0, (%[out]) \n"
"dec %[num] \n"
"jnz 1b \n"
:
: [sched_iv]"r"(m_IVEncryption.GetKeySchedule ()), [sched_l]"r"(m_LayerEncryption.GetKeySchedule ()),
[in]"r"(in), [out]"r"(out), [num]"r"(63) // 63 blocks = 1008 bytes
: "%xmm0", "%xmm1", "cc", "memory"
);
__asm__
(
// encrypt IV
"movups (%[in]), %%xmm0 \n"
EncryptAES256(sched_iv)
"movaps %%xmm0, %%xmm1 \n"
// double IV encryption
EncryptAES256(sched_iv)
"movups %%xmm0, (%[out]) \n"
// encrypt data, IV is xmm1
"1: \n"
"add $16, %[in] \n"
"add $16, %[out] \n"
"movups (%[in]), %%xmm0 \n"
"pxor %%xmm1, %%xmm0 \n"
EncryptAES256(sched_l)
"movaps %%xmm0, %%xmm1 \n"
"movups %%xmm0, (%[out]) \n"
"dec %[num] \n"
"jnz 1b \n"
:
: [sched_iv]"r"(m_IVEncryption.GetKeySchedule ()), [sched_l]"r"(m_LayerEncryption.ECB().GetKeySchedule ()),
[in]"r"(in), [out]"r"(out), [num]"r"(63) // 63 blocks = 1008 bytes
: "%xmm0", "%xmm1", "cc", "memory"
);
#else
m_IVEncryption.Encrypt ((const ChipherBlock *)in, (ChipherBlock *)out); // iv
m_LayerEncryption.SetIV (out);
m_LayerEncryption.Encrypt (in + 16, i2p::tunnel::TUNNEL_DATA_ENCRYPTED_SIZE, out + 16); // data
m_IVEncryption.Encrypt ((ChipherBlock *)out, (ChipherBlock *)out); // double iv
m_IVEncryption.Encrypt ((const ChipherBlock *)in, (ChipherBlock *)out); // iv
m_LayerEncryption.SetIV (out);
m_LayerEncryption.Encrypt (in + 16, i2p::tunnel::TUNNEL_DATA_ENCRYPTED_SIZE, out + 16); // data
m_IVEncryption.Encrypt ((ChipherBlock *)out, (ChipherBlock *)out); // double iv
#endif
}
else
{
m_IVEncryption.Encrypt ((const ChipherBlock *)in, (ChipherBlock *)out); // iv
m_LayerEncryption.SetIV (out);
m_LayerEncryption.Encrypt (in + 16, i2p::tunnel::TUNNEL_DATA_ENCRYPTED_SIZE, out + 16); // data
m_IVEncryption.Encrypt ((ChipherBlock *)out, (ChipherBlock *)out); // double iv
}
}
void TunnelDecryption::Decrypt (const uint8_t * in, uint8_t * out)
{
if(i2p::cpu::aesni)
{
#ifdef AESNI
__asm__
(
// decrypt IV
"movups (%[in]), %%xmm0 \n"
DecryptAES256(sched_iv)
"movaps %%xmm0, %%xmm1 \n"
// double IV encryption
DecryptAES256(sched_iv)
"movups %%xmm0, (%[out]) \n"
// decrypt data, IV is xmm1
"1: \n"
"add $16, %[in] \n"
"add $16, %[out] \n"
"movups (%[in]), %%xmm0 \n"
"movaps %%xmm0, %%xmm2 \n"
DecryptAES256(sched_l)
"pxor %%xmm1, %%xmm0 \n"
"movups %%xmm0, (%[out]) \n"
"movaps %%xmm2, %%xmm1 \n"
"dec %[num] \n"
"jnz 1b \n"
:
: [sched_iv]"r"(m_IVDecryption.GetKeySchedule ()), [sched_l]"r"(m_LayerDecryption.GetKeySchedule ()),
[in]"r"(in), [out]"r"(out), [num]"r"(63) // 63 blocks = 1008 bytes
: "%xmm0", "%xmm1", "%xmm2", "cc", "memory"
);
__asm__
(
// decrypt IV
"movups (%[in]), %%xmm0 \n"
DecryptAES256(sched_iv)
"movaps %%xmm0, %%xmm1 \n"
// double IV encryption
DecryptAES256(sched_iv)
"movups %%xmm0, (%[out]) \n"
// decrypt data, IV is xmm1
"1: \n"
"add $16, %[in] \n"
"add $16, %[out] \n"
"movups (%[in]), %%xmm0 \n"
"movaps %%xmm0, %%xmm2 \n"
DecryptAES256(sched_l)
"pxor %%xmm1, %%xmm0 \n"
"movups %%xmm0, (%[out]) \n"
"movaps %%xmm2, %%xmm1 \n"
"dec %[num] \n"
"jnz 1b \n"
:
: [sched_iv]"r"(m_IVDecryption.GetKeySchedule ()), [sched_l]"r"(m_LayerDecryption.ECB().GetKeySchedule ()),
[in]"r"(in), [out]"r"(out), [num]"r"(63) // 63 blocks = 1008 bytes
: "%xmm0", "%xmm1", "%xmm2", "cc", "memory"
);
#else
m_IVDecryption.Decrypt ((const ChipherBlock *)in, (ChipherBlock *)out); // iv
m_LayerDecryption.SetIV (out);
m_LayerDecryption.Decrypt (in + 16, i2p::tunnel::TUNNEL_DATA_ENCRYPTED_SIZE, out + 16); // data
m_IVDecryption.Decrypt ((ChipherBlock *)out, (ChipherBlock *)out); // double iv
m_IVDecryption.Decrypt ((const ChipherBlock *)in, (ChipherBlock *)out); // iv
m_LayerDecryption.SetIV (out);
m_LayerDecryption.Decrypt (in + 16, i2p::tunnel::TUNNEL_DATA_ENCRYPTED_SIZE, out + 16); // data
m_IVDecryption.Decrypt ((ChipherBlock *)out, (ChipherBlock *)out); // double iv
#endif
}
else
{
m_IVDecryption.Decrypt ((const ChipherBlock *)in, (ChipherBlock *)out); // iv
m_LayerDecryption.SetIV (out);
m_LayerDecryption.Decrypt (in + 16, i2p::tunnel::TUNNEL_DATA_ENCRYPTED_SIZE, out + 16); // data
m_IVDecryption.Decrypt ((ChipherBlock *)out, (ChipherBlock *)out); // double iv
}
}
/* std::vector <std::unique_ptr<std::mutex> > m_OpenSSLMutexes;
@ -904,6 +1040,7 @@ namespace crypto @@ -904,6 +1040,7 @@ namespace crypto
void InitCrypto (bool precomputation)
{
i2p::cpu::Detect ();
SSL_library_init ();
/* auto numLocks = CRYPTO_num_locks();
for (int i = 0; i < numLocks; i++)

123
libi2pd/Crypto.h
Unescape View File

@ -16,6 +16,7 @@ @@ -16,6 +16,7 @@
#include "Base.h"
#include "Tag.h"
#include "CPU.h"
namespace i2p
{
@ -68,33 +69,30 @@ namespace crypto @@ -68,33 +69,30 @@ namespace crypto
void operator^=(const ChipherBlock& other) // XOR
{
#if defined(__AVX__) // AVX
__asm__
(
"vmovups (%[buf]), %%xmm0 \n"
"vmovups (%[other]), %%xmm1 \n"
"vxorps %%xmm0, %%xmm1, %%xmm0 \n"
"vmovups %%xmm0, (%[buf]) \n"
:
: [buf]"r"(buf), [other]"r"(other.buf)
: "%xmm0", "%xmm1", "memory"
);
#elif defined(__SSE__) // SSE
__asm__
(
"movups (%[buf]), %%xmm0 \n"
"movups (%[other]), %%xmm1 \n"
"pxor %%xmm1, %%xmm0 \n"
"movups %%xmm0, (%[buf]) \n"
:
: [buf]"r"(buf), [other]"r"(other.buf)
: "%xmm0", "%xmm1", "memory"
);
if (i2p::cpu::avx)
{
#ifdef AVX
__asm__
(
"vmovups (%[buf]), %%xmm0 \n"
"vmovups (%[other]), %%xmm1 \n"
"vxorps %%xmm0, %%xmm1, %%xmm0 \n"
"vmovups %%xmm0, (%[buf]) \n"
:
: [buf]"r"(buf), [other]"r"(other.buf)
: "%xmm0", "%xmm1", "memory"
);
#else
// TODO: implement it better
for (int i = 0; i < 16; i++)
buf[i] ^= other.buf[i];
for (int i = 0; i < 16; i++)
buf[i] ^= other.buf[i];
#endif
}
else
{
// TODO: implement it better
for (int i = 0; i < 16; i++)
buf[i] ^= other.buf[i];
}
}
};
@ -138,69 +136,40 @@ namespace crypto @@ -138,69 +136,40 @@ namespace crypto
private:
AESAlignedBuffer<240> m_KeySchedule; // 14 rounds for AES-256, 240 bytes
AESAlignedBuffer<240> m_KeySchedule; // 14 rounds for AES-256, 240 bytes
};
#endif
class ECBEncryptionAESNI: public ECBCryptoAESNI
{
public:
void SetKey (const AESKey& key) { ExpandKey (key); };
void Encrypt (const ChipherBlock * in, ChipherBlock * out);
};
class ECBDecryptionAESNI: public ECBCryptoAESNI
{
public:
void SetKey (const AESKey& key);
void Decrypt (const ChipherBlock * in, ChipherBlock * out);
};
typedef ECBEncryptionAESNI ECBEncryption;
typedef ECBDecryptionAESNI ECBDecryption;
#else // use openssl
#ifdef AESNI
class ECBEncryption: public ECBCryptoAESNI
#else
class ECBEncryption
#endif
{
public:
void SetKey (const AESKey& key)
{
AES_set_encrypt_key (key, 256, &m_Key);
}
void Encrypt (const ChipherBlock * in, ChipherBlock * out)
{
AES_encrypt (in->buf, out->buf, &m_Key);
}
private:
void SetKey (const AESKey& key);
void Encrypt(const ChipherBlock * in, ChipherBlock * out);
AES_KEY m_Key;
private:
AES_KEY m_Key;
};
#ifdef AESNI
class ECBDecryption: public ECBCryptoAESNI
#else
class ECBDecryption
#endif
{
public:
void SetKey (const AESKey& key)
{
AES_set_decrypt_key (key, 256, &m_Key);
}
void Decrypt (const ChipherBlock * in, ChipherBlock * out)
{
AES_decrypt (in->buf, out->buf, &m_Key);
}
void SetKey (const AESKey& key);
void Decrypt (const ChipherBlock * in, ChipherBlock * out);
private:
AES_KEY m_Key;
};
#endif
class CBCEncryption
{
public:
@ -214,6 +183,8 @@ namespace crypto @@ -214,6 +183,8 @@ namespace crypto
void Encrypt (const uint8_t * in, std::size_t len, uint8_t * out);
void Encrypt (const uint8_t * in, uint8_t * out); // one block
ECBEncryption & ECB() { return m_ECBEncryption; }
private:
AESAlignedBuffer<16> m_LastBlock;
@ -234,6 +205,8 @@ namespace crypto @@ -234,6 +205,8 @@ namespace crypto
void Decrypt (const uint8_t * in, std::size_t len, uint8_t * out);
void Decrypt (const uint8_t * in, uint8_t * out); // one block
ECBDecryption & ECB() { return m_ECBDecryption; }
private:
AESAlignedBuffer<16> m_IV;
@ -255,11 +228,7 @@ namespace crypto @@ -255,11 +228,7 @@ namespace crypto
private:
ECBEncryption m_IVEncryption;
#ifdef AESNI
ECBEncryption m_LayerEncryption;
#else
CBCEncryption m_LayerEncryption;
#endif
};
class TunnelDecryption // with double IV encryption
@ -277,11 +246,7 @@ namespace crypto @@ -277,11 +246,7 @@ namespace crypto
private:
ECBDecryption m_IVDecryption;
#ifdef AESNI
ECBDecryption m_LayerDecryption;
#else
CBCDecryption m_LayerDecryption;
#endif
};
void InitCrypto (bool precomputation);

1
qt/i2pd_qt/i2pd_qt.pro
Unescape View File

@ -29,6 +29,7 @@ SOURCES += DaemonQT.cpp mainwindow.cpp \ @@ -29,6 +29,7 @@ SOURCES += DaemonQT.cpp mainwindow.cpp \
../../libi2pd/Base.cpp \
../../libi2pd/BloomFilter.cpp \
../../libi2pd/Config.cpp \
../../libi2pd/CPU.cpp \
../../libi2pd/Crypto.cpp \
../../libi2pd/CryptoKey.cpp \
../../libi2pd/Datagram.cpp \

Write Preview
Loading…
Cancel
Save