Merge pull request #1115 from majestrate/aesni-avx-runtime-detect

AESNI/AVX runtime detection [WIP]

build/CMakeLists.txt

@@ -39,6 +39,7 @@ include_directories(${LIBI2PD_CLIENT_SRC_DIR})
 set (LIBI2PD_SRC
   "${LIBI2PD_SRC_DIR}/BloomFilter.cpp"
   "${LIBI2PD_SRC_DIR}/Config.cpp"
+  "${LIBI2PD_SRC_DIR}/CPU.cpp"  
   "${LIBI2PD_SRC_DIR}/Crypto.cpp"
   "${LIBI2PD_SRC_DIR}/CryptoKey.cpp"
   "${LIBI2PD_SRC_DIR}/Garlic.cpp"

libi2pd/CPU.cpp

@@ -0,0 +1,34 @@
+#include "CPU.h"
+#if defined(__x86_64__) || defined(__i386__)
+#include <cpuid.h>
+#endif
+#include "Log.h"
+namespace i2p
+{
+namespace cpu
+{
+	bool aesni = false;
+	bool avx = false;
+
+	void Detect()
+	{
+#if defined(__x86_64__) || defined(__i386__)
+		int info[4];
+		__cpuid(0, info[0], info[1], info[2], info[3]);
+		if (info[0] >= 0x00000001) {
+			__cpuid(0x00000001, info[0], info[1], info[2], info[3]);
+			aesni = info[2] & bit_AES;  // AESNI
+			avx = info[2] & bit_AVX;  // AVX
+		}
+#endif
+		if(aesni)
+		{
+			LogPrint(eLogInfo, "AESNI enabled");
+		}
+		if(avx)
+		{
+			LogPrint(eLogInfo, "AVX enabled");
+		}
+	}
+}
+}

libi2pd/CPU.h

@@ -0,0 +1,15 @@
+#ifndef LIBI2PD_CPU_H
+#define LIBI2PD_CPU_H
+
+namespace i2p
+{
+namespace cpu
+{
+  extern bool aesni;
+  extern bool avx;
+
+  void Detect();
+}
+}
+
+#endif

libi2pd/Crypto.cpp

@@ -479,10 +479,9 @@ namespace crypto
 	const uint64_t IPAD = 0x3636363636363636;
 	const uint64_t OPAD = 0x5C5C5C5C5C5C5C5C;
 
-#if defined(__AVX__)
+
 	static const uint64_t ipads[] = { IPAD, IPAD, IPAD, IPAD };
 	static const uint64_t opads[] = { OPAD, OPAD, OPAD, OPAD };
-#endif
 
 	void HMACMD5Digest (uint8_t * msg, size_t len, const MACKey& key, uint8_t * digest)
 	// key is 32 bytes
@@ -491,7 +490,9 @@ namespace crypto
 	{
 		uint64_t buf[256];
 		uint64_t hash[12]; // 96 bytes
-#if defined(__AVX__) // for AVX
+		if(i2p::cpu::avx)
+		{
+#ifdef AVX
 			__asm__
 				(
 					"vmovups %[key], %%ymm0 \n"
@@ -532,6 +533,30 @@ namespace crypto
 			// fill last 16 bytes with zeros (first hash size assumed 32 bytes in I2P)
 			memset (hash + 10, 0, 16);
 #endif
+		}
+		else
+		{
+			// ikeypad
+			buf[0] = key.GetLL ()[0] ^ IPAD;
+			buf[1] = key.GetLL ()[1] ^ IPAD;
+			buf[2] = key.GetLL ()[2] ^ IPAD;
+			buf[3] = key.GetLL ()[3] ^ IPAD;
+			buf[4] = IPAD;
+			buf[5] = IPAD;
+			buf[6] = IPAD;
+			buf[7] = IPAD;
+			// okeypad
+			hash[0] = key.GetLL ()[0] ^ OPAD;
+			hash[1] = key.GetLL ()[1] ^ OPAD;
+			hash[2] = key.GetLL ()[2] ^ OPAD;
+			hash[3] = key.GetLL ()[3] ^ OPAD;
+			hash[4] = OPAD;
+			hash[5] = OPAD;
+			hash[6] = OPAD;
+			hash[7] = OPAD;
+			// fill last 16 bytes with zeros (first hash size assumed 32 bytes in I2P)
+			memset (hash + 10, 0, 16);
+		}
 
 		// concatenate with msg
 		memcpy (buf + 8, msg, len);
@@ -544,7 +569,6 @@ namespace crypto
 
 // AES
 #ifdef AESNI
-
 	#define KeyExpansion256(round0,round1) \
 		"pshufd	$0xff, %%xmm2, %%xmm2 \n" \
 		"movaps	%%xmm1, %%xmm4 \n" \
@@ -567,7 +591,9 @@ namespace crypto
 		"pxor %%xmm4, %%xmm3 \n" \
 		"pxor %%xmm2, %%xmm3 \n" \
 		"movaps	%%xmm3, "#round1"(%[sched]) \n"
+#endif
 
+#ifdef AESNI
 	void ECBCryptoAESNI::ExpandKey (const AESKey& key)
 	{
 		__asm__
@@ -605,7 +631,10 @@ namespace crypto
 			: "%xmm1", "%xmm2", "%xmm3", "%xmm4", "memory" // clogged
 		);
 	}	
+#endif
 
+
+#if AESNI
 	#define EncryptAES256(sched) \
 		"pxor (%["#sched"]), %%xmm0 \n" \
 		"aesenc	16(%["#sched"]), %%xmm0 \n" \
@@ -622,9 +651,13 @@ namespace crypto
 		"aesenc	192(%["#sched"]), %%xmm0 \n" \
 		"aesenc	208(%["#sched"]), %%xmm0 \n" \
 		"aesenclast	224(%["#sched"]), %%xmm0 \n"
+#endif
 	
-	void ECBEncryptionAESNI::Encrypt (const ChipherBlock * in, ChipherBlock * out)
+	void ECBEncryption::Encrypt (const ChipherBlock * in, ChipherBlock * out)
 	{
+		if(i2p::cpu::aesni)
+		{
+#ifdef AESNI
 			__asm__
 				(
 					"movups	(%[in]), %%xmm0 \n"
@@ -632,8 +665,17 @@ namespace crypto
 					"movups	%%xmm0, (%[out]) \n"
 					: : [sched]"r"(GetKeySchedule ()), [in]"r"(in), [out]"r"(out) : "%xmm0", "memory"
 					);
+#else
+			AES_encrypt (in->buf, out->buf, &m_Key);
+#endif
+		}
+		else
+		{
+			AES_encrypt (in->buf, out->buf, &m_Key);
+		}	
 	}
 
+#ifdef AESNI
 	#define DecryptAES256(sched) \
 		"pxor 224(%["#sched"]), %%xmm0 \n" \
 		"aesdec	208(%["#sched"]), %%xmm0 \n" \
@@ -650,9 +692,13 @@ namespace crypto
 		"aesdec	32(%["#sched"]), %%xmm0 \n" \
 		"aesdec	16(%["#sched"]), %%xmm0 \n" \
 		"aesdeclast (%["#sched"]), %%xmm0 \n"
+#endif
 	
-	void ECBDecryptionAESNI::Decrypt (const ChipherBlock * in, ChipherBlock * out)
+	void ECBDecryption::Decrypt (const ChipherBlock * in, ChipherBlock * out)
 	{
+		if(i2p::cpu::aesni)
+		{
+#ifdef AESNI
 			__asm__
 				(
 					"movups	(%[in]), %%xmm0 \n"
@@ -660,15 +706,44 @@ namespace crypto
 					"movups	%%xmm0, (%[out]) \n"
 					: : [sched]"r"(GetKeySchedule ()), [in]"r"(in), [out]"r"(out) : "%xmm0", "memory"
 					);
+#else
+			AES_decrypt (in->buf, out->buf, &m_Key);
+#endif
+		}
+		else
+		{
+			AES_decrypt (in->buf, out->buf, &m_Key);
+		}
 	}
 
+#ifdef AESNI
 	#define CallAESIMC(offset) \
 		"movaps "#offset"(%[shed]), %%xmm0 \n"	\
 		"aesimc %%xmm0, %%xmm0 \n" \
 		"movaps %%xmm0, "#offset"(%[shed]) \n"
+#endif
 
-	void ECBDecryptionAESNI::SetKey (const AESKey& key)
+	void ECBEncryption::SetKey (const AESKey& key)
 	{
+		if(i2p::cpu::aesni)
+		{
+#ifdef AESNI
+			ExpandKey (key); 
+#else
+			AES_set_encrypt_key (key, 256, &m_Key);
+#endif
+		}
+		else
+		{
+			AES_set_encrypt_key (key, 256, &m_Key);
+		}
+	}
+	
+	void ECBDecryption::SetKey (const AESKey& key)
+	{
+		if(i2p::cpu::aesni)
+		{
+#ifdef AESNI
 			ExpandKey (key); // expand encryption key first
 			// then  invert it using aesimc
 			__asm__
@@ -688,13 +763,21 @@ namespace crypto
 					CallAESIMC(208)
 					: : [shed]"r"(GetKeySchedule ()) : "%xmm0", "memory"
 					);
-	}
-
+#else
+			AES_set_decrypt_key (key, 256, &m_Key);
 #endif
+		}
+		else
+		{
+			AES_set_decrypt_key (key, 256, &m_Key);
+		}
+	}
 
 
 	void CBCEncryption::Encrypt (int numBlocks, const ChipherBlock * in, ChipherBlock * out)
 	{
+		if(i2p::cpu::aesni)
+		{
 #ifdef AESNI
 			__asm__
 				(
@@ -724,6 +807,16 @@ namespace crypto
 			}
 #endif
 		}
+		else
+		{
+			for (int i = 0; i < numBlocks; i++)
+			{
+				*m_LastBlock.GetChipherBlock () ^= in[i];
+				m_ECBEncryption.Encrypt (m_LastBlock.GetChipherBlock (), m_LastBlock.GetChipherBlock ());
+				out[i] = *m_LastBlock.GetChipherBlock ();
+			}
+		}
+	}
 
 	void CBCEncryption::Encrypt (const uint8_t * in, std::size_t len, uint8_t * out)
 	{
@@ -735,6 +828,8 @@ namespace crypto
 
 	void CBCEncryption::Encrypt (const uint8_t * in, uint8_t * out)
 	{
+		if(i2p::cpu::aesni)
+		{
 #ifdef AESNI
 			__asm__
 				(
@@ -753,9 +848,14 @@ namespace crypto
 			Encrypt (1, (const ChipherBlock *)in, (ChipherBlock *)out);
 #endif
 		}
+		else
+			Encrypt (1, (const ChipherBlock *)in, (ChipherBlock *)out);
+	}
 
 	void CBCDecryption::Decrypt (int numBlocks, const ChipherBlock * in, ChipherBlock * out)
 	{
+		if(i2p::cpu::aesni)
+		{
 #ifdef AESNI
 			__asm__
 				(
@@ -787,6 +887,17 @@ namespace crypto
 			}
 #endif
 		}
+		else
+		{
+			for (int i = 0; i < numBlocks; i++)
+			{
+				ChipherBlock tmp = in[i];
+				m_ECBDecryption.Decrypt (in + i, out + i);
+				out[i] ^= *m_IV.GetChipherBlock ();
+				*m_IV.GetChipherBlock () = tmp;
+			}
+		}
+	}
 
 	void CBCDecryption::Decrypt (const uint8_t * in, std::size_t len, uint8_t * out)
 	{
@@ -797,6 +908,8 @@ namespace crypto
 
 	void CBCDecryption::Decrypt (const uint8_t * in, uint8_t * out)
 	{
+		if(i2p::cpu::aesni)
+		{
 #ifdef AESNI
 			__asm__
 				(
@@ -815,9 +928,14 @@ namespace crypto
 			Decrypt (1, (const ChipherBlock *)in, (ChipherBlock *)out);
 #endif
 		}
+		else
+			Decrypt (1, (const ChipherBlock *)in, (ChipherBlock *)out);
+	}
 
 	void TunnelEncryption::Encrypt (const uint8_t * in, uint8_t * out)
 	{
+		if(i2p::cpu::aesni)
+		{
 #ifdef AESNI
 			__asm__
 				(
@@ -840,7 +958,7 @@ namespace crypto
 					"dec %[num] \n"
 					"jnz 1b \n"
 					:
-			: [sched_iv]"r"(m_IVEncryption.GetKeySchedule ()), [sched_l]"r"(m_LayerEncryption.GetKeySchedule ()),
+					: [sched_iv]"r"(m_IVEncryption.GetKeySchedule ()), [sched_l]"r"(m_LayerEncryption.ECB().GetKeySchedule ()),
 						[in]"r"(in), [out]"r"(out), [num]"r"(63) // 63 blocks = 1008 bytes
 					: "%xmm0", "%xmm1", "cc", "memory"
 					);
@@ -851,9 +969,19 @@ namespace crypto
 			m_IVEncryption.Encrypt ((ChipherBlock *)out, (ChipherBlock *)out); // double iv
 #endif
 		}
+		else
+		{
+			m_IVEncryption.Encrypt ((const ChipherBlock *)in, (ChipherBlock *)out); // iv
+			m_LayerEncryption.SetIV (out);
+			m_LayerEncryption.Encrypt (in + 16, i2p::tunnel::TUNNEL_DATA_ENCRYPTED_SIZE, out + 16); // data
+			m_IVEncryption.Encrypt ((ChipherBlock *)out, (ChipherBlock *)out); // double iv
+		}
+	}
 
 	void TunnelDecryption::Decrypt (const uint8_t * in, uint8_t * out)
 	{
+		if(i2p::cpu::aesni)
+		{
 #ifdef AESNI
 			__asm__
 				(
@@ -877,7 +1005,7 @@ namespace crypto
 					"dec %[num] \n"
 					"jnz 1b \n"
 					:
-			: [sched_iv]"r"(m_IVDecryption.GetKeySchedule ()), [sched_l]"r"(m_LayerDecryption.GetKeySchedule ()),
+					: [sched_iv]"r"(m_IVDecryption.GetKeySchedule ()), [sched_l]"r"(m_LayerDecryption.ECB().GetKeySchedule ()),
 						[in]"r"(in), [out]"r"(out), [num]"r"(63) // 63 blocks = 1008 bytes
 					: "%xmm0", "%xmm1", "%xmm2", "cc", "memory"
 					);
@@ -888,6 +1016,14 @@ namespace crypto
 			m_IVDecryption.Decrypt ((ChipherBlock *)out, (ChipherBlock *)out); // double iv
 #endif
 		}
+		else
+		{
+			m_IVDecryption.Decrypt ((const ChipherBlock *)in, (ChipherBlock *)out); // iv
+			m_LayerDecryption.SetIV (out);
+			m_LayerDecryption.Decrypt (in + 16, i2p::tunnel::TUNNEL_DATA_ENCRYPTED_SIZE, out + 16); // data
+			m_IVDecryption.Decrypt ((ChipherBlock *)out, (ChipherBlock *)out); // double iv
+		}
+	}
 
 /*	std::vector <std::unique_ptr<std::mutex> >  m_OpenSSLMutexes;
 	static void OpensslLockingCallback(int mode, int type, const char * file, int line)
@@ -904,6 +1040,7 @@ namespace crypto
 
 	void InitCrypto (bool precomputation)
 	{
+		i2p::cpu::Detect ();
 		SSL_library_init ();
 /*		auto numLocks = CRYPTO_num_locks();
 		for (int i = 0; i < numLocks; i++)

libi2pd/Crypto.h

@@ -16,6 +16,7 @@
 
 #include "Base.h"
 #include "Tag.h"
+#include "CPU.h"
 
 namespace i2p
 {
@@ -68,7 +69,9 @@ namespace crypto
 
 		void operator^=(const ChipherBlock& other) // XOR
 		{
-#if defined(__AVX__) // AVX
+			if (i2p::cpu::avx)
+			{
+#ifdef AVX
 				__asm__
 					(
 						"vmovups (%[buf]), %%xmm0 \n"
@@ -79,23 +82,18 @@ namespace crypto
 						: [buf]"r"(buf), [other]"r"(other.buf)
 						: "%xmm0", "%xmm1", "memory"
 						);
-#elif defined(__SSE__) // SSE
-			__asm__
-			(
-				"movups	(%[buf]), %%xmm0 \n"
-				"movups	(%[other]), %%xmm1 \n"
-				"pxor %%xmm1, %%xmm0 \n"
-				"movups	%%xmm0, (%[buf]) \n"
-				:
-				: [buf]"r"(buf), [other]"r"(other.buf)
-				: "%xmm0", "%xmm1", "memory"
-			);
 #else
-			// TODO: implement it better
 				for (int i = 0; i < 16; i++)
 					buf[i] ^= other.buf[i];
 #endif
 			}
+			else
+			{
+				// TODO: implement it better
+				for (int i = 0; i < 16; i++)
+					buf[i] ^= other.buf[i];
+			}
+		}
 	};
 
 	typedef i2p::data::Tag<32> AESKey;
@@ -140,67 +138,38 @@ namespace crypto
 
 			AESAlignedBuffer<240> m_KeySchedule;	// 14 rounds for AES-256, 240 bytes
 	};
+#endif
 
-	class ECBEncryptionAESNI: public ECBCryptoAESNI
-	{
-		public:
-
-			void SetKey (const AESKey& key) { ExpandKey (key); };
-			void Encrypt (const ChipherBlock * in, ChipherBlock * out);
-	};
-
-	class ECBDecryptionAESNI: public ECBCryptoAESNI
+#ifdef AESNI
+	class ECBEncryption: public ECBCryptoAESNI
+#else
+	class ECBEncryption
+#endif
 	{
 		public:
 
 		void SetKey (const AESKey& key);
-			void Decrypt (const ChipherBlock * in, ChipherBlock * out);
-	};
-
-	typedef ECBEncryptionAESNI ECBEncryption;
-	typedef ECBDecryptionAESNI ECBDecryption;
 		
-#else // use openssl
-
-	class ECBEncryption
-	{
-		public:
-
-			void SetKey (const AESKey& key)
-			{
-				AES_set_encrypt_key (key, 256, &m_Key);
-			}
-			void Encrypt (const ChipherBlock * in, ChipherBlock * out)
-			{
-				AES_encrypt (in->buf, out->buf, &m_Key);
-			}
+		void Encrypt(const ChipherBlock * in, ChipherBlock * out);
 
 	private:
-
 		AES_KEY m_Key;
 	};
 
+#ifdef AESNI
+	class ECBDecryption: public ECBCryptoAESNI
+#else
 	class ECBDecryption
+#endif
 	{
 		public:
 
-			void SetKey (const AESKey& key)
-			{
-				AES_set_decrypt_key (key, 256, &m_Key);
-			}
-			void Decrypt (const ChipherBlock * in, ChipherBlock * out)
-			{
-				AES_decrypt (in->buf, out->buf, &m_Key);
-			}
-
+			void SetKey (const AESKey& key);
+			void Decrypt (const ChipherBlock * in, ChipherBlock * out);
 		private:
-
 			AES_KEY m_Key;
 	};
 
-
-#endif
-
 	class CBCEncryption
 	{
 		public:
@@ -214,6 +183,8 @@ namespace crypto
 			void Encrypt (const uint8_t * in, std::size_t len, uint8_t * out);
 			void Encrypt (const uint8_t * in, uint8_t * out); // one block
 
+			ECBEncryption & ECB() { return m_ECBEncryption; }
+		
 		private:
 
 			AESAlignedBuffer<16> m_LastBlock;
@@ -234,6 +205,8 @@ namespace crypto
 			void Decrypt (const uint8_t * in, std::size_t len, uint8_t * out);
 			void Decrypt (const uint8_t * in, uint8_t * out); // one block
 
+			ECBDecryption & ECB() { return m_ECBDecryption; }
+		
 		private:
 
 			AESAlignedBuffer<16> m_IV;
@@ -255,11 +228,7 @@ namespace crypto
 		private:
 
 			ECBEncryption m_IVEncryption;
-#ifdef AESNI
-			ECBEncryption m_LayerEncryption;
-#else
 			CBCEncryption m_LayerEncryption;
-#endif
 	};
 
 	class TunnelDecryption // with double IV encryption
@@ -277,11 +246,7 @@ namespace crypto
 		private:
 
 			ECBDecryption m_IVDecryption;
-#ifdef AESNI
-			ECBDecryption m_LayerDecryption;
-#else
 			CBCDecryption m_LayerDecryption;
-#endif
 	};
 
 	void InitCrypto (bool precomputation);

qt/i2pd_qt/i2pd_qt.pro

@@ -29,6 +29,7 @@ SOURCES += DaemonQT.cpp mainwindow.cpp \
     ../../libi2pd/Base.cpp \
     ../../libi2pd/BloomFilter.cpp \
     ../../libi2pd/Config.cpp \
+    ../../libi2pd/CPU.cpp \
     ../../libi2pd/Crypto.cpp \
 	../../libi2pd/CryptoKey.cpp \
     ../../libi2pd/Datagram.cpp \