d4708
/
source-engine
mirror of https://github.com/nillerusr/source-engine.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Modified source engine (2017) developed by valve and leaked in 2020. Not for commercial purporses
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
19 Commits
25 Branches
6 Tags
221 MiB
 
 
 
 
 
 
Tree: b7cd3625c8
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from 'b7cd3625c8'
${ noResults }
source-engine/thirdparty/cryptopp/adv_simd.h

1281 lines
54 KiB
Raw Blame History

// adv_simd.h - written and placed in the public domain by Jeffrey Walton
/// \file adv_simd.h
/// \brief Template for AdvancedProcessBlocks and SIMD processing
// The SIMD based implementations for ciphers that use SSE, NEON and Power7
// have a commom pattern. Namely, they have a specialized implementation of
// AdvancedProcessBlocks which processes multiple block using hardware
// acceleration. After several implementations we noticed a lot of copy and
// paste occuring. adv_simd.h provides a template to avoid the copy and paste.
//
// There are 6 templates provided in this file. The number following the
// function name, 128, is the block size in bits. The name following the
// block size is the arrangement and acceleration. For example 4x1_SSE means
// Intel SSE using two encrypt (or decrypt) functions: one that operates on
// 4 SIMD words, and one that operates on 1 SIMD words.
//
// * AdvancedProcessBlocks128_4x1_SSE
// * AdvancedProcessBlocks128_6x2_SSE
// * AdvancedProcessBlocks128_4x1_NEON
// * AdvancedProcessBlocks128_6x1_NEON
// * AdvancedProcessBlocks128_4x1_ALTIVEC
// * AdvancedProcessBlocks128_6x1_ALTIVEC
//
// If an arrangement ends in 2, like 6x2, then the template will handle the
// single block case by padding with 0's and using the two SIMD word
// function. This happens at most one time when processing multiple blocks.
// The extra processing of a zero block is trivial and worth the tradeoff.
//
// The MAYBE_CONST macro present on x86 is a SunCC workaround. Some versions
// of SunCC lose/drop the const-ness in the F1 and F4 functions. It eventually
// results in a failed link due to the const/non-const mismatch.
//
// In July 2020 the library stopped using 64-bit block version of
// AdvancedProcessBlocks. Testing showed unreliable results and failed
// self tests on occasion. Also see Issue 945 and
// https://github.com/weidai11/cryptopp/commit/dd7598e638bb.
#ifndef CRYPTOPP_ADVANCED_SIMD_TEMPLATES
#define CRYPTOPP_ADVANCED_SIMD_TEMPLATES
#include "config.h"
#include "misc.h"
#include "stdcpp.h"
#if (CRYPTOPP_ARM_NEON_HEADER)
# include <arm_neon.h>
#endif
#if (CRYPTOPP_ARM_ACLE_HEADER)
# include <stdint.h>
# include <arm_acle.h>
#endif
#if (CRYPTOPP_SSE2_INTRIN_AVAILABLE)
# include <emmintrin.h>
# include <xmmintrin.h>
#endif
// SunCC needs CRYPTOPP_SSSE3_AVAILABLE, too
#if (CRYPTOPP_SSSE3_AVAILABLE)
# include <emmintrin.h>
# include <pmmintrin.h>
# include <xmmintrin.h>
#endif
#if defined(__ALTIVEC__)
# include "ppc_simd.h"
#endif
// ************************ All block ciphers *********************** //
ANONYMOUS_NAMESPACE_BEGIN
using CryptoPP::BlockTransformation;
CRYPTOPP_CONSTANT(BT_XorInput = BlockTransformation::BT_XorInput);
CRYPTOPP_CONSTANT(BT_AllowParallel = BlockTransformation::BT_AllowParallel);
CRYPTOPP_CONSTANT(BT_InBlockIsCounter = BlockTransformation::BT_InBlockIsCounter);
CRYPTOPP_CONSTANT(BT_ReverseDirection = BlockTransformation::BT_ReverseDirection);
CRYPTOPP_CONSTANT(BT_DontIncrementInOutPointers = BlockTransformation::BT_DontIncrementInOutPointers);
ANONYMOUS_NAMESPACE_END
// *************************** ARM NEON ************************** //
#if (CRYPTOPP_ARM_NEON_AVAILABLE) || (CRYPTOPP_ARM_ASIMD_AVAILABLE) || \
defined(CRYPTOPP_DOXYGEN_PROCESSING)
NAMESPACE_BEGIN(CryptoPP)
/// \brief AdvancedProcessBlocks for 1 and 6 blocks
/// \tparam F1 function to process 1 128-bit block
/// \tparam F6 function to process 6 128-bit blocks
/// \tparam W word type of the subkey table
/// \details AdvancedProcessBlocks128_6x1_NEON processes 6 and 2 NEON SIMD words
/// at a time.
/// \details The subkey type is usually word32 or word64. F1 and F6 must use the
/// same word type.
template <typename F1, typename F6, typename W>
inline size_t AdvancedProcessBlocks128_6x1_NEON(F1 func1, F6 func6,
const W *subKeys, size_t rounds, const byte *inBlocks,
const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
{
CRYPTOPP_ASSERT(subKeys);
CRYPTOPP_ASSERT(inBlocks);
CRYPTOPP_ASSERT(outBlocks);
CRYPTOPP_ASSERT(length >= 16);
const unsigned int w_one[] = {0, 0<<24, 0, 1<<24};
const uint32x4_t s_one = vld1q_u32(w_one);
const size_t blockSize = 16;
// const size_t neonBlockSize = 16;
size_t inIncrement = (flags & (BT_InBlockIsCounter|BT_DontIncrementInOutPointers)) ? 0 : blockSize;
size_t xorIncrement = (xorBlocks != NULLPTR) ? blockSize : 0;
size_t outIncrement = (flags & BT_DontIncrementInOutPointers) ? 0 : blockSize;
// Clang and Coverity are generating findings using xorBlocks as a flag.
const bool xorInput = (xorBlocks != NULLPTR) && (flags & BT_XorInput);
const bool xorOutput = (xorBlocks != NULLPTR) && !(flags & BT_XorInput);
if (flags & BT_ReverseDirection)
{
inBlocks = PtrAdd(inBlocks, length - blockSize);
xorBlocks = PtrAdd(xorBlocks, length - blockSize);
outBlocks = PtrAdd(outBlocks, length - blockSize);
inIncrement = 0-inIncrement;
xorIncrement = 0-xorIncrement;
outIncrement = 0-outIncrement;
}
if (flags & BT_AllowParallel)
{
while (length >= 6*blockSize)
{
uint64x2_t block0, block1, block2, block3, block4, block5;
if (flags & BT_InBlockIsCounter)
{
const uint64x2_t one = vreinterpretq_u64_u32(s_one);
block0 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
block1 = vaddq_u64(block0, one);
block2 = vaddq_u64(block1, one);
block3 = vaddq_u64(block2, one);
block4 = vaddq_u64(block3, one);
block5 = vaddq_u64(block4, one);
vst1q_u8(const_cast<byte*>(inBlocks),
vreinterpretq_u8_u64(vaddq_u64(block5, one)));
}
else
{
block0 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
inBlocks = PtrAdd(inBlocks, inIncrement);
block1 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
inBlocks = PtrAdd(inBlocks, inIncrement);
block2 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
inBlocks = PtrAdd(inBlocks, inIncrement);
block3 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
inBlocks = PtrAdd(inBlocks, inIncrement);
block4 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
inBlocks = PtrAdd(inBlocks, inIncrement);
block5 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
inBlocks = PtrAdd(inBlocks, inIncrement);
}
if (xorInput)
{
block0 = veorq_u64(block0, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block1 = veorq_u64(block1, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block2 = veorq_u64(block2, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block3 = veorq_u64(block3, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block4 = veorq_u64(block4, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block5 = veorq_u64(block5, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
}
func6(block0, block1, block2, block3, block4, block5, subKeys, static_cast<unsigned int>(rounds));
if (xorOutput)
{
block0 = veorq_u64(block0, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block1 = veorq_u64(block1, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block2 = veorq_u64(block2, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block3 = veorq_u64(block3, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block4 = veorq_u64(block4, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block5 = veorq_u64(block5, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
}
vst1q_u8(outBlocks, vreinterpretq_u8_u64(block0));
outBlocks = PtrAdd(outBlocks, outIncrement);
vst1q_u8(outBlocks, vreinterpretq_u8_u64(block1));
outBlocks = PtrAdd(outBlocks, outIncrement);
vst1q_u8(outBlocks, vreinterpretq_u8_u64(block2));
outBlocks = PtrAdd(outBlocks, outIncrement);
vst1q_u8(outBlocks, vreinterpretq_u8_u64(block3));
outBlocks = PtrAdd(outBlocks, outIncrement);
vst1q_u8(outBlocks, vreinterpretq_u8_u64(block4));
outBlocks = PtrAdd(outBlocks, outIncrement);
vst1q_u8(outBlocks, vreinterpretq_u8_u64(block5));
outBlocks = PtrAdd(outBlocks, outIncrement);
length -= 6*blockSize;
}
}
while (length >= blockSize)
{
uint64x2_t block;
block = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
if (xorInput)
block = veorq_u64(block, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
if (flags & BT_InBlockIsCounter)
const_cast<byte *>(inBlocks)[15]++;
func1(block, subKeys, static_cast<unsigned int>(rounds));
if (xorOutput)
block = veorq_u64(block, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
vst1q_u8(outBlocks, vreinterpretq_u8_u64(block));
inBlocks = PtrAdd(inBlocks, inIncrement);
outBlocks = PtrAdd(outBlocks, outIncrement);
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
length -= blockSize;
}
return length;
}
/// \brief AdvancedProcessBlocks for 1 and 4 blocks
/// \tparam F1 function to process 1 128-bit block
/// \tparam F4 function to process 4 128-bit blocks
/// \tparam W word type of the subkey table
/// \details AdvancedProcessBlocks128_4x1_NEON processes 4 and 1 NEON SIMD words
/// at a time.
/// \details The subkey type is usually word32 or word64. V is the vector type and it is
/// usually uint32x4_t or uint32x4_t. F1, F4, and W must use the same word and
/// vector type.
template <typename F1, typename F4, typename W>
inline size_t AdvancedProcessBlocks128_4x1_NEON(F1 func1, F4 func4,
const W *subKeys, size_t rounds, const byte *inBlocks,
const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
{
CRYPTOPP_ASSERT(subKeys);
CRYPTOPP_ASSERT(inBlocks);
CRYPTOPP_ASSERT(outBlocks);
CRYPTOPP_ASSERT(length >= 16);
const unsigned int w_one[] = {0, 0<<24, 0, 1<<24};
const uint32x4_t s_one = vld1q_u32(w_one);
const size_t blockSize = 16;
// const size_t neonBlockSize = 16;
size_t inIncrement = (flags & (BT_InBlockIsCounter|BT_DontIncrementInOutPointers)) ? 0 : blockSize;
size_t xorIncrement = (xorBlocks != NULLPTR) ? blockSize : 0;
size_t outIncrement = (flags & BT_DontIncrementInOutPointers) ? 0 : blockSize;
// Clang and Coverity are generating findings using xorBlocks as a flag.
const bool xorInput = (xorBlocks != NULLPTR) && (flags & BT_XorInput);
const bool xorOutput = (xorBlocks != NULLPTR) && !(flags & BT_XorInput);
if (flags & BT_ReverseDirection)
{
inBlocks = PtrAdd(inBlocks, length - blockSize);
xorBlocks = PtrAdd(xorBlocks, length - blockSize);
outBlocks = PtrAdd(outBlocks, length - blockSize);
inIncrement = 0-inIncrement;
xorIncrement = 0-xorIncrement;
outIncrement = 0-outIncrement;
}
if (flags & BT_AllowParallel)
{
while (length >= 4*blockSize)
{
uint32x4_t block0, block1, block2, block3;
if (flags & BT_InBlockIsCounter)
{
const uint32x4_t one = s_one;
block0 = vreinterpretq_u32_u8(vld1q_u8(inBlocks));
block1 = vreinterpretq_u32_u64(vaddq_u64(vreinterpretq_u64_u32(block0), vreinterpretq_u64_u32(one)));
block2 = vreinterpretq_u32_u64(vaddq_u64(vreinterpretq_u64_u32(block1), vreinterpretq_u64_u32(one)));
block3 = vreinterpretq_u32_u64(vaddq_u64(vreinterpretq_u64_u32(block2), vreinterpretq_u64_u32(one)));
vst1q_u8(const_cast<byte*>(inBlocks), vreinterpretq_u8_u64(vaddq_u64(
vreinterpretq_u64_u32(block3), vreinterpretq_u64_u32(one))));
}
else
{
block0 = vreinterpretq_u32_u8(vld1q_u8(inBlocks));
inBlocks = PtrAdd(inBlocks, inIncrement);
block1 = vreinterpretq_u32_u8(vld1q_u8(inBlocks));
inBlocks = PtrAdd(inBlocks, inIncrement);
block2 = vreinterpretq_u32_u8(vld1q_u8(inBlocks));
inBlocks = PtrAdd(inBlocks, inIncrement);
block3 = vreinterpretq_u32_u8(vld1q_u8(inBlocks));
inBlocks = PtrAdd(inBlocks, inIncrement);
}
if (xorInput)
{
block0 = veorq_u32(block0, vreinterpretq_u32_u8(vld1q_u8(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block1 = veorq_u32(block1, vreinterpretq_u32_u8(vld1q_u8(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block2 = veorq_u32(block2, vreinterpretq_u32_u8(vld1q_u8(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block3 = veorq_u32(block3, vreinterpretq_u32_u8(vld1q_u8(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
}
func4(block0, block1, block2, block3, subKeys, static_cast<unsigned int>(rounds));
if (xorOutput)
{
block0 = veorq_u32(block0, vreinterpretq_u32_u8(vld1q_u8(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block1 = veorq_u32(block1, vreinterpretq_u32_u8(vld1q_u8(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block2 = veorq_u32(block2, vreinterpretq_u32_u8(vld1q_u8(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block3 = veorq_u32(block3, vreinterpretq_u32_u8(vld1q_u8(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
}
vst1q_u8(outBlocks, vreinterpretq_u8_u32(block0));
outBlocks = PtrAdd(outBlocks, outIncrement);
vst1q_u8(outBlocks, vreinterpretq_u8_u32(block1));
outBlocks = PtrAdd(outBlocks, outIncrement);
vst1q_u8(outBlocks, vreinterpretq_u8_u32(block2));
outBlocks = PtrAdd(outBlocks, outIncrement);
vst1q_u8(outBlocks, vreinterpretq_u8_u32(block3));
outBlocks = PtrAdd(outBlocks, outIncrement);
length -= 4*blockSize;
}
}
while (length >= blockSize)
{
uint32x4_t block = vreinterpretq_u32_u8(vld1q_u8(inBlocks));
if (xorInput)
block = veorq_u32(block, vreinterpretq_u32_u8(vld1q_u8(xorBlocks)));
if (flags & BT_InBlockIsCounter)
const_cast<byte *>(inBlocks)[15]++;
func1(block, subKeys, static_cast<unsigned int>(rounds));
if (xorOutput)
block = veorq_u32(block, vreinterpretq_u32_u8(vld1q_u8(xorBlocks)));
vst1q_u8(outBlocks, vreinterpretq_u8_u32(block));
inBlocks = PtrAdd(inBlocks, inIncrement);
outBlocks = PtrAdd(outBlocks, outIncrement);
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
length -= blockSize;
}
return length;
}
/// \brief AdvancedProcessBlocks for 2 and 6 blocks
/// \tparam F2 function to process 2 128-bit blocks
/// \tparam F6 function to process 6 128-bit blocks
/// \tparam W word type of the subkey table
/// \details AdvancedProcessBlocks128_6x2_NEON processes 6 and 2 NEON SIMD words
/// at a time. For a single block the template uses F2 with a zero block.
/// \details The subkey type is usually word32 or word64. F2 and F6 must use the
/// same word type.
template <typename F2, typename F6, typename W>
inline size_t AdvancedProcessBlocks128_6x2_NEON(F2 func2, F6 func6,
const W *subKeys, size_t rounds, const byte *inBlocks,
const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
{
CRYPTOPP_ASSERT(subKeys);
CRYPTOPP_ASSERT(inBlocks);
CRYPTOPP_ASSERT(outBlocks);
CRYPTOPP_ASSERT(length >= 16);
const unsigned int w_one[] = {0, 0<<24, 0, 1<<24};
const uint32x4_t s_one = vld1q_u32(w_one);
const size_t blockSize = 16;
// const size_t neonBlockSize = 16;
size_t inIncrement = (flags & (BT_InBlockIsCounter|BT_DontIncrementInOutPointers)) ? 0 : blockSize;
size_t xorIncrement = (xorBlocks != NULLPTR) ? blockSize : 0;
size_t outIncrement = (flags & BT_DontIncrementInOutPointers) ? 0 : blockSize;
// Clang and Coverity are generating findings using xorBlocks as a flag.
const bool xorInput = (xorBlocks != NULLPTR) && (flags & BT_XorInput);
const bool xorOutput = (xorBlocks != NULLPTR) && !(flags & BT_XorInput);
if (flags & BT_ReverseDirection)
{
inBlocks = PtrAdd(inBlocks, length - blockSize);
xorBlocks = PtrAdd(xorBlocks, length - blockSize);
outBlocks = PtrAdd(outBlocks, length - blockSize);
inIncrement = 0-inIncrement;
xorIncrement = 0-xorIncrement;
outIncrement = 0-outIncrement;
}
if (flags & BT_AllowParallel)
{
while (length >= 6*blockSize)
{
uint64x2_t block0, block1, block2, block3, block4, block5;
if (flags & BT_InBlockIsCounter)
{
const uint64x2_t one = vreinterpretq_u64_u32(s_one);
block0 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
block1 = vaddq_u64(block0, one);
block2 = vaddq_u64(block1, one);
block3 = vaddq_u64(block2, one);
block4 = vaddq_u64(block3, one);
block5 = vaddq_u64(block4, one);
vst1q_u8(const_cast<byte*>(inBlocks),
vreinterpretq_u8_u64(vaddq_u64(block5, one)));
}
else
{
block0 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
inBlocks = PtrAdd(inBlocks, inIncrement);
block1 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
inBlocks = PtrAdd(inBlocks, inIncrement);
block2 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
inBlocks = PtrAdd(inBlocks, inIncrement);
block3 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
inBlocks = PtrAdd(inBlocks, inIncrement);
block4 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
inBlocks = PtrAdd(inBlocks, inIncrement);
block5 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
inBlocks = PtrAdd(inBlocks, inIncrement);
}
if (xorInput)
{
block0 = veorq_u64(block0, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block1 = veorq_u64(block1, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block2 = veorq_u64(block2, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block3 = veorq_u64(block3, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block4 = veorq_u64(block4, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block5 = veorq_u64(block5, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
}
func6(block0, block1, block2, block3, block4, block5, subKeys, static_cast<unsigned int>(rounds));
if (xorOutput)
{
block0 = veorq_u64(block0, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block1 = veorq_u64(block1, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block2 = veorq_u64(block2, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block3 = veorq_u64(block3, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block4 = veorq_u64(block4, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block5 = veorq_u64(block5, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
}
vst1q_u8(outBlocks, vreinterpretq_u8_u64(block0));
outBlocks = PtrAdd(outBlocks, outIncrement);
vst1q_u8(outBlocks, vreinterpretq_u8_u64(block1));
outBlocks = PtrAdd(outBlocks, outIncrement);
vst1q_u8(outBlocks, vreinterpretq_u8_u64(block2));
outBlocks = PtrAdd(outBlocks, outIncrement);
vst1q_u8(outBlocks, vreinterpretq_u8_u64(block3));
outBlocks = PtrAdd(outBlocks, outIncrement);
vst1q_u8(outBlocks, vreinterpretq_u8_u64(block4));
outBlocks = PtrAdd(outBlocks, outIncrement);
vst1q_u8(outBlocks, vreinterpretq_u8_u64(block5));
outBlocks = PtrAdd(outBlocks, outIncrement);
length -= 6*blockSize;
}
while (length >= 2*blockSize)
{
uint64x2_t block0, block1;
if (flags & BT_InBlockIsCounter)
{
const uint64x2_t one = vreinterpretq_u64_u32(s_one);
block0 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
block1 = vaddq_u64(block0, one);
vst1q_u8(const_cast<byte*>(inBlocks),
vreinterpretq_u8_u64(vaddq_u64(block1, one)));
}
else
{
block0 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
inBlocks = PtrAdd(inBlocks, inIncrement);
block1 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
inBlocks = PtrAdd(inBlocks, inIncrement);
}
if (xorInput)
{
block0 = veorq_u64(block0, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block1 = veorq_u64(block1, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
}
func2(block0, block1, subKeys, static_cast<unsigned int>(rounds));
if (xorOutput)
{
block0 = veorq_u64(block0, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block1 = veorq_u64(block1, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
}
vst1q_u8(outBlocks, vreinterpretq_u8_u64(block0));
outBlocks = PtrAdd(outBlocks, outIncrement);
vst1q_u8(outBlocks, vreinterpretq_u8_u64(block1));
outBlocks = PtrAdd(outBlocks, outIncrement);
length -= 2*blockSize;
}
}
while (length >= blockSize)
{
uint64x2_t block, zero = {0,0};
block = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
if (xorInput)
block = veorq_u64(block, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
if (flags & BT_InBlockIsCounter)
const_cast<byte *>(inBlocks)[15]++;
func2(block, zero, subKeys, static_cast<unsigned int>(rounds));
if (xorOutput)
block = veorq_u64(block, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
vst1q_u8(outBlocks, vreinterpretq_u8_u64(block));
inBlocks = PtrAdd(inBlocks, inIncrement);
outBlocks = PtrAdd(outBlocks, outIncrement);
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
length -= blockSize;
}
return length;
}
NAMESPACE_END // CryptoPP
#endif // CRYPTOPP_ARM_NEON_AVAILABLE
// *************************** Intel SSE ************************** //
#if defined(CRYPTOPP_SSSE3_AVAILABLE) || defined(CRYPTOPP_DOXYGEN_PROCESSING)
#if defined(CRYPTOPP_DOXYGEN_PROCESSING)
/// \brief SunCC workaround
/// \details SunCC loses the const on AES_Enc_Block and AES_Dec_Block
/// \sa <A HREF="http://github.com/weidai11/cryptopp/issues/224">Issue
/// 224, SunCC and failed compile for rijndael.cpp</A>
# define MAYBE_CONST const
/// \brief SunCC workaround
/// \details SunCC loses the const on AES_Enc_Block and AES_Dec_Block
/// \sa <A HREF="http://github.com/weidai11/cryptopp/issues/224">Issue
/// 224, SunCC and failed compile for rijndael.cpp</A>
# define MAYBE_UNCONST_CAST(T, x) (x)
#elif (__SUNPRO_CC >= 0x5130)
# define MAYBE_CONST
# define MAYBE_UNCONST_CAST(T, x) const_cast<MAYBE_CONST T>(x)
#else
# define MAYBE_CONST const
# define MAYBE_UNCONST_CAST(T, x) (x)
#endif
#if defined(CRYPTOPP_DOXYGEN_PROCESSING)
/// \brief Clang workaround
/// \details Clang issues spurious alignment warnings
/// \sa <A HREF="http://bugs.llvm.org/show_bug.cgi?id=20670">Issue
/// 20670, _mm_loadu_si128 parameter has wrong type</A>
# define M128_CAST(x) ((__m128i *)(void *)(x))
/// \brief Clang workaround
/// \details Clang issues spurious alignment warnings
/// \sa <A HREF="http://bugs.llvm.org/show_bug.cgi?id=20670">Issue
/// 20670, _mm_loadu_si128 parameter has wrong type</A>
# define CONST_M128_CAST(x) ((const __m128i *)(const void *)(x))
#else
# ifndef M128_CAST
# define M128_CAST(x) ((__m128i *)(void *)(x))
# endif
# ifndef CONST_M128_CAST
# define CONST_M128_CAST(x) ((const __m128i *)(const void *)(x))
# endif
#endif
NAMESPACE_BEGIN(CryptoPP)
/// \brief AdvancedProcessBlocks for 2 and 6 blocks
/// \tparam F2 function to process 2 128-bit blocks
/// \tparam F6 function to process 6 128-bit blocks
/// \tparam W word type of the subkey table
/// \details AdvancedProcessBlocks128_6x2_SSE processes 6 and 2 SSE SIMD words
/// at a time. For a single block the template uses F2 with a zero block.
/// \details The subkey type is usually word32 or word64. F2 and F6 must use the
/// same word type.
template <typename F2, typename F6, typename W>
inline size_t AdvancedProcessBlocks128_6x2_SSE(F2 func2, F6 func6,
MAYBE_CONST W *subKeys, size_t rounds, const byte *inBlocks,
const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
{
CRYPTOPP_ASSERT(subKeys);
CRYPTOPP_ASSERT(inBlocks);
CRYPTOPP_ASSERT(outBlocks);
CRYPTOPP_ASSERT(length >= 16);
const size_t blockSize = 16;
// const size_t xmmBlockSize = 16;
size_t inIncrement = (flags & (BT_InBlockIsCounter|BT_DontIncrementInOutPointers)) ? 0 : blockSize;
size_t xorIncrement = (xorBlocks != NULLPTR) ? blockSize : 0;
size_t outIncrement = (flags & BT_DontIncrementInOutPointers) ? 0 : blockSize;
// Clang and Coverity are generating findings using xorBlocks as a flag.
const bool xorInput = (xorBlocks != NULLPTR) && (flags & BT_XorInput);
const bool xorOutput = (xorBlocks != NULLPTR) && !(flags & BT_XorInput);
if (flags & BT_ReverseDirection)
{
inBlocks = PtrAdd(inBlocks, length - blockSize);
xorBlocks = PtrAdd(xorBlocks, length - blockSize);
outBlocks = PtrAdd(outBlocks, length - blockSize);
inIncrement = 0-inIncrement;
xorIncrement = 0-xorIncrement;
outIncrement = 0-outIncrement;
}
if (flags & BT_AllowParallel)
{
while (length >= 6*blockSize)
{
__m128i block0, block1, block2, block3, block4, block5;
if (flags & BT_InBlockIsCounter)
{
// Increment of 1 in big-endian compatible with the ctr byte array.
const __m128i s_one = _mm_set_epi32(1<<24, 0, 0, 0);
block0 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
block1 = _mm_add_epi32(block0, s_one);
block2 = _mm_add_epi32(block1, s_one);
block3 = _mm_add_epi32(block2, s_one);
block4 = _mm_add_epi32(block3, s_one);
block5 = _mm_add_epi32(block4, s_one);
_mm_storeu_si128(M128_CAST(inBlocks), _mm_add_epi32(block5, s_one));
}
else
{
block0 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
inBlocks = PtrAdd(inBlocks, inIncrement);
block1 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
inBlocks = PtrAdd(inBlocks, inIncrement);
block2 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
inBlocks = PtrAdd(inBlocks, inIncrement);
block3 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
inBlocks = PtrAdd(inBlocks, inIncrement);
block4 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
inBlocks = PtrAdd(inBlocks, inIncrement);
block5 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
inBlocks = PtrAdd(inBlocks, inIncrement);
}
if (xorInput)
{
block0 = _mm_xor_si128(block0, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block1 = _mm_xor_si128(block1, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block2 = _mm_xor_si128(block2, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block3 = _mm_xor_si128(block3, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block4 = _mm_xor_si128(block4, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block5 = _mm_xor_si128(block5, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
}
func6(block0, block1, block2, block3, block4, block5, subKeys, static_cast<unsigned int>(rounds));
if (xorOutput)
{
block0 = _mm_xor_si128(block0, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block1 = _mm_xor_si128(block1, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block2 = _mm_xor_si128(block2, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block3 = _mm_xor_si128(block3, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block4 = _mm_xor_si128(block4, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block5 = _mm_xor_si128(block5, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
}
_mm_storeu_si128(M128_CAST(outBlocks), block0);
outBlocks = PtrAdd(outBlocks, outIncrement);
_mm_storeu_si128(M128_CAST(outBlocks), block1);
outBlocks = PtrAdd(outBlocks, outIncrement);
_mm_storeu_si128(M128_CAST(outBlocks), block2);
outBlocks = PtrAdd(outBlocks, outIncrement);
_mm_storeu_si128(M128_CAST(outBlocks), block3);
outBlocks = PtrAdd(outBlocks, outIncrement);
_mm_storeu_si128(M128_CAST(outBlocks), block4);
outBlocks = PtrAdd(outBlocks, outIncrement);
_mm_storeu_si128(M128_CAST(outBlocks), block5);
outBlocks = PtrAdd(outBlocks, outIncrement);
length -= 6*blockSize;
}
while (length >= 2*blockSize)
{
__m128i block0, block1;
if (flags & BT_InBlockIsCounter)
{
// Increment of 1 in big-endian compatible with the ctr byte array.
const __m128i s_one = _mm_set_epi32(1<<24, 0, 0, 0);
block0 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
block1 = _mm_add_epi32(block0, s_one);
_mm_storeu_si128(M128_CAST(inBlocks), _mm_add_epi32(block1, s_one));
}
else
{
block0 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
inBlocks = PtrAdd(inBlocks, inIncrement);
block1 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
inBlocks = PtrAdd(inBlocks, inIncrement);
}
if (xorInput)
{
block0 = _mm_xor_si128(block0, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block1 = _mm_xor_si128(block1, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
}
func2(block0, block1, subKeys, static_cast<unsigned int>(rounds));
if (xorOutput)
{
block0 = _mm_xor_si128(block0, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block1 = _mm_xor_si128(block1, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
}
_mm_storeu_si128(M128_CAST(outBlocks), block0);
outBlocks = PtrAdd(outBlocks, outIncrement);
_mm_storeu_si128(M128_CAST(outBlocks), block1);
outBlocks = PtrAdd(outBlocks, outIncrement);
length -= 2*blockSize;
}
}
while (length >= blockSize)
{
__m128i block, zero = _mm_setzero_si128();
block = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
if (xorInput)
block = _mm_xor_si128(block, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
if (flags & BT_InBlockIsCounter)
const_cast<byte *>(inBlocks)[15]++;
func2(block, zero, subKeys, static_cast<unsigned int>(rounds));
if (xorOutput)
block = _mm_xor_si128(block, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
_mm_storeu_si128(M128_CAST(outBlocks), block);
inBlocks = PtrAdd(inBlocks, inIncrement);
outBlocks = PtrAdd(outBlocks, outIncrement);
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
length -= blockSize;
}
return length;
}
/// \brief AdvancedProcessBlocks for 1 and 4 blocks
/// \tparam F1 function to process 1 128-bit block
/// \tparam F4 function to process 4 128-bit blocks
/// \tparam W word type of the subkey table
/// \details AdvancedProcessBlocks128_4x1_SSE processes 4 and 1 SSE SIMD words
/// at a time.
/// \details The subkey type is usually word32 or word64. F1 and F4 must use the
/// same word type.
template <typename F1, typename F4, typename W>
inline size_t AdvancedProcessBlocks128_4x1_SSE(F1 func1, F4 func4,
MAYBE_CONST W *subKeys, size_t rounds, const byte *inBlocks,
const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
{
CRYPTOPP_ASSERT(subKeys);
CRYPTOPP_ASSERT(inBlocks);
CRYPTOPP_ASSERT(outBlocks);
CRYPTOPP_ASSERT(length >= 16);
const size_t blockSize = 16;
// const size_t xmmBlockSize = 16;
size_t inIncrement = (flags & (BT_InBlockIsCounter|BT_DontIncrementInOutPointers)) ? 0 : blockSize;
size_t xorIncrement = (xorBlocks != NULLPTR) ? blockSize : 0;
size_t outIncrement = (flags & BT_DontIncrementInOutPointers) ? 0 : blockSize;
// Clang and Coverity are generating findings using xorBlocks as a flag.
const bool xorInput = (xorBlocks != NULLPTR) && (flags & BT_XorInput);
const bool xorOutput = (xorBlocks != NULLPTR) && !(flags & BT_XorInput);
if (flags & BT_ReverseDirection)
{
inBlocks = PtrAdd(inBlocks, length - blockSize);
xorBlocks = PtrAdd(xorBlocks, length - blockSize);
outBlocks = PtrAdd(outBlocks, length - blockSize);
inIncrement = 0-inIncrement;
xorIncrement = 0-xorIncrement;
outIncrement = 0-outIncrement;
}
if (flags & BT_AllowParallel)
{
while (length >= 4*blockSize)
{
__m128i block0, block1, block2, block3;
if (flags & BT_InBlockIsCounter)
{
// Increment of 1 in big-endian compatible with the ctr byte array.
const __m128i s_one = _mm_set_epi32(1<<24, 0, 0, 0);
block0 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
block1 = _mm_add_epi32(block0, s_one);
block2 = _mm_add_epi32(block1, s_one);
block3 = _mm_add_epi32(block2, s_one);
_mm_storeu_si128(M128_CAST(inBlocks), _mm_add_epi32(block3, s_one));
}
else
{
block0 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
inBlocks = PtrAdd(inBlocks, inIncrement);
block1 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
inBlocks = PtrAdd(inBlocks, inIncrement);
block2 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
inBlocks = PtrAdd(inBlocks, inIncrement);
block3 = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
inBlocks = PtrAdd(inBlocks, inIncrement);
}
if (xorInput)
{
block0 = _mm_xor_si128(block0, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block1 = _mm_xor_si128(block1, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block2 = _mm_xor_si128(block2, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block3 = _mm_xor_si128(block3, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
}
func4(block0, block1, block2, block3, subKeys, static_cast<unsigned int>(rounds));
if (xorOutput)
{
block0 = _mm_xor_si128(block0, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block1 = _mm_xor_si128(block1, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block2 = _mm_xor_si128(block2, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block3 = _mm_xor_si128(block3, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
}
_mm_storeu_si128(M128_CAST(outBlocks), block0);
outBlocks = PtrAdd(outBlocks, outIncrement);
_mm_storeu_si128(M128_CAST(outBlocks), block1);
outBlocks = PtrAdd(outBlocks, outIncrement);
_mm_storeu_si128(M128_CAST(outBlocks), block2);
outBlocks = PtrAdd(outBlocks, outIncrement);
_mm_storeu_si128(M128_CAST(outBlocks), block3);
outBlocks = PtrAdd(outBlocks, outIncrement);
length -= 4*blockSize;
}
}
while (length >= blockSize)
{
__m128i block = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
if (xorInput)
block = _mm_xor_si128(block, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
if (flags & BT_InBlockIsCounter)
const_cast<byte *>(inBlocks)[15]++;
func1(block, subKeys, static_cast<unsigned int>(rounds));
if (xorOutput)
block = _mm_xor_si128(block, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
_mm_storeu_si128(M128_CAST(outBlocks), block);
inBlocks = PtrAdd(inBlocks, inIncrement);
outBlocks = PtrAdd(outBlocks, outIncrement);
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
length -= blockSize;
}
return length;
}
NAMESPACE_END // CryptoPP
#endif // CRYPTOPP_SSSE3_AVAILABLE
// ************************** Altivec/Power 4 ************************** //
#if defined(__ALTIVEC__) || defined(CRYPTOPP_DOXYGEN_PROCESSING)
NAMESPACE_BEGIN(CryptoPP)
/// \brief AdvancedProcessBlocks for 1 and 4 blocks
/// \tparam F1 function to process 1 128-bit block
/// \tparam F4 function to process 4 128-bit blocks
/// \tparam W word type of the subkey table
/// \details AdvancedProcessBlocks128_4x1_ALTIVEC processes 4 and 1 Altivec SIMD words
/// at a time.
/// \details The subkey type is usually word32 or word64. F1 and F4 must use the
/// same word type.
template <typename F1, typename F4, typename W>
inline size_t AdvancedProcessBlocks128_4x1_ALTIVEC(F1 func1, F4 func4,
const W *subKeys, size_t rounds, const byte *inBlocks,
const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
{
CRYPTOPP_ASSERT(subKeys);
CRYPTOPP_ASSERT(inBlocks);
CRYPTOPP_ASSERT(outBlocks);
CRYPTOPP_ASSERT(length >= 16);
#if (CRYPTOPP_LITTLE_ENDIAN)
const uint32x4_p s_one = {1,0,0,0};
#else
const uint32x4_p s_one = {0,0,0,1};
#endif
const size_t blockSize = 16;
// const size_t simdBlockSize = 16;
size_t inIncrement = (flags & (BT_InBlockIsCounter|BT_DontIncrementInOutPointers)) ? 0 : blockSize;
size_t xorIncrement = (xorBlocks != NULLPTR) ? blockSize : 0;
size_t outIncrement = (flags & BT_DontIncrementInOutPointers) ? 0 : blockSize;
// Clang and Coverity are generating findings using xorBlocks as a flag.
const bool xorInput = (xorBlocks != NULLPTR) && (flags & BT_XorInput);
const bool xorOutput = (xorBlocks != NULLPTR) && !(flags & BT_XorInput);
if (flags & BT_ReverseDirection)
{
inBlocks = PtrAdd(inBlocks, length - blockSize);
xorBlocks = PtrAdd(xorBlocks, length - blockSize);
outBlocks = PtrAdd(outBlocks, length - blockSize);
inIncrement = 0-inIncrement;
xorIncrement = 0-xorIncrement;
outIncrement = 0-outIncrement;
}
if (flags & BT_AllowParallel)
{
while (length >= 4*blockSize)
{
uint32x4_p block0, block1, block2, block3;
if (flags & BT_InBlockIsCounter)
{
block0 = VecLoadBE(inBlocks);
block1 = VecAdd(block0, s_one);
block2 = VecAdd(block1, s_one);
block3 = VecAdd(block2, s_one);
// Hack due to big-endian loads used by POWER8 (and maybe ARM-BE).
// CTR_ModePolicy::OperateKeystream is wired such that after
// returning from this function CTR_ModePolicy will detect wrap on
// on the last counter byte and increment the next to last byte.
// The problem is, with a big-endian load, inBlocks[15] is really
// located at index 15. The vector addition using a 32-bit element
// generates a carry into inBlocks[14] and then CTR_ModePolicy
// increments inBlocks[14] too.
const_cast<byte*>(inBlocks)[15] += 6;
}
else
{
block0 = VecLoadBE(inBlocks);
inBlocks = PtrAdd(inBlocks, inIncrement);
block1 = VecLoadBE(inBlocks);
inBlocks = PtrAdd(inBlocks, inIncrement);
block2 = VecLoadBE(inBlocks);
inBlocks = PtrAdd(inBlocks, inIncrement);
block3 = VecLoadBE(inBlocks);
inBlocks = PtrAdd(inBlocks, inIncrement);
}
if (xorInput)
{
block0 = VecXor(block0, VecLoadBE(xorBlocks));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block1 = VecXor(block1, VecLoadBE(xorBlocks));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block2 = VecXor(block2, VecLoadBE(xorBlocks));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block3 = VecXor(block3, VecLoadBE(xorBlocks));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
}
func4(block0, block1, block2, block3, subKeys, rounds);
if (xorOutput)
{
block0 = VecXor(block0, VecLoadBE(xorBlocks));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block1 = VecXor(block1, VecLoadBE(xorBlocks));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block2 = VecXor(block2, VecLoadBE(xorBlocks));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block3 = VecXor(block3, VecLoadBE(xorBlocks));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
}
VecStoreBE(block0, outBlocks);
outBlocks = PtrAdd(outBlocks, outIncrement);
VecStoreBE(block1, outBlocks);
outBlocks = PtrAdd(outBlocks, outIncrement);
VecStoreBE(block2, outBlocks);
outBlocks = PtrAdd(outBlocks, outIncrement);
VecStoreBE(block3, outBlocks);
outBlocks = PtrAdd(outBlocks, outIncrement);
length -= 4*blockSize;
}
}
while (length >= blockSize)
{
uint32x4_p block = VecLoadBE(inBlocks);
if (xorInput)
block = VecXor(block, VecLoadBE(xorBlocks));
if (flags & BT_InBlockIsCounter)
const_cast<byte *>(inBlocks)[15]++;
func1(block, subKeys, rounds);
if (xorOutput)
block = VecXor(block, VecLoadBE(xorBlocks));
VecStoreBE(block, outBlocks);
inBlocks = PtrAdd(inBlocks, inIncrement);
outBlocks = PtrAdd(outBlocks, outIncrement);
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
length -= blockSize;
}
return length;
}
/// \brief AdvancedProcessBlocks for 1 and 6 blocks
/// \tparam F1 function to process 1 128-bit block
/// \tparam F6 function to process 6 128-bit blocks
/// \tparam W word type of the subkey table
/// \details AdvancedProcessBlocks128_6x1_ALTIVEC processes 6 and 1 Altivec SIMD words
/// at a time.
/// \details The subkey type is usually word32 or word64. F1 and F6 must use the
/// same word type.
template <typename F1, typename F6, typename W>
inline size_t AdvancedProcessBlocks128_6x1_ALTIVEC(F1 func1, F6 func6,
const W *subKeys, size_t rounds, const byte *inBlocks,
const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
{
CRYPTOPP_ASSERT(subKeys);
CRYPTOPP_ASSERT(inBlocks);
CRYPTOPP_ASSERT(outBlocks);
CRYPTOPP_ASSERT(length >= 16);
#if (CRYPTOPP_LITTLE_ENDIAN)
const uint32x4_p s_one = {1,0,0,0};
#else
const uint32x4_p s_one = {0,0,0,1};
#endif
const size_t blockSize = 16;
// const size_t simdBlockSize = 16;
size_t inIncrement = (flags & (BT_InBlockIsCounter|BT_DontIncrementInOutPointers)) ? 0 : blockSize;
size_t xorIncrement = (xorBlocks != NULLPTR) ? blockSize : 0;
size_t outIncrement = (flags & BT_DontIncrementInOutPointers) ? 0 : blockSize;
// Clang and Coverity are generating findings using xorBlocks as a flag.
const bool xorInput = (xorBlocks != NULLPTR) && (flags & BT_XorInput);
const bool xorOutput = (xorBlocks != NULLPTR) && !(flags & BT_XorInput);
if (flags & BT_ReverseDirection)
{
inBlocks = PtrAdd(inBlocks, length - blockSize);
xorBlocks = PtrAdd(xorBlocks, length - blockSize);
outBlocks = PtrAdd(outBlocks, length - blockSize);
inIncrement = 0-inIncrement;
xorIncrement = 0-xorIncrement;
outIncrement = 0-outIncrement;
}
if (flags & BT_AllowParallel)
{
while (length >= 6*blockSize)
{
uint32x4_p block0, block1, block2, block3, block4, block5;
if (flags & BT_InBlockIsCounter)
{
block0 = VecLoadBE(inBlocks);
block1 = VecAdd(block0, s_one);
block2 = VecAdd(block1, s_one);
block3 = VecAdd(block2, s_one);
block4 = VecAdd(block3, s_one);
block5 = VecAdd(block4, s_one);
// Hack due to big-endian loads used by POWER8 (and maybe ARM-BE).
// CTR_ModePolicy::OperateKeystream is wired such that after
// returning from this function CTR_ModePolicy will detect wrap on
// on the last counter byte and increment the next to last byte.
// The problem is, with a big-endian load, inBlocks[15] is really
// located at index 15. The vector addition using a 32-bit element
// generates a carry into inBlocks[14] and then CTR_ModePolicy
// increments inBlocks[14] too.
//
// To find this bug we needed a test case with a ctr of 0xNN...FA.
// The last octet is 0xFA and adding 6 creates the wrap to trigger
// the issue. If the last octet was 0xFC then 4 would trigger it.
// We dumb-lucked into the test with SPECK-128. The test case of
// interest is the one with IV 348ECA9766C09F04 826520DE47A212FA.
uint8x16_p temp = VecAdd((uint8x16_p)block5, (uint8x16_p)s_one);
VecStoreBE(temp, const_cast<byte*>(inBlocks));
}
else
{
block0 = VecLoadBE(inBlocks);
inBlocks = PtrAdd(inBlocks, inIncrement);
block1 = VecLoadBE(inBlocks);
inBlocks = PtrAdd(inBlocks, inIncrement);
block2 = VecLoadBE(inBlocks);
inBlocks = PtrAdd(inBlocks, inIncrement);
block3 = VecLoadBE(inBlocks);
inBlocks = PtrAdd(inBlocks, inIncrement);
block4 = VecLoadBE(inBlocks);
inBlocks = PtrAdd(inBlocks, inIncrement);
block5 = VecLoadBE(inBlocks);
inBlocks = PtrAdd(inBlocks, inIncrement);
}
if (xorInput)
{
block0 = VecXor(block0, VecLoadBE(xorBlocks));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block1 = VecXor(block1, VecLoadBE(xorBlocks));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block2 = VecXor(block2, VecLoadBE(xorBlocks));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block3 = VecXor(block3, VecLoadBE(xorBlocks));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block4 = VecXor(block4, VecLoadBE(xorBlocks));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block5 = VecXor(block5, VecLoadBE(xorBlocks));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
}
func6(block0, block1, block2, block3, block4, block5, subKeys, rounds);
if (xorOutput)
{
block0 = VecXor(block0, VecLoadBE(xorBlocks));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block1 = VecXor(block1, VecLoadBE(xorBlocks));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block2 = VecXor(block2, VecLoadBE(xorBlocks));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block3 = VecXor(block3, VecLoadBE(xorBlocks));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block4 = VecXor(block4, VecLoadBE(xorBlocks));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
block5 = VecXor(block5, VecLoadBE(xorBlocks));
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
}
VecStoreBE(block0, outBlocks);
outBlocks = PtrAdd(outBlocks, outIncrement);
VecStoreBE(block1, outBlocks);
outBlocks = PtrAdd(outBlocks, outIncrement);
VecStoreBE(block2, outBlocks);
outBlocks = PtrAdd(outBlocks, outIncrement);
VecStoreBE(block3, outBlocks);
outBlocks = PtrAdd(outBlocks, outIncrement);
VecStoreBE(block4, outBlocks);
outBlocks = PtrAdd(outBlocks, outIncrement);
VecStoreBE(block5, outBlocks);
outBlocks = PtrAdd(outBlocks, outIncrement);
length -= 6*blockSize;
}
}
while (length >= blockSize)
{
uint32x4_p block = VecLoadBE(inBlocks);
if (xorInput)
block = VecXor(block, VecLoadBE(xorBlocks));
if (flags & BT_InBlockIsCounter)
const_cast<byte *>(inBlocks)[15]++;
func1(block, subKeys, rounds);
if (xorOutput)
block = VecXor(block, VecLoadBE(xorBlocks));
VecStoreBE(block, outBlocks);
inBlocks = PtrAdd(inBlocks, inIncrement);
outBlocks = PtrAdd(outBlocks, outIncrement);
xorBlocks = PtrAdd(xorBlocks, xorIncrement);
length -= blockSize;
}
return length;
}
NAMESPACE_END // CryptoPP
#endif // __ALTIVEC__
#endif // CRYPTOPP_ADVANCED_SIMD_TEMPLATES