ccminer/lyra2/cuda_lyra2_sm5.cuh

#include <memory.h>

#ifdef __INTELLISENSE__
/* just for vstudio code colors */
#undef __CUDA_ARCH__
#define __CUDA_ARCH__ 500
#endif

#include "cuda_helper.h"

#define TPB50 32

#if __CUDA_ARCH__ == 500
#include "cuda_lyra2_vectors.h"

#define Nrow 8
#define Ncol 8
#define memshift 3

__device__ uint2 *DMatrix;

__device__ __forceinline__ uint2 LD4S(const int index)
{
	extern __shared__ uint2 shared_mem[];

	return shared_mem[(index * blockDim.y + threadIdx.y) * blockDim.x + threadIdx.x];
}

__device__ __forceinline__ void ST4S(const int index, const uint2 data)
{
	extern __shared__ uint2 shared_mem[];

	shared_mem[(index * blockDim.y + threadIdx.y) * blockDim.x + threadIdx.x] = data;
}

#if __CUDA_ARCH__ == 300
__device__ __forceinline__ uint32_t WarpShuffle(uint32_t a, uint32_t b, uint32_t c)
{
	return __shfl(a, b, c);
}

__device__ __forceinline__ uint2 WarpShuffle(uint2 a, uint32_t b, uint32_t c)
{
	return make_uint2(__shfl(a.x, b, c), __shfl(a.y, b, c));
}

__device__ __forceinline__ void WarpShuffle3(uint2 &a1, uint2 &a2, uint2 &a3, uint32_t b1, uint32_t b2, uint32_t b3, uint32_t c)
{
	a1 = WarpShuffle(a1, b1, c);
	a2 = WarpShuffle(a2, b2, c);
	a3 = WarpShuffle(a3, b3, c);
}

#else
__device__ __forceinline__ uint32_t WarpShuffle(uint32_t a, uint32_t b, uint32_t c)
{
	extern __shared__ uint2 shared_mem[];

	const uint32_t thread = blockDim.x * threadIdx.y + threadIdx.x;
	uint32_t *_ptr = (uint32_t*)shared_mem;

	__threadfence_block();
	uint32_t buf = _ptr[thread];

	_ptr[thread] = a;
	__threadfence_block();
	uint32_t result = _ptr[(thread&~(c - 1)) + (b&(c - 1))];

	__threadfence_block();
	_ptr[thread] = buf;

	__threadfence_block();
	return result;
}

__device__ __forceinline__ uint2 WarpShuffle(uint2 a, uint32_t b, uint32_t c)
{
	extern __shared__ uint2 shared_mem[];

	const uint32_t thread = blockDim.x * threadIdx.y + threadIdx.x;

	__threadfence_block();
	uint2 buf = shared_mem[thread];

	shared_mem[thread] = a;
	__threadfence_block();
	uint2 result = shared_mem[(thread&~(c - 1)) + (b&(c - 1))];

	__threadfence_block();
	shared_mem[thread] = buf;

	__threadfence_block();
	return result;
}

__device__ __forceinline__ void WarpShuffle3(uint2 &a1, uint2 &a2, uint2 &a3, uint32_t b1, uint32_t b2, uint32_t b3, uint32_t c)
{
	extern __shared__ uint2 shared_mem[];

	const uint32_t thread = blockDim.x * threadIdx.y + threadIdx.x;

	__threadfence_block();
	uint2 buf = shared_mem[thread];

	shared_mem[thread] = a1;
	__threadfence_block();
	a1 = shared_mem[(thread&~(c - 1)) + (b1&(c - 1))];
	__threadfence_block();
	shared_mem[thread] = a2;
	__threadfence_block();
	a2 = shared_mem[(thread&~(c - 1)) + (b2&(c - 1))];
	__threadfence_block();
	shared_mem[thread] = a3;
	__threadfence_block();
	a3 = shared_mem[(thread&~(c - 1)) + (b3&(c - 1))];

	__threadfence_block();
	shared_mem[thread] = buf;
	__threadfence_block();
}

#endif

#if __CUDA_ARCH__ >= 300
static __device__ __forceinline__
void Gfunc(uint2 &a, uint2 &b, uint2 &c, uint2 &d)
{
	a += b; d ^= a; d = SWAPUINT2(d);
	c += d; b ^= c; b = ROR24(b); //ROR2(b, 24);
	a += b; d ^= a; d = ROR16(d);
	c += d; b ^= c; b = ROR2(b, 63);
}
#endif

__device__ __forceinline__ void round_lyra(uint2 s[4])
{
	Gfunc(s[0], s[1], s[2], s[3]);
	WarpShuffle3(s[1], s[2], s[3], threadIdx.x + 1, threadIdx.x + 2, threadIdx.x + 3, 4);
	Gfunc(s[0], s[1], s[2], s[3]);
	WarpShuffle3(s[1], s[2], s[3], threadIdx.x + 3, threadIdx.x + 2, threadIdx.x + 1, 4);
}

static __device__ __forceinline__
void round_lyra(uint2x4* s)
{
	Gfunc(s[0].x, s[1].x, s[2].x, s[3].x);
	Gfunc(s[0].y, s[1].y, s[2].y, s[3].y);
	Gfunc(s[0].z, s[1].z, s[2].z, s[3].z);
	Gfunc(s[0].w, s[1].w, s[2].w, s[3].w);
	Gfunc(s[0].x, s[1].y, s[2].z, s[3].w);
	Gfunc(s[0].y, s[1].z, s[2].w, s[3].x);
	Gfunc(s[0].z, s[1].w, s[2].x, s[3].y);
	Gfunc(s[0].w, s[1].x, s[2].y, s[3].z);
}

static __device__ __forceinline__
void reduceDuplexV5(uint2 state[4], const uint32_t thread, const uint32_t threads)
{
	uint2 state1[3], state2[3];

	const uint32_t ps0 = (memshift * Ncol * 0 * threads + thread)*blockDim.x + threadIdx.x;
	const uint32_t ps1 = (memshift * Ncol * 1 * threads + thread)*blockDim.x + threadIdx.x;
	const uint32_t ps2 = (memshift * Ncol * 2 * threads + thread)*blockDim.x + threadIdx.x;
	const uint32_t ps3 = (memshift * Ncol * 3 * threads + thread)*blockDim.x + threadIdx.x;
	const uint32_t ps4 = (memshift * Ncol * 4 * threads + thread)*blockDim.x + threadIdx.x;
	const uint32_t ps5 = (memshift * Ncol * 5 * threads + thread)*blockDim.x + threadIdx.x;
	const uint32_t ps6 = (memshift * Ncol * 6 * threads + thread)*blockDim.x + threadIdx.x;
	const uint32_t ps7 = (memshift * Ncol * 7 * threads + thread)*blockDim.x + threadIdx.x;

	for (int i = 0; i < 8; i++)
	{
		const uint32_t s0 = memshift * Ncol * 0 + (Ncol - 1 - i) * memshift;
		#pragma unroll
		for (int j = 0; j < 3; j++)
			ST4S(s0 + j, state[j]);
		round_lyra(state);
	}

	for (int i = 0; i < 8; i++)
	{
		const uint32_t s0 = memshift * Ncol * 0 + i * memshift;
		const uint32_t s1 = ps1 + (7 - i)*memshift* threads*blockDim.x;
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state1[j] = LD4S(s0 + j);
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state[j] ^= state1[j];

		round_lyra(state);

		#pragma unroll
		for (int j = 0; j < 3; j++)
			*(DMatrix + s1 + j*threads*blockDim.x) = state1[j] ^ state[j];
	}

	// 1, 0, 2
	for (int i = 0; i < 8; i++)
	{
		const uint32_t s0 = memshift * Ncol * 0 + i * memshift;
		const uint32_t s1 = ps1 + i * memshift* threads*blockDim.x;
		const uint32_t s2 = ps2 + (7 - i)*memshift* threads*blockDim.x;
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state1[j] = *(DMatrix + s1 + j*threads*blockDim.x);
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state2[j] = LD4S(s0 + j);
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state[j] ^= state1[j] + state2[j];

		round_lyra(state);

		#pragma unroll
		for (int j = 0; j < 3; j++)
			*(DMatrix + s2 + j*threads*blockDim.x) = state1[j] ^ state[j];

		// simultaneously receive data from preceding thread and send data to following thread
		uint2 Data0 = state[0];
		uint2 Data1 = state[1];
		uint2 Data2 = state[2];
		WarpShuffle3(Data0, Data1, Data2, threadIdx.x - 1, threadIdx.x - 1, threadIdx.x - 1, 4);

		if (threadIdx.x == 0)
		{
			state2[0] ^= Data2;
			state2[1] ^= Data0;
			state2[2] ^= Data1;
		}
		else
		{
			state2[0] ^= Data0;
			state2[1] ^= Data1;
			state2[2] ^= Data2;
		}

		#pragma unroll
		for (int j = 0; j < 3; j++)
			ST4S(s0 + j, state2[j]);
	}

	// 2, 1, 3
	for (int i = 0; i < 8; i++)
	{
		const uint32_t s1 = ps1 + i * memshift* threads*blockDim.x;
		const uint32_t s2 = ps2 + i * memshift* threads*blockDim.x;
		const uint32_t s3 = ps3 + (7 - i)*memshift* threads*blockDim.x;
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state1[j] = *(DMatrix + s2 + j*threads*blockDim.x);
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state2[j] = *(DMatrix + s1 + j*threads*blockDim.x);
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state[j] ^= state1[j] + state2[j];

		round_lyra(state);

		#pragma unroll
		for (int j = 0; j < 3; j++)
			*(DMatrix + s3 + j*threads*blockDim.x) = state1[j] ^ state[j];

		// simultaneously receive data from preceding thread and send data to following thread
		uint2 Data0 = state[0];
		uint2 Data1 = state[1];
		uint2 Data2 = state[2];
		WarpShuffle3(Data0, Data1, Data2, threadIdx.x - 1, threadIdx.x - 1, threadIdx.x - 1, 4);

		if (threadIdx.x == 0)
		{
			state2[0] ^= Data2;
			state2[1] ^= Data0;
			state2[2] ^= Data1;
		} else  {
			state2[0] ^= Data0;
			state2[1] ^= Data1;
			state2[2] ^= Data2;
		}

		#pragma unroll
		for (int j = 0; j < 3; j++)
			*(DMatrix + s1 + j*threads*blockDim.x) = state2[j];
	}

	// 3, 0, 4
	for (int i = 0; i < 8; i++)
	{
		const uint32_t ls0 = memshift * Ncol * 0 + i * memshift;
		const uint32_t s0 = ps0 + i * memshift* threads*blockDim.x;
		const uint32_t s3 = ps3 + i * memshift* threads*blockDim.x;
		const uint32_t s4 = ps4 + (7 - i)*memshift* threads*blockDim.x;
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state1[j] = *(DMatrix + s3 + j*threads*blockDim.x);
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state2[j] = LD4S(ls0 + j);
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state[j] ^= state1[j] + state2[j];

		round_lyra(state);

		#pragma unroll
		for (int j = 0; j < 3; j++)
			*(DMatrix + s4 + j*threads*blockDim.x) = state1[j] ^ state[j];

		// simultaneously receive data from preceding thread and send data to following thread
		uint2 Data0 = state[0];
		uint2 Data1 = state[1];
		uint2 Data2 = state[2];
		WarpShuffle3(Data0, Data1, Data2, threadIdx.x - 1, threadIdx.x - 1, threadIdx.x - 1, 4);

		if (threadIdx.x == 0)
		{
			state2[0] ^= Data2;
			state2[1] ^= Data0;
			state2[2] ^= Data1;
		} else {
			state2[0] ^= Data0;
			state2[1] ^= Data1;
			state2[2] ^= Data2;
		}

		#pragma unroll
		for (int j = 0; j < 3; j++)
			*(DMatrix + s0 + j*threads*blockDim.x) = state2[j];
	}

	// 4, 3, 5
	for (int i = 0; i < 8; i++)
	{
		const uint32_t s3 = ps3 + i * memshift* threads*blockDim.x;
		const uint32_t s4 = ps4 + i * memshift* threads*blockDim.x;
		const uint32_t s5 = ps5 + (7 - i)*memshift* threads*blockDim.x;
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state1[j] = *(DMatrix + s4 + j*threads*blockDim.x);
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state2[j] = *(DMatrix + s3 + j*threads*blockDim.x);
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state[j] ^= state1[j] + state2[j];

		round_lyra(state);

		#pragma unroll
		for (int j = 0; j < 3; j++)
			*(DMatrix + s5 + j*threads*blockDim.x) = state1[j] ^ state[j];

		// simultaneously receive data from preceding thread and send data to following thread
		uint2 Data0 = state[0];
		uint2 Data1 = state[1];
		uint2 Data2 = state[2];
		WarpShuffle3(Data0, Data1, Data2, threadIdx.x - 1, threadIdx.x - 1, threadIdx.x - 1, 4);

		if (threadIdx.x == 0)
		{
			state2[0] ^= Data2;
			state2[1] ^= Data0;
			state2[2] ^= Data1;
		}
		else
		{
			state2[0] ^= Data0;
			state2[1] ^= Data1;
			state2[2] ^= Data2;
		}

		#pragma unroll
		for (int j = 0; j < 3; j++)
			*(DMatrix + s3 + j*threads*blockDim.x) = state2[j];
	}

	// 5, 2, 6
	for (int i = 0; i < 8; i++)
	{
		const uint32_t s2 = ps2 + i * memshift* threads*blockDim.x;
		const uint32_t s5 = ps5 + i * memshift* threads*blockDim.x;
		const uint32_t s6 = ps6 + (7 - i)*memshift* threads*blockDim.x;
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state1[j] = *(DMatrix + s5 + j*threads*blockDim.x);
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state2[j] = *(DMatrix + s2 + j*threads*blockDim.x);
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state[j] ^= state1[j] + state2[j];

		round_lyra(state);

		#pragma unroll
		for (int j = 0; j < 3; j++)
			*(DMatrix + s6 + j*threads*blockDim.x) = state1[j] ^ state[j];

		// simultaneously receive data from preceding thread and send data to following thread
		uint2 Data0 = state[0];
		uint2 Data1 = state[1];
		uint2 Data2 = state[2];
		WarpShuffle3(Data0, Data1, Data2, threadIdx.x - 1, threadIdx.x - 1, threadIdx.x - 1, 4);

		if (threadIdx.x == 0)
		{
			state2[0] ^= Data2;
			state2[1] ^= Data0;
			state2[2] ^= Data1;
		}
		else
		{
			state2[0] ^= Data0;
			state2[1] ^= Data1;
			state2[2] ^= Data2;
		}

		#pragma unroll
		for (int j = 0; j < 3; j++)
			*(DMatrix + s2 + j*threads*blockDim.x) = state2[j];
	}

	// 6, 1, 7
	for (int i = 0; i < 8; i++)
	{
		const uint32_t s1 = ps1 + i * memshift* threads*blockDim.x;
		const uint32_t s6 = ps6 + i * memshift* threads*blockDim.x;
		const uint32_t s7 = ps7 + (7 - i)*memshift* threads*blockDim.x;
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state1[j] = *(DMatrix + s6 + j*threads*blockDim.x);
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state2[j] = *(DMatrix + s1 + j*threads*blockDim.x);
		#pragma unroll
		for (int j = 0; j < 3; j++)
			state[j] ^= state1[j] + state2[j];

		round_lyra(state);

		#pragma unroll
		for (int j = 0; j < 3; j++)
			*(DMatrix + s7 + j*threads*blockDim.x) = state1[j] ^ state[j];

		// simultaneously receive data from preceding thread and send data to following thread
		uint2 Data0 = state[0];
		uint2 Data1 = state[1];
		uint2 Data2 = state[2];
		WarpShuffle3(Data0, Data1, Data2, threadIdx.x - 1, threadIdx.x - 1, threadIdx.x - 1, 4);

		if (threadIdx.x == 0)
		{
			state2[0] ^= Data2;
			state2[1] ^= Data0;
			state2[2] ^= Data1;
		} else {
			state2[0] ^= Data0;
			state2[1] ^= Data1;
			state2[2] ^= Data2;
		}

		#pragma unroll
		for (int j = 0; j < 3; j++)
			*(DMatrix + s1 + j*threads*blockDim.x) = state2[j];
	}
}

static __device__ __forceinline__
void reduceDuplexRowV50(const int rowIn, const int rowInOut, const int rowOut, uint2 state[4], const uint32_t thread, const uint32_t threads)
{
	const uint32_t ps1 = (memshift * Ncol * rowIn*threads + thread)*blockDim.x + threadIdx.x;
	const uint32_t ps2 = (memshift * Ncol * rowInOut *threads + thread)*blockDim.x + threadIdx.x;
	const uint32_t ps3 = (memshift * Ncol * rowOut*threads + thread)*blockDim.x + threadIdx.x;

	#pragma unroll 1
	for (int i = 0; i < 8; i++)
	{
		uint2 state1[3], state2[3];

		const uint32_t s1 = ps1 + i*memshift*threads *blockDim.x;
		const uint32_t s2 = ps2 + i*memshift*threads *blockDim.x;
		const uint32_t s3 = ps3 + i*memshift*threads *blockDim.x;

		#pragma unroll
		for (int j = 0; j < 3; j++) {
			state1[j] = *(DMatrix + s1 + j*threads*blockDim.x);
			state2[j] = *(DMatrix + s2 + j*threads*blockDim.x);
		}

		#pragma unroll
		for (int j = 0; j < 3; j++) {
			state1[j] += state2[j];
			state[j] ^= state1[j];
		}

		round_lyra(state);

		// simultaneously receive data from preceding thread and send data to following thread
		uint2 Data0 = state[0];
		uint2 Data1 = state[1];
		uint2 Data2 = state[2];
		WarpShuffle3(Data0, Data1, Data2, threadIdx.x - 1, threadIdx.x - 1, threadIdx.x - 1, 4);

		if (threadIdx.x == 0)
		{
			state2[0] ^= Data2;
			state2[1] ^= Data0;
			state2[2] ^= Data1;
		} else {
			state2[0] ^= Data0;
			state2[1] ^= Data1;
			state2[2] ^= Data2;
		}

		#pragma unroll
		for (int j = 0; j < 3; j++)
		{
			*(DMatrix + s2 + j*threads*blockDim.x) = state2[j];
			*(DMatrix + s3 + j*threads*blockDim.x) ^= state[j];
		}
	}
}

static __device__ __forceinline__
void reduceDuplexRowV50_8(const int rowInOut, uint2 state[4], const uint32_t thread, const uint32_t threads)
{
	const uint32_t ps1 = (memshift * Ncol * 2*threads + thread)*blockDim.x + threadIdx.x;
	const uint32_t ps2 = (memshift * Ncol * rowInOut *threads + thread)*blockDim.x + threadIdx.x;
	// const uint32_t ps3 = (memshift * Ncol * 5*threads + thread)*blockDim.x + threadIdx.x;

	uint2 state1[3], last[3];

	#pragma unroll
	for (int j = 0; j < 3; j++) {
		state1[j] = *(DMatrix + ps1 + j*threads*blockDim.x);
		last[j] = *(DMatrix + ps2 + j*threads*blockDim.x);
	}

	#pragma unroll
	for (int j = 0; j < 3; j++) {
		state1[j] += last[j];
		state[j] ^= state1[j];
	}

	round_lyra(state);

	// simultaneously receive data from preceding thread and send data to following thread
	uint2 Data0 = state[0];
	uint2 Data1 = state[1];
	uint2 Data2 = state[2];
	WarpShuffle3(Data0, Data1, Data2, threadIdx.x - 1, threadIdx.x - 1, threadIdx.x - 1, 4);

	if (threadIdx.x == 0)
	{
		last[0] ^= Data2;
		last[1] ^= Data0;
		last[2] ^= Data1;
	} else {
		last[0] ^= Data0;
		last[1] ^= Data1;
		last[2] ^= Data2;
	}

	if (rowInOut == 5)
	{
		#pragma unroll
		for (int j = 0; j < 3; j++)
			last[j] ^= state[j];
	}

	for (int i = 1; i < 8; i++)
	{
		const uint32_t s1 = ps1 + i*memshift*threads *blockDim.x;
		const uint32_t s2 = ps2 + i*memshift*threads *blockDim.x;

		#pragma unroll
		for (int j = 0; j < 3; j++)
			state[j] ^= *(DMatrix + s1 + j*threads*blockDim.x) + *(DMatrix + s2 + j*threads*blockDim.x);

		round_lyra(state);
	}


#pragma unroll
	for (int j = 0; j < 3; j++)
		state[j] ^= last[j];

}

__global__ __launch_bounds__(64, 1)
void lyra2_gpu_hash_32_1_sm5(uint32_t threads, uint2 *g_hash)
{
	const uint2x4 blake2b_IV[2] = {
		{ { 0xf3bcc908, 0x6a09e667 }, { 0x84caa73b, 0xbb67ae85 }, { 0xfe94f82b, 0x3c6ef372 }, { 0x5f1d36f1, 0xa54ff53a } },
		{ { 0xade682d1, 0x510e527f }, { 0x2b3e6c1f, 0x9b05688c }, { 0xfb41bd6b, 0x1f83d9ab }, { 0x137e2179, 0x5be0cd19 } }
	};

	const uint32_t thread = (blockDim.x * blockIdx.x + threadIdx.x);
	if (thread < threads)
	{
		uint2x4 state[4];

		((uint2*)state)[0] = __ldg(&g_hash[thread]);
		((uint2*)state)[1] = __ldg(&g_hash[thread + threads]);
		((uint2*)state)[2] = __ldg(&g_hash[thread + threads * 2]);
		((uint2*)state)[3] = __ldg(&g_hash[thread + threads * 3]);

		state[1] = state[0];
		state[2] = blake2b_IV[0];
		state[3] = blake2b_IV[1];

		for (int i = 0; i < 24; i++)
			round_lyra(state); //because 12 is not enough

		((uint2x4*)DMatrix)[0 * threads + thread] = state[0];
		((uint2x4*)DMatrix)[1 * threads + thread] = state[1];
		((uint2x4*)DMatrix)[2 * threads + thread] = state[2];
		((uint2x4*)DMatrix)[3 * threads + thread] = state[3];
	}
}

__global__ __launch_bounds__(TPB50, 1)
void lyra2_gpu_hash_32_2_sm5(uint32_t threads, uint2 *g_hash)
{
	const uint32_t thread = (blockDim.y * blockIdx.x + threadIdx.y);

	if (thread < threads)
	{
		uint2 state[4];
		state[0] = __ldg(&DMatrix[(0 * threads + thread)*blockDim.x + threadIdx.x]);
		state[1] = __ldg(&DMatrix[(1 * threads + thread)*blockDim.x + threadIdx.x]);
		state[2] = __ldg(&DMatrix[(2 * threads + thread)*blockDim.x + threadIdx.x]);
		state[3] = __ldg(&DMatrix[(3 * threads + thread)*blockDim.x + threadIdx.x]);

		reduceDuplexV5(state, thread, threads);

		uint32_t rowa = WarpShuffle(state[0].x, 0, 4) & 7;
		reduceDuplexRowV50(7, rowa, 0, state, thread, threads);
		rowa = WarpShuffle(state[0].x, 0, 4) & 7;
		reduceDuplexRowV50(0, rowa, 3, state, thread, threads);
		rowa = WarpShuffle(state[0].x, 0, 4) & 7;
		reduceDuplexRowV50(3, rowa, 6, state, thread, threads);
		rowa = WarpShuffle(state[0].x, 0, 4) & 7;
		reduceDuplexRowV50(6, rowa, 1, state, thread, threads);
		rowa = WarpShuffle(state[0].x, 0, 4) & 7;
		reduceDuplexRowV50(1, rowa, 4, state, thread, threads);
		rowa = WarpShuffle(state[0].x, 0, 4) & 7;
		reduceDuplexRowV50(4, rowa, 7, state, thread, threads);
		rowa = WarpShuffle(state[0].x, 0, 4) & 7;
		reduceDuplexRowV50(7, rowa, 2, state, thread, threads);
		rowa = WarpShuffle(state[0].x, 0, 4) & 7;
		reduceDuplexRowV50_8(rowa, state, thread, threads);

		DMatrix[(0 * threads + thread)*blockDim.x + threadIdx.x] = state[0];
		DMatrix[(1 * threads + thread)*blockDim.x + threadIdx.x] = state[1];
		DMatrix[(2 * threads + thread)*blockDim.x + threadIdx.x] = state[2];
		DMatrix[(3 * threads + thread)*blockDim.x + threadIdx.x] = state[3];
	}
}

__global__ __launch_bounds__(64, 1)
void lyra2_gpu_hash_32_3_sm5(uint32_t threads, uint2 *g_hash)
{
	const uint32_t thread = (blockDim.x * blockIdx.x + threadIdx.x);

	if (thread < threads)
	{
		uint2x4 state[4];
		state[0] = __ldg4(&((uint2x4*)DMatrix)[0 * threads + thread]);
		state[1] = __ldg4(&((uint2x4*)DMatrix)[1 * threads + thread]);
		state[2] = __ldg4(&((uint2x4*)DMatrix)[2 * threads + thread]);
		state[3] = __ldg4(&((uint2x4*)DMatrix)[3 * threads + thread]);

		for (int i = 0; i < 12; i++)
			round_lyra(state);

		g_hash[thread] = ((uint2*)state)[0];
		g_hash[thread + threads] = ((uint2*)state)[1];
		g_hash[thread + threads * 2] = ((uint2*)state)[2];
		g_hash[thread + threads * 3] = ((uint2*)state)[3];
	}
}

__global__ __launch_bounds__(64, 1)
void lyra2_gpu_hash_64_1_sm5(uint32_t threads, uint2* const d_hash_512, const uint32_t round)
{
	const uint2x4 blake2b_IV[2] = {
		{ { 0xf3bcc908, 0x6a09e667 }, { 0x84caa73b, 0xbb67ae85 }, { 0xfe94f82b, 0x3c6ef372 }, { 0x5f1d36f1, 0xa54ff53a } },
		{ { 0xade682d1, 0x510e527f }, { 0x2b3e6c1f, 0x9b05688c }, { 0xfb41bd6b, 0x1f83d9ab }, { 0x137e2179, 0x5be0cd19 } }
	};
	// This kernel loads 2x 256-bits hashes from 512-bits chain offsets in 2 steps
	const uint32_t thread = (blockDim.x * blockIdx.x + threadIdx.x);
	if (thread < threads)
	{
		uint2x4 state[4];
		const size_t offset = (size_t)8 * thread + (round * 4U);
		uint2 *psrc = (uint2*)(&d_hash_512[offset]);
		state[0].x = state[1].x = __ldg(&psrc[0]);
		state[0].y = state[1].y = __ldg(&psrc[1]);
		state[0].z = state[1].z = __ldg(&psrc[2]);
		state[0].w = state[1].w = __ldg(&psrc[3]);

		state[1] = state[0];
		state[2] = blake2b_IV[0];
		state[3] = blake2b_IV[1];

		for (int i = 0; i<24; i++)
			round_lyra(state);

		((uint2x4*)DMatrix)[threads * 0 + thread] = state[0];
		((uint2x4*)DMatrix)[threads * 1 + thread] = state[1];
		((uint2x4*)DMatrix)[threads * 2 + thread] = state[2];
		((uint2x4*)DMatrix)[threads * 3 + thread] = state[3];
	}
}

__global__ __launch_bounds__(64, 1)
void lyra2_gpu_hash_64_3_sm5(uint32_t threads, uint2 *d_hash_512, const uint32_t round)
{
	// This kernel outputs 2x 256-bits hashes in 512-bits chain offsets in 2 steps
	const uint32_t thread = blockDim.x * blockIdx.x + threadIdx.x;
	if (thread < threads)
	{
		uint2x4 state[4];
		state[0] = __ldg4(&((uint2x4*)DMatrix)[threads * 0 + thread]);
		state[1] = __ldg4(&((uint2x4*)DMatrix)[threads * 1 + thread]);
		state[2] = __ldg4(&((uint2x4*)DMatrix)[threads * 2 + thread]);
		state[3] = __ldg4(&((uint2x4*)DMatrix)[threads * 3 + thread]);

		for (int i = 0; i < 12; i++)
			round_lyra(state);

		const size_t offset = (size_t)8 * thread + (round * 4U);
		uint2 *pdst = (uint2*)(&d_hash_512[offset]);
		pdst[0] = state[0].x;
		pdst[1] = state[0].y;
		pdst[2] = state[0].z;
		pdst[3] = state[0].w;
	}
}
#else
/* if __CUDA_ARCH__ != 500 .. host */
__global__ void lyra2_gpu_hash_32_1_sm5(uint32_t threads, uint2 *g_hash) {}
__global__ void lyra2_gpu_hash_32_2_sm5(uint32_t threads, uint2 *g_hash) {}
__global__ void lyra2_gpu_hash_32_3_sm5(uint32_t threads, uint2 *g_hash) {}
__global__ void lyra2_gpu_hash_64_1_sm5(uint32_t threads, uint2* const d_hash_512, const uint32_t round) {}
__global__ void lyra2_gpu_hash_64_3_sm5(uint32_t threads, uint2 *d_hash_512, const uint32_t round) {}
#endif