You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
161 lines
5.8 KiB
161 lines
5.8 KiB
#ifndef CUDA_HELPER_H |
|
#define CUDA_HELPER_H |
|
|
|
#include <cuda.h> |
|
#include <cuda_runtime.h> |
|
|
|
#if defined(_MSC_VER) |
|
/* reduce warnings */ |
|
#include <device_functions.h> |
|
#include <device_launch_parameters.h> |
|
#endif |
|
|
|
#include <stdint.h> |
|
|
|
extern __device__ __device_builtin__ void __syncthreads(void); |
|
|
|
#ifndef __CUDA_ARCH__ |
|
// define blockDim and threadIdx for host |
|
extern const dim3 blockDim; |
|
extern const uint3 threadIdx; |
|
#endif |
|
|
|
#ifndef SPH_C32 |
|
#define SPH_C32(x) ((uint32_t)(x ## U)) |
|
#endif |
|
|
|
#ifndef SPH_C64 |
|
#define SPH_C64(x) ((uint64_t)(x ## ULL)) |
|
#endif |
|
|
|
#define SPH_T32(x) ((x) & SPH_C32(0xFFFFFFFF)) |
|
|
|
#if __CUDA_ARCH__ < 350 |
|
// Kepler (Compute 3.0) |
|
#define ROTL32(x, n) SPH_T32(((x) << (n)) | ((x) >> (32 - (n)))) |
|
#else |
|
// Kepler (Compute 3.5, 5.0) |
|
#define ROTL32(x, n) __funnelshift_l( (x), (x), (n) ) |
|
#endif |
|
|
|
__device__ __forceinline__ unsigned long long MAKE_ULONGLONG(uint32_t LO, uint32_t HI) |
|
{ |
|
#if __CUDA_ARCH__ >= 130 |
|
return __double_as_longlong(__hiloint2double(HI, LO)); |
|
#else |
|
return (unsigned long long)LO | (((unsigned long long)HI) << 32); |
|
#endif |
|
} |
|
|
|
// das Hi Word in einem 64 Bit Typen ersetzen |
|
__device__ __forceinline__ uint64_t REPLACE_HIWORD(const uint64_t &x, const uint32_t &y) { |
|
return (x & 0xFFFFFFFFULL) | (((uint64_t)y) << 32ULL); |
|
} |
|
|
|
// das Lo Word in einem 64 Bit Typen ersetzen |
|
__device__ __forceinline__ uint64_t REPLACE_LOWORD(const uint64_t &x, const uint32_t &y) { |
|
return (x & 0xFFFFFFFF00000000ULL) | ((uint64_t)y); |
|
} |
|
|
|
// Endian Drehung für 32 Bit Typen |
|
#ifdef __CUDA_ARCH__ |
|
__device__ __forceinline__ uint32_t cuda_swab32(uint32_t x) |
|
{ |
|
/* device */ |
|
return __byte_perm(x, x, 0x0123); |
|
} |
|
#else |
|
/* host */ |
|
#define cuda_swab32(x) \ |
|
((((x) << 24) & 0xff000000u) | (((x) << 8) & 0x00ff0000u) | \ |
|
(((x) >> 8) & 0x0000ff00u) | (((x) >> 24) & 0x000000ffu)) |
|
#endif |
|
|
|
// das Lo Word aus einem 64 Bit Typen extrahieren |
|
__device__ __forceinline__ uint32_t _LOWORD(const uint64_t &x) { |
|
#if __CUDA_ARCH__ >= 130 |
|
return (uint32_t)__double2loint(__longlong_as_double(x)); |
|
#else |
|
return (uint32_t)(x & 0xFFFFFFFFULL); |
|
#endif |
|
} |
|
|
|
// das Hi Word aus einem 64 Bit Typen extrahieren |
|
__device__ __forceinline__ uint32_t _HIWORD(const uint64_t &x) { |
|
#if __CUDA_ARCH__ >= 130 |
|
return (uint32_t)__double2hiint(__longlong_as_double(x)); |
|
#else |
|
return (uint32_t)(x >> 32); |
|
#endif |
|
} |
|
|
|
#ifdef __CUDA_ARCH__ |
|
__device__ __forceinline__ uint64_t cuda_swab64(uint64_t x) |
|
{ |
|
// Input: 77665544 33221100 |
|
// Output: 00112233 44556677 |
|
uint64_t temp[2]; |
|
temp[0] = __byte_perm(_HIWORD(x), 0, 0x0123); |
|
temp[1] = __byte_perm(_LOWORD(x), 0, 0x0123); |
|
|
|
return temp[0] | (temp[1]<<32); |
|
} |
|
#else |
|
/* host */ |
|
#define cuda_swab64(x) \ |
|
((uint64_t)((((uint64_t)(x) & 0xff00000000000000ULL) >> 56) | \ |
|
(((uint64_t)(x) & 0x00ff000000000000ULL) >> 40) | \ |
|
(((uint64_t)(x) & 0x0000ff0000000000ULL) >> 24) | \ |
|
(((uint64_t)(x) & 0x000000ff00000000ULL) >> 8) | \ |
|
(((uint64_t)(x) & 0x00000000ff000000ULL) << 8) | \ |
|
(((uint64_t)(x) & 0x0000000000ff0000ULL) << 24) | \ |
|
(((uint64_t)(x) & 0x000000000000ff00ULL) << 40) | \ |
|
(((uint64_t)(x) & 0x00000000000000ffULL) << 56))) |
|
#endif |
|
|
|
// diese 64 Bit Rotates werden unter Compute 3.5 (und besser) mit dem Funnel Shifter beschleunigt |
|
#if __CUDA_ARCH__ >= 350 |
|
__device__ __forceinline__ uint64_t ROTR64(const uint64_t value, const int offset) { |
|
uint2 result; |
|
if(offset < 32) { |
|
asm("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(result.x) : "r"(__double2loint(__longlong_as_double(value))), "r"(__double2hiint(__longlong_as_double(value))), "r"(offset)); |
|
asm("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(result.y) : "r"(__double2hiint(__longlong_as_double(value))), "r"(__double2loint(__longlong_as_double(value))), "r"(offset)); |
|
} else { |
|
asm("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(result.x) : "r"(__double2hiint(__longlong_as_double(value))), "r"(__double2loint(__longlong_as_double(value))), "r"(offset)); |
|
asm("shf.r.wrap.b32 %0, %1, %2, %3;" : "=r"(result.y) : "r"(__double2loint(__longlong_as_double(value))), "r"(__double2hiint(__longlong_as_double(value))), "r"(offset)); |
|
} |
|
return __double_as_longlong(__hiloint2double(result.y, result.x)); |
|
} |
|
#else |
|
#define ROTR64(x, n) (((x) >> (n)) | ((x) << (64 - (n)))) |
|
#endif |
|
|
|
// diese 64 Bit Rotates werden unter Compute 3.5 (und besser) mit dem Funnel Shifter beschleunigt |
|
#if __CUDA_ARCH__ >= 350 |
|
__device__ __forceinline__ uint64_t ROTL64(const uint64_t value, const int offset) { |
|
uint2 result; |
|
if(offset >= 32) { |
|
asm("shf.l.wrap.b32 %0, %1, %2, %3;" : "=r"(result.x) : "r"(__double2loint(__longlong_as_double(value))), "r"(__double2hiint(__longlong_as_double(value))), "r"(offset)); |
|
asm("shf.l.wrap.b32 %0, %1, %2, %3;" : "=r"(result.y) : "r"(__double2hiint(__longlong_as_double(value))), "r"(__double2loint(__longlong_as_double(value))), "r"(offset)); |
|
} else { |
|
asm("shf.l.wrap.b32 %0, %1, %2, %3;" : "=r"(result.x) : "r"(__double2hiint(__longlong_as_double(value))), "r"(__double2loint(__longlong_as_double(value))), "r"(offset)); |
|
asm("shf.l.wrap.b32 %0, %1, %2, %3;" : "=r"(result.y) : "r"(__double2loint(__longlong_as_double(value))), "r"(__double2hiint(__longlong_as_double(value))), "r"(offset)); |
|
} |
|
return __double_as_longlong(__hiloint2double(result.y, result.x)); |
|
} |
|
#else |
|
#define ROTL64(x, n) (((x) << (n)) | ((x) >> (64 - (n)))) |
|
#endif |
|
|
|
// Macro to catch CUDA errors in CUDA runtime calls |
|
#define CUDA_SAFE_CALL(call) \ |
|
do { \ |
|
cudaError_t err = call; \ |
|
if (cudaSuccess != err) { \ |
|
fprintf (stderr, "Cuda error in file '%s' in line %i : %s.\n",\ |
|
__FILE__, __LINE__, cudaGetErrorString(err) ); \ |
|
exit(EXIT_FAILURE); \ |
|
} \ |
|
} while (0) |
|
|
|
#endif // #ifndef CUDA_HELPER_H
|
|
|