// Auto-generated by EmbedCudaKernels.cmake.  Do not edit.
namespace cuda_kernels {

// from compute_sp.cu
const char *compute_sp_name = "compute_sp.cu";
const char *compute_sp_src = R"CLPEAK_KDELIM(
// Single-precision MAD-chain throughput.  Structure mirrors
// src/shaders/compute_sp_v1.comp -- ping-pong x/y feedback so the compiler
// can't hoist invariants and so each FMA depends on the previous, exposing
// ILP across the two parallel accumulator chains.
//
// 128 outer iters * 16 FMAs * 2 ops = 4096 ops per thread (= COMPUTE_FP_WORK_PER_WI).

#define MAD_4(x, y)  x = fmaf(y, x, y); y = fmaf(x, y, x); x = fmaf(y, x, y); y = fmaf(x, y, x);
#define MAD_16(x, y) MAD_4(x, y) MAD_4(x, y) MAD_4(x, y) MAD_4(x, y)

extern "C" __global__ void compute_sp(float *out, float A)
{
    float x = A;
    float y = (float)threadIdx.x;

    #pragma unroll 1
    for (int i = 0; i < 128; i++)
    {
        MAD_16(x, y)
    }

    out[blockIdx.x * blockDim.x + threadIdx.x] = y;
}
)CLPEAK_KDELIM";

// from compute_hp.cu
const char *compute_hp_name = "compute_hp.cu";
const char *compute_hp_src = R"CLPEAK_KDELIM(
// FP16 MAD-chain throughput.  Two variants:
//   compute_hp   -- scalar __half FMA, 2 parallel chains
//   compute_hp2  -- __half2 packed HFMA2 (NVIDIA's 2x FP32 fp16 peak from
//                   sm_53+; one instruction issues two fp16 FMAs/cycle).
//
// compute_hp uses 2 parallel (x,y) chains (rather than the 1 chain in the
// SP/MP/BF16 kernels) because RTX 5060 single-chain scalar __hfma
// measured at 60% of compute_sp -- the scalar HFMA pipe is latency-bound
// at 1 chain.  Confirmed empirically: 1 chain hit 10.6 TFLOPS, 2 chains
// hit 21.1 TFLOPS, matching the half2 packed peak.
//
// Op accounting matches compute_hp.cu prior version: 4096 fp16 ops/thread
// either way.  hp scalar = 64 outer * 16 FMAs * 2 chains * 2 ops; hp2 =
// 64 outer * 16 HFMA2 * 4 ops.

#include <cuda_fp16.h>

#define MAD_4(x, y)  x = __hfma(y, x, y); y = __hfma(x, y, x); x = __hfma(y, x, y); y = __hfma(x, y, x);
#define MAD_16(x, y) MAD_4(x, y) MAD_4(x, y) MAD_4(x, y) MAD_4(x, y)

extern "C" __global__ void compute_hp(float *out, float A)
{
    __half x0 = __float2half(A);
    __half y0 = __float2half((float)threadIdx.x);
    __half x1 = __float2half(A + 1.0f);
    __half y1 = __float2half((float)threadIdx.x + 7.0f);

    #pragma unroll 1
    for (int i = 0; i < 64; i++)
    {
        MAD_16(x0, y0)
        MAD_16(x1, y1)
    }

    out[blockIdx.x * blockDim.x + threadIdx.x] =
        __half2float(__hadd(__hadd(x0, y0), __hadd(x1, y1)));
}

#define MAD2_4(x, y)  x = __hfma2(y, x, y); y = __hfma2(x, y, x); x = __hfma2(y, x, y); y = __hfma2(x, y, x);
#define MAD2_16(x, y) MAD2_4(x, y) MAD2_4(x, y) MAD2_4(x, y) MAD2_4(x, y)

extern "C" __global__ void compute_hp2(float *out, float A)
{
    __half2 x = __float2half2_rn(A);
    __half2 y = __float2half2_rn((float)threadIdx.x);

    #pragma unroll 1
    for (int i = 0; i < 64; i++)
    {
        MAD2_16(x, y)
    }

    __half2 r = __hadd2(x, y);
    out[blockIdx.x * blockDim.x + threadIdx.x] =
        __half2float(__low2half(r)) + __half2float(__high2half(r));
}
)CLPEAK_KDELIM";

// from compute_dp.cu
const char *compute_dp_name = "compute_dp.cu";
const char *compute_dp_src = R"CLPEAK_KDELIM(
// Double-precision MAD-chain throughput.  Identical shape to compute_sp.cu
// with double / fma.  RTX consumer cards have a 1:32 fp64 ratio; expect
// numbers ~3% of the SP peak.

#define MAD_4(x, y)  x = fma(y, x, y); y = fma(x, y, x); x = fma(y, x, y); y = fma(x, y, x);
#define MAD_16(x, y) MAD_4(x, y) MAD_4(x, y) MAD_4(x, y) MAD_4(x, y)

extern "C" __global__ void compute_dp(double *out, double A)
{
    double x = A;
    double y = (double)threadIdx.x;

    #pragma unroll 1
    for (int i = 0; i < 128; i++)
    {
        MAD_16(x, y)
    }

    out[blockIdx.x * blockDim.x + threadIdx.x] = y;
}
)CLPEAK_KDELIM";

// from compute_mp.cu
const char *compute_mp_name = "compute_mp.cu";
const char *compute_mp_src = R"CLPEAK_KDELIM(
// Mixed-precision MAC: fp16 inputs, fp32 accumulator.  Mirrors
// src/shaders/compute_mp_v1.comp's structure -- the inner FMA chain is
// pure FP32 (because fp16xfp16+fp32 emits FFMA on NVIDIA shader cores at
// FP32 issue rate); the fp16 cast happens once per outer iter so the data
// path actually exercises the fp16 conversion units.
//
// Earlier sketch with __float2half / __half2float inside the FMA macro
// chained 2-3 conversions onto the critical path of every FMA -- that is
// exactly the lowering the Vulkan MP shader was rewritten to avoid (commit
// f6ea4c4); the same fix applies here.

#include <cuda_fp16.h>

#define MAD_4(x, y)  x = fmaf(y, x, y); y = fmaf(x, y, x); x = fmaf(y, x, y); y = fmaf(x, y, x);
#define MAD_16(x, y) MAD_4(x, y) MAD_4(x, y) MAD_4(x, y) MAD_4(x, y)

extern "C" __global__ void compute_mp(float *out, float A)
{
    // Roundtrip through __half once at init + once per outer iter to force
    // the fp16 type into the data path while keeping the inner loop FFMA-only.
    float x = __half2float(__float2half(A));
    float y = __half2float(__float2half((float)threadIdx.x));

    #pragma unroll 1
    for (int i = 0; i < 128; i++)
    {
        MAD_16(x, y)
        x = __half2float(__float2half(x));
        y = __half2float(__float2half(y));
    }

    out[blockIdx.x * blockDim.x + threadIdx.x] = x + y;
}
)CLPEAK_KDELIM";

// from compute_bf16.cu
const char *compute_bf16_name = "compute_bf16.cu";
const char *compute_bf16_src = R"CLPEAK_KDELIM(
// BF16 x BF16 + FP32 shader-core MAD path.  Same shape as compute_mp.cu but
// with __nv_bfloat16 cast instead of __half.  Mirrors compute_bf16_v1.comp:
// deeper inner FMA chain (MAD_128 instead of MAD_16) because bf16<->fp32
// casts are emulated on some drivers and we need to amortise their cost.
//
// 16 outer iters * 128 FMAs * 2 ops = 4096 ops per thread (= COMPUTE_FP_WORK_PER_WI).

#include <cuda_bf16.h>

#define MAD_4(x, y)   x = fmaf(y, x, y); y = fmaf(x, y, x); x = fmaf(y, x, y); y = fmaf(x, y, x);
#define MAD_16(x, y)  MAD_4(x, y) MAD_4(x, y) MAD_4(x, y) MAD_4(x, y)
#define MAD_64(x, y)  MAD_16(x, y) MAD_16(x, y) MAD_16(x, y) MAD_16(x, y)
#define MAD_128(x, y) MAD_64(x, y) MAD_64(x, y)

extern "C" __global__ void compute_bf16(float *out, float A)
{
    float x = __bfloat162float(__float2bfloat16(A));
    float y = __bfloat162float(__float2bfloat16((float)threadIdx.x));

    #pragma unroll 1
    for (int i = 0; i < 16; i++)
    {
        MAD_128(x, y)
        x = __bfloat162float(__float2bfloat16(x));
        y = __bfloat162float(__float2bfloat16(y));
    }

    out[blockIdx.x * blockDim.x + threadIdx.x] = x + y;
}
)CLPEAK_KDELIM";

// from compute_int8_dp.cu
const char *compute_int8_dp_name = "compute_int8_dp.cu";
const char *compute_int8_dp_src = R"CLPEAK_KDELIM(
// INT8 dot-product throughput via __dp4a -- the canonical NVIDIA
// shader-core INT8 path on Pascal+ (sm_61+).  Each __dp4a call performs
// dot(char4, char4) into an int32 accumulator: 4 INT8 multiply-adds
// = 8 INT8 ops.
//
// Four variants for ILP scaling:
//   compute_int8_dp   -- 1 dependent chain (issue-rate floor)
//   compute_int8_dp2  -- 2 independent chains
//   compute_int8_dp4  -- 4 independent chains (matches Vulkan int8_dp4)
//   compute_int8_dp8  -- 8 independent chains.  Probed whether v4's
//                        plateau was chain-count bound or hardware
//                        pinned: v8 measured at the same ~34 TIOPS,
//                        confirming __dp4a is hardware-capped at ~half
//                        of theoretical SP rate on consumer Blackwell.
//                        The variant is kept because the v1..v8 series
//                        is itself the documentation of that ceiling.
//
// Op accounting: 8192 ops/thread = 1024 dp4a calls.  v1 = 64*16, v2 = each
// chain does 32*16 = 512 calls => 1024 total per thread.  v4 = each chain
// does 16*16 = 256 => 1024 total.

#define STEP(x, y, a)    a = __dp4a(x, y, a); y ^= a;
#define STEP_4(x, y, a)  STEP(x, y, a) STEP(x, y, a) STEP(x, y, a) STEP(x, y, a)
#define STEP_16(x, y, a) STEP_4(x, y, a) STEP_4(x, y, a) STEP_4(x, y, a) STEP_4(x, y, a)

extern "C" __global__ void compute_int8_dp(int *out, int A)
{
    int x = (A & 0xff)
          | (((A + 1) & 0xff) << 8)
          | (((A + 2) & 0xff) << 16)
          | (((A + 3) & 0xff) << 24);
    int y = (int)threadIdx.x;
    int a = (int)threadIdx.x;

    #pragma unroll 1
    for (int i = 0; i < 64; i++)
    {
        STEP_16(x, y, a)
    }

    out[blockIdx.x * blockDim.x + threadIdx.x] = a;
}

extern "C" __global__ void compute_int8_dp2(int *out, int A)
{
    int x  = (A & 0xff) | (((A+1)&0xff)<<8) | (((A+2)&0xff)<<16) | (((A+3)&0xff)<<24);
    int y0 = (int)threadIdx.x;
    int y1 = (int)threadIdx.x + 1;
    int a0 = (int)threadIdx.x;
    int a1 = (int)threadIdx.x + 7;

    #pragma unroll 1
    for (int i = 0; i < 32; i++)
    {
        STEP_16(x, y0, a0)
        STEP_16(x, y1, a1)
    }

    out[blockIdx.x * blockDim.x + threadIdx.x] = a0 + a1;
}

extern "C" __global__ void compute_int8_dp4(int *out, int A)
{
    int x  = (A & 0xff) | (((A+1)&0xff)<<8) | (((A+2)&0xff)<<16) | (((A+3)&0xff)<<24);
    int y0 = (int)threadIdx.x;
    int y1 = (int)threadIdx.x + 1;
    int y2 = (int)threadIdx.x + 2;
    int y3 = (int)threadIdx.x + 3;
    int a0 = (int)threadIdx.x;
    int a1 = (int)threadIdx.x + 7;
    int a2 = (int)threadIdx.x + 13;
    int a3 = (int)threadIdx.x + 19;

    #pragma unroll 1
    for (int i = 0; i < 16; i++)
    {
        STEP_16(x, y0, a0)
        STEP_16(x, y1, a1)
        STEP_16(x, y2, a2)
        STEP_16(x, y3, a3)
    }

    out[blockIdx.x * blockDim.x + threadIdx.x] = a0 + a1 + a2 + a3;
}

extern "C" __global__ void compute_int8_dp8(int *out, int A)
{
    int x  = (A & 0xff) | (((A+1)&0xff)<<8) | (((A+2)&0xff)<<16) | (((A+3)&0xff)<<24);
    int y0 = (int)threadIdx.x;
    int y1 = (int)threadIdx.x + 1;
    int y2 = (int)threadIdx.x + 2;
    int y3 = (int)threadIdx.x + 3;
    int y4 = (int)threadIdx.x + 4;
    int y5 = (int)threadIdx.x + 5;
    int y6 = (int)threadIdx.x + 6;
    int y7 = (int)threadIdx.x + 7;
    int a0 = (int)threadIdx.x;
    int a1 = (int)threadIdx.x + 11;
    int a2 = (int)threadIdx.x + 17;
    int a3 = (int)threadIdx.x + 23;
    int a4 = (int)threadIdx.x + 29;
    int a5 = (int)threadIdx.x + 31;
    int a6 = (int)threadIdx.x + 37;
    int a7 = (int)threadIdx.x + 41;

    // 8 outer * (16 dots/chain * 8 ops) * 8 chains = 8192 ops/thread.
    #pragma unroll 1
    for (int i = 0; i < 8; i++)
    {
        STEP_16(x, y0, a0)
        STEP_16(x, y1, a1)
        STEP_16(x, y2, a2)
        STEP_16(x, y3, a3)
        STEP_16(x, y4, a4)
        STEP_16(x, y5, a5)
        STEP_16(x, y6, a6)
        STEP_16(x, y7, a7)
    }

    out[blockIdx.x * blockDim.x + threadIdx.x] =
        a0 + a1 + a2 + a3 + a4 + a5 + a6 + a7;
}
)CLPEAK_KDELIM";

// from compute_int4_packed.cu
const char *compute_int4_packed_name = "compute_int4_packed.cu";
const char *compute_int4_packed_src = R"CLPEAK_KDELIM(
// Packed-INT4 emulated MAD throughput.  Two signed 4-bit lanes per signed
// 8-bit container; each MAC sign-extends both lanes, multiplies and
// accumulates in int32, then masks back to 4 bits and repacks.  Models the
// dequant-on-the-fly cost in 4-bit weight kernels; clearly NOT a hardware
// peak.  Numbers are expected to be lower than INT8-DP on the same device.
//
// 64 outer iters * 16 MACs * 4 int4 ops = 4096 int4 ops per thread
// (= COMPUTE_INT4_PACKED_WORK_PER_WI).

#define INT4_MAC(dst, src) { \
    int _d = (int)(dst); \
    int _s = (int)(src); \
    int _dl = (_d << 28) >> 28; \
    int _dh = _d >> 4; \
    int _sl = (_s << 28) >> 28; \
    int _sh = _s >> 4; \
    _dl = _sl * _dl + _sl; \
    _dh = _sh * _dh + _sh; \
    dst = ((_dl & 0x0F) | ((_dh & 0x0F) << 4)); \
}

#define MAD_4(x, y)  INT4_MAC(x, y) INT4_MAC(y, x) INT4_MAC(x, y) INT4_MAC(y, x)
#define MAD_16(x, y) MAD_4(x, y) MAD_4(x, y) MAD_4(x, y) MAD_4(x, y)

extern "C" __global__ void compute_int4_packed(int *out, int A)
{
    int x = A;
    int y = (int)threadIdx.x;

    #pragma unroll 1
    for (int i = 0; i < 64; i++)
    {
        MAD_16(x, y)
    }

    out[blockIdx.x * blockDim.x + threadIdx.x] = x + y;
}
)CLPEAK_KDELIM";

// from compute_int32.cu
const char *compute_int32_name = "compute_int32.cu";
const char *compute_int32_src = R"CLPEAK_KDELIM(
// 32-bit integer IMAD-chain throughput.  Mirrors compute_sp.cu's two-chain
// ping-pong structure so the compiler can't hoist invariants and each MAD
// depends on the previous, exposing ILP across the (x, y) accumulators.
//
// Distinct hardware path from compute_int8_dp (__dp4a) and the int4-packed
// emulation: this is the shader-core IMAD pipe used by ordinary `int`
// arithmetic.  Reported in GOPS.
//
// 128 outer iters * 16 MADs * 2 ops = 4096 ops per thread (= COMPUTE_FP_WORK_PER_WI).

#define MAD_4(x, y)  x = y * x + y; y = x * y + x; x = y * x + y; y = x * y + x;
#define MAD_16(x, y) MAD_4(x, y) MAD_4(x, y) MAD_4(x, y) MAD_4(x, y)

extern "C" __global__ void compute_int32(int *out, int A)
{
    int x = A;
    int y = (int)threadIdx.x;

    #pragma unroll 1
    for (int i = 0; i < 128; i++)
    {
        MAD_16(x, y)
    }

    out[blockIdx.x * blockDim.x + threadIdx.x] = y;
}
)CLPEAK_KDELIM";

// from global_bandwidth.cu
const char *global_bandwidth_name = "global_bandwidth.cu";
const char *global_bandwidth_src = R"CLPEAK_KDELIM(
// Global memory read bandwidth.  Per-thread fetches FETCH_PER_WI elements
// of width V (V = 1, 2, 4 floats); per-block stride layout from
// global_bandwidth_v1.comp so adjacent blocks own disjoint regions and we
// measure VRAM rather than L2.  CUDA's widest native load is LDG.128
// (float4), so float8/float16 would lower to multiple float4 loads with
// no new HW path -- we stop at float4.

#define FETCH_PER_WI 16

extern "C" __global__ void global_bandwidth_v1(const float *in, float *out)
{
    unsigned int gid = blockIdx.x * blockDim.x + threadIdx.x;
    unsigned int lid = threadIdx.x;
    unsigned int wid = blockIdx.x;
    unsigned int lsz = blockDim.x;
    unsigned int offset = wid * lsz * FETCH_PER_WI + lid;

    float sum = 0.0f;
    #pragma unroll
    for (int i = 0; i < FETCH_PER_WI; i++)
    {
        sum += in[offset];
        offset += lsz;
    }
    out[gid] = sum;
}

extern "C" __global__ void global_bandwidth_v2(const float2 *in, float *out)
{
    unsigned int gid = blockIdx.x * blockDim.x + threadIdx.x;
    unsigned int lid = threadIdx.x;
    unsigned int wid = blockIdx.x;
    unsigned int lsz = blockDim.x;
    unsigned int offset = wid * lsz * FETCH_PER_WI + lid;

    float2 sum = make_float2(0.0f, 0.0f);
    #pragma unroll
    for (int i = 0; i < FETCH_PER_WI; i++)
    {
        float2 v = in[offset];
        sum.x += v.x; sum.y += v.y;
        offset += lsz;
    }
    out[gid] = sum.x + sum.y;
}

extern "C" __global__ void global_bandwidth_v4(const float4 *in, float *out)
{
    unsigned int gid = blockIdx.x * blockDim.x + threadIdx.x;
    unsigned int lid = threadIdx.x;
    unsigned int wid = blockIdx.x;
    unsigned int lsz = blockDim.x;
    unsigned int offset = wid * lsz * FETCH_PER_WI + lid;

    float4 sum = make_float4(0.0f, 0.0f, 0.0f, 0.0f);
    #pragma unroll
    for (int i = 0; i < FETCH_PER_WI; i++)
    {
        float4 v = in[offset];
        sum.x += v.x; sum.y += v.y; sum.z += v.z; sum.w += v.w;
        offset += lsz;
    }
    out[gid] = sum.x + sum.y + sum.z + sum.w;
}
)CLPEAK_KDELIM";

// from kernel_latency.cu
const char *kernel_latency_name = "kernel_latency.cu";
const char *kernel_latency_src = R"CLPEAK_KDELIM(
// Empty kernel for measuring launch+complete round-trip overhead.  Single
// block, single thread, no work -- whatever ms/launch this measures is
// pure CUDA dispatch + driver scheduling latency.

extern "C" __global__ void kernel_latency_noop()
{
}
)CLPEAK_KDELIM";

// from wmma_fp16.cu
const char *wmma_fp16_name = "wmma_fp16.cu";
const char *wmma_fp16_src = R"CLPEAK_KDELIM(
// WMMA fp16xfp16+fp32 m16n16k16 -- the canonical Volta+ tensor-core
// throughput test.  Mirrors src/shaders/coopmat_fp16.comp at the same
// tile shape and iteration count.
//
// Four independent accumulator chains (c0..c3) so the tensor unit's
// multi-cycle pipeline can issue back-to-back MMAs instead of stalling
// on the c -> c dependency every iter.  Single-chain on RTX 5060 topped
// out at ~42 TFLOPS = 35% of the spec peak; same fix we used for Metal
// simdgroup_matrix and the Vulkan int8_dp4 variant.
//
// Per warp ops = 256 outer * 4 chains * (16*16*16*2) = 8,388,608;
// per thread = 262,144 (= 4 * COOPMAT_WORK_PER_WI).

#include <mma.h>
#include <cuda_fp16.h>

extern "C" __global__ void wmma_fp16(float *out, float A)
{
    using namespace nvcuda::wmma;

    fragment<matrix_a, 16, 16, 16, __half, row_major> a;
    fragment<matrix_b, 16, 16, 16, __half, col_major> b;
    fragment<accumulator, 16, 16, 16, float> c0, c1, c2, c3;

    fill_fragment(a, __float2half(A));
    fill_fragment(b, __float2half(A));
    fill_fragment(c0, 0.0f);
    fill_fragment(c1, 0.0f);
    fill_fragment(c2, 0.0f);
    fill_fragment(c3, 0.0f);

    #pragma unroll 1
    for (int i = 0; i < 256; i++)
    {
        mma_sync(c0, a, b, c0);
        mma_sync(c1, a, b, c1);
        mma_sync(c2, a, b, c2);
        mma_sync(c3, a, b, c3);
    }

    // Fold so every accumulator contributes to the stored tile -- prevents
    // the compiler from dropping any chain it sees as dead.  3 extra MMAs
    // is ~0.3% noise on the measured TFLOPS.
    mma_sync(c0, a, b, c1);
    mma_sync(c2, a, b, c3);
    mma_sync(c0, a, b, c2);

    store_matrix_sync(out + blockIdx.x * 16 * 16, c0, 16, mem_row_major);
}
)CLPEAK_KDELIM";

// from wmma_bf16.cu
const char *wmma_bf16_name = "wmma_bf16.cu";
const char *wmma_bf16_src = R"CLPEAK_KDELIM(
// WMMA bf16xbf16+fp32 m16n16k16 -- Ampere+ tensor-core throughput test.
// Same 4-chain ILP structure as wmma_fp16; only the input fragment type
// differs.

#include <mma.h>
#include <cuda_bf16.h>

extern "C" __global__ void wmma_bf16(float *out, float A)
{
    using namespace nvcuda::wmma;

    fragment<matrix_a, 16, 16, 16, __nv_bfloat16, row_major> a;
    fragment<matrix_b, 16, 16, 16, __nv_bfloat16, col_major> b;
    fragment<accumulator, 16, 16, 16, float> c0, c1, c2, c3;

    fill_fragment(a, __float2bfloat16(A));
    fill_fragment(b, __float2bfloat16(A));
    fill_fragment(c0, 0.0f);
    fill_fragment(c1, 0.0f);
    fill_fragment(c2, 0.0f);
    fill_fragment(c3, 0.0f);

    #pragma unroll 1
    for (int i = 0; i < 256; i++)
    {
        mma_sync(c0, a, b, c0);
        mma_sync(c1, a, b, c1);
        mma_sync(c2, a, b, c2);
        mma_sync(c3, a, b, c3);
    }

    mma_sync(c0, a, b, c1);
    mma_sync(c2, a, b, c3);
    mma_sync(c0, a, b, c2);

    store_matrix_sync(out + blockIdx.x * 16 * 16, c0, 16, mem_row_major);
}
)CLPEAK_KDELIM";

// from wmma_int8.cu
const char *wmma_int8_name = "wmma_int8.cu";
const char *wmma_int8_src = R"CLPEAK_KDELIM(
// WMMA int8xint8+int32 m16n16k16 -- Turing+ tensor-core INT8 throughput.
// Same 4-chain ILP structure as wmma_fp16.
//
// Note: NVIDIA tensor cores do INT8 most efficiently at K=32 (the natural
// native tile width); m16n16k16 here halves the K dim, so this kernel
// captures the WMMA-fragment INT8 path but a separate K=32 path via
// mma.sync inline PTX is the way to reach the device's true INT8 peak.

#include <mma.h>

extern "C" __global__ void wmma_int8(int *out, int A)
{
    using namespace nvcuda::wmma;

    fragment<matrix_a, 16, 16, 16, signed char, row_major> a;
    fragment<matrix_b, 16, 16, 16, signed char, col_major> b;
    fragment<accumulator, 16, 16, 16, int> c0, c1, c2, c3;

    fill_fragment(a, (signed char)(A & 0x7f));
    fill_fragment(b, (signed char)(A & 0x7f));
    fill_fragment(c0, 0);
    fill_fragment(c1, 0);
    fill_fragment(c2, 0);
    fill_fragment(c3, 0);

    #pragma unroll 1
    for (int i = 0; i < 256; i++)
    {
        mma_sync(c0, a, b, c0);
        mma_sync(c1, a, b, c1);
        mma_sync(c2, a, b, c2);
        mma_sync(c3, a, b, c3);
    }

    mma_sync(c0, a, b, c1);
    mma_sync(c2, a, b, c3);
    mma_sync(c0, a, b, c2);

    store_matrix_sync(out + blockIdx.x * 16 * 16, c0, 16, mem_row_major);
}
)CLPEAK_KDELIM";

// from wmma_int8_k32.cu
const char *wmma_int8_k32_name = "wmma_int8_k32.cu";
const char *wmma_int8_k32_src = R"CLPEAK_KDELIM(
// INT8 tensor-core throughput at the m16n8k32 tile via inline mma.sync
// PTX.  Originally added on the hypothesis that the wmma.h K=16 path
// (m16n16k16) was halving INT8 tensor throughput; the actual measurement
// flipped that.
//
// RTX 5060 (sm_120): wmma_int8 (K=16) hits 327 TIOPS; this kernel
// (K=32, m16n8k32, 4 chains, single non-volatile asm block) hits 165
// TIOPS -- exactly HALF the wmma path.  Likely cause: nvcc lowers wmma
// m16n16k16 INT8 to multiple mma.sync.m16n8k16 instructions issued
// back-to-back, exposing more ILP for ptxas to interleave than our
// explicit K=32 chain.  Or m16n8k32 INT8 has half the issue rate of
// m16n8k16 on this arch.
//
// We keep this kernel because the K=16 vs K=32 throughput delta IS
// useful data -- it documents that the wmma fragment API is the right
// path for INT8 on consumer Blackwell, not raw mma.sync.  Datacenter
// (sm_100) parts may behave differently.
//
// Per-thread fragment layout (32 threads/warp, A=row-major, B=col-major):
//   A: m16 x k32 = 512 bytes / 32 threads = 16 bytes/thread = 4 x .b32
//   B: k32 x  n8 = 256 bytes / 32 threads =  8 bytes/thread = 2 x .b32
//   C/D: m16 x n8 = 128 int32 / 32 threads = 4 int32/thread per accumulator
//
// 4 independent accumulator chains, packed into one non-volatile asm
// block (same pattern as wmma_fp8_e4m3.cu).
//
// Per-thread fragment layout (32 threads/warp, A=row-major, B=col-major):
//   A: m16 x k32 = 512 bytes / 32 threads = 16 bytes/thread = 4 x .b32
//   B: k32 x  n8 = 256 bytes / 32 threads =  8 bytes/thread = 2 x .b32
//   C/D: m16 x n8 = 128 int32 / 32 threads = 4 int32/thread per accumulator
//
// 4 independent accumulator chains, packed into one non-volatile asm
// block (same fix as wmma_fp8_e4m3.cu).
//
// Per warp ops = 256 outer * 4 chains * (16*8*32*2) = 8,388,608;
// per thread = 262,144 (= 4 * COOPMAT_WORK_PER_WI).

extern "C" __global__ void wmma_int8_k32(int *out, int A)
{
    unsigned int packed = (A & 0xff)
                        | (((A + 1) & 0xff) << 8)
                        | (((A + 2) & 0xff) << 16)
                        | (((A + 3) & 0xff) << 24);
    unsigned int a0 = packed, a1 = packed, a2 = packed, a3 = packed;
    unsigned int b0 = packed, b1 = packed;

    int c00=0,c01=0,c02=0,c03=0;
    int c10=0,c11=0,c12=0,c13=0;
    int c20=0,c21=0,c22=0,c23=0;
    int c30=0,c31=0,c32=0,c33=0;

    #pragma unroll 1
    for (int i = 0; i < 256; i++)
    {
        asm(
          "mma.sync.aligned.m16n8k32.row.col.s32.s8.s8.s32 "
              "{%0,%1,%2,%3}, {%16,%17,%18,%19}, {%20,%21}, {%0,%1,%2,%3};\n"
          "mma.sync.aligned.m16n8k32.row.col.s32.s8.s8.s32 "
              "{%4,%5,%6,%7}, {%16,%17,%18,%19}, {%20,%21}, {%4,%5,%6,%7};\n"
          "mma.sync.aligned.m16n8k32.row.col.s32.s8.s8.s32 "
              "{%8,%9,%10,%11}, {%16,%17,%18,%19}, {%20,%21}, {%8,%9,%10,%11};\n"
          "mma.sync.aligned.m16n8k32.row.col.s32.s8.s8.s32 "
              "{%12,%13,%14,%15}, {%16,%17,%18,%19}, {%20,%21}, {%12,%13,%14,%15};\n"
          : "+r"(c00),"+r"(c01),"+r"(c02),"+r"(c03),
            "+r"(c10),"+r"(c11),"+r"(c12),"+r"(c13),
            "+r"(c20),"+r"(c21),"+r"(c22),"+r"(c23),
            "+r"(c30),"+r"(c31),"+r"(c32),"+r"(c33)
          : "r"(a0),"r"(a1),"r"(a2),"r"(a3), "r"(b0),"r"(b1));
    }

    c00 += c10+c20+c30;
    c01 += c11+c21+c31;
    c02 += c12+c22+c32;
    c03 += c13+c23+c33;

    unsigned int base = blockIdx.x * 16u * 8u + threadIdx.x * 4u;
    out[base + 0] = c00;
    out[base + 1] = c01;
    out[base + 2] = c02;
    out[base + 3] = c03;
}
)CLPEAK_KDELIM";

// from wmma_fp8_e4m3.cu
const char *wmma_fp8_e4m3_name = "wmma_fp8_e4m3.cu";
const char *wmma_fp8_e4m3_src = R"CLPEAK_KDELIM(
// FP8 (E4M3) tensor-core throughput via inline mma.sync PTX.
// Tile: m16n8k32 with fp32 accumulator.  Available on sm_89+ (Ada) and
// sm_90+ (Hopper); built into PTX ISA 8.0 (CUDA 12+).
//
// Per-thread fragment layout (32 threads/warp, A=row-major, B=col-major):
//   A: m16 x k32 = 512 bytes / 32 threads = 16 bytes/thread = 4 x .b32
//   B: k32 x  n8 = 256 bytes / 32 threads =  8 bytes/thread = 2 x .b32
//   C/D: m16 x n8 = 128 fp32 / 32 threads = 4 fp32/thread per accumulator
//
// EIGHT independent accumulator chains (vs the 4 used for WMMA): the FP8
// tensor pipeline on consumer Blackwell has a deeper latency than INT8
// WMMA, so 4 chains was still latency-bound (84 TFLOPS, same as
// single-chain).  Bumping to 8 unlocks the issue rate.
//
// All 8 mma.sync instructions live in a SINGLE inline-asm block (no
// `volatile`) so ptxas is free to interleave the per-chain ops.  Volatile
// blocks would force in-order emission, defeating the ILP gain even with
// independent register operands.
//
// Per warp ops = 256 outer * 8 chains * (16*8*32*2) = 16,777,216;
// per thread = 524,288 (= 8 * COOPMAT_WORK_PER_WI).
//
// Consumer-Blackwell ceiling note: this kernel measures ~85 TFLOPS on
// RTX 5060 regardless of chain count or asm-block packing, so 85 TFLOPS
// appears to be the hardware ceiling for dense `mma.sync` FP8 on
// sm_120 -- roughly 1/4 of the WMMA INT8 number on the same device.
// Higher FP8 throughput would require `wgmma.mma_async` (Hopper+
// warpgroup-scope MMA, 4x larger tiles) or `tcgen05.mma` (Blackwell-only,
// uses separate tensor memory).  Both are major undertakings; the
// simpler mma.sync path is what we ship.  The m16n8k64 shape only
// supports INT8 / sub-byte types -- ptxas rejects it for FP8.

extern "C" __global__ void wmma_fp8_e4m3(float *out, float A)
{
    unsigned int packed = 0x3c3c3c3cu ^ (__float_as_uint(A) & 0x0f0f0f0fu);
    unsigned int a0 = packed, a1 = packed, a2 = packed, a3 = packed;
    unsigned int b0 = packed, b1 = packed;

    float c00=0,c01=0,c02=0,c03=0;
    float c10=0,c11=0,c12=0,c13=0;
    float c20=0,c21=0,c22=0,c23=0;
    float c30=0,c31=0,c32=0,c33=0;
    float c40=0,c41=0,c42=0,c43=0;
    float c50=0,c51=0,c52=0,c53=0;
    float c60=0,c61=0,c62=0,c63=0;
    float c70=0,c71=0,c72=0,c73=0;

    #pragma unroll 1
    for (int i = 0; i < 256; i++)
    {
        // 8 mma.sync in one asm block, non-volatile -- ptxas may interleave
        // the per-chain ops to saturate the FP8 issue rate.
        asm(
          "mma.sync.aligned.m16n8k32.row.col.f32.e4m3.e4m3.f32 "
              "{%0,%1,%2,%3}, {%32,%33,%34,%35}, {%36,%37}, {%0,%1,%2,%3};\n"
          "mma.sync.aligned.m16n8k32.row.col.f32.e4m3.e4m3.f32 "
              "{%4,%5,%6,%7}, {%32,%33,%34,%35}, {%36,%37}, {%4,%5,%6,%7};\n"
          "mma.sync.aligned.m16n8k32.row.col.f32.e4m3.e4m3.f32 "
              "{%8,%9,%10,%11}, {%32,%33,%34,%35}, {%36,%37}, {%8,%9,%10,%11};\n"
          "mma.sync.aligned.m16n8k32.row.col.f32.e4m3.e4m3.f32 "
              "{%12,%13,%14,%15}, {%32,%33,%34,%35}, {%36,%37}, {%12,%13,%14,%15};\n"
          "mma.sync.aligned.m16n8k32.row.col.f32.e4m3.e4m3.f32 "
              "{%16,%17,%18,%19}, {%32,%33,%34,%35}, {%36,%37}, {%16,%17,%18,%19};\n"
          "mma.sync.aligned.m16n8k32.row.col.f32.e4m3.e4m3.f32 "
              "{%20,%21,%22,%23}, {%32,%33,%34,%35}, {%36,%37}, {%20,%21,%22,%23};\n"
          "mma.sync.aligned.m16n8k32.row.col.f32.e4m3.e4m3.f32 "
              "{%24,%25,%26,%27}, {%32,%33,%34,%35}, {%36,%37}, {%24,%25,%26,%27};\n"
          "mma.sync.aligned.m16n8k32.row.col.f32.e4m3.e4m3.f32 "
              "{%28,%29,%30,%31}, {%32,%33,%34,%35}, {%36,%37}, {%28,%29,%30,%31};\n"
          : "+f"(c00),"+f"(c01),"+f"(c02),"+f"(c03),
            "+f"(c10),"+f"(c11),"+f"(c12),"+f"(c13),
            "+f"(c20),"+f"(c21),"+f"(c22),"+f"(c23),
            "+f"(c30),"+f"(c31),"+f"(c32),"+f"(c33),
            "+f"(c40),"+f"(c41),"+f"(c42),"+f"(c43),
            "+f"(c50),"+f"(c51),"+f"(c52),"+f"(c53),
            "+f"(c60),"+f"(c61),"+f"(c62),"+f"(c63),
            "+f"(c70),"+f"(c71),"+f"(c72),"+f"(c73)
          : "r"(a0),"r"(a1),"r"(a2),"r"(a3), "r"(b0),"r"(b1));
    }

    // Fold all 8 accumulators into c0 so every chain is live at the store.
    c00 += c10+c20+c30+c40+c50+c60+c70;
    c01 += c11+c21+c31+c41+c51+c61+c71;
    c02 += c12+c22+c32+c42+c52+c62+c72;
    c03 += c13+c23+c33+c43+c53+c63+c73;

    unsigned int base = blockIdx.x * 16u * 8u + threadIdx.x * 4u;
    out[base + 0] = c00;
    out[base + 1] = c01;
    out[base + 2] = c02;
    out[base + 3] = c03;
}
)CLPEAK_KDELIM";

// from wmma_fp8_e5m2.cu
const char *wmma_fp8_e5m2_name = "wmma_fp8_e5m2.cu";
const char *wmma_fp8_e5m2_src = R"CLPEAK_KDELIM(
// FP8 (E5M2) tensor-core throughput via inline mma.sync PTX.  Identical
// shape to wmma_fp8_e4m3.cu; only the dtype mnemonic on the mma.sync
// instruction changes.  Same 8-chain / single-asm-block structure for
// the same reason (consumer-Blackwell FP8 issue pipeline is deeper than
// INT8's; 4 chains in volatile blocks plateaued at ~84 TFLOPS).  See
// wmma_fp8_e4m3.cu for the consumer-Blackwell FP8 ceiling note.

extern "C" __global__ void wmma_fp8_e5m2(float *out, float A)
{
    unsigned int packed = 0x3c3c3c3cu ^ (__float_as_uint(A) & 0x0f0f0f0fu);
    unsigned int a0 = packed, a1 = packed, a2 = packed, a3 = packed;
    unsigned int b0 = packed, b1 = packed;

    float c00=0,c01=0,c02=0,c03=0;
    float c10=0,c11=0,c12=0,c13=0;
    float c20=0,c21=0,c22=0,c23=0;
    float c30=0,c31=0,c32=0,c33=0;
    float c40=0,c41=0,c42=0,c43=0;
    float c50=0,c51=0,c52=0,c53=0;
    float c60=0,c61=0,c62=0,c63=0;
    float c70=0,c71=0,c72=0,c73=0;

    #pragma unroll 1
    for (int i = 0; i < 256; i++)
    {
        asm(
          "mma.sync.aligned.m16n8k32.row.col.f32.e5m2.e5m2.f32 "
              "{%0,%1,%2,%3}, {%32,%33,%34,%35}, {%36,%37}, {%0,%1,%2,%3};\n"
          "mma.sync.aligned.m16n8k32.row.col.f32.e5m2.e5m2.f32 "
              "{%4,%5,%6,%7}, {%32,%33,%34,%35}, {%36,%37}, {%4,%5,%6,%7};\n"
          "mma.sync.aligned.m16n8k32.row.col.f32.e5m2.e5m2.f32 "
              "{%8,%9,%10,%11}, {%32,%33,%34,%35}, {%36,%37}, {%8,%9,%10,%11};\n"
          "mma.sync.aligned.m16n8k32.row.col.f32.e5m2.e5m2.f32 "
              "{%12,%13,%14,%15}, {%32,%33,%34,%35}, {%36,%37}, {%12,%13,%14,%15};\n"
          "mma.sync.aligned.m16n8k32.row.col.f32.e5m2.e5m2.f32 "
              "{%16,%17,%18,%19}, {%32,%33,%34,%35}, {%36,%37}, {%16,%17,%18,%19};\n"
          "mma.sync.aligned.m16n8k32.row.col.f32.e5m2.e5m2.f32 "
              "{%20,%21,%22,%23}, {%32,%33,%34,%35}, {%36,%37}, {%20,%21,%22,%23};\n"
          "mma.sync.aligned.m16n8k32.row.col.f32.e5m2.e5m2.f32 "
              "{%24,%25,%26,%27}, {%32,%33,%34,%35}, {%36,%37}, {%24,%25,%26,%27};\n"
          "mma.sync.aligned.m16n8k32.row.col.f32.e5m2.e5m2.f32 "
              "{%28,%29,%30,%31}, {%32,%33,%34,%35}, {%36,%37}, {%28,%29,%30,%31};\n"
          : "+f"(c00),"+f"(c01),"+f"(c02),"+f"(c03),
            "+f"(c10),"+f"(c11),"+f"(c12),"+f"(c13),
            "+f"(c20),"+f"(c21),"+f"(c22),"+f"(c23),
            "+f"(c30),"+f"(c31),"+f"(c32),"+f"(c33),
            "+f"(c40),"+f"(c41),"+f"(c42),"+f"(c43),
            "+f"(c50),"+f"(c51),"+f"(c52),"+f"(c53),
            "+f"(c60),"+f"(c61),"+f"(c62),"+f"(c63),
            "+f"(c70),"+f"(c71),"+f"(c72),"+f"(c73)
          : "r"(a0),"r"(a1),"r"(a2),"r"(a3), "r"(b0),"r"(b1));
    }

    c00 += c10+c20+c30+c40+c50+c60+c70;
    c01 += c11+c21+c31+c41+c51+c61+c71;
    c02 += c12+c22+c32+c42+c52+c62+c72;
    c03 += c13+c23+c33+c43+c53+c63+c73;

    unsigned int base = blockIdx.x * 16u * 8u + threadIdx.x * 4u;
    out[base + 0] = c00;
    out[base + 1] = c01;
    out[base + 2] = c02;
    out[base + 3] = c03;
}
)CLPEAK_KDELIM";

// from wmma_tf32.cu
const char *wmma_tf32_name = "wmma_tf32.cu";
const char *wmma_tf32_src = R"CLPEAK_KDELIM(
// WMMA tf32xtf32+fp32 m16n16k8 -- Ampere+ tensor-core TF32 throughput.
// Mirrors wmma_fp16.cu's 4-chain ILP structure; only the fragment dtype
// (precision::tf32) and K dimension (8 instead of 16) change.
//
// Per warp ops = 256 outer * 4 chains * (16*16*8*2) = 4,194,304;
// per thread = 131,072 (= 2 * COOPMAT_WORK_PER_WI).

#include <mma.h>

extern "C" __global__ void wmma_tf32(float *out, float A)
{
    using namespace nvcuda::wmma;

    fragment<matrix_a, 16, 16, 8, precision::tf32, row_major> a;
    fragment<matrix_b, 16, 16, 8, precision::tf32, col_major> b;
    fragment<accumulator, 16, 16, 8, float> c0, c1, c2, c3;

    fill_fragment(a, __float_to_tf32(A));
    fill_fragment(b, __float_to_tf32(A));
    fill_fragment(c0, 0.0f);
    fill_fragment(c1, 0.0f);
    fill_fragment(c2, 0.0f);
    fill_fragment(c3, 0.0f);

    #pragma unroll 1
    for (int i = 0; i < 256; i++)
    {
        mma_sync(c0, a, b, c0);
        mma_sync(c1, a, b, c1);
        mma_sync(c2, a, b, c2);
        mma_sync(c3, a, b, c3);
    }

    mma_sync(c0, a, b, c1);
    mma_sync(c2, a, b, c3);
    mma_sync(c0, a, b, c2);

    store_matrix_sync(out + blockIdx.x * 16 * 16, c0, 16, mem_row_major);
}
)CLPEAK_KDELIM";

// from wmma_fp64.cu
const char *wmma_fp64_name = "wmma_fp64.cu";
const char *wmma_fp64_src = R"CLPEAK_KDELIM(
// WMMA fp64xfp64+fp64 m8n8k4 -- Ampere+ DP tensor-core throughput.
// Native double tensor cores were added on sm_80; consumer Ada/Blackwell
// support the same fragment shape.  Same 4-chain ILP structure as the
// FP16/BF16 kernels.
//
// The m8n8k4 tile is 4x smaller than m16n16k16, so we run 4x more outer
// iterations (1024 vs 256) to keep the per-thread op budget aligned with
// the convention used by the rest of the WMMA suite.
//
// Per warp ops = 1024 outer * 4 chains * (8*8*4*2) = 2,097,152;
// per thread = 65,536 (= 1 * COOPMAT_WORK_PER_WI).

#include <mma.h>

extern "C" __global__ void wmma_fp64(double *out, double A)
{
    using namespace nvcuda::wmma;

    fragment<matrix_a, 8, 8, 4, double, row_major> a;
    fragment<matrix_b, 8, 8, 4, double, col_major> b;
    fragment<accumulator, 8, 8, 4, double> c0, c1, c2, c3;

    fill_fragment(a, A);
    fill_fragment(b, A);
    fill_fragment(c0, 0.0);
    fill_fragment(c1, 0.0);
    fill_fragment(c2, 0.0);
    fill_fragment(c3, 0.0);

    #pragma unroll 1
    for (int i = 0; i < 1024; i++)
    {
        mma_sync(c0, a, b, c0);
        mma_sync(c1, a, b, c1);
        mma_sync(c2, a, b, c2);
        mma_sync(c3, a, b, c3);
    }

    mma_sync(c0, a, b, c1);
    mma_sync(c2, a, b, c3);
    mma_sync(c0, a, b, c2);

    store_matrix_sync(out + blockIdx.x * 8 * 8, c0, 8, mem_row_major);
}
)CLPEAK_KDELIM";

// from wmma_int4.cu
const char *wmma_int4_name = "wmma_int4.cu";
const char *wmma_int4_src = R"CLPEAK_KDELIM(
// INT4 (s4) tensor-core throughput at the m8n8k32 tile via inline mma.sync
// PTX.  s4 IMMA was added on Turing (sm_75) and kept through Ada (sm_89);
// removed on Hopper+ (sm_90+).  The runtime gate is set in CudaDevice::init.
//
// Per-thread fragment layout (32 threads/warp, A=row-major, B=col-major):
//   A: m8  x k32 = 256 nibbles = 128 bytes / 32 threads = 4 bytes  = 1 x .b32
//   B: k32 x n8  = same                                            = 1 x .b32
//   C/D: m8 x n8 = 64 int32 / 32 threads = 2 int32/thread per accumulator
//
// 4 independent accumulator chains, single non-volatile asm block (same
// pattern as wmma_int8_k32.cu).
//
// Per warp ops = 256 outer * 4 chains * (8*8*32*2) = 4,194,304;
// per thread = 131,072 (= 2 * COOPMAT_WORK_PER_WI).  Reported in TOPS.

extern "C" __global__ void wmma_int4(int *out, int A)
{
    // Pack 8 nibbles into one int32 register (s4).
    unsigned int packed = ((A & 0xf) <<  0) | ((A & 0xf) <<  4)
                        | ((A & 0xf) <<  8) | ((A & 0xf) << 12)
                        | ((A & 0xf) << 16) | ((A & 0xf) << 20)
                        | ((A & 0xf) << 24) | ((A & 0xf) << 28);
    unsigned int a0 = packed;
    unsigned int b0 = packed;

    int c00=0,c01=0;
    int c10=0,c11=0;
    int c20=0,c21=0;
    int c30=0,c31=0;

    #pragma unroll 1
    for (int i = 0; i < 256; i++)
    {
        asm(
          "mma.sync.aligned.m8n8k32.row.col.s32.s4.s4.s32 "
              "{%0,%1}, {%8}, {%9}, {%0,%1};\n"
          "mma.sync.aligned.m8n8k32.row.col.s32.s4.s4.s32 "
              "{%2,%3}, {%8}, {%9}, {%2,%3};\n"
          "mma.sync.aligned.m8n8k32.row.col.s32.s4.s4.s32 "
              "{%4,%5}, {%8}, {%9}, {%4,%5};\n"
          "mma.sync.aligned.m8n8k32.row.col.s32.s4.s4.s32 "
              "{%6,%7}, {%8}, {%9}, {%6,%7};\n"
          : "+r"(c00),"+r"(c01),
            "+r"(c10),"+r"(c11),
            "+r"(c20),"+r"(c21),
            "+r"(c30),"+r"(c31)
          : "r"(a0), "r"(b0));
    }

    c00 += c10 + c20 + c30;
    c01 += c11 + c21 + c31;

    unsigned int base = blockIdx.x * 8u * 8u + threadIdx.x * 2u;
    out[base + 0] = c00;
    out[base + 1] = c01;
}
)CLPEAK_KDELIM";

// from wmma_bmma_b1.cu
const char *wmma_bmma_b1_name = "wmma_bmma_b1.cu";
const char *wmma_bmma_b1_src = R"CLPEAK_KDELIM(
// Binary (1-bit) BMMA tensor-core throughput at the m8n8k128 tile via
// inline mma.sync PTX with XOR-popcount semantics.  Available sm_75+
// (Turing).  Reported in TOPS following the conventional 2*m*n*k op
// count for binary tensor cores.
//
// Per-thread fragment layout (32 threads/warp, A=row-major, B=col-major):
//   A: m8  x k128 = 1024 bits = 128 bytes / 32 threads = 4 bytes = 1 x .b32
//   B: k128 x n8  = same                                         = 1 x .b32
//   C/D: m8 x n8  = 64 int32 / 32 threads = 2 int32/thread per accumulator
//
// 4 independent accumulator chains, single non-volatile asm block.
//
// Per warp ops = 256 outer * 4 chains * (8*8*128*2) = 16,777,216;
// per thread = 524,288 (= 8 * COOPMAT_WORK_PER_WI).

extern "C" __global__ void wmma_bmma_b1(int *out, int A)
{
    unsigned int a0 = (unsigned int)A;
    unsigned int b0 = (unsigned int)A;

    int c00=0,c01=0;
    int c10=0,c11=0;
    int c20=0,c21=0;
    int c30=0,c31=0;

    #pragma unroll 1
    for (int i = 0; i < 256; i++)
    {
        asm(
          "mma.sync.aligned.m8n8k128.row.col.s32.b1.b1.s32.xor.popc "
              "{%0,%1}, {%8}, {%9}, {%0,%1};\n"
          "mma.sync.aligned.m8n8k128.row.col.s32.b1.b1.s32.xor.popc "
              "{%2,%3}, {%8}, {%9}, {%2,%3};\n"
          "mma.sync.aligned.m8n8k128.row.col.s32.b1.b1.s32.xor.popc "
              "{%4,%5}, {%8}, {%9}, {%4,%5};\n"
          "mma.sync.aligned.m8n8k128.row.col.s32.b1.b1.s32.xor.popc "
              "{%6,%7}, {%8}, {%9}, {%6,%7};\n"
          : "+r"(c00),"+r"(c01),
            "+r"(c10),"+r"(c11),
            "+r"(c20),"+r"(c21),
            "+r"(c30),"+r"(c31)
          : "r"(a0), "r"(b0));
    }

    c00 += c10 + c20 + c30;
    c01 += c11 + c21 + c31;

    unsigned int base = blockIdx.x * 8u * 8u + threadIdx.x * 2u;
    out[base + 0] = c00;
    out[base + 1] = c01;
}
)CLPEAK_KDELIM";

// from local_bandwidth.cu
const char *local_bandwidth_name = "local_bandwidth.cu";
const char *local_bandwidth_src = R"CLPEAK_KDELIM(
// Local memory (__shared__) bandwidth -- ping-pong write/read with two
// __syncthreads per rep.  Mirrors local_bandwidth_kernels.cl::v1..v8.
//
// 64 reps * (1 write + 1 read) = 128 accesses per WI per kernel call.
// Bytes = LMEM_REPS * 2 * width * sizeof(float) * globalThreads.

extern "C" __global__ void local_bandwidth_v1(float *out)
{
    __shared__ float scratch[256];
    unsigned int lid   = threadIdx.x;
    unsigned int lsize = blockDim.x;
    unsigned int gid   = blockIdx.x * lsize + lid;
    unsigned int next  = (lid + 1u) % lsize;

    float sum = (float)gid;
    #pragma unroll 1
    for (int i = 0; i < 64; i++)
    {
        scratch[lid] = sum;
        __syncthreads();
        sum = scratch[next];
        __syncthreads();
    }
    out[gid] = sum;
}

extern "C" __global__ void local_bandwidth_v2(float *out)
{
    __shared__ float2 scratch[256];
    unsigned int lid   = threadIdx.x;
    unsigned int lsize = blockDim.x;
    unsigned int gid   = blockIdx.x * lsize + lid;
    unsigned int next  = (lid + 1u) % lsize;

    float2 sum = make_float2((float)gid, (float)(gid + 1u));
    #pragma unroll 1
    for (int i = 0; i < 64; i++)
    {
        scratch[lid] = sum;
        __syncthreads();
        sum = scratch[next];
        __syncthreads();
    }
    out[gid] = sum.x + sum.y;
}

extern "C" __global__ void local_bandwidth_v4(float *out)
{
    __shared__ float4 scratch[256];
    unsigned int lid   = threadIdx.x;
    unsigned int lsize = blockDim.x;
    unsigned int gid   = blockIdx.x * lsize + lid;
    unsigned int next  = (lid + 1u) % lsize;

    float4 sum = make_float4((float)gid,     (float)(gid + 1u),
                             (float)(gid + 2u), (float)(gid + 3u));
    #pragma unroll 1
    for (int i = 0; i < 64; i++)
    {
        scratch[lid] = sum;
        __syncthreads();
        sum = scratch[next];
        __syncthreads();
    }
    out[gid] = sum.x + sum.y + sum.z + sum.w;
}

)CLPEAK_KDELIM";

// from image_bandwidth.cu
const char *image_bandwidth_name = "image_bandwidth.cu";
const char *image_bandwidth_src = R"CLPEAK_KDELIM(
// Image (texture) bandwidth via cudaTextureObject_t / tex2D fetch.
// Each thread reads 16 RGBA float pixels with nearest-neighbour sampling.
//
// Bytes = IMAGE_FETCH_PER_WI * 4 * sizeof(float) * globalThreads.

extern "C" __global__ void image_bandwidth(cudaTextureObject_t tex, float *out, int width, int height)
{
    int gid   = (int)(blockIdx.x * blockDim.x + threadIdx.x);
    int gsize = (int)(gridDim.x * blockDim.x);
    int total = width * height;

    float4 sum = make_float4(0.0f, 0.0f, 0.0f, 0.0f);
    #pragma unroll
    for (int i = 0; i < 16; i++)
    {
        int pixel = (gid + i * gsize) % total;
        int x = pixel % width;
        int y = pixel / width;
        float4 v = tex2D<float4>(tex, x, y);
        sum.x += v.x; sum.y += v.y; sum.z += v.z; sum.w += v.w;
    }
    out[gid] = sum.x + sum.y + sum.z + sum.w;
}
)CLPEAK_KDELIM";

// from atomic_throughput.cu
const char *atomic_throughput_name = "atomic_throughput.cu";
const char *atomic_throughput_src = R"CLPEAK_KDELIM(
// Atomic throughput -- two patterns:
//   global: each thread hammers its own counter in global memory (no
//           cross-thread contention).  Measures peak global-atomic rate.
//   local:  every thread in the block contends on a single __shared__
//           counter (histogram / reduction pattern).
//
// 256 atomicAdd calls per thread (= ATOMIC_REPS); the loop dependency
// prevents the compiler from hoisting / eliminating.

extern "C" __global__ void atomic_throughput_global(int *counter)
{
    int *cnt = counter + blockIdx.x * blockDim.x + threadIdx.x;
    #pragma unroll 1
    for (int i = 0; i < 256; i++)
    {
        atomicAdd(cnt, 1);
    }
}

extern "C" __global__ void atomic_throughput_local(int *out)
{
    __shared__ int scratch;
    if (threadIdx.x == 0) scratch = 0;
    __syncthreads();

    #pragma unroll 1
    for (int i = 0; i < 256; i++)
    {
        atomicAdd(&scratch, 1);
    }

    __syncthreads();
    if (threadIdx.x == 0) out[blockIdx.x] = scratch;
}
)CLPEAK_KDELIM";

} // namespace cuda_kernels
