```python
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the license found in the
# LICENSE file in the root directory of this source tree.

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.cpp_extension import load_inline

# Allow TF32 tensor-core math for FP32 matmuls (explicitly permitted by the
# precision rule). Storage and compute type stay FP32.
torch.backends.cuda.matmul.allow_tf32 = True
try:
    torch.set_float32_matmul_precision("high")
except Exception:
    pass

# ---------------------------------------------------------------------------
# Fully fused implementation of:  out = min(x @ W.T + bias, c) - c
#
# For the benchmark shape (M=128, K=N=16384) the op is dominated by streaming
# the 1 GiB FP32 weight matrix from HBM exactly once. This kernel is built to
# be HBM-bandwidth-bound:
#   * TF32 tensor-core GEMM (wmma) on a 128x128 tile, BK=32.
#   * 4-stage cp.async pipeline (~96 KiB in flight per SM) so even a partial
#     wave of blocks can saturate HBM.
#   * L2 eviction policies: the tiny activation matrix (8 MiB) is marked
#     evict_last (read once from HBM, then served from L2 by all blocks) while
#     the streamed weights are marked evict_first (no L2 thrash).
#   * A runtime cost model picks the split-K factor S; for the target shape it
#     picks S=1 so the epilogue (bias add, min, subtract) is fused directly
#     into the GEMM with zero extra passes over the output. For S>1 a fused
#     reduction kernel applies the epilogue.
# ---------------------------------------------------------------------------
gemm_source = r"""
#include <torch/extension.h>
#include <ATen/cuda/CUDAContext.h>
#include <cuda_runtime.h>
#include <mma.h>
#include <algorithm>
#include <cmath>

using namespace nvcuda;

#define BM 128
#define BN 128
#define BK 32
#define STAGES 4
#define LDS (BK + 4)
#define NTHREADS 256
#define SMEM_BYTES (STAGES * (BM + BN) * LDS * 4)

__device__ __forceinline__ void cp_async16(void* smem_ptr, const void* gmem_ptr,
                                           unsigned long long policy) {
    unsigned saddr = static_cast<unsigned>(__cvta_generic_to_shared(smem_ptr));
    asm volatile("cp.async.cg.shared.global.L2::cache_hint [%0], [%1], 16, %2;\n"
                 :: "r"(saddr), "l"(gmem_ptr), "l"(policy));
}
__device__ __forceinline__ void cp_async_commit() {
    asm volatile("cp.async.commit_group;\n");
}
template <int N>
__device__ __forceinline__ void cp_async_wait() {
    asm volatile("cp.async.wait_group %0;\n" :: "n"(N));
}

// Computes a 128x128 output tile of C = A[128,K] @ W[N,K]^T over a K-chunk.
// If S == 1 the epilogue is fused: out = min(C + bias, c) - c.
// Otherwise the raw partial sums are written to ws[s].
__global__ void __launch_bounds__(NTHREADS)
gemm_tf32_fused_kernel(const float* __restrict__ A,
                       const float* __restrict__ W,
                       const float* __restrict__ bias,
                       float* __restrict__ out,
                       float* __restrict__ ws,
                       const int gridN, const int K, const int N,
                       const int base_iters, const int rem,
                       const int S, const float c) {
    unsigned long long pol_a, pol_w;
    asm volatile("createpolicy.fractional.L2::evict_last.b64 %0, 1.0;\n" : "=l"(pol_a));
    asm volatile("createpolicy.fractional.L2::evict_first.b64 %0, 1.0;\n" : "=l"(pol_w));

    const int s  = blockIdx.x / gridN;
    const int nt = blockIdx.x % gridN;
    const int n0 = nt * BN;

    const int kiters = base_iters + (s < rem ? 1 : 0);
    const int kstart = s * base_iters + min(s, rem);

    extern __shared__ float smem[];
    float* As[STAGES];
    float* Ws[STAGES];
    #pragma unroll
    for (int st = 0; st < STAGES; ++st) {
        As[st] = smem + st * (BM + BN) * LDS;
        Ws[st] = As[st] + BM * LDS;
    }

    const int tid  = threadIdx.x;
    const int warp = tid >> 5;
    const int warpM = warp >> 2;   // 0..1  -> 64 rows each
    const int warpN = warp & 3;    // 0..3  -> 32 cols each

    wmma::fragment<wmma::accumulator, 16, 16, 8, float> acc[4][2];
    #pragma unroll
    for (int mi = 0; mi < 4; ++mi)
        #pragma unroll
        for (int ni = 0; ni < 2; ++ni)
            wmma::fill_fragment(acc[mi][ni], 0.0f);

    const long Kstride = K;

    auto prefetch = [&](int iter, int buf) {
        const int k0 = (kstart + iter) * BK;
        // A tile: 128 rows x BK floats (8 float4 per row) -> 1024 chunks
        #pragma unroll
        for (int t = tid; t < BM * (BK / 4); t += NTHREADS) {
            const int row = t >> 3;
            const int ch  = t & 7;
            cp_async16(As[buf] + row * LDS + ch * 4,
                       A + (long)row * Kstride + k0 + ch * 4, pol_a);
        }
        // W tile: rows n0..n0+127 (streamed, evict_first)
        #pragma unroll
        for (int t = tid; t < BN * (BK / 4); t += NTHREADS) {
            const int row = t >> 3;
            const int ch  = t & 7;
            cp_async16(Ws[buf] + row * LDS + ch * 4,
                       W + (long)(n0 + row) * Kstride + k0 + ch * 4, pol_w);
        }
        cp_async_commit();
    };

    // Prologue: fill the first STAGES-1 pipeline stages.
    #pragma unroll
    for (int st = 0; st < STAGES - 1; ++st) {
        if (st < kiters) prefetch(st, st);
        else cp_async_commit();
    }

    for (int i = 0; i < kiters; ++i) {
        cp_async_wait<STAGES - 2>();
        __syncthreads();

        const int nxt = i + STAGES - 1;
        if (nxt < kiters) prefetch(nxt, nxt & (STAGES - 1));
        else cp_async_commit();

        const float* a = As[i & (STAGES - 1)];
        const float* w = Ws[i & (STAGES - 1)];
        #pragma unroll
        for (int kk = 0; kk < BK / 8; ++kk) {
            wmma::fragment<wmma::matrix_a, 16, 16, 8, wmma::precision::tf32, wmma::row_major> fa[4];
            wmma::fragment<wmma::matrix_b, 16, 16, 8, wmma::precision::tf32, wmma::col_major> fb[2];
            #pragma unroll
            for (int mi = 0; mi < 4; ++mi) {
                wmma::load_matrix_sync(fa[mi], a + (warpM * 64 + mi * 16) * LDS + kk * 8, LDS);
                #pragma unroll
                for (int e = 0; e < fa[mi].num_elements; ++e)
                    fa[mi].x[e] = wmma::__float_to_tf32(fa[mi].x[e]);
            }
            #pragma unroll
            for (int ni = 0; ni < 2; ++ni) {
                wmma::load_matrix_sync(fb[ni], w + (warpN * 32 + ni * 16) * LDS + kk * 8, LDS);
                #pragma unroll
                for (int e = 0; e < fb[ni].num_elements; ++e)
                    fb[ni].x[e] = wmma::__float_to_tf32(fb[ni].x[e]);
            }
            #pragma unroll
            for (int mi = 0; mi < 4; ++mi)
                #pragma unroll
                for (int ni = 0; ni < 2; ++ni)
                    wmma::mma_sync(acc[mi][ni], fa[mi], fb[ni], acc[mi][ni]);
        }
    }

    cp_async_wait<0>();
    __syncthreads();

    // Stage the accumulator tile through shared memory for fully coalesced
    // vectorized global accesses.
    float* smC = smem;  // >= 128*132 floats available
    #pragma unroll
    for (int mi = 0; mi < 4; ++mi)
        #pragma unroll
        for (int ni = 0; ni < 2; ++ni)
            wmma::store_matrix_sync(
                smC + (warpM * 64 + mi * 16) * 132 + warpN * 32 + ni * 16,
                acc[mi][ni], 132, wmma::mem_row_major);
    __syncthreads();

    if (S == 1) {
        // Fused epilogue: out = min(C + bias, c) - c
        float* out_base = out + n0;
        const float4* bias4 = reinterpret_cast<const float4*>(bias + n0);
        #pragma unroll 4
        for (int t = tid; t < BM * (BN / 4); t += NTHREADS) {
            const int row = t >> 5;
            const int c4  = t & 31;
            float4 v = *reinterpret_cast<const float4*>(smC + row * 132 + c4 * 4);
            float4 bv = bias4[c4];
            v.x = fminf(v.x + bv.x, c) - c;
            v.y = fminf(v.y + bv.y, c) - c;
            v.z = fminf(v.z + bv.z, c) - c;
            v.w = fminf(v.w + bv.w, c) - c;
            *reinterpret_cast<float4*>(out_base + (long)row * N + c4 * 4) = v;
        }
    } else {
        float* ws_base = ws + ((long)s * BM) * N + n0;
        #pragma unroll 4
        for (int t = tid; t < BM * (BN / 4); t += NTHREADS) {
            const int row = t >> 5;
            const int c4  = t & 31;
            *reinterpret_cast<float4*>(ws_base + (long)row * N + c4 * 4) =
                *reinterpret_cast<const float4*>(smC + row * 132 + c4 * 4);
        }
    }
}

// Split-K reduction with fused epilogue:
// out = min(sum_s ws[s] + bias, c) - c   (vectorized float4)
__global__ void reduce_min_sub_kernel(const float4* __restrict__ ws,
                                      const float4* __restrict__ bias,
                                      float4* __restrict__ out,
                                      const int S,
                                      const long n4,
                                      const long biasN4,
                                      const float c) {
    const long stride = (long)gridDim.x * blockDim.x;
    for (long i = blockIdx.x * (long)blockDim.x + threadIdx.x; i < n4; i += stride) {
        float4 acc = bias[i % biasN4];
        const float4* p = ws + i;
        for (int sp = 0; sp < S; ++sp) {
            float4 v = p[(long)sp * n4];
            acc.x += v.x; acc.y += v.y; acc.z += v.z; acc.w += v.w;
        }
        acc.x = fminf(acc.x, c) - c;
        acc.y = fminf(acc.y, c) - c;
        acc.z = fminf(acc.z, c) - c;
        acc.w = fminf(acc.w, c) - c;
        out[i] = acc;
    }
}

torch::Tensor gemm_min_sub_cuda(torch::Tensor x, torch::Tensor w, torch::Tensor b, double c) {
    TORCH_CHECK(x.is_cuda() && w.is_cuda() && b.is_cuda(), "tensors must be CUDA");
    TORCH_CHECK(x.scalar_type() == at::kFloat, "x must be float32");

    const long M = x.size(0);
    const long K = x.size(1);
    const long N = w.size(0);
    TORCH_CHECK(M == BM, "M must be 128");
    TORCH_CHECK(K % BK == 0, "K must be a multiple of 32");
    TORCH_CHECK(N % BN == 0, "N must be a multiple of 128");

    auto xc = x.contiguous();
    auto wc = w.contiguous();
    auto bc = b.contiguous();

    const int gridN = (int)(N / BN);
    const int total_iters = (int)(K / BK);

    int dev = 0;
    cudaGetDevice(&dev);
    int sms = 0;
    cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev);
    if (sms <= 0) sms = 108;

    // Split-K cost model. Each block streams W through cp.async with ~96 KiB
    // in flight, so even a partially filled wave can sustain close to peak
    // HBM bandwidth; model the GEMM as pure bandwidth time per wave with a
    // per-SM bandwidth cap, plus the split-K reduction traffic.
    const double hbm_bw  = 1.8e12;    // effective HBM bytes/s
    const double persm_bw = 110e9;    // sustainable bytes/s per SM
    const double w_bytes  = (double)N * (double)K * 4.0;
    const double tile_mb  = (double)M * (double)N * 4.0;

    int bestS = 1;
    double bestT = 1e300;
    const int maxS = std::min(16, total_iters);
    for (int s = 1; s <= maxS; ++s) {
        const long blocks = (long)gridN * s;
        const double per_block = w_bytes / (double)blocks;
        double t = 0.0;
        long remaining = blocks;
        while (remaining > 0) {
            const long b = std::min<long>(remaining, sms);
            const double rate = std::min(hbm_bw, persm_bw * (double)b);
            t += per_block * (double)b / rate;
            remaining -= b;
        }
        if (s > 1) t += (double)s * tile_mb * 2.0 / hbm_bw;  // ws write+read
        t += tile_mb / hbm_bw;                               // output write
        if (t < bestT - 1e-12) { bestT = t; bestS = s; }
    }
    const int S = bestS;
    const int base_iters = total_iters / S;
    const int rem = total_iters % S;

    auto out = at::empty({M, N}, x.options());
    torch::Tensor ws;
    float* ws_ptr = nullptr;
    if (S > 1) {
        ws = at::empty({(long)S, M, N}, x.options());
        ws_ptr = ws.data_ptr<float>();
    }

    static bool attr_set = false;
    if (!attr_set) {
        cudaFuncSetAttribute(gemm_tf32_fused_kernel,
                             cudaFuncAttributeMaxDynamicSharedMemorySize,
                             SMEM_BYTES);
        attr_set = true;
    }

    cudaStream_t stream = at::cuda::getCurrentCUDAStream();
    dim3 grid((unsigned)((long)gridN * S));
    gemm_tf32_fused_kernel<<<grid, NTHREADS, SMEM_BYTES, stream>>>(
        xc.data_ptr<float>(), wc.data_ptr<float>(), bc.data_ptr<float>(),
        out.data_ptr<float>(), ws_ptr,
        gridN, (int)K, (int)N, base_iters, rem, S, static_cast<float>(c));

    if (S > 1) {
        const long n4 = (M * N) >> 2;
        const long biasN4 = N >> 2;
        const long blocks_needed = (n4 + NTHREADS - 1) / NTHREADS;
        const int rgrid = (int)std::min<long>(blocks_needed, 8192L);
        reduce_min_sub_kernel<<<rgrid, NTHREADS, 0, stream>>>(
            reinterpret_cast<const float4*>(ws_ptr),
            reinterpret_cast<const float4*>(bc.data_ptr<float>()),
            reinterpret_cast<float4*>(out.data_ptr<float>()),
            S, n4, biasN4, static_cast<float>(c));
    }

    return out;
}
"""

gemm_cpp_source = (
    "torch::Tensor gemm_min_sub_cuda(torch::Tensor x, torch::Tensor w, torch::Tensor b, double c);"
)

fused_gemm = load_inline(
    name="fused_gemm_tf32_min_sub_v2",
    cpp_sources=gemm_cpp_source,
    cuda_sources=gemm_source,
    functions=["gemm_min_sub_cuda"],
    verbose=False,
    extra_cuda_cflags=["-O3", "--use_fast_math"],
)


class ModelNew(nn.Module):
    """
    Optimized version of Model.

    Everything (GEMM + bias + min + subtract) runs in a single custom TF32
    tensor-core CUDA kernel for the benchmark shape:
      * the 1 GiB FP32 weight matrix is streamed from HBM exactly once via a
        4-stage cp.async pipeline (evict_first so it never pollutes L2),
      * the 8 MiB activation matrix is pinned in L2 (evict_last) and served
        to all blocks from cache,
      * the epilogue (bias add, torch.min, subtraction) is fused into the
        GEMM, so the output is written exactly once with no extra passes.

    Falls back to the standard PyTorch path for any other shape/device.
    """
    def __init__(self, in_features, out_features, constant):
        super(ModelNew, self).__init__()
        self.linear = nn.Linear(in_features, out_features)
        self.constant = nn.Parameter(torch.tensor(constant))
        self._const_val = float(constant)

    def forward(self, x):
        if (x.is_cuda and x.dtype == torch.float32 and x.dim() == 2
                and x.size(0) == 128
                and self.linear.in_features % 32 == 0
                and self.linear.out_features % 128 == 0):
            return fused_gemm.gemm_min_sub_cuda(
                x, self.linear.weight, self.linear.bias, self._const_val)
        out = F.linear(x, self.linear.weight, self.linear.bias)
        return torch.min(out, self.constant) - self.constant
```