nbxtensor 0.0.1__py3-none-any.whl

nbxtensor/ops/screen_oracle.py

@@ -0,0 +1,479 @@
+"""The fp64 oracle the autotune screen consults, computed from the LIVE args.
+
+The screen's consensus asks the candidates whether they agree with each other.
+That question has one blind spot, and it is not academic: when the defect is in
+the BACKEND'S EMISSION rather than in the choice of tile, every candidate is
+wrong in the same way, the vote is unanimous, and the screen seats a wrong
+configuration in silence.
+
+Measured 2026-09-11 on four `addmm` shapes the catalogue demands: the emitted
+MSL declared the float scalars `alpha`/`beta` as `int`, so the bit pattern of
+`1.0f` multiplied the accumulator — wrong by a factor of 1065353216. Eight
+runs, four shapes, two arms: the bare vote SEATED a configuration every time;
+the same screen with this oracle REFUSED every time.
+
+## Where the cost is paid, and why it is affordable
+
+A CERTIFIED entry is served without a sweep, without a screen and without an
+oracle — zero cost. Only an UNCERTIFIED key sweeps, and only then is the screen
+run and the oracle computed, once for the key and not once per candidate.
+
+**The cost is therefore paid exactly where the risk lives, and never
+elsewhere** — and the certified directory gains a second reason to exist that
+was not visible before: it does not only remove the sweep, it removes the
+oracle's cost with it.
+
+## What this covers, and what it says when it does not
+
+An oracle exists for the kernels whose mathematics can be recomputed in float64
+on the host from the live operands. For anything else this returns None, and
+the screen falls back to the consensus — LOUDLY, once per process, naming the
+key, because a screen running without an oracle is exactly the situation the
+`addmm` measurement showed to be dangerous.
+"""
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+import numpy as np
+
+#: Keys already announced as running without an oracle. Once per process: a
+#: warning repeated per launch is a warning nobody reads.
+_ANNOUNCED: set = set()
+
+_NP = {"fp16": np.float16, "float16": np.float16,
+       "fp32": np.float32, "float32": np.float32,
+       "fp64": np.float64, "float64": np.float64}
+
+
+def _to_f64(t) -> Optional[np.ndarray]:
+    """A live operand as float64, or None when it cannot be read here."""
+    try:
+        from nbxtensor.dtype.nbx_dtype import NBXDtype
+        # ONE path to host memory, for every dtype. The bf16 case used to read
+        # the device pointer directly with `ctypes.string_at`, which crosses NO
+        # BARRIER -- so a buffer a kernel had just written was read before the
+        # write landed. Measured 2026-09-12 on a kernel-written buffer: 4093 of
+        # 4096 elements differed from the copy path, and agreed after an
+        # explicit synchronisation.
+        #
+        # That produced the symptom it is easiest to misread: every candidate
+        # compared against the same stale bytes, so the fp64 oracle contradicted
+        # all ten and the screen refused the whole space. A reference read at
+        # the wrong moment contradicts every correct answer exactly as loudly
+        # as a wrong one. Eighteen recorded refusals rested on it.
+        #
+        # The defect had the shape of a PATH, not of a dtype: bf16 was merely
+        # the only traffic on it. So the raw path is gone rather than repaired,
+        # and a dtype routed here later cannot inherit it. Removing it also
+        # removes the second failure mode `string_at` carries -- it hands its
+        # size to `PyBytes_FromStringAndSize` as a C int, so any buffer of
+        # 2 GiB or more returns "Negative size", confirmed live on this machine.
+        host = t.to_cpu() if getattr(t, "_device", "cpu") != "cpu" else t
+        if getattr(t, "_dtype", None) == NBXDtype.bfloat16:
+            # numpy has no bf16; the bits are the top half of an fp32 and
+            # travel in a uint16 container.
+            bits = np.asarray(host.numpy()).view(np.uint16).astype(np.uint32)
+            return ((bits << np.uint32(16)).view(np.float32)
+                    .reshape(tuple(t.shape)).astype(np.float64))
+        return np.asarray(host.numpy(), dtype=np.float64)
+    except Exception:                                  # noqa: BLE001
+        return None
+
+
+def _mm(named: Dict[str, Any], rows: Optional[tuple] = None) -> Optional[np.ndarray]:
+    """The fp64 reference for `mm`/`addmm`, optionally on a ROW WINDOW only.
+
+    `rows=(r0, r1)` computes `a[r0:r1] @ b` instead of the whole product. That is the rack's
+    row-windowed oracle technique (autotune_certify.RowWindowedOracle), brought to the screen
+    so a shape over the screening budget is verified on named windows instead of not at all.
+    A whole fp64 reference for a 4 194 304 x 540 output is 18 GB; three row windows of it are
+    a few megabytes, and the per-element comparison is identical on them."""
+    a_live = named.get("a_ptr")
+    if rows is not None:
+        # The rows are cut WHERE THE OPERAND LIVES, before it crosses. Cut after the read,
+        # the whole operand came to the host and was cast whole to float64 once per window
+        # (measured 2026-09-28: 4 096 MiB on the host for 64 rows of a 1 048 576 x 256 fp32
+        # operand; the certifier's footprint spiked by 3.7 GB per matmul key), so the window
+        # saved nothing where it was meant to. A device slice is a view whose strides no flat
+        # reader follows; it is made contiguous on the device first (`_materialise`'s rule).
+        r0, r1 = rows
+        if getattr(a_live, "_device", "cpu") != "cpu" and hasattr(a_live, "__getitem__"):
+            piece = a_live[r0:r1]
+            contiguous = getattr(piece, "contiguous", None)
+            a_live = contiguous() if callable(contiguous) else piece
+    a, b = _to_f64(a_live), _to_f64(named.get("b_ptr"))
+    if a is None or b is None:
+        return None
+    if rows is not None and a.shape[0] != (r1 - r0):
+        a = a[r0:r1]                                    # a host operand: cut after the read
+    out = a @ b
+    bias = named.get("bias_ptr")
+    if bias is not None:
+        bv = _to_f64(bias)
+        if bv is None:
+            return None
+        alpha = float(named.get("alpha", 1.0) or 1.0)
+        beta = float(named.get("beta", 1.0) or 1.0)
+        if bv.ndim == 2 and rows is not None and bv.shape[0] != out.shape[0]:
+            bv = bv[r0:r1]                              # a per-row bias follows the window
+        out = alpha * out + beta * (bv[None, :] if bv.ndim == 1 else bv)
+    return out
+
+
+def _baddbmm(named: Dict[str, Any]) -> Optional[np.ndarray]:
+    a, b = _to_f64(named.get("A_ptr")), _to_f64(named.get("B_ptr"))
+    if a is None or b is None:
+        return None
+    out = a @ b
+    alpha = float(named.get("alpha", 1.0) or 1.0)
+    beta = float(named.get("beta", 1.0) or 1.0)
+    out = alpha * out
+    if named.get("HAS_BIAS") and named.get("bias_ptr") is not None:
+        bv = _to_f64(named["bias_ptr"])
+        if bv is None:
+            return None
+        out = out + beta * bv
+    return out
+
+
+#: kernel short name -> (reference for its OUTPUT from live args, output arg).
+#: A kernel absent from here has no oracle and the screen says so out loud.
+#:
+#: The OUTPUT ARGUMENT NAME is not decoration. The first version of this
+#: provider matched the output buffer by BYTE LENGTH, and for a matmul at
+#: (19, 2048, 2048) the operand `a` and the output `c` are both 19x2048 fp32 —
+#: the same 155648 bytes. The oracle was therefore placed on `a`, an INPUT, so
+#: every candidate "disagreed" with it and the screen refused all ten
+#: candidates of a shape whose certified deviation is 1.7e-06.
+#:
+#: An oracle that refuses correct configurations is worse than no oracle: it
+#: turns a silent wrong into a loud stop on healthy work, and it would have
+#: been believed because a refusal reads as vigilance. Matched by ADDRESS now.
+ORACLES = {
+    "matmul_kernel": (_mm, "c_ptr"),
+    "addmm_kernel": (_mm, "c_ptr"),
+    "baddbmm_kernel": (_baddbmm, "out_ptr"),}
+
+# The convolution family — the 11.5 % the table above did not cover (owed-proofs
+# item 3: 822 of 7 158 keys on this rack). Its reference lives in its own module
+# with its own tests against torch at 1e-12; until 2026-09-13 that module's
+# table was never joined to THIS one, so the live screen kept announcing "no
+# oracle for this kernel" on every conv key while the docstring next door said
+# the family was covered. Register entry 46.
+from nbxtensor.ops.oracles import conv2d_fp64 as _conv  # noqa: E402
+
+ORACLES.update({
+    name: ((lambda named, _fn=fn: _fn(named, _to_f64)), out_name)
+    for name, (fn, out_name) in _conv.ORACLES.items()
+})
+
+
+#: Kernels whose oracle can be computed on a ROW WINDOW of the output — the output read as
+#: `[prod(leading dims), last dim]`, which for the matrix kernels is the matrix and for the
+#: convolution family is `[N*Cout*Ho, Wo]`. The convolution rows map to input rows through the
+#: receptive field (`conv2d_fp64.receptive_slab`, the certifier's arithmetic since 2026-09-07);
+#: until 2026-09-26 the family was absent here and an over-budget conv key fell to windowed
+#: CONSENSUS or to no screen at all, while an under-budget one read its WHOLE input in float64
+#: (a 128-channel 2048x1024 input: 2.1 GB, the Mac's 27.5 GB footprint). A kernel absent from
+#: this set falls back to windowed consensus, announced, rather than to nothing.
+ROW_WINDOWABLE = {"matmul_kernel", "addmm_kernel"} | set(_conv.ROW_ORACLES)
+
+
+def _slab_reader(tensor, shape):
+    """Read `[n0:n1, :, u0:u1, v0:v1]` of a contiguous NCHW device tensor to the host in float64
+    — ONLY that slab crosses, cut where the tensor lives. `shape` is the `[N, C, H, W]` the
+    kernel's named extents declare; the live argument may be flat."""
+    def slab(n0, n1, u0, u1, v0, v1):
+        view = tensor.view(*shape) if tuple(tensor.shape) != tuple(shape) else tensor
+        piece = view[n0:n1, :, u0:u1, v0:v1].contiguous()
+        return _to_f64(piece)
+    return slab
+
+
+def oracle_row_bytes(tuner, meta) -> Optional[int]:
+    """What one flat output row costs THIS kernel's float64 reference on the host, or None for
+    a kernel the row table does not window (the default: the row's own elements in float64).
+    Raises `ValueError` for a kernel the table windows whose cost cannot be formed — the
+    launcher then seats the key unscreened and says why, rather than sizing the window to the
+    output row."""
+    name = getattr(getattr(tuner, "base_fn", None), "__name__", "") or ""
+    named = {**dict(getattr(tuner, "nargs", None) or {}), **dict(meta or {})}
+    return _conv.oracle_row_bytes(name, named)
+
+
+def fp64_footprint(buffers) -> int:
+    """The host bytes a FULL reference would hold for these device buffers: every element in
+    float64, whatever it is on the device. The screening budget is a rule about cost, and the
+    oracle's cost is this number, not the buffers' device bytes — a 537 MB bf16 input is 2.1 GB
+    of float64 before the first tap is computed. The element size comes from `NBXDtype`, the
+    one authority (`_writable_buffers` records `dtype.name`, the enum's own name); a name the
+    enum does not know is a refusal, never a guess."""
+    from nbxtensor.dtype.nbx_dtype import NBXDtype, dtype_size, parse_dtype
+    total = 0
+    for _addr, nbytes, dtype in buffers:
+        # `_writable_buffers` records `value.dtype.name`, and `NBXTensor.dtype` is the TRITON
+        # dtype (`fp16`, `bf16`, `fp32`, ...) — the engine's own parser knows those spellings
+        # beside the enum's; anything neither knows is refused, never guessed.
+        name = str(dtype)
+        try:
+            nbx = NBXDtype[name] if name in NBXDtype.__members__ else parse_dtype(name)
+        except (KeyError, ValueError):
+            raise ValueError(f"the screen cannot size a buffer of dtype {dtype!r}: not an NBXDtype") from None
+        total += int(nbytes) * 8 // dtype_size(nbx)
+    return total
+
+
+def windowed_reference(tuner, meta, rows) -> Optional[np.ndarray]:
+    """The fp64 reference for flat `rows` of the output, or None if this kernel cannot be windowed.
+
+    Returns the ARRAY, not bytes: the launcher slices the candidates' own bytes to the same
+    rows and compares, so both sides are the same rows of the same tensor. A refusal by a
+    row oracle is recorded through `set_last_refusal`, so the launcher's seat line can say
+    "the oracle refused, and why" instead of "no oracle covers this kernel"."""
+    name = getattr(getattr(tuner, "base_fn", None), "__name__", "") or ""
+    if name not in ROW_WINDOWABLE:
+        return None
+    named = {**dict(getattr(tuner, "nargs", None) or {}), **dict(meta or {})}
+    if not named:
+        return None
+    entry = _conv.ROW_ORACLES.get(name)
+    if entry is None:
+        try:
+            return _mm(named, rows=rows)
+        except Exception:                               # noqa: BLE001
+            return None
+    try:
+        x, w = named.get(entry.input_name), named.get(entry.weight_name)
+        if x is None or w is None:
+            raise ValueError(f"its arguments {entry.input_name!r}/{entry.weight_name!r} are not among the live arguments")
+        weight = _to_f64(w)
+        if weight is None:
+            raise ValueError("its weight could not be read here")
+        return entry.rows(named, _slab_reader(x, entry.input_shape(named)), weight, rows)
+    except (KeyError, TypeError, ValueError) as exc:
+        set_last_refusal(f"{name}: the row-windowed oracle refused — {exc}")
+        return None
+
+
+def _span_bytes(logical, tensor, nbytes: int) -> Optional[bytes]:
+    """`logical` (the tensor's elements in row-major order) laid out as the `nbytes` span the screen snapshots from
+    the tensor's `data_ptr()`, or None when they cannot be the same buffer.
+
+    The screen compares a candidate's effect on the PHYSICAL span of every buffer (`_writable_buffers` keeps a dense
+    permutation, a conv input read NHWC as NCHW, and a stride-0 broadcast). A reference in logical order is that span
+    only for a contiguous tensor: for a permuted fp32 conv input it is the same elements in another order, every
+    candidate "changed" the input, and the screen refused the whole space of a correct kernel (2026-10-29)."""
+    stride = getattr(tensor, "stride", None)
+    item = logical.dtype.itemsize
+    if (not callable(stride) or getattr(tensor, "is_contiguous", lambda: True)()
+            or logical.ndim == 0):
+        return logical.tobytes() if logical.nbytes == nbytes else None
+    if nbytes % item:
+        return None
+    steps = tuple(int(s) for s in stride())
+    if any(s < 0 for s in steps):
+        return None
+    reach = sum((int(n) - 1) * s for n, s in zip(logical.shape, steps)) + 1 if logical.size else 0
+    if reach * item != nbytes:
+        return None
+    span = np.zeros(reach, dtype=logical.dtype)
+    np.lib.stride_tricks.as_strided(span, shape=logical.shape, strides=tuple(s * item for s in steps))[...] = logical
+    return span.tobytes()
+
+
+def provider(tuner, key, buffers, meta=None) -> Optional[List[bytes]]:
+    """The correct contents of every screened buffer, or None.
+
+    The screen snapshots EVERY writable buffer, not just the output — four of
+    them for `addmm` (`a`, `b`, `bias`, `c`). The correct content of an INPUT
+    after the kernel is its content unchanged; only the output has an oracle.
+    Returning the oracle for all of them compares an input to an output.
+    """
+
+    name = getattr(getattr(tuner, "base_fn", None), "__name__", "") or ""
+    entry = ORACLES.get(name)
+    if entry is None:
+        announce_no_oracle(name or str(tuner), key)
+        return None
+    fn, out_name = entry
+    # Positional arguments from the autotuner, constexpr arguments from the
+    # launch kwargs: an oracle that needs `kernel_height` finds it only here.
+    named = {**dict(getattr(tuner, "nargs", None) or {}), **dict(meta or {})}
+    if not named:
+        return None
+    out_tensor = named.get(out_name)
+    out_addr = int(out_tensor.data_ptr()) if out_tensor is not None else None
+    if out_addr is None:
+        announce_no_oracle(name, key,
+                           why=f"its output argument {out_name!r} is not among "
+                               f"the live arguments")
+        return None
+    try:
+        import time as _time
+        _t0 = _time.perf_counter()
+        reference = fn(named)
+        # The oracle's own CPU wall time is the one number measured for this
+        # key BEFORE any candidate runs, so it is the bootstrap base for the
+        # first candidate's time budget: a GPU configuration slower than the
+        # single-threaded float64 CPU computation of the same mathematics is
+        # not a configuration, whatever else it is.
+        record_oracle_ms(key, (_time.perf_counter() - _t0) * 1e3)
+    except Exception:                                  # noqa: BLE001
+        reference = None
+    if reference is None:
+        announce_no_oracle(name, key, why="its operands could not be read here")
+        return None
+
+    # A reference built from non-finite operands vouches for nothing. NaN
+    # compares unequal to everything including itself, so every candidate
+    # "disagrees" and the screen either refuses the whole space or — worse, when
+    # the deviation is computed on bf16 bits — lets a wrong configuration
+    # through wearing a `screened: true` stamp. Measured on swin2SR (M4 Pro,
+    # 2026-09-17): a conv whose input carried 63181 NaN and 356 Inf produced
+    # "the fp64 oracle contradicts EVERY candidate (18 of 18)".
+    #
+    # Refusing here routes the key through `_seat_unscreened`, which records it
+    # `screened: false` with this reason and keeps it out of the certified
+    # directory. The engine still runs; it just stops claiming the run was
+    # validated.
+    bad = int(np.count_nonzero(~np.isfinite(reference)))
+    if bad:
+        set_last_refusal(
+            f"the fp64 reference is not finite ({bad} of {reference.size} "
+            f"elements NaN or Inf), so it cannot adjudicate any candidate — "
+            f"the operands read for this key were themselves not finite")
+        announce_no_oracle(name, key, why=last_refusal())
+        return None
+
+    # Every live tensor by its address, so an INPUT is read through the copy
+    # path like everything else. The line this replaces read the device
+    # pointer with `ctypes.string_at` and justified it as "an input,
+    # unchanged" -- an assumption this repository's own measurement destroyed
+    # the same day: the bf16 operand of eighteen refusals was the BIAS, an
+    # input produced by a conversion kernel earlier in the graph. An input can
+    # be exactly as stale as an output, and "unchanged" says nothing about
+    # WHEN it was written.
+    #
+    # The size failure mode does not even need that argument: `string_at`
+    # hands its length to `PyBytes_FromStringAndSize` as a C int, so any
+    # buffer of 2 GiB or more returns "Negative size" whoever wrote it --
+    # confirmed live here at 2 GiB + 1 -- and a matmul input passes 2 GiB
+    # without difficulty on the shapes that matter.
+    by_addr = {}
+    for _v in named.values():
+        try:
+            by_addr[int(_v.data_ptr())] = _v
+        except Exception:                              # noqa: BLE001
+            continue
+
+    out: List[bytes] = []
+    for addr, nbytes, dtype_name in buffers:
+        if int(addr) == out_addr:                      # the output, BY ADDRESS
+            if dtype_name in ("bf16", "bfloat16"):
+                # bfloat16 is not a numpy dtype, so `_NP.get` fell through to
+                # fp32 -- and the reference then measured 4 bytes/element while
+                # the bf16 output buffer is 2, so `want.nbytes` was exactly
+                # double and the oracle was DISCARDED on every bf16 output.
+                # Measured 2026-09-14: conv2d at (1,180,448,448) bf16 --
+                # reference 144506880 vs buffer 72253440 -- so the whole conv
+                # family (and any bf16-output kernel) sat on the bare vote while
+                # the fp64 oracle was wired and computing. The output holds the
+                # top 16 bits of fp32; round the reference to bf16 the same way
+                # (`f32_to_bf16_bits`, round-to-nearest-even) so it compares in
+                # the buffer's own width. `_deviation` decodes bf16 bytes, so
+                # the tolerance comparison is unchanged. A dtype fix, not a
+                # backend one.
+                from nbxtensor.dtype.convert import f32_to_bf16_bits
+                want = np.ascontiguousarray(
+                    f32_to_bf16_bits(np.ascontiguousarray(reference, dtype=np.float32)))
+            else:
+                want = np.ascontiguousarray(
+                    reference.astype(_NP.get(dtype_name, np.float32)))
+            if want.size == int(np.prod(getattr(out_tensor, "shape", (want.size,)))):
+                want = want.reshape(tuple(out_tensor.shape))
+            raw = _span_bytes(want, out_tensor, int(nbytes))
+            if raw is None:
+                announce_no_oracle(
+                    name, key,
+                    why=f"the reference is {want.nbytes} bytes and the output "
+                        f"buffer is {nbytes}")
+                return None
+            out.append(raw)
+        else:
+            # An input, unchanged -- read through the tensor's own copy to the
+            # host, which crosses the barrier a raw pointer read does not.
+            src = by_addr.get(int(addr))
+            if src is None:
+                announce_no_oracle(
+                    name, key,
+                    why=f"an input buffer at {addr:#x} is not among the live "
+                        f"arguments, so it can only be read by raw pointer, "
+                        f"which crosses no barrier and caps at 2 GiB")
+                return None
+            host = src.to_cpu() if getattr(src, "_device", "cpu") != "cpu" else src
+            logical = np.ascontiguousarray(host.numpy())
+            raw = _span_bytes(logical, src, int(nbytes))
+            if raw is None:
+                announce_no_oracle(
+                    name, key,
+                    why=f"the input at {addr:#x} copies to {logical.nbytes} bytes "
+                        f"and the buffer is {nbytes}")
+                return None
+            out.append(raw)
+    return out
+
+
+#: Oracle CPU wall time by key text, for the sweep's time budget. In-process,
+#: like `_SCREEN_CACHE`: the budget protects a sweep that runs in this very
+#: process, so persisting these would only let a stale machine state speak.
+_ORACLE_MS: dict = {}
+
+
+#: Why the oracle last declined, so the record can say it instead of "no
+#: reference". Set immediately before returning None; read by the launcher when
+#: it stamps the key unscreened.
+_LAST_REFUSAL: Optional[str] = None
+
+
+def set_last_refusal(why: str) -> None:
+    global _LAST_REFUSAL
+    _LAST_REFUSAL = why
+
+
+def last_refusal() -> Optional[str]:
+    return _LAST_REFUSAL
+
+
+def record_oracle_ms(key, ms: float) -> None:
+    _ORACLE_MS[str(key)] = float(ms)
+
+
+def oracle_ms_of(key):
+    """The oracle's measured CPU time for this key, or None if it never ran."""
+    return _ORACLE_MS.get(str(key))
+
+
+def announce_no_oracle(kernel_name: str, key, why: str = "no oracle is registered "
+                                                        "for this kernel") -> None:
+    """Say, once per process, that a screen ran on a consensus alone.
+
+    A mechanism that is complete and switched off is the most expensive form of
+    a vacuous guard: it costs the price of writing it and returns nothing. The
+    only thing worse is one that is silent about being off.
+    """
+    token = (kernel_name, str(key))
+    if token in _ANNOUNCED:
+        return
+    _ANNOUNCED.add(token)
+    print(f"[AUTOTUNE_ORACLE] {kernel_name}: screening at key {key} on the "
+          f"CONSENSUS ALONE — {why}. A consensus cannot see a defect that is "
+          f"in the backend's emission, because every candidate then shares it "
+          f"and the vote is unanimous (measured: addmm wrong by 1e9 with a "
+          f"unanimous vote, 2026-09-11).", flush=True)
+
+
+def install() -> None:
+    """Make the screen consult this oracle."""
+    from nbxtensor.backend import launcher as L
+    L.set_screen_oracle(provider)

nbxtensor/ops/selu.py

@@ -0,0 +1,22 @@
+"""SELU — pure @triton.jit kernel."""
+
+import triton
+import triton.language as tl
+
+@triton.jit
+def selu_forward_kernel(
+    input_ptr, output_ptr,
+    n_elements,
+    BLOCK_SIZE: tl.constexpr,
+):
+    pid = tl.program_id(0).to(tl.int64)
+    offset = pid.to(tl.int64) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)   # 64-bit from the program id: the product itself wraps past 2^31 elements (register 58)
+    mask = offset < n_elements
+
+    scale = 1.0507009873554804934193349852946
+    alpha = 1.6732632423543772848170429916717
+
+    x = tl.load(input_ptr + offset, mask=mask)
+    x_fp32 = x.to(tl.float32)
+    out = scale * (tl.maximum(0, x_fp32) + tl.minimum(0, alpha * (tl.exp(x_fp32) - 1.0)))
+    tl.store(output_ptr + offset, out, mask=mask)