wgblas 0.1.2 → 1.0.0

package/src/sgemv/sgemv.d.mts

@@ -0,0 +1,118 @@
+import { GpuVector } from "../classes/GpuVector.mjs";
+import { GpuMatrix } from "../classes/GpuMatrix.mjs";
+
+/**
+ * Performs the matrix-vector operation y = alpha * op(A) * x + beta * y
+ *
+ * - `trans='no-transpose'`: op(A) = A,   x is length n, y is length m
+ * - `trans='transpose'`:    op(A) = A^T, x is length m, y is length n
+ *
+ * A is an m×n matrix stored in row-major order. `lda` is the leading dimension
+ * (number of floats between the start of consecutive rows — must be >= n).
+ *
+ * {@includeCode ../../examples/sgemv/sgemv.js}
+ *
+ * **Browser (standalone HTML):**
+ * {@includeCode ../../examples/sgemv/web/sgemv.html}
+ *
+ * @param device - GPUDevice from `init()`
+ * @param trans  - `'no-transpose'` for A, `'transpose'` for A^T
+ * @param m      - number of rows in A
+ * @param n      - number of columns in A
+ * @param alpha  - scalar multiplier for op(A)*x
+ * @param A      - Float32Array or GpuMatrix, row-major, at least (m-1)*lda+n elements
+ * @param lda    - leading dimension of A (>= n)
+ * @param x      - Float32Array input vector
+ * @param incx   - stride for x (must be a positive integer)
+ * @param beta   - scalar multiplier for y
+ * @param y      - Float32Array input/output vector
+ * @param incy   - stride for y (must be a positive integer)
+ * @see <a href="https://github.com/manit2004/wgblas/blob/main/src/sgemv/sgemv.mjs#L16">Source code: sgemv.mjs (L16)</a>
+ * @category BLAS Level 2
+ */
+export declare function sgemv(
+  device: GPUDevice,
+  trans: 'no-transpose' | 'transpose',
+  m: number,
+  n: number,
+  alpha: number,
+  A: Float32Array | GpuMatrix,
+  lda: number,
+  x: Float32Array,
+  incx: number,
+  beta: number,
+  y: Float32Array,
+  incy: number,
+): Promise<{ y: Float32Array; gpuTimeMs?: number }>;
+
+/**
+ * Performs the matrix-vector operation y = alpha * op(A) * x + beta * y
+ *
+ * A is kept GPU-resident; x and y are CPU Float32Arrays.
+ *
+ * @param device - GPUDevice from `init()`
+ * @param trans  - `'no-transpose'` for A, `'transpose'` for A^T
+ * @param m      - number of rows in A
+ * @param n      - number of columns in A
+ * @param alpha  - scalar multiplier for op(A)*x
+ * @param A      - GpuMatrix, row-major, GPU-resident
+ * @param lda    - leading dimension of A (must equal A.lda)
+ * @param x      - Float32Array input vector
+ * @param incx   - stride for x (must be a positive integer)
+ * @param beta   - scalar multiplier for y
+ * @param y      - Float32Array input/output vector
+ * @param incy   - stride for y (must be a positive integer)
+ * @see <a href="https://github.com/manit2004/wgblas/blob/main/src/sgemv/sgemv.mjs#L16">Source code: sgemv.mjs (L16)</a>
+ * @category BLAS Level 2
+ */
+export declare function sgemv(
+  device: GPUDevice,
+  trans: 'no-transpose' | 'transpose',
+  m: number,
+  n: number,
+  alpha: number,
+  A: GpuMatrix,
+  lda: number,
+  x: Float32Array,
+  incx: number,
+  beta: number,
+  y: Float32Array,
+  incy: number,
+): Promise<{ y: Float32Array; gpuTimeMs?: number }>;
+
+/**
+ * Performs the matrix-vector operation y = alpha * op(A) * x + beta * y
+ *
+ * x and y are kept resident on the GPU. A must be a GpuMatrix.
+ *
+ * {@includeCode ../../examples/sgemv/gpuvec.sgemv.js}
+ *
+ * @param device - GPUDevice from `init()`
+ * @param trans  - `'no-transpose'` for A, `'transpose'` for A^T
+ * @param m      - number of rows in A
+ * @param n      - number of columns in A
+ * @param alpha  - scalar multiplier for op(A)*x
+ * @param A      - GpuMatrix, row-major
+ * @param lda    - leading dimension of A (>= n)
+ * @param x      - GpuVector input vector (not mutated)
+ * @param incx   - stride for x (must be a positive integer)
+ * @param beta   - scalar multiplier for y
+ * @param y      - GpuVector input/output vector (mutated in place)
+ * @param incy   - stride for y (must be a positive integer)
+ * @see <a href="https://github.com/manit2004/wgblas/blob/main/src/sgemv/sgemv.mjs#L16">Source code: sgemv.mjs (L16)</a>
+ * @category BLAS Level 2
+ */
+export declare function sgemv(
+  device: GPUDevice,
+  trans: 'no-transpose' | 'transpose',
+  m: number,
+  n: number,
+  alpha: number,
+  A: GpuMatrix,
+  lda: number,
+  x: GpuVector,
+  incx: number,
+  beta: number,
+  y: GpuVector,
+  incy: number,
+): Promise<{ gpuTimeMs?: number }>;

package/src/sgemv/sgemv.mjs

@@ -0,0 +1,141 @@
+import {
+  uploadBuffer,
+  createParamsBuffer,
+  stageReadback,
+  destroyBuffers,
+} from "../util/buffer.mjs";
+import { createBindGroup } from "../util/bindgroup.mjs";
+import { runComputePass, submit } from "../util/compute.mjs";
+import { extractResult } from "../util/result.mjs";
+import { extractTimestamp } from "../util/benchmark.mjs";
+import { getPipeline } from "../util/pipeline.mjs";
+import { calcWorkgroups } from "../util/workgroup.mjs";
+import { GpuVector } from "../classes/GpuVector.mjs";
+import { GpuMatrix } from "../classes/GpuMatrix.mjs";
+
+export async function sgemv(device, trans, m, n, alpha, A, lda, x, incx, beta, y, incy) {
+  const xIsGpu = x instanceof GpuVector;
+  const yIsGpu = y instanceof GpuVector;
+  const AIsGpu = A instanceof GpuMatrix;
+  const isNoTrans = trans === "no-transpose";
+
+  if (!(device instanceof GPUDevice))
+    throw new Error("device must be a GPUDevice.");
+  if (!isNoTrans && trans !== "transpose")
+    throw new Error("trans must be 'no-transpose' or 'transpose'.");
+  if (
+    !Number.isInteger(m) ||
+    !Number.isInteger(n) ||
+    !Number.isInteger(incx) ||
+    !Number.isInteger(incy) ||
+    !Number.isInteger(lda)
+  )
+    throw new Error("m, n, incx, incy, and lda must be integers.");
+  if (typeof alpha !== "number")
+    throw new Error("alpha must be a number.");
+  if (Number.isNaN(alpha)) throw new Error("alpha must not be NaN.");
+  if (!Number.isFinite(alpha)) throw new Error("alpha must be finite.");
+  if (typeof beta !== "number")
+    throw new Error("beta must be a number.");
+  if (Number.isNaN(beta)) throw new Error("beta must not be NaN.");
+  if (!Number.isFinite(beta)) throw new Error("beta must be finite.");
+  if (incx <= 0 || incy <= 0)
+    throw new Error("incx and incy must be positive.");
+  if (lda < n) throw new Error("lda must be >= n.");
+  if (!AIsGpu && !(A instanceof Float32Array))
+    throw new Error("A must be a Float32Array or GpuMatrix.");
+  if (!xIsGpu && !(x instanceof Float32Array))
+    throw new Error("x must be a Float32Array or GpuVector.");
+  if (!yIsGpu && !(y instanceof Float32Array))
+    throw new Error("y must be a Float32Array or GpuVector.");
+  if (xIsGpu !== yIsGpu)
+    throw new Error(
+      "x and y must be the same type (both Float32Array or both GpuVector).",
+    );
+  if (xIsGpu && !AIsGpu)
+    throw new Error(
+      "A must be a GpuMatrix when x and y are GpuVectors.",
+    );
+  if (xIsGpu && x._buf === y._buf)
+    throw new Error("x and y must not reference the same GPU buffer when both are GpuVectors.");
+  if (AIsGpu && lda !== A.lda)
+    throw new Error("lda must match A.lda when A is a GpuMatrix.");
+  if (AIsGpu && (A.rows < m || A.cols < n))
+    throw new Error("A is too small for the given m and n.");
+  if (m < 0 || n < 0) throw new Error("m and n must be non-negative.");
+  if (m === 0 || n === 0) return yIsGpu ? {} : { y };
+
+  // NoTrans: x has n elements, y has m elements
+  // Trans:   x has m elements, y has n elements
+  const xLen = isNoTrans ? n : m;
+  const yLen = isNoTrans ? m : n;
+
+  if (!AIsGpu && A.length < (m - 1) * lda + n)
+    throw new Error(
+      "A does not have enough elements for the given m, n, and lda.",
+    );
+  if (x.length < (xLen - 1) * incx + 1)
+    throw new Error(
+      "x does not have enough elements for the given dimensions and incx.",
+    );
+  if (y.length < (yLen - 1) * incy + 1)
+    throw new Error(
+      "y does not have enough elements for the given dimensions and incy.",
+    );
+
+  const shaderName = isNoTrans ? "sgemv_n" : "sgemv_t";
+  const pipeline   = await getPipeline(device, shaderName);
+
+  const ABuffer = AIsGpu ? A._buf : uploadBuffer(A, "sgemv-A", false);
+  const xBuffer = xIsGpu ? x._buf : uploadBuffer(x, "sgemv-x", false);
+  const yBuffer = yIsGpu ? y._buf : uploadBuffer(y, "sgemv-y", true);
+  const paramsBuffer = createParamsBuffer(
+    [
+      { value: m,     type: "u32" },
+      { value: n,     type: "u32" },
+      { value: alpha, type: "f32" },
+      { value: beta,  type: "f32" },
+      { value: incx,  type: "u32" },
+      { value: incy,  type: "u32" },
+      { value: lda,   type: "u32" },
+    ],
+    "sgemv-params",
+  );
+
+  try {
+    const bindGroup = createBindGroup(pipeline.getBindGroupLayout(0), [
+      ABuffer,
+      xBuffer,
+      yBuffer,
+      paramsBuffer,
+    ]);
+
+    // NoTrans: one workgroup per row; clamped to device limit — the shader's
+    // grid-stride loop handles remaining rows when m > dispatch count.
+    // Trans:   one thread per output column → dispatch ceil(n/64)
+    const wgCount = isNoTrans
+      ? Math.min(m, device.limits.maxComputeWorkgroupsPerDimension)
+      : calcWorkgroups(yLen);
+    const { commandEncoder, ts } = runComputePass(pipeline, bindGroup, wgCount);
+    const readBuffer = yIsGpu ? null : stageReadback(commandEncoder, yBuffer);
+
+    submit(commandEncoder);
+
+    const gpuTimeMs = await extractTimestamp(ts);
+
+    if (yIsGpu) {
+      if (gpuTimeMs !== undefined) return { gpuTimeMs };
+      return {};
+    }
+
+    const result = await extractResult(readBuffer, Float32Array);
+    if (gpuTimeMs !== undefined) return { y: result, gpuTimeMs };
+    return { y: result };
+  } finally {
+    if (!AIsGpu) destroyBuffers(ABuffer);
+    if (!xIsGpu) destroyBuffers(xBuffer);
+    if (!yIsGpu) destroyBuffers(yBuffer);
+    destroyBuffers(paramsBuffer);
+
+  }
+}

package/src/shaders/browser-shaders.mjs

@@ -1,5 +1,6 @@
 import argmax from "./reduction/argmax.wgsl";
 import sum from "./reduction/sum.wgsl";
+import sumF64 from "./reduction/sumF64.wgsl";
 import sscal from "./sscal.wgsl";
 import sswap from "./sswap.wgsl";
 import saxpy from "./saxpy.wgsl";
@@ -10,10 +11,20 @@ import snrm2 from "./snrm2.wgsl";
 import srot from "./srot.wgsl";
 import srotm from "./srotm.wgsl";
 import isamax from "./isamax.wgsl";
+import sgemv_n from "./sgemv_n.wgsl";
+import sgemv_t from "./sgemv_t.wgsl";
+import ssymv from "./ssymv.wgsl";
+import strmv from "./strmv.wgsl";
+import f64add from "./f64add.wgsl";
+import dasum from "./dasum.wgsl";
+import strsv_invert_block from "./strsv_invert_block.wgsl";
+import strsv_apply_inverse from "./strsv_apply_inverse.wgsl";
+import strsv_update from "./strsv_update.wgsl";
 
 export const shaderSources = {
   "reduction/argmax": argmax,
   "reduction/sum": sum,
+  "reduction/sumF64": sumF64,
   sscal,
   sswap,
   saxpy,
@@ -24,4 +35,13 @@ export const shaderSources = {
   srot,
   srotm,
   isamax,
+  sgemv_n,
+  sgemv_t,
+  ssymv,
+  strmv,
+  f64add,
+  dasum,
+  strsv_invert_block,
+  strsv_apply_inverse,
+  strsv_update,
 };

package/src/shaders/dasum.wgsl

@@ -0,0 +1,98 @@
+// dasum: result = sum(|x[i]|)
+// pass 1 dispatches exactly 2 * WGS workgroups; pass 2 uses reduction/sumF64.wgsl.
+// Same structure as sasum.wgsl — every value is now a [main, aux] pair
+// (see src/util/f64pack.mjs) and every `+`/`+=` is computeSum via addPair
+// instead of plain f32 addition. Concatenated after f64add.wgsl by
+// getPipeline (WGSL has no #include), reusing its decode/encode/computeSum/
+// addFields and Packed struct — f64add.wgsl declares no bindings and no entry
+// point of its own (just helper functions), so bindings here start at 0 and
+// the entry point is simply `dasum_main`.
+//
+// xAux/partialsAux are array<u32>, not array<f32> — aux's bits must never
+// pass through an f32-typed storage slot (NaN-bit-pattern corruption risk,
+// see f64pack.mjs and the Packed struct comment above decode()/encode() in
+// f64add.wgsl); Packed (from f64add.wgsl) keeps aux as u32 in registers/
+// workgroup memory too.
+//
+// Per-thread accumulation (acc0..acc3) stays in DECODED Fields form for the
+// entire strided loop below, via addFields — not re-encoded to Packed and
+// re-decoded on every single element like a naive version would. Only the
+// freshly-loaded x[idx] needs decoding each iteration (unavoidable, it's new
+// data every time); the running total never leaves Fields form until the
+// four accumulators are combined and encoded exactly once, right before
+// writing into workgroup-shared `tile`. The cross-thread reduction tree
+// after that still goes through Packed per level (unavoidable — each level
+// combines values that live in different threads' registers via shared
+// memory), but that's a fixed 6 levels regardless of n, unlike the strided
+// loop above whose iteration count scales with n.
+
+@group(0) @binding(0) var<storage, read>       xMain:        array<f32>;
+@group(0) @binding(1) var<storage, read>       xAux:         array<u32>;
+@group(0) @binding(2) var<storage, read_write> partialsMain: array<f32>;
+@group(0) @binding(3) var<storage, read_write> partialsAux:  array<u32>;
+@group(0) @binding(4) var<uniform>             params:       Params;
+
+struct Params {
+  n:     u32,
+  x_inc: u32,
+}
+
+const WGS: u32 = 64;
+
+var<workgroup> tile: array<Packed, 64>;
+
+// a + b, where a/b are [main, aux] pairs — computeSum takes decoded Fields.
+// Only used for the cross-thread reduction tree below; the per-thread
+// strided loop uses addFields directly instead (see module comment).
+fn addPair(a: Packed, b: Packed) -> Packed {
+  return computeSum(decode(bitcast<u32>(a.main), a.aux), decode(bitcast<u32>(b.main), b.aux));
+}
+
+// |x| for a packed double is abs(main) with aux untouched — only main's
+// sign bit carries the double's sign (see fieldsToPacked() in f64pack.mjs).
+// Returns decoded Fields directly (not Packed) for the per-thread loop.
+fn absFields(idx: u32) -> Fields {
+  return decode(bitcast<u32>(abs(xMain[idx])), xAux[idx]);
+}
+
+@compute @workgroup_size(64)
+fn dasum_main(
+  @builtin(global_invocation_id) gid:    vec3u,
+  @builtin(local_invocation_id)  lid:    vec3u,
+  @builtin(workgroup_id)         wgid:   vec3u,
+  @builtin(num_workgroups)       num_wg: vec3u,
+) {
+  var acc0: Fields = Fields(0u, 0u, 0u, 0u);
+  var acc1: Fields = Fields(0u, 0u, 0u, 0u);
+  var acc2: Fields = Fields(0u, 0u, 0u, 0u);
+  var acc3: Fields = Fields(0u, 0u, 0u, 0u);
+
+  let stride   = num_wg.x * WGS;
+  let n4_floor = (params.n / (4u * stride)) * (4u * stride);
+
+  for (var id = gid.x; id < n4_floor; id += 4u * stride) {
+    acc0 = addFields(acc0, absFields( id                * params.x_inc));
+    acc1 = addFields(acc1, absFields((id +      stride) * params.x_inc));
+    acc2 = addFields(acc2, absFields((id + 2u * stride) * params.x_inc));
+    acc3 = addFields(acc3, absFields((id + 3u * stride) * params.x_inc));
+  }
+  for (var id = n4_floor + gid.x; id < params.n; id += stride) {
+    acc0 = addFields(acc0, absFields(id * params.x_inc));
+  }
+
+  // Combine the 4 per-thread accumulators in Fields form too — still no
+  // encode/decode needed, since none of them have touched Packed yet.
+  let combined = addFields(addFields(acc0, acc1), addFields(acc2, acc3));
+  tile[lid.x] = encode(combined.sign, combined.rawExp, combined.mantissaHi, combined.lo);
+  workgroupBarrier();
+
+  for (var s = WGS / 2u; s > 0u; s >>= 1u) {
+    if (lid.x < s) { tile[lid.x] = addPair(tile[lid.x], tile[lid.x + s]); }
+    workgroupBarrier();
+  }
+
+  if (lid.x == 0u) {
+    partialsMain[wgid.x] = tile[0].main;
+    partialsAux[wgid.x] = tile[0].aux;
+  }
+}

package/src/shaders/f64add.wgsl

@@ -0,0 +1,281 @@
+// f64add: adds two doubles, each packed as a [main, aux] pair (main: f32
+// value, aux: raw u32 bits — see src/util/f64pack.mjs; decode()/encode()
+// below are the WGSL mirror of that file's packedToFields()/fieldsToPacked()),
+// producing the sum as another [main, aux] pair.
+//
+// Implements IEEE-754 binary64 addition (align, add/subtract significands,
+// normalize, round-to-nearest-even) using only u32 bitwise/integer
+// arithmetic — WGSL has no 64-bit integer type or arbitrary-precision
+// integers, so each operand's 53-bit significand is carried as a two-word
+// (hi, lo) pair, widened by 3 bits at the bottom to hold guard/round/sticky
+// information while aligning exponents.
+
+const EXP_ALL_ONES: u32 = 0x7ffu;
+const BIAS: i32 = 1023;
+const QUIET_NAN_MANTISSA_HI: u32 = 1u << 19u; // bit51 of the 52-bit mantissa -> canonical quiet NaN
+
+struct Fields {
+  sign: u32,
+  rawExp: u32,
+  mantissaHi: u32, // 20 bits
+  lo: u32,         // 32 bits
+}
+
+// A packed [main, aux] result — aux stays a raw u32; it must never be stored
+// as an array<f32>/treated as a real float (bit pattern can land on a NaN/
+// Infinity exponent for perfectly ordinary doubles — an f32-typed storage
+// slot canonicalizes/corrupts that on any round-trip). See f64pack.mjs's
+// comment above fieldsToPacked, and dasum.wgsl's xAux/partialsAux bindings.
+struct Packed {
+  main: f32,
+  aux:  u32,
+}
+
+// Mirrors packedToFields() in f64pack.mjs.
+fn decode(mainBits: u32, auxBits: u32) -> Fields {
+  let sign = mainBits >> 31u;
+  let expMain = (mainBits >> 23u) & 0xffu;
+  let mantMain = mainBits & 0x7fffffu;
+
+  let auxSign = auxBits >> 31u;
+  let auxExp8 = (auxBits >> 23u) & 0xffu;
+  let auxMant23 = auxBits & 0x7fffffu;
+
+  let expExtra = (auxSign << 2u) | (auxExp8 >> 6u);
+  let mantExtra29 = ((auxExp8 & 0x3fu) << 23u) | auxMant23;
+
+  let rawExp = (expMain << 3u) | expExtra;
+  let mantissaHi = mantMain >> 3u;
+  let mantTop3 = mantMain & 0x7u;
+  let lo = (mantTop3 << 29u) | mantExtra29;
+
+  return Fields(sign, rawExp, mantissaHi, lo);
+}
+
+// Mirrors fieldsToPacked() in f64pack.mjs.
+fn encode(sign: u32, rawExp: u32, mantissaHi: u32, lo: u32) -> Packed {
+  let expMain = rawExp >> 3u;
+  let expExtra = rawExp & 0x7u;
+
+  let mantTop3 = lo >> 29u;
+  let mantMain = (mantissaHi << 3u) | mantTop3;
+  let mantExtra29 = lo & 0x1fffffffu;
+
+  let mainBits = (sign << 31u) | (expMain << 23u) | mantMain;
+
+  let auxSign = (expExtra >> 2u) & 0x1u;
+  let auxExpTop2 = expExtra & 0x3u;
+  let auxExpBot6 = mantExtra29 >> 23u;
+  let auxMant23 = mantExtra29 & 0x7fffffu;
+  let auxExp8 = (auxExpTop2 << 6u) | auxExpBot6;
+
+  let auxBits = (auxSign << 31u) | (auxExp8 << 23u) | auxMant23;
+
+  return Packed(bitcast<f32>(mainBits), auxBits);
+}
+
+struct Pair { hi: u32, lo: u32 }
+struct Shifted { hi: u32, lo: u32, sticky: u32 }
+
+// Two-word right shift by 0..64+ bits, folding every shifted-out 1 bit into
+// a returned sticky flag — used only for the (potentially huge) exponent
+// alignment shift, where exact bits can't all be kept.
+fn shr_sticky(hi: u32, lo: u32, n: u32) -> Shifted {
+  if (n == 0u) {
+    return Shifted(hi, lo, 0u);
+  }
+  if (n >= 64u) {
+    return Shifted(0u, 0u, select(0u, 1u, hi != 0u || lo != 0u));
+  }
+  if (n < 32u) {
+    let stickyBits = lo & ((1u << n) - 1u);
+    let newLo = (lo >> n) | (hi << (32u - n));
+    let newHi = hi >> n;
+    return Shifted(newHi, newLo, select(0u, 1u, stickyBits != 0u));
+  }
+  if (n == 32u) {
+    return Shifted(0u, hi, select(0u, 1u, lo != 0u));
+  }
+  let m = n - 32u;
+  let stickyBits = lo | (hi & ((1u << m) - 1u));
+  let newLo = hi >> m;
+  return Shifted(0u, newLo, select(0u, 1u, stickyBits != 0u));
+}
+
+// Two-word left shift by 0..63 bits — used only to renormalize after
+// cancellation, by an amount that exactly matches the leading-zero count,
+// so nothing meaningful is ever lost off the top.
+fn shl(hi: u32, lo: u32, n: u32) -> Pair {
+  if (n == 0u) {
+    return Pair(hi, lo);
+  }
+  if (n < 32u) {
+    let newHi = (hi << n) | (lo >> (32u - n));
+    let newLo = lo << n;
+    return Pair(newHi, newLo);
+  }
+  if (n == 32u) {
+    return Pair(lo, 0u);
+  }
+  let m = n - 32u;
+  return Pair(lo << m, 0u);
+}
+
+fn add64(aHi: u32, aLo: u32, bHi: u32, bLo: u32) -> Pair {
+  let sumLo = aLo + bLo;
+  let carry = select(0u, 1u, sumLo < aLo); // wrapped around -> there was a carry
+  let sumHi = aHi + bHi + carry;
+  return Pair(sumHi, sumLo);
+}
+
+// Assumes (aHi:aLo) >= (bHi:bLo) — callers guarantee this so no sign handling is needed.
+fn sub64(aHi: u32, aLo: u32, bHi: u32, bLo: u32) -> Pair {
+  let borrow = select(0u, 1u, aLo < bLo);
+  let diffLo = aLo - bLo;
+  let diffHi = aHi - bHi - borrow;
+  return Pair(diffHi, diffLo);
+}
+
+fn ge64(aHi: u32, aLo: u32, bHi: u32, bLo: u32) -> bool {
+  return aHi > bHi || (aHi == bHi && aLo >= bLo);
+}
+
+// The actual IEEE-754 addition, returning decoded Fields rather than an
+// encoded Packed pair — lets a caller that's accumulating many values in a
+// row (e.g. dasum.wgsl's per-thread reduction loop) keep the running total
+// in Fields form the whole time, only encoding once at the very end, instead
+// of paying a decode+encode round-trip on every single addition. computeSum
+// (below) is the Packed-in/Packed-out convenience wrapper around this.
+fn addFields(a: Fields, b: Fields) -> Fields {
+  let aIsNaN = a.rawExp == EXP_ALL_ONES && (a.mantissaHi != 0u || a.lo != 0u);
+  let bIsNaN = b.rawExp == EXP_ALL_ONES && (b.mantissaHi != 0u || b.lo != 0u);
+  if (aIsNaN || bIsNaN) {
+    return Fields(0u, EXP_ALL_ONES, QUIET_NAN_MANTISSA_HI, 0u);
+  }
+
+  let aIsInf = a.rawExp == EXP_ALL_ONES; // mantissa==0 here since NaN is excluded above
+  let bIsInf = b.rawExp == EXP_ALL_ONES;
+  if (aIsInf && bIsInf) {
+    if (a.sign != b.sign) {
+      return Fields(0u, EXP_ALL_ONES, QUIET_NAN_MANTISSA_HI, 0u);
+    }
+    return Fields(a.sign, EXP_ALL_ONES, 0u, 0u);
+  }
+  if (aIsInf) { return Fields(a.sign, EXP_ALL_ONES, 0u, 0u); }
+  if (bIsInf) { return Fields(b.sign, EXP_ALL_ONES, 0u, 0u); }
+
+  let aIsZero = a.rawExp == 0u && a.mantissaHi == 0u && a.lo == 0u;
+  let bIsZero = b.rawExp == 0u && b.mantissaHi == 0u && b.lo == 0u;
+  if (aIsZero && bIsZero) {
+    return Fields(a.sign & b.sign, 0u, 0u, 0u);
+  }
+  if (aIsZero) { return Fields(b.sign, b.rawExp, b.mantissaHi, b.lo); }
+  if (bIsZero) { return Fields(a.sign, a.rawExp, a.mantissaHi, a.lo); }
+
+  // Effective (unbiased) exponent — subnormals share the smallest normal
+  // exponent for alignment purposes and have no implicit leading 1.
+  var expA = i32(a.rawExp) - BIAS;
+  if (a.rawExp == 0u) { expA = 1 - BIAS; }
+  var expB = i32(b.rawExp) - BIAS;
+  if (b.rawExp == 0u) { expB = 1 - BIAS; }
+
+  let implicitA = select(0u, 1u, a.rawExp != 0u);
+  let implicitB = select(0u, 1u, b.rawExp != 0u);
+
+  // Widen each 53-bit significand (implicit + 52-bit mantissa) by 3 zero
+  // bits at the bottom — room for guard/round/sticky once alignment shifts happen.
+  let sigHiA = (implicitA << 23u) | (a.mantissaHi << 3u) | (a.lo >> 29u);
+  let sigLoA = a.lo << 3u;
+  let sigHiB = (implicitB << 23u) | (b.mantissaHi << 3u) | (b.lo >> 29u);
+  let sigLoB = b.lo << 3u;
+
+  // P = the operand with the larger exponent (Q = the other); on a tie, P =
+  // whichever has the larger significand — keeps subtraction below always
+  // non-negative without needing signed magnitudes.
+  var signP: u32; var expP: i32; var sigHiP: u32; var sigLoP: u32;
+  var signQ: u32; var expQ: i32; var sigHiQ: u32; var sigLoQ: u32;
+  if (expA > expB || (expA == expB && ge64(sigHiA, sigLoA, sigHiB, sigLoB))) {
+    signP = a.sign; expP = expA; sigHiP = sigHiA; sigLoP = sigLoA;
+    signQ = b.sign; expQ = expB; sigHiQ = sigHiB; sigLoQ = sigLoB;
+  } else {
+    signP = b.sign; expP = expB; sigHiP = sigHiB; sigLoP = sigLoB;
+    signQ = a.sign; expQ = expA; sigHiQ = sigHiA; sigLoQ = sigLoA;
+  }
+
+  let diff = u32(expP - expQ);
+  let shiftedQ = shr_sticky(sigHiQ, sigLoQ, diff);
+  let alignedHiQ = shiftedQ.hi;
+  let alignedLoQ = shiftedQ.lo | shiftedQ.sticky; // fold sticky into bit0
+
+  var sumHi: u32; var sumLo: u32;
+  if (signP == signQ) {
+    let s = add64(sigHiP, sigLoP, alignedHiQ, alignedLoQ);
+    sumHi = s.hi; sumLo = s.lo;
+  } else {
+    let s = sub64(sigHiP, sigLoP, alignedHiQ, alignedLoQ); // P >= Q(aligned) by construction
+    sumHi = s.hi; sumLo = s.lo;
+  }
+
+  if (sumHi == 0u && sumLo == 0u) {
+    return Fields(0u, 0u, 0u, 0u); // exact cancellation -> +0
+  }
+
+  // commonExp2: per-bit scale of sumLo's bit0 in the widened representation.
+  let commonExp2 = expP - 55;
+
+  var leadPos: i32;
+  if (sumHi != 0u) {
+    leadPos = 32 + i32(31u - countLeadingZeros(sumHi));
+  } else {
+    leadPos = i32(31u - countLeadingZeros(sumLo));
+  }
+  let tentativeExp = leadPos + commonExp2;
+  var targetLSBScale = tentativeExp - 52;
+  if (tentativeExp < -1022) { targetLSBScale = -1074; }
+  let shiftAmt = targetLSBScale - commonExp2;
+
+  var keepHi: u32; var keepLo: u32;
+  if (shiftAmt <= 0) {
+    let sh = shl(sumHi, sumLo, u32(-shiftAmt)); // exact — cancellation only, never loses bits
+    keepHi = sh.hi; keepLo = sh.lo;
+  } else {
+    // Only reached without cancellation (same-sign add, or a tied-exponent
+    // subtract with no shrinkage) — shiftAmt here is always exactly 3 or 4,
+    // so the dropped bits are fully known from sumLo directly (no sticky
+    // approximation needed, unlike the Q-alignment shift above).
+    let n = u32(shiftAmt);
+    let remainder = sumLo & ((1u << n) - 1u);
+    let halfway = 1u << (n - 1u);
+    let sh = shr_sticky(sumHi, sumLo, n);
+    keepHi = sh.hi; keepLo = sh.lo;
+    if (remainder > halfway || (remainder == halfway && (keepLo & 1u) != 0u)) {
+      let inc = add64(keepHi, keepLo, 0u, 1u);
+      keepHi = inc.hi; keepLo = inc.lo;
+    }
+  }
+
+  var resultExpBase = targetLSBScale;
+  if ((keepHi & (1u << 21u)) != 0u) { // rounding carried past the 53-bit budget
+    let sh = shr_sticky(keepHi, keepLo, 1u); // dropped bit is guaranteed 0 here
+    keepHi = sh.hi; keepLo = sh.lo;
+    resultExpBase = resultExpBase + 1;
+  }
+
+  let resultSign = signP;
+  if ((keepHi & (1u << 20u)) != 0u) { // normal-shaped result
+    let unbiasedExp = 52 + resultExpBase;
+    let rawExpFinal = unbiasedExp + BIAS;
+    if (rawExpFinal >= 2047) {
+      return Fields(resultSign, EXP_ALL_ONES, 0u, 0u); // overflow -> Infinity
+    }
+    return Fields(resultSign, u32(rawExpFinal), keepHi & 0xfffffu, keepLo);
+  }
+  return Fields(resultSign, 0u, keepHi & 0xfffffu, keepLo); // subnormal result
+}
+
+// Packed-in/Packed-out convenience wrapper around addFields — encodes once,
+// after the math, rather than addFields itself needing to know about Packed.
+fn computeSum(a: Fields, b: Fields) -> Packed {
+  let f = addFields(a, b);
+  return encode(f.sign, f.rawExp, f.mantissaHi, f.lo);
+}