wgblas 2.0.0 → 2.2.0

package/src/sgemv/sgemv.mjs

@@ -9,27 +9,40 @@ import { runComputePass, submit } from "../util/compute.mjs";
 import { extractResult } from "../util/result.mjs";
 import { extractTimestamp } from "../util/benchmark.mjs";
 import { getPipeline } from "../util/pipeline.mjs";
-import { calcWorkgroups } from "../util/workgroup.mjs";
+import { requireWorkgroups } from "../util/workgroup.mjs";
 import { GpuVector } from "../classes/GpuVector.mjs";
 import { GpuMatrix } from "../classes/GpuMatrix.mjs";
+import { requireGpuDevice, requireSameDevice } from "../util/device.mjs";
 
-export async function sgemv(device, trans, m, n, alpha, A, lda, x, incx, beta, y, incy, layout = "row-major") {
+export async function sgemv(
+  device,
+  trans,
+  m,
+  n,
+  alpha,
+  A,
+  lda,
+  x,
+  incx,
+  beta,
+  y,
+  incy,
+  layout = "row-major",
+) {
   const AIsGpu = A instanceof GpuMatrix;
   const xIsGpu = x instanceof GpuVector;
   const yIsGpu = y instanceof GpuVector;
 
-  if (!(device instanceof GPUDevice))
-    throw new Error("device must be a GPUDevice.");
+  requireGpuDevice(device);
+  requireSameDevice(device, "sgemv", { A, x, y });
   if (trans !== "no-transpose" && trans !== "transpose")
     throw new Error("trans must be 'no-transpose' or 'transpose'.");
   if (layout !== "row-major" && layout !== "column-major")
     throw new Error("layout must be 'row-major' or 'column-major'.");
-  if (typeof alpha !== "number")
-    throw new Error("alpha must be a number.");
+  if (typeof alpha !== "number") throw new Error("alpha must be a number.");
   if (Number.isNaN(alpha)) throw new Error("alpha must not be NaN.");
   if (!Number.isFinite(alpha)) throw new Error("alpha must be finite.");
-  if (typeof beta !== "number")
-    throw new Error("beta must be a number.");
+  if (typeof beta !== "number") throw new Error("beta must be a number.");
   if (Number.isNaN(beta)) throw new Error("beta must not be NaN.");
   if (!Number.isFinite(beta)) throw new Error("beta must be finite.");
   if (
@@ -53,15 +66,15 @@ export async function sgemv(device, trans, m, n, alpha, A, lda, x, incx, beta, y
       "x and y must be the same type (both Float32Array or both GpuVector).",
     );
   if (xIsGpu && !AIsGpu)
-    throw new Error(
-      "A must be a GpuMatrix when x and y are GpuVectors.",
-    );
+    throw new Error("A must be a GpuMatrix when x and y are GpuVectors.");
   if (AIsGpu && !xIsGpu)
+    throw new Error("x and y must be GpuVectors when A is a GpuMatrix.");
+  if (xIsGpu && x._buf === y._buf)
     throw new Error(
-      "x and y must be GpuVectors when A is a GpuMatrix.",
+      "x and y must not reference the same GPU buffer when both are GpuVectors.",
     );
-  if (xIsGpu && x._buf === y._buf)
-    throw new Error("x and y must not reference the same GPU buffer when both are GpuVectors.");
+  if (AIsGpu && yIsGpu && A._buf === y._buf)
+    throw new Error("A and y must not reference the same GPU buffer.");
   if (AIsGpu && lda !== A.lda)
     throw new Error("lda must match A.lda when A is a GpuMatrix.");
   if (AIsGpu && (A.rows < m || A.cols < n))
@@ -97,40 +110,56 @@ export async function sgemv(device, trans, m, n, alpha, A, lda, x, incx, beta, y
     );
 
   const shaderName = isNoTrans ? "sgemv_n" : "sgemv_t";
-  const pipeline   = await getPipeline(device, shaderName);
+  const pipeline = await getPipeline(device, shaderName);
 
-  const ABuffer = AIsGpu ? A._buf : uploadBuffer(A, "sgemv-A", false);
-  const xBuffer = xIsGpu ? x._buf : uploadBuffer(x, "sgemv-x", false);
-  const yBuffer = yIsGpu ? y._buf : uploadBuffer(y, "sgemv-y", true);
-  const paramsBuffer = createParamsBuffer(
-    [
-      { value: m,     type: "u32" },
-      { value: n,     type: "u32" },
-      { value: alpha, type: "f32" },
-      { value: beta,  type: "f32" },
-      { value: incx,  type: "u32" },
-      { value: incy,  type: "u32" },
-      { value: lda,   type: "u32" },
-    ],
-    "sgemv-params",
-  );
+  let ABuffer = null;
+  let xBuffer = null;
+  let yBuffer = null;
+  let paramsBuffer = null;
 
   try {
-    const bindGroup = createBindGroup(pipeline.getBindGroupLayout(0), [
+    ABuffer = AIsGpu ? A._buf : uploadBuffer(device, A, "sgemv-A", false);
+    xBuffer = xIsGpu ? x._buf : uploadBuffer(device, x, "sgemv-x", false);
+    yBuffer = yIsGpu ? y._buf : uploadBuffer(device, y, "sgemv-y", true);
+    paramsBuffer = createParamsBuffer(
+      device,
+      [
+        { value: m, type: "u32" },
+        { value: n, type: "u32" },
+        { value: alpha, type: "f32" },
+        { value: beta, type: "f32" },
+        { value: incx, type: "u32" },
+        { value: incy, type: "u32" },
+        { value: lda, type: "u32" },
+      ],
+      "sgemv-params",
+    );
+
+    const bindGroup = createBindGroup(device, pipeline.getBindGroupLayout(0), [
       ABuffer,
       xBuffer,
       yBuffer,
       paramsBuffer,
     ]);
 
-    // NoTrans: one workgroup per row (grid-stride handles overflow); Trans: one thread per output column.
+    // NoTrans: one workgroup per row — sgemv_n.wgsl is a grid-stride loop, so
+    // clamping here only costs parallelism. Trans: one thread per output
+    // column, and sgemv_t.wgsl indexes straight off global_invocation_id with
+    // no fallback, so an over-limit dispatch must be refused, not truncated.
     const wgCount = isNoTrans
       ? Math.min(m, device.limits.maxComputeWorkgroupsPerDimension)
-      : calcWorkgroups(yLen);
-    const { commandEncoder, ts } = runComputePass(pipeline, bindGroup, wgCount);
-    const readBuffer = yIsGpu ? null : stageReadback(commandEncoder, yBuffer);
+      : requireWorkgroups(device, "sgemv", yLen);
+    const { commandEncoder, ts } = runComputePass(
+      device,
+      pipeline,
+      bindGroup,
+      wgCount,
+    );
+    const readBuffer = yIsGpu
+      ? null
+      : stageReadback(device, commandEncoder, yBuffer);
 
-    submit(commandEncoder);
+    submit(device, commandEncoder);
 
     const gpuTimeMs = await extractTimestamp(ts);
 
@@ -143,10 +172,9 @@ export async function sgemv(device, trans, m, n, alpha, A, lda, x, incx, beta, y
     if (gpuTimeMs !== undefined) return { y: result, gpuTimeMs };
     return { y: result };
   } finally {
-    if (!AIsGpu) destroyBuffers(ABuffer);
-    if (!xIsGpu) destroyBuffers(xBuffer);
-    if (!yIsGpu) destroyBuffers(yBuffer);
-    destroyBuffers(paramsBuffer);
-
+    if (!AIsGpu && ABuffer) destroyBuffers(ABuffer);
+    if (!xIsGpu && xBuffer) destroyBuffers(xBuffer);
+    if (!yIsGpu && yBuffer) destroyBuffers(yBuffer);
+    if (paramsBuffer) destroyBuffers(paramsBuffer);
   }
 }

package/src/sger/sger.d.mts

@@ -2,7 +2,7 @@ import { GpuVector } from "../classes/GpuVector.mjs";
 import { GpuMatrix } from "../classes/GpuMatrix.mjs";
 
 /**
- * Performs the rank-1 update A = alpha * x * y^T + A
+ * Performs the rank-1 update $$A \leftarrow \alpha x y^{T} + A$$
  *
  * A is an m×n matrix stored in row-major order, updated in place. `lda` is
  * the leading dimension (number of floats between the start of consecutive
@@ -43,7 +43,7 @@ export declare function sger(
 ): Promise<{ A: Float32Array; gpuTimeMs?: number }>;
 
 /**
- * Performs the rank-1 update A = alpha * x * y^T + A
+ * Performs the rank-1 update $$A \leftarrow \alpha x y^{T} + A$$
  *
  * x, y, and A are all kept resident on the GPU. `A`'s own `layout` (set at
  * `GpuMatrix.from` time) determines the operation — there is no separate

package/src/sger/sger.mjs

@@ -11,16 +11,28 @@ import { extractTimestamp } from "../util/benchmark.mjs";
 import { getPipeline } from "../util/pipeline.mjs";
 import { GpuVector } from "../classes/GpuVector.mjs";
 import { GpuMatrix } from "../classes/GpuMatrix.mjs";
+import { requireGpuDevice, requireSameDevice } from "../util/device.mjs";
 
-export async function sger(device, m, n, alpha, x, incx, y, incy, A, lda, layout = "row-major") {
+export async function sger(
+  device,
+  m,
+  n,
+  alpha,
+  x,
+  incx,
+  y,
+  incy,
+  A,
+  lda,
+  layout = "row-major",
+) {
   const AIsGpu = A instanceof GpuMatrix;
 
-  if (!(device instanceof GPUDevice))
-    throw new Error("device must be a GPUDevice.");
+  requireGpuDevice(device);
+  requireSameDevice(device, "sger", { A, x, y });
   if (layout !== "row-major" && layout !== "column-major")
     throw new Error("layout must be 'row-major' or 'column-major'.");
-  if (typeof alpha !== "number")
-    throw new Error("alpha must be a number.");
+  if (typeof alpha !== "number") throw new Error("alpha must be a number.");
   if (Number.isNaN(alpha)) throw new Error("alpha must not be NaN.");
   if (!Number.isFinite(alpha)) throw new Error("alpha must be finite.");
   if (
@@ -77,9 +89,13 @@ export async function sger(device, m, n, alpha, x, incx, y, incy, A, lda, layout
       "A does not have enough elements for the given m, n, and lda.",
     );
   if (x.length < (m - 1) * incx + 1)
-    throw new Error("x does not have enough elements for the given m and incx.");
+    throw new Error(
+      "x does not have enough elements for the given m and incx.",
+    );
   if (y.length < (n - 1) * incy + 1)
-    throw new Error("y does not have enough elements for the given n and incy.");
+    throw new Error(
+      "y does not have enough elements for the given n and incy.",
+    );
 
   const pipeline = await getPipeline(device, "sger");
 
@@ -89,22 +105,23 @@ export async function sger(device, m, n, alpha, x, incx, y, incy, A, lda, layout
   let paramsBuffer = null;
 
   try {
-    xBuffer = xIsGpu ? x._buf : uploadBuffer(x, "sger-x", false);
-    yBuffer = yIsGpu ? y._buf : uploadBuffer(y, "sger-y", false);
-    ABuffer = AIsGpu ? A._buf : uploadBuffer(A, "sger-A", true);
+    xBuffer = xIsGpu ? x._buf : uploadBuffer(device, x, "sger-x", false);
+    yBuffer = yIsGpu ? y._buf : uploadBuffer(device, y, "sger-y", false);
+    ABuffer = AIsGpu ? A._buf : uploadBuffer(device, A, "sger-A", true);
     paramsBuffer = createParamsBuffer(
+      device,
       [
-        { value: m,     type: "u32" },
-        { value: n,     type: "u32" },
+        { value: m, type: "u32" },
+        { value: n, type: "u32" },
         { value: alpha, type: "f32" },
-        { value: incx,  type: "u32" },
-        { value: incy,  type: "u32" },
-        { value: lda,   type: "u32" },
+        { value: incx, type: "u32" },
+        { value: incy, type: "u32" },
+        { value: lda, type: "u32" },
       ],
       "sger-params",
     );
 
-    const bindGroup = createBindGroup(pipeline.getBindGroupLayout(0), [
+    const bindGroup = createBindGroup(device, pipeline.getBindGroupLayout(0), [
       xBuffer,
       yBuffer,
       ABuffer,
@@ -114,10 +131,17 @@ export async function sger(device, m, n, alpha, x, incx, y, incy, A, lda, layout
     // One workgroup per row of A; clamped to device limit — the shader's
     // grid-stride loop handles remaining rows when m > dispatch count.
     const wgCount = Math.min(m, device.limits.maxComputeWorkgroupsPerDimension);
-    const { commandEncoder, ts } = runComputePass(pipeline, bindGroup, wgCount);
-    const readBuffer = AIsGpu ? null : stageReadback(commandEncoder, ABuffer);
+    const { commandEncoder, ts } = runComputePass(
+      device,
+      pipeline,
+      bindGroup,
+      wgCount,
+    );
+    const readBuffer = AIsGpu
+      ? null
+      : stageReadback(device, commandEncoder, ABuffer);
 
-    submit(commandEncoder);
+    submit(device, commandEncoder);
 
     const gpuTimeMs = await extractTimestamp(ts);
 

package/src/shaders/__test_pipeline_a.wgsl

@@ -0,0 +1,3 @@
+// Fixture: valid shader, used only by test.pipeline.js's compile-error
+// line-mapping regression test. Not referenced by any routine or by
+// shaders/index.mjs's browser-bundle mapping.

package/src/shaders/__test_pipeline_b.wgsl

@@ -0,0 +1,2 @@
+// Fixture: shader with a deliberate syntax error on the next line.
+this is not valid wgsl syntax;

package/src/shaders/cscal.wgsl

@@ -0,0 +1,33 @@
+// cscal: x := alpha * x, complex. x is one interleaved f32 array
+// (re0, im0, re1, im1, ...), matching Complex32Array/GpuVector's storage
+// (and cuBLAS's cuComplex / stdlib's Complex64Array) — no repacking needed
+// between JS and GPU.
+//   (alphaRe + i*alphaIm)(re + i*im) = (alphaRe*re - alphaIm*im) + i*(alphaRe*im + alphaIm*re)
+
+@group(0) @binding(0) var<storage, read_write> x: array<f32>;
+
+struct Params {
+  n:       u32,
+  alphaRe: f32,
+  alphaIm: f32,
+  x_inc:   u32,
+}
+
+@group(0) @binding(1) var<uniform> params: Params;
+
+const WGS: u32 = 64;
+
+@compute @workgroup_size(64)
+fn main(
+  @builtin(global_invocation_id) gid: vec3u,
+  @builtin(num_workgroups) num_wg: vec3u,
+) {
+  for (var id = gid.x; id < params.n; id += num_wg.x * WGS) {
+    let base = 2u * id * params.x_inc;
+    // Both new parts need both old parts, so capture them before either write.
+    let re = x[base];
+    let im = x[base + 1u];
+    x[base]      = params.alphaRe * re - params.alphaIm * im;
+    x[base + 1u] = params.alphaRe * im + params.alphaIm * re;
+  }
+}

package/src/shaders/daxpy.wgsl

@@ -0,0 +1,66 @@
+// daxpy: y := alpha * x + y, double-double (Dekker) f64 emulation of saxpy.
+// Each element costs one ddMulProtected (alpha*x[i]) then one ddAddProtected
+// (+ y[i]) — the same two-protected-op shape ddot spends per term, applied
+// straight to the output instead of folded into a reduction. See dscal.wgsl
+// for why this is a uniform main pass plus a ragged, select-masked tail
+// rather than a plain `id < params.n` grid-stride loop.
+
+@group(0) @binding(0) var<storage, read>       xHi: array<f32>;
+@group(0) @binding(1) var<storage, read>       xLo: array<f32>;
+@group(0) @binding(2) var<storage, read_write> yHi: array<f32>;
+@group(0) @binding(3) var<storage, read_write> yLo: array<f32>;
+@group(0) @binding(4) var<uniform> params: Params;
+
+struct Params {
+  n:       u32,
+  alphaHi: f32,
+  alphaLo: f32,
+  x_inc:   u32,
+  y_inc:   u32,
+}
+
+const WGS: u32 = 64;
+
+@compute @workgroup_size(64)
+fn daxpy_main(
+  @builtin(global_invocation_id) gid:    vec3u,
+  @builtin(local_invocation_id)  lid:    vec3u,
+  @builtin(workgroup_id)         wgid:   vec3u,
+  @builtin(num_workgroups)       num_wg: vec3u,
+) {
+  let alpha = DD(params.alphaHi, params.alphaLo);
+  let stride = num_wg.x * WGS;
+
+  let n_floor = (params.n / stride) * stride;
+  let mainIters = n_floor / stride;
+  for (var iter = 0u; iter < mainIters; iter++) {
+    let id = gid.x + iter * stride;
+    let ix = id * params.x_inc;
+    let iy = id * params.y_inc;
+    let prod = ddMulProtected(alpha, DD(xHi[ix], xLo[ix]), lid.x);
+    let result = ddAddProtected(prod, DD(yHi[iy], yLo[iy]), lid.x);
+    yHi[iy] = result.hi;
+    yLo[iy] = result.lo;
+  }
+
+  // Tail is ragged (0 or 1 extra per thread) — pad to this workgroup's worst
+  // case so every thread still calls ddMulProtected/ddAddProtected the same
+  // number of times (their barriers need that), masking only the write.
+  let wgBaseGid = wgid.x * WGS;
+  var tailIters = 0u;
+  if (n_floor + wgBaseGid < params.n) {
+    tailIters = (params.n - 1u - n_floor - wgBaseGid) / stride + 1u;
+  }
+  for (var iter = 0u; iter < tailIters; iter++) {
+    let id = n_floor + gid.x + iter * stride;
+    let valid = id < params.n;
+    let ix = select(0u, id * params.x_inc, valid); // index 0 always in-bounds
+    let iy = select(0u, id * params.y_inc, valid);
+    let prod = ddMulProtected(alpha, DD(xHi[ix], xLo[ix]), lid.x);
+    let result = ddAddProtected(prod, DD(yHi[iy], yLo[iy]), lid.x);
+    if (valid) {
+      yHi[iy] = result.hi;
+      yLo[iy] = result.lo;
+    }
+  }
+}

package/src/shaders/dcopy.wgsl

@@ -0,0 +1,34 @@
+// dcopy: y = x, double-double (Dekker) f64 emulation of scopy. A copy is
+// pure data movement — hi and lo are transferred verbatim, with no
+// arithmetic at all — so (unlike dscal/daxpy/ddot) this needs no
+// ddMulProtected/ddAddProtected renormalizing barrier, and so no
+// ragged-tail-with-barrier split; a plain grid-stride loop is safe, same
+// shape as scopy.wgsl itself.
+
+@group(0) @binding(0) var<storage, read>       xHi: array<f32>;
+@group(0) @binding(1) var<storage, read>       xLo: array<f32>;
+@group(0) @binding(2) var<storage, read_write> yHi: array<f32>;
+@group(0) @binding(3) var<storage, read_write> yLo: array<f32>;
+
+struct Params {
+  n:     u32,
+  x_inc: u32,
+  y_inc: u32,
+}
+
+@group(0) @binding(4) var<uniform> params: Params;
+
+const WGS: u32 = 64;
+
+@compute @workgroup_size(64)
+fn main(
+  @builtin(global_invocation_id) gid: vec3u,
+  @builtin(num_workgroups) num_wg: vec3u,
+) {
+  for (var id = gid.x; id < params.n; id += num_wg.x * WGS) {
+    let ix = id * params.x_inc;
+    let iy = id * params.y_inc;
+    yHi[iy] = xHi[ix];
+    yLo[iy] = xLo[ix];
+  }
+}

package/src/shaders/ddot.wgsl

@@ -0,0 +1,106 @@
+// ddot: sum(x[i] * y[i]), double-double (Dekker). Same ILP=4 shape as
+// dasum.wgsl, which this mirrors closely — the only structural difference is
+// a second input vector and a product where dasum takes an absolute value.
+//
+// See f64/utils/add.wgsl for ddAddProtected and why plain ddAdd isn't safe,
+// and f64/utils/multiply.wgsl for ddMulProtected. The multiply itself
+// (twoProdBit) needs no barrier; only its final renormalisation does, which
+// is why each element costs two protected ops here against dasum's one.
+
+@group(0) @binding(0) var<storage, read>       xHi:        array<f32>;
+@group(0) @binding(1) var<storage, read>       xLo:        array<f32>;
+@group(0) @binding(2) var<storage, read>       yHi:        array<f32>;
+@group(0) @binding(3) var<storage, read>       yLo:        array<f32>;
+@group(0) @binding(4) var<storage, read_write> partialsHi: array<f32>;
+@group(0) @binding(5) var<storage, read_write> partialsLo: array<f32>;
+@group(0) @binding(6) var<uniform>             params:     Params;
+
+struct Params {
+  n:     u32,
+  x_inc: u32,
+  y_inc: u32,
+}
+
+const WGS: u32 = 64;
+
+var<workgroup> tile: array<DD, 64>;
+
+@compute @workgroup_size(64)
+fn ddot_main(
+  @builtin(global_invocation_id) gid:    vec3u,
+  @builtin(local_invocation_id)  lid:    vec3u,
+  @builtin(workgroup_id)         wgid:   vec3u,
+  @builtin(num_workgroups)       num_wg: vec3u,
+) {
+  var acc0 = DD(0.0, 0.0);
+  var acc1 = DD(0.0, 0.0);
+  var acc2 = DD(0.0, 0.0);
+  var acc3 = DD(0.0, 0.0);
+
+  let stride   = num_wg.x * WGS;
+  let n4_floor = (params.n / (4u * stride)) * (4u * stride);
+
+  // Same trip count for every thread, but driven by a counter, not `id`
+  // itself (the protected ops' barriers need a provably-uniform loop bound).
+  let mainIters = n4_floor / (4u * stride);
+  for (var iter = 0u; iter < mainIters; iter++) {
+    let id =  gid.x + iter * 4u * stride;
+    let d0 =  id;
+    let d1 =  id +      stride;
+    let d2 =  id + 2u * stride;
+    let d3 =  id + 3u * stride;
+
+    let p0 = ddMulProtected(DD(xHi[d0 * params.x_inc], xLo[d0 * params.x_inc]),
+                            DD(yHi[d0 * params.y_inc], yLo[d0 * params.y_inc]), lid.x);
+    let p1 = ddMulProtected(DD(xHi[d1 * params.x_inc], xLo[d1 * params.x_inc]),
+                            DD(yHi[d1 * params.y_inc], yLo[d1 * params.y_inc]), lid.x);
+    let p2 = ddMulProtected(DD(xHi[d2 * params.x_inc], xLo[d2 * params.x_inc]),
+                            DD(yHi[d2 * params.y_inc], yLo[d2 * params.y_inc]), lid.x);
+    let p3 = ddMulProtected(DD(xHi[d3 * params.x_inc], xLo[d3 * params.x_inc]),
+                            DD(yHi[d3 * params.y_inc], yLo[d3 * params.y_inc]), lid.x);
+
+    acc0 = ddAddProtected(acc0, p0, lid.x);
+    acc1 = ddAddProtected(acc1, p1, lid.x);
+    acc2 = ddAddProtected(acc2, p2, lid.x);
+    acc3 = ddAddProtected(acc3, p3, lid.x);
+  }
+
+  // Tail is ragged (0-3 extra per thread) — pad to this workgroup's worst case.
+  // Out-of-range lanes still run the multiply (it carries a barrier, so every
+  // thread must reach it) against index 0, then mask the result to zero.
+  let wgBaseGid = wgid.x * WGS;
+  var tailIters = 0u;
+  if (n4_floor + wgBaseGid < params.n) {
+    tailIters = (params.n - 1u - n4_floor - wgBaseGid) / stride + 1u;
+  }
+  for (var iter = 0u; iter < tailIters; iter++) {
+    let id    = n4_floor + gid.x + iter * stride;
+    let valid = id < params.n;
+    let ix    = select(0u, id * params.x_inc, valid);
+    let iy    = select(0u, id * params.y_inc, valid);
+    let prod  = ddMulProtected(DD(xHi[ix], xLo[ix]), DD(yHi[iy], yLo[iy]), lid.x);
+    // select() has no DD overload
+    let contribution = DD(select(0.0, prod.hi, valid), select(0.0, prod.lo, valid));
+    acc0 = ddAddProtected(acc0, contribution, lid.x);
+  }
+
+  let combined01 = ddAddProtected(acc0, acc1, lid.x);
+  let combined23 = ddAddProtected(acc2, acc3, lid.x);
+  tile[lid.x] = ddAddProtected(combined01, combined23, lid.x);
+  workgroupBarrier();
+
+  // Inactive threads combine against a throwaway partner and discard it
+  // (ddAddProtected must be called unconditionally by every thread).
+  for (var s = WGS / 2u; s > 0u; s >>= 1u) {
+    let partner = select(lid.x, lid.x + s, lid.x < s);
+    let combined = ddAddProtected(tile[lid.x], tile[partner], lid.x);
+    workgroupBarrier(); // all threads must read tile[] above before any write below
+    if (lid.x < s) { tile[lid.x] = combined; }
+    workgroupBarrier();
+  }
+
+  if (lid.x == 0u) {
+    partialsHi[wgid.x] = tile[0].hi;
+    partialsLo[wgid.x] = tile[0].lo;
+  }
+}