wgblas 1.2.1 → 2.1.0

package/src/sgemmtr/sgemmtr.mjs

@@ -0,0 +1,204 @@
+import {
+  uploadBuffer,
+  createParamsBuffer,
+  stageReadback,
+  destroyBuffers,
+} from "../util/buffer.mjs";
+import { createBindGroup } from "../util/bindgroup.mjs";
+import { runComputePass, submit } from "../util/compute.mjs";
+import { extractResult } from "../util/result.mjs";
+import { extractTimestamp } from "../util/benchmark.mjs";
+import { getPipeline } from "../util/pipeline.mjs";
+import { GpuMatrix } from "../classes/GpuMatrix.mjs";
+import { requireWorkgroupCount } from "../util/workgroup.mjs";
+import { BM_SMALL, BN_SMALL, BM_LARGE, BN_LARGE, LARGE_TILE_WORKGROUP_THRESHOLD } from "../util/constants.mjs";
+import { requireSameDevice } from "../util/device.mjs";
+
+
+export async function sgemmtr(
+  device, uplo, transA, transB, m, n, k, alpha, A, lda, B, ldb, beta, C, ldc, layout = "row-major",
+) {
+  let AIsGpu = A instanceof GpuMatrix;
+  let BIsGpu = B instanceof GpuMatrix;
+  const CIsGpu = C instanceof GpuMatrix;
+
+  if (!(device instanceof GPUDevice))
+    throw new Error("device must be a GPUDevice.");
+  requireSameDevice(device, "sgemmtr", { A, B, C });
+  if (uplo !== "lower" && uplo !== "upper")
+    throw new Error("uplo must be 'lower' or 'upper'.");
+  if (transA !== "no-transpose" && transA !== "transpose")
+    throw new Error("transA must be 'no-transpose' or 'transpose'.");
+  if (transB !== "no-transpose" && transB !== "transpose")
+    throw new Error("transB must be 'no-transpose' or 'transpose'.");
+  if (layout !== "row-major" && layout !== "column-major")
+    throw new Error("layout must be 'row-major' or 'column-major'.");
+  if (typeof alpha !== "number")
+    throw new Error("alpha must be a number.");
+  if (Number.isNaN(alpha)) throw new Error("alpha must not be NaN.");
+  if (!Number.isFinite(alpha)) throw new Error("alpha must be finite.");
+  if (typeof beta !== "number")
+    throw new Error("beta must be a number.");
+  if (Number.isNaN(beta)) throw new Error("beta must not be NaN.");
+  if (!Number.isFinite(beta)) throw new Error("beta must be finite.");
+  if (
+    !Number.isInteger(m) ||
+    !Number.isInteger(n) ||
+    !Number.isInteger(k) ||
+    !Number.isInteger(lda) ||
+    !Number.isInteger(ldb) ||
+    !Number.isInteger(ldc)
+  )
+    throw new Error("m, n, k, lda, ldb, and ldc must be integers.");
+  if (!AIsGpu && !(A instanceof Float32Array))
+    throw new Error("A must be a Float32Array or GpuMatrix.");
+  if (!BIsGpu && !(B instanceof Float32Array))
+    throw new Error("B must be a Float32Array or GpuMatrix.");
+  if (!CIsGpu && !(C instanceof Float32Array))
+    throw new Error("C must be a Float32Array or GpuMatrix.");
+  if ((AIsGpu || BIsGpu) && !CIsGpu)
+    throw new Error("C must be a GpuMatrix when A or B is a GpuMatrix.");
+  if (CIsGpu && (!AIsGpu || !BIsGpu))
+    throw new Error("A and B must be GpuMatrix when C is a GpuMatrix.");
+  if (m < 0 || n < 0 || k < 0) throw new Error("m, n, and k must be non-negative.");
+  if (m === 0 || n === 0) return CIsGpu ? {} : { C };
+
+  // GpuMatrix's own .layout wins over the shared `layout` argument.
+  const effLayoutA = AIsGpu ? A.layout : layout;
+  const effLayoutB = BIsGpu ? B.layout : layout;
+  const effLayoutC = CIsGpu ? C.layout : layout;
+
+  // Shape validation, before any layout-driven swapping below.
+  // A: op(A) is m x k; A itself is m x k or k x m depending on transA.
+  const aRows = effLayoutA === "column-major" ? k : m;
+  const aCols = effLayoutA === "column-major" ? m : k;
+  const aOuter = transA === "no-transpose" ? aRows : aCols; // # of stored chunks
+  const aInner = transA === "no-transpose" ? aCols : aRows; // required chunk length
+  if (lda < aInner)
+    throw new Error(`lda must be >= ${effLayoutA === "column-major" ? "rows" : "cols"} of A as stored.`);
+  if (AIsGpu) {
+    if (lda !== A.lda) throw new Error("lda must match A.lda when A is a GpuMatrix.");
+    const [aLogRows, aLogCols] = transA === "no-transpose" ? [m, k] : [k, m];
+    if (A.rows < aLogRows || A.cols < aLogCols)
+      throw new Error("A is too small for the given m, k, and transA.");
+  } else if (A.length < (aOuter - 1) * lda + aInner) {
+    throw new Error("A does not have enough elements for the given dimensions and lda.");
+  }
+
+  // B: same reasoning as A, with op(B) = k x n.
+  const bRows = effLayoutB === "column-major" ? n : k;
+  const bCols = effLayoutB === "column-major" ? k : n;
+  const bOuter = transB === "no-transpose" ? bRows : bCols;
+  const bInner = transB === "no-transpose" ? bCols : bRows;
+  if (ldb < bInner)
+    throw new Error(`ldb must be >= ${effLayoutB === "column-major" ? "rows" : "cols"} of B as stored.`);
+  if (BIsGpu) {
+    if (ldb !== B.lda) throw new Error("ldb must match B.lda when B is a GpuMatrix.");
+    const [bLogRows, bLogCols] = transB === "no-transpose" ? [k, n] : [n, k];
+    if (B.rows < bLogRows || B.cols < bLogCols)
+      throw new Error("B is too small for the given n, k, and transB.");
+  } else if (B.length < (bOuter - 1) * ldb + bInner) {
+    throw new Error("B does not have enough elements for the given dimensions and ldb.");
+  }
+
+  // C: always m x n (no trans flag) — layout only affects lda/storage order.
+  const cOuter = effLayoutC === "column-major" ? n : m;
+  const cInner = effLayoutC === "column-major" ? m : n;
+  if (ldc < cInner)
+    throw new Error(`ldc must be >= ${effLayoutC === "column-major" ? "rows" : "cols"} of C as stored.`);
+  if (CIsGpu) {
+    if (ldc !== C.lda) throw new Error("ldc must match C.lda when C is a GpuMatrix.");
+    if (C.rows < m || C.cols < n) throw new Error("C is too small for the given m and n.");
+  } else if (C.length < (cOuter - 1) * ldc + cInner) {
+    throw new Error("C does not have enough elements for the given dimensions and ldc.");
+  }
+
+  // Column-major A/B reinterpreted row-major is A^T/B^T — flip the trans flag.
+  if (effLayoutA === "column-major")
+    transA = transA === "no-transpose" ? "transpose" : "no-transpose";
+  if (effLayoutB === "column-major")
+    transB = transB === "no-transpose" ? "transpose" : "no-transpose";
+
+  // Column-major C: compute C^T = op(B)^T * op(A)^T instead (swap A/B, flip trans, swap m<->n)
+  // — same trick sgemm uses. uplo(C) in row/col terms becomes uplo(C^T) with row/col swapped,
+  // i.e. the opposite triangle, so uplo must flip here too (same reasoning ssyr's isLower flip
+  // uses for column-major A) — nothing else that's uplo-specific needs to change, since the
+  // shader's row/col test is applied to whatever (m, n) grid it's actually given.
+  if (effLayoutC === "column-major") {
+    [A, B] = [B, A];
+    [AIsGpu, BIsGpu] = [BIsGpu, AIsGpu];
+    [lda, ldb] = [ldb, lda];
+    [transA, transB] = [
+      transB === "no-transpose" ? "transpose" : "no-transpose",
+      transA === "no-transpose" ? "transpose" : "no-transpose",
+    ];
+    [m, n] = [n, m];
+    uplo = uplo === "lower" ? "upper" : "lower";
+  }
+
+  // Shape-based auto-select — see sgemmtr_small.wgsl/sgemmtr_large.wgsl.
+  const largeWgX = Math.ceil(n / BN_LARGE);
+  const largeWgY = Math.ceil(m / BM_LARGE);
+  const useLargeTile = largeWgX * largeWgY >= LARGE_TILE_WORKGROUP_THRESHOLD;
+
+  const pipeline = await getPipeline(device, useLargeTile ? "sgemmtr_large" : "sgemmtr_small");
+
+  const ABuffer = AIsGpu ? A._buf : uploadBuffer(device, A, "sgemmtr-A", false);
+  const BBuffer = BIsGpu ? B._buf : uploadBuffer(device, B, "sgemmtr-B", false);
+  const CBuffer = CIsGpu ? C._buf : uploadBuffer(device, C, "sgemmtr-C", true);
+  const paramsBuffer = createParamsBuffer(device,
+    [
+      { value: m,   type: "u32" },
+      { value: n,   type: "u32" },
+      { value: k,   type: "u32" },
+      { value: alpha, type: "f32" },
+      { value: beta,  type: "f32" },
+      { value: lda, type: "u32" },
+      { value: ldb, type: "u32" },
+      { value: ldc, type: "u32" },
+      { value: transA === "transpose" ? 1 : 0, type: "u32" },
+      { value: transB === "transpose" ? 1 : 0, type: "u32" },
+      { value: uplo === "upper" ? 1 : 0, type: "u32" },
+    ],
+    "sgemmtr-params",
+  );
+
+  try {
+    const bindGroup = createBindGroup(device, pipeline.getBindGroupLayout(0), [
+      ABuffer,
+      BBuffer,
+      CBuffer,
+      paramsBuffer,
+    ]);
+
+    const wgCount = useLargeTile
+      ? {
+        x: requireWorkgroupCount(device, largeWgX, "sgemmtr", "x"),
+        y: requireWorkgroupCount(device, largeWgY, "sgemmtr", "y"),
+      }
+      : {
+        x: requireWorkgroupCount(device, Math.ceil(n / BN_SMALL), "sgemmtr", "x"),
+        y: requireWorkgroupCount(device, Math.ceil(m / BM_SMALL), "sgemmtr", "y"),
+      };
+    const { commandEncoder, ts } = runComputePass(device, pipeline, bindGroup, wgCount);
+    const readBuffer = CIsGpu ? null : stageReadback(device, commandEncoder, CBuffer);
+
+    submit(device, commandEncoder);
+
+    const gpuTimeMs = await extractTimestamp(ts);
+
+    if (CIsGpu) {
+      if (gpuTimeMs !== undefined) return { gpuTimeMs };
+      return {};
+    }
+
+    const result = await extractResult(readBuffer, Float32Array);
+    if (gpuTimeMs !== undefined) return { C: result, gpuTimeMs };
+    return { C: result };
+  } finally {
+    if (!AIsGpu) destroyBuffers(ABuffer);
+    if (!BIsGpu) destroyBuffers(BBuffer);
+    if (!CIsGpu) destroyBuffers(CBuffer);
+    destroyBuffers(paramsBuffer);
+  }
+}

package/src/sgemv/sgemv.d.mts

@@ -50,43 +50,6 @@ export declare function sgemv(
   layout?: 'row-major' | 'column-major',
 ): Promise<{ y: Float32Array; gpuTimeMs?: number }>;
 
-/**
- * Performs the matrix-vector operation y = alpha * op(A) * x + beta * y
- *
- * A is kept GPU-resident; x and y are CPU Float32Arrays. `A`'s own `layout`
- * (set at `GpuMatrix.from` time) determines the operation — there is no
- * separate `layout` argument here.
- *
- * @param device - GPUDevice from `init()`
- * @param trans  - `'no-transpose'` for A, `'transpose'` for A^T
- * @param m      - number of rows in A
- * @param n      - number of columns in A
- * @param alpha  - scalar multiplier for op(A)*x
- * @param A      - GpuMatrix, GPU-resident
- * @param lda    - leading dimension of A (must equal A.lda)
- * @param x      - Float32Array input vector
- * @param incx   - stride for x (must be a positive integer)
- * @param beta   - scalar multiplier for y
- * @param y      - Float32Array input/output vector
- * @param incy   - stride for y (must be a positive integer)
- * @see <a href="https://github.com/manit2004/wgblas/blob/main/src/sgemv/sgemv.mjs#L16">Source code: sgemv.mjs (L16)</a>
- * @category BLAS Level 2
- */
-export declare function sgemv(
-  device: GPUDevice,
-  trans: 'no-transpose' | 'transpose',
-  m: number,
-  n: number,
-  alpha: number,
-  A: GpuMatrix,
-  lda: number,
-  x: Float32Array,
-  incx: number,
-  beta: number,
-  y: Float32Array,
-  incy: number,
-): Promise<{ y: Float32Array; gpuTimeMs?: number }>;
-
 /**
  * Performs the matrix-vector operation y = alpha * op(A) * x + beta * y
  *
@@ -94,7 +57,7 @@ export declare function sgemv(
  * `layout` (set at `GpuMatrix.from` time) determines the operation — there is
  * no separate `layout` argument here.
  *
- * {@includeCode ../../examples/sgemv/gpuvec.sgemv.js}
+ * {@includeCode ../../examples/sgemv/gpu.sgemv.js}
  *
  * @param device - GPUDevice from `init()`
  * @param trans  - `'no-transpose'` for A, `'transpose'` for A^T

package/src/sgemv/sgemv.mjs

@@ -9,9 +9,10 @@ import { runComputePass, submit } from "../util/compute.mjs";
 import { extractResult } from "../util/result.mjs";
 import { extractTimestamp } from "../util/benchmark.mjs";
 import { getPipeline } from "../util/pipeline.mjs";
-import { calcWorkgroups } from "../util/workgroup.mjs";
+import { requireWorkgroups } from "../util/workgroup.mjs";
 import { GpuVector } from "../classes/GpuVector.mjs";
 import { GpuMatrix } from "../classes/GpuMatrix.mjs";
+import { requireSameDevice } from "../util/device.mjs";
 
 export async function sgemv(device, trans, m, n, alpha, A, lda, x, incx, beta, y, incy, layout = "row-major") {
   const AIsGpu = A instanceof GpuMatrix;
@@ -20,6 +21,7 @@ export async function sgemv(device, trans, m, n, alpha, A, lda, x, incx, beta, y
 
   if (!(device instanceof GPUDevice))
     throw new Error("device must be a GPUDevice.");
+  requireSameDevice(device, "sgemv", { A, x, y });
   if (trans !== "no-transpose" && trans !== "transpose")
     throw new Error("trans must be 'no-transpose' or 'transpose'.");
   if (layout !== "row-major" && layout !== "column-major")
@@ -56,8 +58,14 @@ export async function sgemv(device, trans, m, n, alpha, A, lda, x, incx, beta, y
     throw new Error(
       "A must be a GpuMatrix when x and y are GpuVectors.",
     );
+  if (AIsGpu && !xIsGpu)
+    throw new Error(
+      "x and y must be GpuVectors when A is a GpuMatrix.",
+    );
   if (xIsGpu && x._buf === y._buf)
     throw new Error("x and y must not reference the same GPU buffer when both are GpuVectors.");
+  if (AIsGpu && yIsGpu && A._buf === y._buf)
+    throw new Error("A and y must not reference the same GPU buffer.");
   if (AIsGpu && lda !== A.lda)
     throw new Error("lda must match A.lda when A is a GpuMatrix.");
   if (AIsGpu && (A.rows < m || A.cols < n))
@@ -95,38 +103,46 @@ export async function sgemv(device, trans, m, n, alpha, A, lda, x, incx, beta, y
   const shaderName = isNoTrans ? "sgemv_n" : "sgemv_t";
   const pipeline   = await getPipeline(device, shaderName);
 
-  const ABuffer = AIsGpu ? A._buf : uploadBuffer(A, "sgemv-A", false);
-  const xBuffer = xIsGpu ? x._buf : uploadBuffer(x, "sgemv-x", false);
-  const yBuffer = yIsGpu ? y._buf : uploadBuffer(y, "sgemv-y", true);
-  const paramsBuffer = createParamsBuffer(
-    [
-      { value: m,     type: "u32" },
-      { value: n,     type: "u32" },
-      { value: alpha, type: "f32" },
-      { value: beta,  type: "f32" },
-      { value: incx,  type: "u32" },
-      { value: incy,  type: "u32" },
-      { value: lda,   type: "u32" },
-    ],
-    "sgemv-params",
-  );
+  let ABuffer = null;
+  let xBuffer = null;
+  let yBuffer = null;
+  let paramsBuffer = null;
 
   try {
-    const bindGroup = createBindGroup(pipeline.getBindGroupLayout(0), [
+    ABuffer = AIsGpu ? A._buf : uploadBuffer(device, A, "sgemv-A", false);
+    xBuffer = xIsGpu ? x._buf : uploadBuffer(device, x, "sgemv-x", false);
+    yBuffer = yIsGpu ? y._buf : uploadBuffer(device, y, "sgemv-y", true);
+    paramsBuffer = createParamsBuffer(device,
+      [
+        { value: m,     type: "u32" },
+        { value: n,     type: "u32" },
+        { value: alpha, type: "f32" },
+        { value: beta,  type: "f32" },
+        { value: incx,  type: "u32" },
+        { value: incy,  type: "u32" },
+        { value: lda,   type: "u32" },
+      ],
+      "sgemv-params",
+    );
+
+    const bindGroup = createBindGroup(device, pipeline.getBindGroupLayout(0), [
       ABuffer,
       xBuffer,
       yBuffer,
       paramsBuffer,
     ]);
 
-    // NoTrans: one workgroup per row (grid-stride handles overflow); Trans: one thread per output column.
+    // NoTrans: one workgroup per row — sgemv_n.wgsl is a grid-stride loop, so
+    // clamping here only costs parallelism. Trans: one thread per output
+    // column, and sgemv_t.wgsl indexes straight off global_invocation_id with
+    // no fallback, so an over-limit dispatch must be refused, not truncated.
     const wgCount = isNoTrans
       ? Math.min(m, device.limits.maxComputeWorkgroupsPerDimension)
-      : calcWorkgroups(yLen);
-    const { commandEncoder, ts } = runComputePass(pipeline, bindGroup, wgCount);
-    const readBuffer = yIsGpu ? null : stageReadback(commandEncoder, yBuffer);
+      : requireWorkgroups(device, "sgemv", yLen);
+    const { commandEncoder, ts } = runComputePass(device, pipeline, bindGroup, wgCount);
+    const readBuffer = yIsGpu ? null : stageReadback(device, commandEncoder, yBuffer);
 
-    submit(commandEncoder);
+    submit(device, commandEncoder);
 
     const gpuTimeMs = await extractTimestamp(ts);
 
@@ -139,10 +155,10 @@ export async function sgemv(device, trans, m, n, alpha, A, lda, x, incx, beta, y
     if (gpuTimeMs !== undefined) return { y: result, gpuTimeMs };
     return { y: result };
   } finally {
-    if (!AIsGpu) destroyBuffers(ABuffer);
-    if (!xIsGpu) destroyBuffers(xBuffer);
-    if (!yIsGpu) destroyBuffers(yBuffer);
-    destroyBuffers(paramsBuffer);
+    if (!AIsGpu && ABuffer) destroyBuffers(ABuffer);
+    if (!xIsGpu && xBuffer) destroyBuffers(xBuffer);
+    if (!yIsGpu && yBuffer) destroyBuffers(yBuffer);
+    if (paramsBuffer) destroyBuffers(paramsBuffer);
 
   }
 }

package/src/sger/sger.d.mts

@@ -42,39 +42,6 @@ export declare function sger(
   layout?: 'row-major' | 'column-major',
 ): Promise<{ A: Float32Array; gpuTimeMs?: number }>;
 
-/**
- * Performs the rank-1 update A = alpha * x * y^T + A
- *
- * A is kept GPU-resident; x and y are CPU Float32Arrays. `A`'s own `layout`
- * (set at `GpuMatrix.from` time) determines the operation — there is no
- * separate `layout` argument here.
- *
- * @param device - GPUDevice from `init()`
- * @param m      - number of rows in A
- * @param n      - number of columns in A
- * @param alpha  - scalar multiplier for x*y^T
- * @param x      - Float32Array input vector
- * @param incx   - stride for x (must be a positive integer)
- * @param y      - Float32Array input vector
- * @param incy   - stride for y (must be a positive integer)
- * @param A      - GpuMatrix, GPU-resident
- * @param lda    - leading dimension of A (must equal A.lda)
- * @see <a href="https://github.com/manit2004/wgblas/blob/main/src/sger/sger.mjs#L15">Source code: sger.mjs (L15)</a>
- * @category BLAS Level 2
- */
-export declare function sger(
-  device: GPUDevice,
-  m: number,
-  n: number,
-  alpha: number,
-  x: Float32Array,
-  incx: number,
-  y: Float32Array,
-  incy: number,
-  A: GpuMatrix,
-  lda: number,
-): Promise<{ gpuTimeMs?: number }>;
-
 /**
  * Performs the rank-1 update A = alpha * x * y^T + A
  *
@@ -82,7 +49,7 @@ export declare function sger(
  * `GpuMatrix.from` time) determines the operation — there is no separate
  * `layout` argument here.
  *
- * {@includeCode ../../examples/sger/gpuvec.sger.js}
+ * {@includeCode ../../examples/sger/gpu.sger.js}
  *
  * @param device - GPUDevice from `init()`
  * @param m      - number of rows in A

package/src/sger/sger.mjs

@@ -11,12 +11,14 @@ import { extractTimestamp } from "../util/benchmark.mjs";
 import { getPipeline } from "../util/pipeline.mjs";
 import { GpuVector } from "../classes/GpuVector.mjs";
 import { GpuMatrix } from "../classes/GpuMatrix.mjs";
+import { requireSameDevice } from "../util/device.mjs";
 
 export async function sger(device, m, n, alpha, x, incx, y, incy, A, lda, layout = "row-major") {
   const AIsGpu = A instanceof GpuMatrix;
 
   if (!(device instanceof GPUDevice))
     throw new Error("device must be a GPUDevice.");
+  requireSameDevice(device, "sger", { A, x, y });
   if (layout !== "row-major" && layout !== "column-major")
     throw new Error("layout must be 'row-major' or 'column-major'.");
   if (typeof alpha !== "number")
@@ -63,6 +65,8 @@ export async function sger(device, m, n, alpha, x, incx, y, incy, A, lda, layout
     );
   if (xIsGpu && !AIsGpu)
     throw new Error("A must be a GpuMatrix when x and y are GpuVectors.");
+  if (AIsGpu && !xIsGpu)
+    throw new Error("x and y must be GpuVectors when A is a GpuMatrix.");
   if (AIsGpu && xIsGpu && A._buf === x._buf)
     throw new Error("A and x must not reference the same GPU buffer.");
   if (AIsGpu && yIsGpu && A._buf === y._buf)
@@ -87,10 +91,10 @@ export async function sger(device, m, n, alpha, x, incx, y, incy, A, lda, layout
   let paramsBuffer = null;
 
   try {
-    xBuffer = xIsGpu ? x._buf : uploadBuffer(x, "sger-x", false);
-    yBuffer = yIsGpu ? y._buf : uploadBuffer(y, "sger-y", false);
-    ABuffer = AIsGpu ? A._buf : uploadBuffer(A, "sger-A", true);
-    paramsBuffer = createParamsBuffer(
+    xBuffer = xIsGpu ? x._buf : uploadBuffer(device, x, "sger-x", false);
+    yBuffer = yIsGpu ? y._buf : uploadBuffer(device, y, "sger-y", false);
+    ABuffer = AIsGpu ? A._buf : uploadBuffer(device, A, "sger-A", true);
+    paramsBuffer = createParamsBuffer(device,
       [
         { value: m,     type: "u32" },
         { value: n,     type: "u32" },
@@ -102,7 +106,7 @@ export async function sger(device, m, n, alpha, x, incx, y, incy, A, lda, layout
       "sger-params",
     );
 
-    const bindGroup = createBindGroup(pipeline.getBindGroupLayout(0), [
+    const bindGroup = createBindGroup(device, pipeline.getBindGroupLayout(0), [
       xBuffer,
       yBuffer,
       ABuffer,
@@ -112,10 +116,10 @@ export async function sger(device, m, n, alpha, x, incx, y, incy, A, lda, layout
     // One workgroup per row of A; clamped to device limit — the shader's
     // grid-stride loop handles remaining rows when m > dispatch count.
     const wgCount = Math.min(m, device.limits.maxComputeWorkgroupsPerDimension);
-    const { commandEncoder, ts } = runComputePass(pipeline, bindGroup, wgCount);
-    const readBuffer = AIsGpu ? null : stageReadback(commandEncoder, ABuffer);
+    const { commandEncoder, ts } = runComputePass(device, pipeline, bindGroup, wgCount);
+    const readBuffer = AIsGpu ? null : stageReadback(device, commandEncoder, ABuffer);
 
-    submit(commandEncoder);
+    submit(device, commandEncoder);
 
     const gpuTimeMs = await extractTimestamp(ts);
 

package/src/shaders/block_transfer.wgsl

@@ -0,0 +1,42 @@
+// block_transfer: gather/scatter/scatter-subtract between a tight (blockLen
+// x otherLen) block and a sub-range of a strided (any ld, row/col-major)
+// buffer — needed since block offsets aren't 256-byte-aligned and block
+// rows/cols aren't always one contiguous range for copyBufferToBuffer.
+
+@group(0) @binding(0) var<storage, read_write> block:   array<f32>; // blockLen x otherLen, block[i*otherLen+j]
+@group(0) @binding(1) var<storage, read_write> strided: array<f32>; // B's or A's own buffer
+
+struct Params {
+  blockStart: u32,
+  blockLen:   u32,
+  otherStart: u32,
+  otherLen:   u32,
+  ld:         u32,
+  isColMajor: u32, // 0 = row-major addressing, 1 = column-major (row/col swapped)
+  blockIsRow: u32, // 1 = blockStart indexes strided's rows, 0 = its columns
+  mode:       u32, // 0 = scatter (strided := block), 1 = scatter_sub (strided -= block), 2 = gather (block := strided)
+}
+
+@group(0) @binding(2) var<uniform> params: Params;
+
+@compute @workgroup_size(8, 8)
+fn main(@builtin(global_invocation_id) gid: vec3u) {
+  let i = gid.y; // index along the blocked axis, within the block
+  let j = gid.x; // index along the other axis, within the block
+  if (i >= params.blockLen || j >= params.otherLen) {
+    return;
+  }
+
+  let row = select(params.otherStart + j, params.blockStart + i, params.blockIsRow == 1u);
+  let col = select(params.blockStart + i, params.otherStart + j, params.blockIsRow == 1u);
+  let stridedIdx = select(row * params.ld + col, col * params.ld + row, params.isColMajor == 1u);
+  let blockIdx = i * params.otherLen + j;
+
+  if (params.mode == 2u) {
+    block[blockIdx] = strided[stridedIdx];
+  } else if (params.mode == 1u) {
+    strided[stridedIdx] -= block[blockIdx];
+  } else {
+    strided[stridedIdx] = block[blockIdx];
+  }
+}

package/src/shaders/dasum.wgsl

@@ -1,6 +1,7 @@
 // dasum: sum(|x[i]|), double-double (Dekker). Same ILP=4 shape as sasum.wgsl;
-// see f64/dekker.wgsl for ddAddProtected and why plain ddAdd isn't safe.
-// GpuVector input isn't pre-abs'd, so ddAbs() applies unconditionally below.
+// see f64/utils/add.wgsl for ddAddProtected and why plain ddAdd isn't safe.
+// GpuVector input isn't pre-abs'd, so ddAbs() (f64/utils/abs.wgsl) applies
+// unconditionally below.
 
 @group(0) @binding(0) var<storage, read>       xHi:        array<f32>;
 @group(0) @binding(1) var<storage, read>       xLo:        array<f32>;

package/src/shaders/f64/dekker.wgsl

@@ -7,93 +7,12 @@
 //
 // No bindings, no entry point — a helper library, concatenated with a
 // consumer's own bindings/entry point by getPipeline (WGSL has no #include).
+// The DD struct lives here — abs.wgsl/add.wgsl/greater.wgsl/equal.wgsl all
+// use it but don't redefine it (WGSL errors on duplicate struct definitions
+// once concatenated), so any consumer using those must concatenate this
+// file too, first.
 
 struct DD {
   hi: f32,
   lo: f32,
 }
-
-// |a| for a double-double pair. Negation is exact (no rounding), so this is
-// just a sign flip on both components — hi alone determines the pair's sign.
-fn ddAbs(a: DD) -> DD {
-  if (a.hi < 0.0) {
-    return DD(-a.hi, -a.lo);
-  }
-  return a;
-}
-
-// ── A real compiler bug — read before touching anything below ──────────────
-//
-// twoSum/fastTwoSum's error term `e` should be nonzero (that's the point —
-// `s` is rounded). A front-end-optimizer bug zeros it anyway, on both NVIDIA
-// and Mesa (ANV + llvmpipe), via two different mechanisms needing two fixes:
-// bitcast-based subtraction (`fsub`/`negf`, fixes NVIDIA) and materializing
-// the sum through workgroup memory + workgroupBarrier() (fixes Mesa). Only
-// both together (ddAddProtected) is verified correct everywhere — the plain
-// twoSum/fastTwoSum/ddAdd below are reference-only, not safe to use.
-fn negf(x: f32) -> f32 {
-  return bitcast<f32>(bitcast<u32>(x) ^ 0x80000000u);
-}
-fn fsub(a: f32, b: f32) -> f32 {
-  return a + negf(b);
-}
-
-// Knuth/Møller's TwoSum: s = fl(a+b), e = exact rounding error, a+b == s+e.
-// Works for any a, b. UNPROTECTED — see header above.
-fn twoSum(a: f32, b: f32) -> DD {
-  let s = a + b;
-  let v = s - a;
-  let e = (a - (s - v)) + (b - v);
-  return DD(s, e);
-}
-
-// Dekker's Fast-Two-Sum: same contract, but only correct when |a| >= |b|.
-// UNPROTECTED — see header above.
-fn fastTwoSum(a: f32, b: f32) -> DD {
-  let s = a + b;
-  let e = b - (s - a);
-  return DD(s, e);
-}
-
-// Double-double addition (Dekker's Add2). UNPROTECTED — see header above.
-fn ddAdd(a: DD, b: DD) -> DD {
-  let s = twoSum(a.hi, b.hi);
-  let loSum = a.lo + b.lo;
-  return fastTwoSum(s.hi, s.lo + loSum);
-}
-
-// ── Protected variants — use these ──────────────────────────────────────────
-//
-// Bitcast subtraction + workgroup-barrier materialization, verified correct
-// on all three backends tested. Costs a real barrier: fine for O(1)-per-
-// thread or O(log n) reduction use, not a long per-element loop. A
-// workgroupBarrier() requires uniform control flow, so:
-//   - `threadSlot` must be unique per concurrent caller (e.g. local_invocation_index).
-//   - Every thread in the workgroup must call this the same number of times
-//     — including ones whose result gets discarded. Compute unconditionally;
-//     only the write-back should be conditional.
-var<workgroup> dekkerScratch: array<f32, 64>;
-
-fn twoSumProtected(a: f32, b: f32, threadSlot: u32) -> DD {
-  dekkerScratch[threadSlot] = a + b;
-  workgroupBarrier();
-  let s = dekkerScratch[threadSlot];
-  let v = fsub(s, a);
-  let e = fsub(a, fsub(s, v)) + fsub(b, v);
-  return DD(s, e);
-}
-
-fn fastTwoSumProtected(a: f32, b: f32, threadSlot: u32) -> DD {
-  dekkerScratch[threadSlot] = a + b;
-  workgroupBarrier();
-  let s = dekkerScratch[threadSlot];
-  let e = fsub(b, fsub(s, a));
-  return DD(s, e);
-}
-
-// Protected double-double addition — same contract as ddAdd, but exact.
-fn ddAddProtected(a: DD, b: DD, threadSlot: u32) -> DD {
-  let s = twoSumProtected(a.hi, b.hi, threadSlot);
-  let loSum = a.lo + b.lo;
-  return fastTwoSumProtected(s.hi, s.lo + loSum, threadSlot);
-}

package/src/shaders/f64/utils/abs.wgsl

@@ -0,0 +1,10 @@
+// Requires f64/dekker.wgsl concatenated first for the DD struct.
+
+// |a| for a double-double pair. Negation is exact (no rounding), so this is
+// just a sign flip on both components — hi alone determines the pair's sign.
+fn ddAbs(a: DD) -> DD {
+  if (a.hi < 0.0) {
+    return DD(-a.hi, -a.lo);
+  }
+  return a;
+}