wgblas 0.1.2 → 1.0.0

package/src/shaders/isamax.wgsl

@@ -25,19 +25,42 @@ fn main(
 ) {
   // -1.0 is a safe sentinel: any |x[i]| >= 0 beats it,
   // so workgroups with no elements lose gracefully in the epilogue.
-  var best_val: f32 = -1.0;
-  var best_idx: u32 = 0u;
+  var best_val0: f32 = -1.0; var best_idx0: u32 = 0u;
+  var best_val1: f32 = -1.0; var best_idx1: u32 = 0u;
+  var best_val2: f32 = -1.0; var best_idx2: u32 = 0u;
+  var best_val3: f32 = -1.0; var best_idx3: u32 = 0u;
 
-  for (var id = gid.x; id < params.n; id += num_wg.x * WGS) {
+  let stride   = num_wg.x * WGS;
+  let n4_floor = (params.n / (4u * stride)) * (4u * stride);
+
+  for (var id = gid.x; id < n4_floor; id += 4u * stride) {
+    let v0 = abs(x[ id                * params.x_inc]);
+    let v1 = abs(x[(id +      stride) * params.x_inc]);
+    let v2 = abs(x[(id + 2u * stride) * params.x_inc]);
+    let v3 = abs(x[(id + 3u * stride) * params.x_inc]);
+    if (v0 > best_val0) { best_val0 = v0; best_idx0 = id; }
+    if (v1 > best_val1) { best_val1 = v1; best_idx1 = id +      stride; }
+    if (v2 > best_val2) { best_val2 = v2; best_idx2 = id + 2u * stride; }
+    if (v3 > best_val3) { best_val3 = v3; best_idx3 = id + 3u * stride; }
+  }
+  for (var id = n4_floor + gid.x; id < params.n; id += stride) {
     let v = abs(x[id * params.x_inc]);
-    if (v > best_val) {
-      best_val = v;
-      best_idx = id;
-    }
+    if (v > best_val0) { best_val0 = v; best_idx0 = id; }
+  }
+
+  // merge 4 independent lanes; prefer lower index on tie (first occurrence wins)
+  if (best_val1 > best_val0 || (best_val1 == best_val0 && best_idx1 < best_idx0)) {
+    best_val0 = best_val1; best_idx0 = best_idx1;
+  }
+  if (best_val2 > best_val0 || (best_val2 == best_val0 && best_idx2 < best_idx0)) {
+    best_val0 = best_val2; best_idx0 = best_idx2;
+  }
+  if (best_val3 > best_val0 || (best_val3 == best_val0 && best_idx3 < best_idx0)) {
+    best_val0 = best_val3; best_idx0 = best_idx3;
   }
 
-  tile_val[lid.x] = best_val;
-  tile_idx[lid.x] = best_idx;
+  tile_val[lid.x] = best_val0;
+  tile_idx[lid.x] = best_idx0;
   workgroupBarrier();
 
   for (var s = WGS / 2u; s > 0u; s >>= 1u) {

package/src/shaders/reduction/sumF64.wgsl

@@ -0,0 +1,49 @@
+// sum reduction (f64): collapses 2*WGS partial [main, aux] pairs into one,
+// using computeSum instead of plain f32 `+` (see reduction/sum.wgsl for the
+// f32 original this mirrors).
+// dispatch: 1 workgroup of WGS threads. partialsMain/partialsAux must have
+// exactly 2*WGS entries each.
+//
+// Concatenated after f64add.wgsl by getPipeline (WGSL has no #include),
+// reusing its decode/encode/computeSum and Packed struct — f64add.wgsl
+// declares no bindings and no entry point of its own (just helper functions),
+// so bindings here start at 0 and the entry point is simply `reduce_f64`.
+//
+// partialsAux/result's aux slot are array<u32>, not array<f32> — aux's bits
+// must never pass through an f32-typed storage slot (NaN-bit-pattern
+// corruption risk, see f64pack.mjs and the Packed struct comment above
+// decode()/encode() in f64add.wgsl).
+
+@group(0) @binding(0) var<storage, read>       partialsMain: array<f32>;
+@group(0) @binding(1) var<storage, read>       partialsAux:  array<u32>;
+@group(0) @binding(2) var<storage, read_write> resultMain:   array<f32, 1>;
+@group(0) @binding(3) var<storage, read_write> resultAux:    array<u32, 1>;
+
+const WGS: u32 = 64;
+
+var<workgroup> tile: array<Packed, 64>;
+
+fn addPair(a: Packed, b: Packed) -> Packed {
+  return computeSum(decode(bitcast<u32>(a.main), a.aux), decode(bitcast<u32>(b.main), b.aux));
+}
+
+@compute @workgroup_size(64)
+fn reduce_f64(
+  @builtin(local_invocation_id) lid: vec3u,
+) {
+  let i = lid.x;
+  let a = Packed(partialsMain[i], partialsAux[i]);
+  let b = Packed(partialsMain[i + WGS], partialsAux[i + WGS]);
+  tile[i] = addPair(a, b);
+  workgroupBarrier();
+
+  for (var s = WGS / 2u; s > 0u; s >>= 1u) {
+    if (i < s) { tile[i] = addPair(tile[i], tile[i + s]); }
+    workgroupBarrier();
+  }
+
+  if (i == 0u) {
+    resultMain[0] = tile[0].main;
+    resultAux[0] = tile[0].aux;
+  }
+}

package/src/shaders/sasum.wgsl

@@ -21,11 +21,25 @@ fn main(
   @builtin(workgroup_id)         wgid:   vec3u,
   @builtin(num_workgroups)       num_wg: vec3u,
 ) {
-  var acc: f32 = 0.0;
-  for (var id = gid.x; id < params.n; id += num_wg.x * WGS) {
-    acc += abs(x[id * params.x_inc]);
+  var acc0: f32 = 0.0;
+  var acc1: f32 = 0.0;
+  var acc2: f32 = 0.0;
+  var acc3: f32 = 0.0;
+
+  let stride   = num_wg.x * WGS;
+  let n4_floor = (params.n / (4u * stride)) * (4u * stride);
+
+  for (var id = gid.x; id < n4_floor; id += 4u * stride) {
+    acc0 += abs(x[ id                * params.x_inc]);
+    acc1 += abs(x[(id +      stride) * params.x_inc]);
+    acc2 += abs(x[(id + 2u * stride) * params.x_inc]);
+    acc3 += abs(x[(id + 3u * stride) * params.x_inc]);
   }
-  tile[lid.x] = acc;
+  for (var id = n4_floor + gid.x; id < params.n; id += stride) {
+    acc0 += abs(x[id * params.x_inc]);
+  }
+
+  tile[lid.x] = acc0 + acc1 + acc2 + acc3;
   workgroupBarrier();
 
   for (var s = WGS / 2u; s > 0u; s >>= 1u) {

package/src/shaders/sdot.wgsl

@@ -23,11 +23,25 @@ fn main(
   @builtin(workgroup_id)         wgid:   vec3u,
   @builtin(num_workgroups)       num_wg: vec3u,
 ) {
-  var acc: f32 = 0.0;
-  for (var id = gid.x; id < params.n; id += num_wg.x * WGS) {
-    acc += x[id * params.x_inc] * y[id * params.y_inc];
+  var acc0: f32 = 0.0;
+  var acc1: f32 = 0.0;
+  var acc2: f32 = 0.0;
+  var acc3: f32 = 0.0;
+
+  let stride   = num_wg.x * WGS;
+  let n4_floor = (params.n / (4u * stride)) * (4u * stride);
+
+  for (var id = gid.x; id < n4_floor; id += 4u * stride) {
+    acc0 += x[ id                * params.x_inc] * y[ id                * params.y_inc];
+    acc1 += x[(id +      stride) * params.x_inc] * y[(id +      stride) * params.y_inc];
+    acc2 += x[(id + 2u * stride) * params.x_inc] * y[(id + 2u * stride) * params.y_inc];
+    acc3 += x[(id + 3u * stride) * params.x_inc] * y[(id + 3u * stride) * params.y_inc];
   }
-  tile[lid.x] = acc;
+  for (var id = n4_floor + gid.x; id < params.n; id += stride) {
+    acc0 += x[id * params.x_inc] * y[id * params.y_inc];
+  }
+
+  tile[lid.x] = acc0 + acc1 + acc2 + acc3;
   workgroupBarrier();
 
   for (var s = WGS / 2u; s > 0u; s >>= 1u) {

package/src/shaders/sgemv_n.wgsl

@@ -0,0 +1,75 @@
+// sgemv_n: y = alpha * A * x + beta * y  (A is m×n row-major, no-transpose)
+//
+// One workgroup per output row, with a grid-stride outer loop so the shader
+// still covers all rows when m exceeds maxComputeWorkgroupsPerDimension.
+// Threads stride through A[row, :] and x with coalesced reads (consecutive
+// threads → consecutive addresses). Four independent accumulators let the GPU
+// pipeline memory requests across iterations (ILP=4), hiding the
+// global-memory latency.
+
+@group(0) @binding(0) var<storage, read>       A: array<f32>;
+@group(0) @binding(1) var<storage, read>       x: array<f32>;
+@group(0) @binding(2) var<storage, read_write> y: array<f32>;
+
+struct Params {
+  m:     u32,
+  n:     u32,
+  alpha: f32,
+  beta:  f32,
+  incx:  u32,
+  incy:  u32,
+  lda:   u32,
+}
+
+@group(0) @binding(3) var<uniform> params: Params;
+
+const WGS: u32 = 64u;
+var<workgroup> scratch: array<f32, 64>;
+
+@compute @workgroup_size(64)
+fn main(
+  @builtin(workgroup_id)        wgid: vec3u,
+  @builtin(local_invocation_id) lid:  vec3u,
+  @builtin(num_workgroups)      nwg:  vec3u,
+) {
+  // Grid-stride loop: each workgroup handles ceil(m / nwg.x) rows.
+  for (var row = wgid.x; row < params.m; row += nwg.x) {
+    let row_base = row * params.lda;
+    var acc0: f32 = 0.0;
+    var acc1: f32 = 0.0;
+    var acc2: f32 = 0.0;
+    var acc3: f32 = 0.0;
+
+    // 4-unrolled loop: each iteration issues 4 independent loads for A and x.
+    // The accumulators are independent so the GPU can overlap the memory
+    // requests rather than serialising them behind a dependency chain.
+    let n4_floor = (params.n / (4u * WGS)) * (4u * WGS);
+    for (var j: u32 = lid.x; j < n4_floor; j += 4u * WGS) {
+      acc0 += A[row_base + j            ] * x[ j             * params.incx];
+      acc1 += A[row_base + j +     WGS  ] * x[(j +     WGS)  * params.incx];
+      acc2 += A[row_base + j + 2u * WGS ] * x[(j + 2u * WGS) * params.incx];
+      acc3 += A[row_base + j + 3u * WGS ] * x[(j + 3u * WGS) * params.incx];
+    }
+    // Scalar tail: at most 3*WGS elements left after the unrolled block.
+    for (var j: u32 = n4_floor + lid.x; j < params.n; j += WGS) {
+      acc0 += A[row_base + j] * x[j * params.incx];
+    }
+
+    // Parallel reduction: 64 → 32 → 16 → 8 → 4 → 2 → 1
+    scratch[lid.x] = acc0 + acc1 + acc2 + acc3;
+    workgroupBarrier();
+    for (var stride = WGS >> 1u; stride > 0u; stride >>= 1u) {
+      if lid.x < stride {
+        scratch[lid.x] += scratch[lid.x + stride];
+      }
+      workgroupBarrier();
+    }
+
+    if lid.x == 0u {
+      let yi = row * params.incy;
+      y[yi] = params.alpha * scratch[0] + params.beta * y[yi];
+    }
+    // All 64 threads must agree before the next row reuses scratch[].
+    workgroupBarrier();
+  }
+}

package/src/shaders/sgemv_t.wgsl

@@ -0,0 +1,65 @@
+// sgemv_t: y = alpha * A^T * x + beta * y  (A is m×n row-major, transposed)
+// each thread owns one column of A → one element of y (length n)
+// tiles over x (length m) using shared memory; four independent accumulators
+// let the GPU pipeline A reads across j within each tile (ILP=4)
+
+@group(0) @binding(0) var<storage, read>       A: array<f32>;
+@group(0) @binding(1) var<storage, read>       x: array<f32>;
+@group(0) @binding(2) var<storage, read_write> y: array<f32>;
+
+struct Params {
+  m:     u32,
+  n:     u32,
+  alpha: f32,
+  beta:  f32,
+  incx:  u32,
+  incy:  u32,
+  lda:   u32,
+}
+
+@group(0) @binding(3) var<uniform> params: Params;
+
+const WGS: u32 = 64u;
+var<workgroup> x_tile: array<f32, 64>;
+
+@compute @workgroup_size(64)
+fn main(
+  @builtin(global_invocation_id) gid: vec3u,
+  @builtin(local_invocation_id)  lid: vec3u,
+) {
+  // each thread owns column col of A → output y[col]
+  let col     = gid.x;
+  // tile over x (length m, the rows of A)
+  let m_floor = (params.m / WGS) * WGS;
+  var acc0: f32 = 0.0;
+  var acc1: f32 = 0.0;
+  var acc2: f32 = 0.0;
+  var acc3: f32 = 0.0;
+
+  for (var base = 0u; base < m_floor; base += WGS) {
+    // cooperative load: all 64 threads fill x_tile with x[base..base+WGS]
+    x_tile[lid.x] = x[(base + lid.x) * params.incx];
+    workgroupBarrier();
+
+    // 4-unrolled inner loop: 4 independent A reads let the GPU pipeline
+    // global-memory requests within each tile. WGS=64 divides by 4 exactly.
+    if (col < params.n) {
+      for (var j = 0u; j < WGS; j += 4u) {
+        acc0 += A[(base + j    ) * params.lda + col] * x_tile[j    ];
+        acc1 += A[(base + j + 1) * params.lda + col] * x_tile[j + 1];
+        acc2 += A[(base + j + 2) * params.lda + col] * x_tile[j + 2];
+        acc3 += A[(base + j + 3) * params.lda + col] * x_tile[j + 3];
+      }
+    }
+    workgroupBarrier();
+  }
+
+  if (col < params.n) {
+    // remainder: m not divisible by WGS — short loop, single accumulator fine
+    for (var k = m_floor; k < params.m; k++) {
+      acc0 += A[k * params.lda + col] * x[k * params.incx];
+    }
+    let yi = col * params.incy;
+    y[yi] = params.alpha * (acc0 + acc1 + acc2 + acc3) + params.beta * y[yi];
+  }
+}

package/src/shaders/snrm2.wgsl

@@ -21,12 +21,30 @@ fn main(
   @builtin(workgroup_id)         wgid:   vec3u,
   @builtin(num_workgroups)       num_wg: vec3u,
 ) {
-  var acc: f32 = 0.0;
-  for (var id = gid.x; id < params.n; id += num_wg.x * WGS) {
+  var acc0: f32 = 0.0;
+  var acc1: f32 = 0.0;
+  var acc2: f32 = 0.0;
+  var acc3: f32 = 0.0;
+
+  let stride   = num_wg.x * WGS;
+  let n4_floor = (params.n / (4u * stride)) * (4u * stride);
+
+  for (var id = gid.x; id < n4_floor; id += 4u * stride) {
+    let v0 = x[ id                * params.x_inc];
+    let v1 = x[(id +      stride) * params.x_inc];
+    let v2 = x[(id + 2u * stride) * params.x_inc];
+    let v3 = x[(id + 3u * stride) * params.x_inc];
+    acc0 += v0 * v0;
+    acc1 += v1 * v1;
+    acc2 += v2 * v2;
+    acc3 += v3 * v3;
+  }
+  for (var id = n4_floor + gid.x; id < params.n; id += stride) {
     let v = x[id * params.x_inc];
-    acc += v * v;
+    acc0 += v * v;
   }
-  tile[lid.x] = acc;
+
+  tile[lid.x] = acc0 + acc1 + acc2 + acc3;
   workgroupBarrier();
 
   for (var s = WGS / 2u; s > 0u; s >>= 1u) {

package/src/shaders/ssymv.wgsl

@@ -0,0 +1,69 @@
+// ssymv: y = alpha * A * x + beta * y
+// A is n×n symmetric, lower (uplo=0) or upper (uplo=1) triangle stored.
+// The logical matrix is fully dense (symmetric), so each row's dot product
+// sums over all n columns; entries on the unstored side of the diagonal are
+// fetched from their mirror position (A[i,j] == A[j,i]).
+// One workgroup per row, grid-stride outer loop.
+
+@group(0) @binding(0) var<storage, read>       A: array<f32>;
+@group(0) @binding(1) var<storage, read>       x: array<f32>;
+@group(0) @binding(2) var<storage, read_write> y: array<f32>;
+
+struct Params {
+  n:     u32,
+  alpha: f32,
+  beta:  f32,
+  incx:  u32,
+  incy:  u32,
+  lda:   u32,
+  uplo:  u32,  // 0 = lower, 1 = upper
+}
+
+@group(0) @binding(3) var<uniform> params: Params;
+
+const WGS: u32 = 64u;
+var<workgroup> scratch: array<f32, 64>;
+
+@compute @workgroup_size(64)
+fn main(
+  @builtin(workgroup_id)        wgid: vec3u,
+  @builtin(local_invocation_id) lid:  vec3u,
+  @builtin(num_workgroups)      nwg:  vec3u,
+) {
+  for (var i = wgid.x; i < params.n; i += nwg.x) {
+    var acc = 0.0f;
+
+    // y[i] = Σ_j A[i,j] * x[j]
+    for (var j = lid.x; j < params.n; j += WGS) {
+      var aVal: f32;
+      if params.uplo == 0u {
+        // Lower: A[i,j] stored at A[i*lda+j] for j ≤ i, mirrored from A[j*lda+i] otherwise
+        if j <= i {
+          aVal = A[i * params.lda + j];
+        } else {
+          aVal = A[j * params.lda + i];
+        }
+      } else {
+        // Upper: A[i,j] stored at A[i*lda+j] for j ≥ i, mirrored from A[j*lda+i] otherwise
+        if j >= i {
+          aVal = A[i * params.lda + j];
+        } else {
+          aVal = A[j * params.lda + i];
+        }
+      }
+      acc += aVal * x[j * params.incx];
+    }
+
+    // Parallel reduction: 64 → 1
+    scratch[lid.x] = acc;
+    workgroupBarrier();
+    for (var stride = WGS >> 1u; stride > 0u; stride >>= 1u) {
+      if lid.x < stride { scratch[lid.x] += scratch[lid.x + stride]; }
+      workgroupBarrier();
+    }
+
+    if lid.x == 0u {
+      y[i * params.incy] = params.alpha * scratch[0] + params.beta * y[i * params.incy];
+    }
+  }
+}

package/src/shaders/strmv.wgsl

@@ -0,0 +1,103 @@
+// strmv: y = op(A) * x
+// A is n×n triangular, lower (uplo=0) or upper (uplo=1) triangle stored.
+// op(A) is A (trans=0) or A^T (trans=1).
+// diag=1 (unit) treats the diagonal as 1 without reading A's diagonal values.
+// One workgroup per row, grid-stride outer loop.
+
+@group(0) @binding(0) var<storage, read>       A: array<f32>;
+@group(0) @binding(1) var<storage, read>       x: array<f32>;
+@group(0) @binding(2) var<storage, read_write> y: array<f32>;
+
+struct Params {
+  n:     u32,
+  incx:  u32,
+  incy:  u32,
+  lda:   u32,
+  trans: u32,  // 0 = no-transpose, 1 = transpose
+  uplo:  u32,  // 0 = lower, 1 = upper
+  diag:  u32,  // 0 = non-unit, 1 = unit
+}
+
+@group(0) @binding(3) var<uniform> params: Params;
+
+const WGS: u32 = 64u;
+var<workgroup> scratch: array<f32, 64>;
+
+@compute @workgroup_size(64)
+fn main(
+  @builtin(workgroup_id)        wgid: vec3u,
+  @builtin(local_invocation_id) lid:  vec3u,
+  @builtin(num_workgroups)      nwg:  vec3u,
+) {
+  for (var i = wgid.x; i < params.n; i += nwg.x) {
+    var acc = 0.0f;
+
+    if params.trans == 0u {
+      // No-transpose: y[i] = Σ_j A[i,j] * x[j]
+      if params.uplo == 0u {
+        // Lower: A[i,j] stored at A[i*lda+j] for j ≤ i
+        for (var j = lid.x; j <= i; j += WGS) {
+          var aVal: f32;
+          // unit diagonal: use 1 instead of A's actual diagonal value
+          if params.diag == 1u && j == i {
+            aVal = 1.0;
+          } else if ( j <= i ) {
+            aVal = A[i * params.lda + j];
+          }
+          acc += aVal * x[j * params.incx];
+        }
+      } else {
+        // Upper: A[i,j] stored at A[i*lda+j] for j ≥ i
+        for (var j = i + lid.x; j < params.n; j += WGS) {
+          var aVal: f32;
+          // unit diagonal: use 1 instead of A's actual diagonal value
+          if params.diag == 1u && j == i {
+            aVal = 1.0;
+          } else if ( j >= i ) {
+            aVal = A[i * params.lda + j];
+          }
+          acc += aVal * x[j * params.incx];
+        }
+      }
+    } else {
+      // Transpose: y[i] = Σ_j A[j,i] * x[j]
+      if params.uplo == 0u {
+        // Lower: A[j,i] stored at A[j*lda+i] for j ≥ i
+        for (var j = i + lid.x; j < params.n; j += WGS) {
+          var aVal: f32;
+          // unit diagonal: use 1 instead of A's actual diagonal value
+          if params.diag == 1u && j == i {
+            aVal = 1.0;
+          } else if ( j >= i ) {
+            aVal = A[j * params.lda + i];
+          }
+          acc += aVal * x[j * params.incx];
+        }
+      } else {
+        // Upper: A[j,i] stored at A[j*lda+i] for j ≤ i
+        for (var j = lid.x; j <= i; j += WGS) {
+          var aVal: f32;
+          // unit diagonal: use 1 instead of A's actual diagonal value
+          if params.diag == 1u && j == i {
+            aVal = 1.0;
+          } else if ( j <= i ) {
+            aVal = A[j * params.lda + i];
+          }
+          acc += aVal * x[j * params.incx];
+        }
+      }
+    }
+
+    // Parallel reduction: 64 → 1
+    scratch[lid.x] = acc;
+    workgroupBarrier();
+    for (var stride = WGS >> 1u; stride > 0u; stride >>= 1u) {
+      if lid.x < stride { scratch[lid.x] += scratch[lid.x + stride]; }
+      workgroupBarrier();
+    }
+
+    if lid.x == 0u {
+      y[ i * params.incy ] = scratch[0];
+    }
+  }
+}

package/src/shaders/strsv_apply_inverse.wgsl

@@ -0,0 +1,47 @@
+// strsv_apply_inverse: given a precomputed block inverse (from
+// strsv_invert_block.wgsl), computes this block's solution as a dense
+// matrix-vector multiply against the block's current remainder in x —
+// replacing what the old strsv_block.wgsl did via a genuinely sequential,
+// barrier-per-row substitution.
+//
+// All blockLen rows are computed in parallel within a single workgroup: the
+// remainder is loaded into workgroup-shared memory once, then each thread
+// independently computes one full row's dot product from that shared copy.
+// No further synchronization is needed after the load — every thread only
+// reads shared memory from then on (never written again within this call)
+// and writes a distinct element of x, so there's no cross-thread hazard to
+// guard against.
+
+@group(0) @binding(0) var<storage, read>       Ainv: array<f32>;
+@group(0) @binding(1) var<storage, read_write> x: array<f32>;
+
+struct Params {
+  incx:       u32,
+  blockIndex: u32,
+  blockStart: u32,
+  blockEnd:   u32,
+}
+
+@group(0) @binding(2) var<uniform> params: Params;
+
+const BLOCK_SIZE: u32 = 64u;
+var<workgroup> xLocal: array<f32, 64>;
+
+@compute @workgroup_size(64)
+fn strsv_apply_inverse_main(@builtin(local_invocation_id) lid: vec3u) {
+  let blockLen = params.blockEnd - params.blockStart;
+
+  if (lid.x < blockLen) {
+    xLocal[lid.x] = x[(params.blockStart + lid.x) * params.incx];
+  }
+  workgroupBarrier();
+
+  if (lid.x >= blockLen) { return; }
+
+  let ainvBase = params.blockIndex * BLOCK_SIZE * BLOCK_SIZE;
+  var acc = 0.0f;
+  for (var j = 0u; j < blockLen; j++) {
+    acc += Ainv[ainvBase + lid.x * BLOCK_SIZE + j] * xLocal[j];
+  }
+  x[(params.blockStart + lid.x) * params.incx] = acc;
+}

package/src/shaders/strsv_invert_block.wgsl

@@ -0,0 +1,109 @@
+// strsv_invert_block: computes ONE column (workgroup_id.x) of ONE block's
+// (workgroup_id.y) explicit inverse, via the same one-row-at-a-time
+// substitution as strsv_block.wgsl, but solving against a unit basis vector
+// e_col instead of the real right-hand side, and writing to a dense
+// (BLOCK_SIZE x BLOCK_SIZE, row-major) scratch buffer per block instead of
+// mutating x. Dispatched once for the whole matrix (2D: BLOCK_SIZE columns x
+// numBlocks), fully in parallel -- unlike the sequential per-block main
+// loop in strsv.mjs, no block's inverse depends on any other block or on x.
+//
+// A triangular block's inverse is itself triangular: forward (effectively-
+// lower, e.g. no-trans+lower) blocks have inverse column col nonzero only
+// for row>=col, solved in increasing row order; backward (effectively-
+// upper) blocks have it nonzero only for row<=col, solved in decreasing
+// order. Rows outside a column's nonzero range are written as literal 0 —
+// strsv_apply_inverse.wgsl's dense matvec depends on that, not just on
+// those entries being mathematically implied zero.
+
+@group(0) @binding(0) var<storage, read>       A: array<f32>;
+@group(0) @binding(1) var<storage, read_write> Ainv: array<f32>;
+
+struct Params {
+  n:     u32,
+  lda:   u32,
+  trans: u32,  // 0 = no-transpose, 1 = transpose
+  uplo:  u32,  // 0 = lower, 1 = upper
+  diag:  u32,  // 0 = non-unit, 1 = unit
+}
+
+@group(0) @binding(2) var<uniform> params: Params;
+
+const WGS: u32 = 64u;
+const BLOCK_SIZE: u32 = 64u;
+var<workgroup> scratch: array<f32, 64>;
+
+fn readA(i: u32, j: u32) -> f32 {
+  if params.trans == 0u {
+    return A[i * params.lda + j];
+  } else {
+    return A[j * params.lda + i];
+  }
+}
+
+@compute @workgroup_size(64)
+fn strsv_invert_block_main(
+  @builtin(workgroup_id)        wgid: vec3u,
+  @builtin(local_invocation_id) lid:  vec3u,
+) {
+  let col = wgid.x;
+  let blockIndex = wgid.y;
+  let blockStart = blockIndex * BLOCK_SIZE;
+  var blockEnd = blockStart + BLOCK_SIZE;
+  if (blockEnd > params.n) { blockEnd = params.n; }
+  let blockLen = blockEnd - blockStart;
+
+  if (col >= blockLen) { return; }
+
+  let ainvBase = blockIndex * BLOCK_SIZE * BLOCK_SIZE;
+  let forward = (params.trans == 0u) == (params.uplo == 0u);
+
+  if forward {
+    for (var r = lid.x; r < col; r += WGS) {
+      Ainv[ainvBase + r * BLOCK_SIZE + col] = 0.0;
+    }
+  } else {
+    for (var r = col + 1u + lid.x; r < blockLen; r += WGS) {
+      Ainv[ainvBase + r * BLOCK_SIZE + col] = 0.0;
+    }
+  }
+  storageBarrier();
+  workgroupBarrier();
+
+  let numSteps = select(col + 1u, blockLen - col, forward);
+  for (var step = 0u; step < numSteps; step++) {
+    let localRow = select(col - step, col + step, forward);
+    let i = blockStart + localRow;
+
+    var acc = 0.0f;
+    if forward {
+      for (var lj = col + lid.x; lj < localRow; lj += WGS) {
+        acc += readA(i, blockStart + lj) * Ainv[ainvBase + lj * BLOCK_SIZE + col];
+      }
+    } else {
+      for (var lj = localRow + 1u + lid.x; lj <= col; lj += WGS) {
+        acc += readA(i, blockStart + lj) * Ainv[ainvBase + lj * BLOCK_SIZE + col];
+      }
+    }
+
+    scratch[lid.x] = acc;
+    workgroupBarrier();
+    for (var stride = WGS >> 1u; stride > 0u; stride >>= 1u) {
+      if lid.x < stride { scratch[lid.x] += scratch[lid.x + stride]; }
+      workgroupBarrier();
+    }
+
+    if lid.x == 0u {
+      let e = select(0.0, 1.0, localRow == col);
+      let rhs = e - scratch[0];
+      var val: f32;
+      if params.diag == 1u {
+        val = rhs;
+      } else {
+        val = rhs / A[i * params.lda + i];
+      }
+      Ainv[ainvBase + localRow * BLOCK_SIZE + col] = val;
+    }
+    storageBarrier();
+    workgroupBarrier();
+  }
+}