wgblas 2.2.1 → 2.3.0

package/src/shaders/dgemv_n.wgsl

@@ -0,0 +1,112 @@
+// dgemv_n: y := alpha * A * x + beta * y, double-double (Dekker) f64
+// emulation of sgemv_n (matrix-vector product, no transpose). A, x, y,
+// alpha, and beta are each split into an f32 (hi, lo) pair; WGSL has no f64
+// type. Same one-workgroup-per-output-row shape as sgemv_n.wgsl (grid-stride
+// over rows), but the per-row dot product is reduced via a full
+// within-workgroup DD tree reduction — the same reduction ddot.wgsl uses to
+// combine 64 lanes down to one — rather than sgemv_n.wgsl's 4-way ILP
+// unroll + shared-memory tree (ddMulProtected/ddAddProtected each carry a
+// workgroupBarrier(), so unrolling would pay that barrier four times over
+// per iteration for no benefit). Since each row is handled by exactly one
+// workgroup, no separate cross-workgroup reduction pass (ddot's reduce_f64)
+// is needed here — the tree reduction alone finishes the row.
+
+@group(0) @binding(0) var<storage, read>       AHi: array<f32>;
+@group(0) @binding(1) var<storage, read>       ALo: array<f32>;
+@group(0) @binding(2) var<storage, read>       xHi: array<f32>;
+@group(0) @binding(3) var<storage, read>       xLo: array<f32>;
+@group(0) @binding(4) var<storage, read_write> yHi: array<f32>;
+@group(0) @binding(5) var<storage, read_write> yLo: array<f32>;
+
+struct Params {
+  m:       u32,
+  n:       u32,
+  alphaHi: f32,
+  alphaLo: f32,
+  betaHi:  f32,
+  betaLo:  f32,
+  incx:    u32,
+  incy:    u32,
+  lda:     u32,
+}
+
+@group(0) @binding(6) var<uniform> params: Params;
+
+const WGS: u32 = 64u;
+var<workgroup> tile: array<DD, 64>;
+
+@compute @workgroup_size(64)
+fn dgemv_n_main(
+  @builtin(workgroup_id)        wgid: vec3u,
+  @builtin(local_invocation_id) lid:  vec3u,
+  @builtin(num_workgroups)      nwg:  vec3u,
+) {
+  let alpha = DD(params.alphaHi, params.alphaLo);
+  let beta  = DD(params.betaHi, params.betaLo);
+  let isBetaNonzero = params.betaHi != 0.0 || params.betaLo != 0.0;
+
+  // Column loop's trip count depends only on n and WGS, not on row — same
+  // main/tail split applies uniformly to every row (see dger.wgsl).
+  let n_floor   = (params.n / WGS) * WGS;
+  let mainIters = n_floor / WGS;
+  let hasTail   = select(0u, 1u, n_floor < params.n);
+
+  for (var row = wgid.x; row < params.m; row += nwg.x) {
+    let row_base = row * params.lda;
+    var acc = DD(0.0, 0.0);
+
+    for (var iter = 0u; iter < mainIters; iter++) {
+      let j  = lid.x + iter * WGS;
+      let ia = row_base + j;
+      let ix = j * params.incx;
+      let prod = ddMulProtected(DD(AHi[ia], ALo[ia]), DD(xHi[ix], xLo[ix]), lid.x);
+      acc = ddAddProtected(acc, prod, lid.x);
+    }
+
+    for (var iter = 0u; iter < hasTail; iter++) {
+      let j     = n_floor + lid.x;
+      let valid = j < params.n;
+      let ia    = select(0u, row_base + j, valid);
+      let ix    = select(0u, j * params.incx, valid);
+      let prod  = ddMulProtected(DD(AHi[ia], ALo[ia]), DD(xHi[ix], xLo[ix]), lid.x);
+      let contribution = DD(select(0.0, prod.hi, valid), select(0.0, prod.lo, valid));
+      acc = ddAddProtected(acc, contribution, lid.x);
+    }
+
+    tile[lid.x] = acc;
+    workgroupBarrier();
+
+    // Inactive threads combine against a throwaway partner and discard it
+    // (ddAddProtected must be called unconditionally by every thread).
+    for (var s = WGS / 2u; s > 0u; s >>= 1u) {
+      let partner = select(lid.x, lid.x + s, lid.x < s);
+      let combined = ddAddProtected(tile[lid.x], tile[partner], lid.x);
+      workgroupBarrier(); // all threads must read tile[] above before any write below
+      if (lid.x < s) { tile[lid.x] = combined; }
+      workgroupBarrier();
+    }
+
+    // tile[0] now holds the full row dot product, visible to every lane —
+    // every thread redundantly finishes the O(1) alpha/beta combine so the
+    // protected ops below stay uniformly called (only the write is gated).
+    let dot = tile[0];
+    let scaled = ddMulProtected(alpha, dot, lid.x);
+    let iy = row * params.incy;
+    let yVal = DD(yHi[iy], yLo[iy]);
+    let betaTimesY = ddMulProtected(beta, yVal, lid.x);
+    let withBeta = ddAddProtected(scaled, betaTimesY, lid.x);
+    // BLAS beta==0 semantics: y is written, not accumulated — the result
+    // must not depend on y's prior value (though it is still read above).
+    let result = DD(
+      select(scaled.hi, withBeta.hi, isBetaNonzero),
+      select(scaled.lo, withBeta.lo, isBetaNonzero),
+    );
+
+    if (lid.x == 0u) {
+      yHi[iy] = result.hi;
+      yLo[iy] = result.lo;
+    }
+    // All 64 threads must agree before the next row reuses tile[].
+    workgroupBarrier();
+  }
+}

package/src/shaders/dgemv_t.wgsl

@@ -0,0 +1,102 @@
+// dgemv_t: y := alpha * A^T * x + beta * y, double-double (Dekker) f64
+// emulation of sgemv_t (matrix-vector product, transposed). A, x, y, alpha,
+// and beta are each split into an f32 (hi, lo) pair; WGSL has no f64 type.
+// Same one-thread-per-output-column shape as sgemv_t.wgsl, tiling over x
+// (length m) via shared memory. Because ddMulProtected/ddAddProtected each
+// carry a workgroupBarrier(), every thread in the workgroup — including ones
+// whose column is out of range (n not a multiple of WGS) — must call them
+// the same number of times: unlike sgemv_t.wgsl's `if (col < n)` guard
+// around the whole accumulation, out-of-range threads here compute against
+// a clamped dummy column unconditionally and simply never write their
+// result (same technique dger.wgsl/ddot.wgsl use for their ragged tails).
+
+@group(0) @binding(0) var<storage, read>       AHi: array<f32>;
+@group(0) @binding(1) var<storage, read>       ALo: array<f32>;
+@group(0) @binding(2) var<storage, read>       xHi: array<f32>;
+@group(0) @binding(3) var<storage, read>       xLo: array<f32>;
+@group(0) @binding(4) var<storage, read_write> yHi: array<f32>;
+@group(0) @binding(5) var<storage, read_write> yLo: array<f32>;
+
+struct Params {
+  m:       u32,
+  n:       u32,
+  alphaHi: f32,
+  alphaLo: f32,
+  betaHi:  f32,
+  betaLo:  f32,
+  incx:    u32,
+  incy:    u32,
+  lda:     u32,
+}
+
+@group(0) @binding(6) var<uniform> params: Params;
+
+const WGS: u32 = 64u;
+var<workgroup> xTile: array<DD, 64>;
+
+@compute @workgroup_size(64)
+fn dgemv_t_main(
+  @builtin(global_invocation_id) gid: vec3u,
+  @builtin(local_invocation_id)  lid: vec3u,
+) {
+  let col      = gid.x;
+  let colValid = col < params.n;
+  // Clamp so out-of-range threads still index safely — their contribution
+  // is computed but never written back.
+  let safeCol  = select(0u, col, colValid);
+
+  var acc = DD(0.0, 0.0);
+
+  let m_floor   = (params.m / WGS) * WGS;
+  let mainIters = m_floor / WGS;
+
+  for (var iter = 0u; iter < mainIters; iter++) {
+    let base = iter * WGS;
+    // Cooperative load: all 64 threads fill xTile with x[base..base+WGS),
+    // independent of col — safe regardless of whether this thread's column
+    // is in range.
+    let ix = (base + lid.x) * params.incx;
+    xTile[lid.x] = DD(xHi[ix], xLo[ix]);
+    workgroupBarrier();
+
+    for (var j = 0u; j < WGS; j++) {
+      let ia = (base + j) * params.lda + safeCol;
+      let prod = ddMulProtected(DD(AHi[ia], ALo[ia]), xTile[j], lid.x);
+      acc = ddAddProtected(acc, prod, lid.x);
+    }
+    workgroupBarrier();
+  }
+
+  // Remainder rows (m not a multiple of WGS): the count is the same for
+  // every thread regardless of col, so this loop is already barrier-safe
+  // without needing a second tiling pass.
+  let remCount = params.m - m_floor;
+  for (var k = 0u; k < remCount; k++) {
+    let row = m_floor + k;
+    let ia  = row * params.lda + safeCol;
+    let ix  = row * params.incx;
+    let prod = ddMulProtected(DD(AHi[ia], ALo[ia]), DD(xHi[ix], xLo[ix]), lid.x);
+    acc = ddAddProtected(acc, prod, lid.x);
+  }
+
+  let alpha = DD(params.alphaHi, params.alphaLo);
+  let beta  = DD(params.betaHi, params.betaLo);
+  let scaled = ddMulProtected(alpha, acc, lid.x);
+
+  let iy = safeCol * params.incy;
+  let yVal = DD(yHi[iy], yLo[iy]);
+  let betaTimesY = ddMulProtected(beta, yVal, lid.x);
+  let withBeta = ddAddProtected(scaled, betaTimesY, lid.x);
+  let isBetaNonzero = params.betaHi != 0.0 || params.betaLo != 0.0;
+  // BLAS beta==0 semantics: y is written, not accumulated — the result
+  // must not depend on y's prior value (though it is still read above).
+  let result = DD(
+    select(scaled.hi, withBeta.hi, isBetaNonzero),
+    select(scaled.lo, withBeta.lo, isBetaNonzero),
+  );
+
+  if (colValid) {
+    yHi[iy] = result.hi;
+    yLo[iy] = result.lo;
+  }
+}

package/src/shaders/dger.wgsl

@@ -0,0 +1,77 @@
+// dger: A := alpha * x * y^T + A, double-double (Dekker) f64 emulation of
+// sger (rank-1 update). x, y, A, and alpha are each split into an f32
+// (hi, lo) pair; WGSL has no f64 type. One workgroup per row of A
+// (grid-stride over rows); within a row, ddMulProtected/ddAddProtected each
+// carry a workgroupBarrier(), so the column loop uses the same
+// uniform-main + ragged-tail split dscal.wgsl uses, rather than sger.wgsl's
+// 4-way ILP unroll — unrolling would pay the barrier round-trip four times
+// over per iteration for no benefit here.
+
+@group(0) @binding(0) var<storage, read>       xHi: array<f32>;
+@group(0) @binding(1) var<storage, read>       xLo: array<f32>;
+@group(0) @binding(2) var<storage, read>       yHi: array<f32>;
+@group(0) @binding(3) var<storage, read>       yLo: array<f32>;
+@group(0) @binding(4) var<storage, read_write> AHi: array<f32>;
+@group(0) @binding(5) var<storage, read_write> ALo: array<f32>;
+
+struct Params {
+  m:       u32,
+  n:       u32,
+  alphaHi: f32,
+  alphaLo: f32,
+  x_inc:   u32,
+  y_inc:   u32,
+  lda:     u32,
+}
+
+@group(0) @binding(6) var<uniform> params: Params;
+
+const WGS: u32 = 64u;
+
+@compute @workgroup_size(64)
+fn dger_main(
+  @builtin(workgroup_id)        wgid: vec3u,
+  @builtin(local_invocation_id) lid:  vec3u,
+  @builtin(num_workgroups)      nwg:  vec3u,
+) {
+  let alpha = DD(params.alphaHi, params.alphaLo);
+
+  // Column loop's trip count depends only on n and WGS, not on row — the
+  // same main/tail split applies uniformly to every row, so every thread
+  // reaches each protected call the same number of times overall.
+  let n_floor = (params.n / WGS) * WGS;
+  let mainIters = n_floor / WGS;
+  let hasTail = select(0u, 1u, n_floor < params.n);
+
+  for (var row = wgid.x; row < params.m; row += nwg.x) {
+    // Every thread in the workgroup redundantly computes the same alpha*x[row]
+    // — wasteful but harmless, and keeps the per-row protected-call count
+    // trivially identical across threads.
+    let ix = row * params.x_inc;
+    let xi = ddMulProtected(alpha, DD(xHi[ix], xLo[ix]), lid.x);
+    let row_base = row * params.lda;
+
+    for (var iter = 0u; iter < mainIters; iter++) {
+      let col = lid.x + iter * WGS;
+      let idx = row_base + col;
+      let iy = col * params.y_inc;
+      let prod = ddMulProtected(xi, DD(yHi[iy], yLo[iy]), lid.x);
+      let result = ddAddProtected(prod, DD(AHi[idx], ALo[idx]), lid.x);
+      AHi[idx] = result.hi;
+      ALo[idx] = result.lo;
+    }
+
+    for (var iter = 0u; iter < hasTail; iter++) {
+      let col = n_floor + lid.x;
+      let valid = col < params.n;
+      let idx = select(0u, row_base + col, valid);
+      let iy = select(0u, col * params.y_inc, valid);
+      let prod = ddMulProtected(xi, DD(yHi[iy], yLo[iy]), lid.x);
+      let result = ddAddProtected(prod, DD(AHi[idx], ALo[idx]), lid.x);
+      if (valid) {
+        AHi[idx] = result.hi;
+        ALo[idx] = result.lo;
+      }
+    }
+  }
+}

package/src/shaders/dsymv.wgsl

@@ -0,0 +1,125 @@
+// dsymv: y := alpha * A * x + beta * y, double-double (Dekker) f64 emulation
+// of ssymv (symmetric matrix-vector product). A, x, y, alpha, and beta are
+// each split into an f32 (hi, lo) pair; WGSL has no f64 type. A is n×n
+// symmetric — only the triangle specified by uplo is physically stored, so
+// entries on the unstored side of the diagonal are fetched from their
+// mirror position (A[i,j] == A[j,i]), same as ssymv.wgsl. Same
+// one-workgroup-per-row + full within-workgroup DD tree reduction shape as
+// dgemv_n.wgsl (every row sums over all n columns here, unlike dsyr's
+// triangle-restricted range, since the logical matrix is fully dense).
+
+@group(0) @binding(0) var<storage, read>       AHi: array<f32>;
+@group(0) @binding(1) var<storage, read>       ALo: array<f32>;
+@group(0) @binding(2) var<storage, read>       xHi: array<f32>;
+@group(0) @binding(3) var<storage, read>       xLo: array<f32>;
+@group(0) @binding(4) var<storage, read_write> yHi: array<f32>;
+@group(0) @binding(5) var<storage, read_write> yLo: array<f32>;
+
+struct Params {
+  n:       u32,
+  alphaHi: f32,
+  alphaLo: f32,
+  betaHi:  f32,
+  betaLo:  f32,
+  incx:    u32,
+  incy:    u32,
+  lda:     u32,
+  uplo:    u32,  // 0 = lower, 1 = upper
+}
+
+@group(0) @binding(6) var<uniform> params: Params;
+
+const WGS: u32 = 64u;
+var<workgroup> tile: array<DD, 64>;
+
+// A[i,j] flat index for the symmetric matrix — stored position if (i,j) is
+// on the uplo side of the diagonal, mirrored from (j,i) otherwise. Pure
+// addressing (no protected op inside), so branching here is safe.
+fn symIdx(i: u32, j: u32) -> u32 {
+  var row: u32;
+  var col: u32;
+  if params.uplo == 0u {
+    // Lower: stored at (i,j) for j <= i, mirrored from (j,i) otherwise.
+    if j <= i { row = i; col = j; } else { row = j; col = i; }
+  } else {
+    // Upper: stored at (i,j) for j >= i, mirrored from (j,i) otherwise.
+    if j >= i { row = i; col = j; } else { row = j; col = i; }
+  }
+  return row * params.lda + col;
+}
+
+@compute @workgroup_size(64)
+fn dsymv_main(
+  @builtin(workgroup_id)        wgid: vec3u,
+  @builtin(local_invocation_id) lid:  vec3u,
+  @builtin(num_workgroups)      nwg:  vec3u,
+) {
+  let alpha = DD(params.alphaHi, params.alphaLo);
+  let beta  = DD(params.betaHi, params.betaLo);
+  let isBetaNonzero = params.betaHi != 0.0 || params.betaLo != 0.0;
+
+  // Column loop's trip count depends only on n and WGS, not on row — same
+  // main/tail split applies uniformly to every row (see dger.wgsl).
+  let n_floor   = (params.n / WGS) * WGS;
+  let mainIters = n_floor / WGS;
+  let hasTail   = select(0u, 1u, n_floor < params.n);
+
+  for (var i = wgid.x; i < params.n; i += nwg.x) {
+    var acc = DD(0.0, 0.0);
+
+    for (var iter = 0u; iter < mainIters; iter++) {
+      let j   = lid.x + iter * WGS;
+      let idx = symIdx(i, j);
+      let ix  = j * params.incx;
+      let prod = ddMulProtected(DD(AHi[idx], ALo[idx]), DD(xHi[ix], xLo[ix]), lid.x);
+      acc = ddAddProtected(acc, prod, lid.x);
+    }
+
+    for (var iter = 0u; iter < hasTail; iter++) {
+      let j     = n_floor + lid.x;
+      let valid = j < params.n;
+      let safeJ = select(0u, j, valid);
+      let idx   = symIdx(i, safeJ);
+      let ix    = safeJ * params.incx;
+      let prod  = ddMulProtected(DD(AHi[idx], ALo[idx]), DD(xHi[ix], xLo[ix]), lid.x);
+      let contribution = DD(select(0.0, prod.hi, valid), select(0.0, prod.lo, valid));
+      acc = ddAddProtected(acc, contribution, lid.x);
+    }
+
+    tile[lid.x] = acc;
+    workgroupBarrier();
+
+    // Inactive threads combine against a throwaway partner and discard it
+    // (ddAddProtected must be called unconditionally by every thread).
+    for (var s = WGS / 2u; s > 0u; s >>= 1u) {
+      let partner = select(lid.x, lid.x + s, lid.x < s);
+      let combined = ddAddProtected(tile[lid.x], tile[partner], lid.x);
+      workgroupBarrier(); // all threads must read tile[] above before any write below
+      if (lid.x < s) { tile[lid.x] = combined; }
+      workgroupBarrier();
+    }
+
+    // tile[0] now holds the full row dot product, visible to every lane —
+    // every thread redundantly finishes the O(1) alpha/beta combine so the
+    // protected ops below stay uniformly called (only the write is gated).
+    let dot = tile[0];
+    let scaled = ddMulProtected(alpha, dot, lid.x);
+    let iy = i * params.incy;
+    let yVal = DD(yHi[iy], yLo[iy]);
+    let betaTimesY = ddMulProtected(beta, yVal, lid.x);
+    let withBeta = ddAddProtected(scaled, betaTimesY, lid.x);
+    // BLAS beta==0 semantics: y is written, not accumulated — the result
+    // must not depend on y's prior value (though it is still read above).
+    let result = DD(
+      select(scaled.hi, withBeta.hi, isBetaNonzero),
+      select(scaled.lo, withBeta.lo, isBetaNonzero),
+    );
+
+    if (lid.x == 0u) {
+      yHi[iy] = result.hi;
+      yLo[iy] = result.lo;
+    }
+    // All 64 threads must agree before the next row reuses tile[].
+    workgroupBarrier();
+  }
+}

package/src/shaders/dsyr.wgsl

@@ -0,0 +1,84 @@
+// dsyr: A := alpha * x * x^T + A, double-double (Dekker) f64 emulation of
+// ssyr (symmetric rank-1 update). x, A, and alpha are each split into an f32
+// (hi, lo) pair; WGSL has no f64 type. Only the triangle specified by uplo
+// is referenced/updated. One workgroup per row of A (grid-stride over
+// rows); within a row, ddMulProtected/ddAddProtected each carry a
+// workgroupBarrier(), so the column loop uses the same uniform-main +
+// ragged-tail split dger.wgsl uses over its *stored* range, rather than
+// ssyr.wgsl's 4-way ILP unroll.
+
+@group(0) @binding(0) var<storage, read>       xHi: array<f32>;
+@group(0) @binding(1) var<storage, read>       xLo: array<f32>;
+@group(0) @binding(2) var<storage, read_write> AHi: array<f32>;
+@group(0) @binding(3) var<storage, read_write> ALo: array<f32>;
+
+struct Params {
+  n:     u32,
+  alphaHi: f32,
+  alphaLo: f32,
+  incx:  u32,
+  lda:   u32,
+  uplo:  u32,  // 0 = lower, 1 = upper
+}
+
+@group(0) @binding(4) var<uniform> params: Params;
+
+const WGS: u32 = 64u;
+
+@compute @workgroup_size(64)
+fn dsyr_main(
+  @builtin(workgroup_id)        wgid: vec3u,
+  @builtin(local_invocation_id) lid:  vec3u,
+  @builtin(num_workgroups)      nwg:  vec3u,
+) {
+  let alpha = DD(params.alphaHi, params.alphaLo);
+
+  for (var row = wgid.x; row < params.n; row += nwg.x) {
+    let ix = row * params.incx;
+    let xi = ddMulProtected(alpha, DD(xHi[ix], xLo[ix]), lid.x);
+    let row_base = row * params.lda;
+
+    // Stored-triangle column range for this row: lower [0,row], upper [row,n).
+    var colStart: u32;
+    var colEnd: u32;
+    if params.uplo == 1u {
+      colStart = row;
+      colEnd = params.n;
+    } else {
+      colStart = 0u;
+      colEnd = row + 1u;
+    }
+
+    // Range length varies per row, but every thread in the workgroup runs
+    // the SAME row at the same time (the outer loop is shared), so the
+    // main/tail split computed from colStart/colEnd is already uniform
+    // across threads for this row — just not the same across different rows.
+    let rangeLen = colEnd - colStart;
+    let range_floor = colStart + (rangeLen / WGS) * WGS;
+    let mainIters = (range_floor - colStart) / WGS;
+    let hasTail = select(0u, 1u, range_floor < colEnd);
+
+    for (var iter = 0u; iter < mainIters; iter++) {
+      let col = colStart + lid.x + iter * WGS;
+      let idx = row_base + col;
+      let ic = col * params.incx;
+      let prod = ddMulProtected(xi, DD(xHi[ic], xLo[ic]), lid.x);
+      let result = ddAddProtected(prod, DD(AHi[idx], ALo[idx]), lid.x);
+      AHi[idx] = result.hi;
+      ALo[idx] = result.lo;
+    }
+
+    for (var iter = 0u; iter < hasTail; iter++) {
+      let col = range_floor + lid.x;
+      let valid = col < colEnd;
+      let idx = select(0u, row_base + col, valid);
+      let ic = select(0u, col * params.incx, valid);
+      let prod = ddMulProtected(xi, DD(xHi[ic], xLo[ic]), lid.x);
+      let result = ddAddProtected(prod, DD(AHi[idx], ALo[idx]), lid.x);
+      if (valid) {
+        AHi[idx] = result.hi;
+        ALo[idx] = result.lo;
+      }
+    }
+  }
+}

package/src/shaders/dsyr2.wgsl

@@ -0,0 +1,95 @@
+// dsyr2: A := alpha * x * y^T + alpha * y * x^T + A, double-double (Dekker)
+// f64 emulation of ssyr2 (symmetric rank-2 update). x, y, A, and alpha are
+// each split into an f32 (hi, lo) pair; WGSL has no f64 type. Only the
+// triangle specified by uplo is referenced/updated. One workgroup per row
+// of A (grid-stride over rows); within a row, ddMulProtected/ddAddProtected
+// each carry a workgroupBarrier(), so the column loop uses the same
+// uniform-main + ragged-tail split dsyr.wgsl uses over its *stored* range,
+// rather than ssyr2.wgsl's 4-way ILP unroll.
+
+@group(0) @binding(0) var<storage, read>       xHi: array<f32>;
+@group(0) @binding(1) var<storage, read>       xLo: array<f32>;
+@group(0) @binding(2) var<storage, read>       yHi: array<f32>;
+@group(0) @binding(3) var<storage, read>       yLo: array<f32>;
+@group(0) @binding(4) var<storage, read_write> AHi: array<f32>;
+@group(0) @binding(5) var<storage, read_write> ALo: array<f32>;
+
+struct Params {
+  n:       u32,
+  alphaHi: f32,
+  alphaLo: f32,
+  incx:    u32,
+  incy:    u32,
+  lda:     u32,
+  uplo:    u32,  // 0 = lower, 1 = upper
+}
+
+@group(0) @binding(6) var<uniform> params: Params;
+
+const WGS: u32 = 64u;
+
+@compute @workgroup_size(64)
+fn dsyr2_main(
+  @builtin(workgroup_id)        wgid: vec3u,
+  @builtin(local_invocation_id) lid:  vec3u,
+  @builtin(num_workgroups)      nwg:  vec3u,
+) {
+  let alpha = DD(params.alphaHi, params.alphaLo);
+
+  for (var row = wgid.x; row < params.n; row += nwg.x) {
+    let ix = row * params.incx;
+    let iy = row * params.incy;
+    let xi = ddMulProtected(alpha, DD(xHi[ix], xLo[ix]), lid.x);
+    let yi = ddMulProtected(alpha, DD(yHi[iy], yLo[iy]), lid.x);
+    let row_base = row * params.lda;
+
+    // Stored-triangle column range for this row: lower [0,row], upper [row,n).
+    var colStart: u32;
+    var colEnd: u32;
+    if params.uplo == 1u {
+      colStart = row;
+      colEnd = params.n;
+    } else {
+      colStart = 0u;
+      colEnd = row + 1u;
+    }
+
+    // Range length varies per row, but every thread in the workgroup runs
+    // the SAME row at the same time (the outer loop is shared), so the
+    // main/tail split computed from colStart/colEnd is already uniform
+    // across threads for this row — just not the same across different rows.
+    let rangeLen = colEnd - colStart;
+    let range_floor = colStart + (rangeLen / WGS) * WGS;
+    let mainIters = (range_floor - colStart) / WGS;
+    let hasTail = select(0u, 1u, range_floor < colEnd);
+
+    for (var iter = 0u; iter < mainIters; iter++) {
+      let col = colStart + lid.x + iter * WGS;
+      let idx = row_base + col;
+      let jc_x = col * params.incx;
+      let jc_y = col * params.incy;
+      let prod1 = ddMulProtected(xi, DD(yHi[jc_y], yLo[jc_y]), lid.x);
+      let prod2 = ddMulProtected(yi, DD(xHi[jc_x], xLo[jc_x]), lid.x);
+      let sum1 = ddAddProtected(prod1, prod2, lid.x);
+      let result = ddAddProtected(sum1, DD(AHi[idx], ALo[idx]), lid.x);
+      AHi[idx] = result.hi;
+      ALo[idx] = result.lo;
+    }
+
+    for (var iter = 0u; iter < hasTail; iter++) {
+      let col = range_floor + lid.x;
+      let valid = col < colEnd;
+      let idx = select(0u, row_base + col, valid);
+      let jc_x = select(0u, col * params.incx, valid);
+      let jc_y = select(0u, col * params.incy, valid);
+      let prod1 = ddMulProtected(xi, DD(yHi[jc_y], yLo[jc_y]), lid.x);
+      let prod2 = ddMulProtected(yi, DD(xHi[jc_x], xLo[jc_x]), lid.x);
+      let sum1 = ddAddProtected(prod1, prod2, lid.x);
+      let result = ddAddProtected(sum1, DD(AHi[idx], ALo[idx]), lid.x);
+      if (valid) {
+        AHi[idx] = result.hi;
+        ALo[idx] = result.lo;
+      }
+    }
+  }
+}