wgblas 2.0.0 → 2.2.0

package/src/shaders/dnrm2.wgsl

@@ -0,0 +1,167 @@
+// dnrm2: result = sqrt(sum(x[i] * x[i])), double-double (Dekker) f64
+// emulation of snrm2 — same scaled accumulation (Blue's algorithm), just
+// with `scale`/`ssq` as DD pairs (via ddDivProtected/ddMulProtected/
+// ddAddProtected/ddSqrtProtected) instead of plain f32. Squaring still
+// saturates an f32 hi component above ~1.8e19 regardless of DD precision
+// (DD widens the mantissa, not the exponent range), so the scaling is
+// still needed here for the same reason it was in snrm2.
+//
+// snrm2.wgsl's ssqAccum/ssqMerge each branch on which operand is bigger —
+// can't carry over directly, since a protected op's workgroupBarrier()
+// needs every thread to reach the same call site, and here different
+// threads could take different branches. Both formulas are computed
+// unconditionally below; only the final combine (`ddSelect`) differs per
+// thread — same fix shape as drot's/drotm's own per-dispatch flags, just
+// applied to a per-element branch instead.
+//
+// pass 1 dispatches 2*WGS workgroups; pass 2 (reduction/scaledSumF64.wgsl)
+// duplicates ssqAccumProtected/ssqMergeProtected rather than sharing them,
+// same as the plain-f32 pair already does.
+
+@group(0) @binding(0) var<storage, read>       xHi:           array<f32>;
+@group(0) @binding(1) var<storage, read>       xLo:           array<f32>;
+@group(0) @binding(2) var<storage, read_write> partialsScaleHi: array<f32>;
+@group(0) @binding(3) var<storage, read_write> partialsScaleLo: array<f32>;
+@group(0) @binding(4) var<storage, read_write> partialsSsqHi:   array<f32>;
+@group(0) @binding(5) var<storage, read_write> partialsSsqLo:   array<f32>;
+@group(0) @binding(6) var<uniform>             params:        Params;
+
+struct Params {
+  n:     u32,
+  x_inc: u32,
+}
+
+const WGS: u32 = 64;
+
+struct ScaleSsq {
+  scale: DD,
+  ssq:   DD,
+}
+
+fn ddSelect(a: DD, b: DD, cond: bool) -> DD {
+  return DD(select(a.hi, b.hi, cond), select(a.lo, b.lo, cond));
+}
+
+// Folds one more |value| (DD) into a running (scale, ssq) pair — branch-free,
+// see file header. `bigger`/`smaller` name the two operands by magnitude
+// (not by which one was "acc" vs "new"), and biggerIsZero==true only when
+// both scale and absxi are still exactly zero (the very first zero
+// elements, before any nonzero value has been seen) — substituting a safe
+// denominator there avoids a 0/0 without needing a separate branch/return;
+// the arithmetic already reduces to a correct no-op in that case.
+fn ssqAccumProtected(acc: ScaleSsq, absxi: DD, threadSlot: u32) -> ScaleSsq {
+  let isBigger = ddGreater(absxi, acc.scale);
+  let bigger = ddSelect(acc.scale, absxi, isBigger);
+  let smaller = ddSelect(absxi, acc.scale, isBigger);
+  let biggerIsZero = bigger.hi == 0.0;
+  let safeBigger = ddSelect(bigger, DD(1.0, 0.0), biggerIsZero);
+  let r = ddDivProtected(smaller, safeBigger, threadSlot);
+  let rsq = ddMulProtected(r, r, threadSlot);
+  let ssqTimesRsq = ddMulProtected(acc.ssq, rsq, threadSlot);
+  let sumIfBigger = ddAddProtected(DD(1.0, 0.0), ssqTimesRsq, threadSlot);
+  let sumIfNotBigger = ddAddProtected(acc.ssq, rsq, threadSlot);
+  let newSsq = ddSelect(sumIfNotBigger, sumIfBigger, isBigger);
+  return ScaleSsq(bigger, newSsq);
+}
+
+// Associative merge of two independent (scale, ssq) partials — same
+// branch-free shape, for combining ILP lanes and the tree reduction.
+fn ssqMergeProtected(a: ScaleSsq, b: ScaleSsq, threadSlot: u32) -> ScaleSsq {
+  let isBigger = !ddGreater(b.scale, a.scale); // a.scale >= b.scale
+  let bigger = ddSelect(b.scale, a.scale, isBigger);
+  let smaller = ddSelect(a.scale, b.scale, isBigger);
+  let biggerSsq = ddSelect(b.ssq, a.ssq, isBigger);
+  let smallerSsq = ddSelect(a.ssq, b.ssq, isBigger);
+  let biggerIsZero = bigger.hi == 0.0;
+  let safeBigger = ddSelect(bigger, DD(1.0, 0.0), biggerIsZero);
+  let r = ddDivProtected(smaller, safeBigger, threadSlot);
+  let rsq = ddMulProtected(r, r, threadSlot);
+  let smallerSsqTimesRsq = ddMulProtected(smallerSsq, rsq, threadSlot);
+  let newSsq = ddAddProtected(biggerSsq, smallerSsqTimesRsq, threadSlot);
+  return ScaleSsq(bigger, newSsq);
+}
+
+var<workgroup> tileScaleHi: array<f32, 64>;
+var<workgroup> tileScaleLo: array<f32, 64>;
+var<workgroup> tileSsqHi:   array<f32, 64>;
+var<workgroup> tileSsqLo:   array<f32, 64>;
+
+@compute @workgroup_size(64)
+fn dnrm2_main(
+  @builtin(global_invocation_id) gid:    vec3u,
+  @builtin(local_invocation_id)  lid:    vec3u,
+  @builtin(workgroup_id)         wgid:   vec3u,
+  @builtin(num_workgroups)       num_wg: vec3u,
+) {
+  var acc0 = ScaleSsq(DD(0.0, 0.0), DD(1.0, 0.0));
+  var acc1 = ScaleSsq(DD(0.0, 0.0), DD(1.0, 0.0));
+  var acc2 = ScaleSsq(DD(0.0, 0.0), DD(1.0, 0.0));
+  var acc3 = ScaleSsq(DD(0.0, 0.0), DD(1.0, 0.0));
+
+  let stride   = num_wg.x * WGS;
+  let n4_floor = (params.n / (4u * stride)) * (4u * stride);
+
+  // Same trip count for every thread, driven by a counter (protected ops'
+  // barriers need a provably-uniform loop bound) — see dasum.wgsl.
+  let mainIters = n4_floor / (4u * stride);
+  for (var iter = 0u; iter < mainIters; iter++) {
+    let id =  gid.x + iter * 4u * stride;
+    let i0 =  id                * params.x_inc;
+    let i1 = (id +      stride) * params.x_inc;
+    let i2 = (id + 2u * stride) * params.x_inc;
+    let i3 = (id + 3u * stride) * params.x_inc;
+    acc0 = ssqAccumProtected(acc0, ddAbs(DD(xHi[i0], xLo[i0])), lid.x);
+    acc1 = ssqAccumProtected(acc1, ddAbs(DD(xHi[i1], xLo[i1])), lid.x);
+    acc2 = ssqAccumProtected(acc2, ddAbs(DD(xHi[i2], xLo[i2])), lid.x);
+    acc3 = ssqAccumProtected(acc3, ddAbs(DD(xHi[i3], xLo[i3])), lid.x);
+  }
+
+  // Tail is ragged (0-3 extra per thread) — pad to this workgroup's worst
+  // case, masking an invalid element to exactly 0 (contributes nothing).
+  let wgBaseGid = wgid.x * WGS;
+  var tailIters = 0u;
+  if (n4_floor + wgBaseGid < params.n) {
+    tailIters = (params.n - 1u - n4_floor - wgBaseGid) / stride + 1u;
+  }
+  for (var iter = 0u; iter < tailIters; iter++) {
+    let id    = n4_floor + gid.x + iter * stride;
+    let valid = id < params.n;
+    let i     = select(0u, id * params.x_inc, valid); // index 0 always in-bounds
+    let loaded = ddAbs(DD(xHi[i], xLo[i]));
+    let contribution = DD(select(0.0, loaded.hi, valid), select(0.0, loaded.lo, valid));
+    acc0 = ssqAccumProtected(acc0, contribution, lid.x);
+  }
+
+  let combined01 = ssqMergeProtected(acc0, acc1, lid.x);
+  let combined23 = ssqMergeProtected(acc2, acc3, lid.x);
+  let combined = ssqMergeProtected(combined01, combined23, lid.x);
+  tileScaleHi[lid.x] = combined.scale.hi;
+  tileScaleLo[lid.x] = combined.scale.lo;
+  tileSsqHi[lid.x]   = combined.ssq.hi;
+  tileSsqLo[lid.x]   = combined.ssq.lo;
+  workgroupBarrier();
+
+  // Inactive threads merge against a throwaway partner and discard it
+  // (ssqMergeProtected must be called unconditionally by every thread).
+  for (var s = WGS / 2u; s > 0u; s >>= 1u) {
+    let partner = select(lid.x, lid.x + s, lid.x < s);
+    let a = ScaleSsq(DD(tileScaleHi[lid.x], tileScaleLo[lid.x]), DD(tileSsqHi[lid.x], tileSsqLo[lid.x]));
+    let b = ScaleSsq(DD(tileScaleHi[partner], tileScaleLo[partner]), DD(tileSsqHi[partner], tileSsqLo[partner]));
+    let merged = ssqMergeProtected(a, b, lid.x);
+    workgroupBarrier(); // all threads must read tile[] above before any write below
+    if (lid.x < s) {
+      tileScaleHi[lid.x] = merged.scale.hi;
+      tileScaleLo[lid.x] = merged.scale.lo;
+      tileSsqHi[lid.x]   = merged.ssq.hi;
+      tileSsqLo[lid.x]   = merged.ssq.lo;
+    }
+    workgroupBarrier();
+  }
+
+  if (lid.x == 0u) {
+    partialsScaleHi[wgid.x] = tileScaleHi[0];
+    partialsScaleLo[wgid.x] = tileScaleLo[0];
+    partialsSsqHi[wgid.x]   = tileSsqHi[0];
+    partialsSsqLo[wgid.x]   = tileSsqLo[0];
+  }
+}

package/src/shaders/drot.wgsl

@@ -0,0 +1,81 @@
+// drot: x = c*x + s*y, y = -s*x + c*y — double-double (Dekker) f64 emulation
+// of srot. c, s, x, and y are each split into an f32 (hi, lo) pair. Each
+// element costs four ddMulProtected (c*x, s*y, -s*x, c*y) then two
+// ddAddProtected (the two sums) — negS is computed once outside the loop
+// via bitcast negation (exact, no rounding, so no barrier needed there)
+// rather than adding a DD-subtract helper. See dscal.wgsl for why this is a
+// uniform main pass plus a ragged, select-masked tail rather than a plain
+// `id < params.n` grid-stride loop.
+
+@group(0) @binding(0) var<storage, read_write> xHi: array<f32>;
+@group(0) @binding(1) var<storage, read_write> xLo: array<f32>;
+@group(0) @binding(2) var<storage, read_write> yHi: array<f32>;
+@group(0) @binding(3) var<storage, read_write> yLo: array<f32>;
+@group(0) @binding(4) var<uniform> params: Params;
+
+struct Params {
+  n:     u32,
+  cHi:   f32,
+  cLo:   f32,
+  sHi:   f32,
+  sLo:   f32,
+  x_inc: u32,
+  y_inc: u32,
+}
+
+const WGS: u32 = 64;
+
+@compute @workgroup_size(64)
+fn drot_main(
+  @builtin(global_invocation_id) gid:    vec3u,
+  @builtin(local_invocation_id)  lid:    vec3u,
+  @builtin(workgroup_id)         wgid:   vec3u,
+  @builtin(num_workgroups)       num_wg: vec3u,
+) {
+  let c = DD(params.cHi, params.cLo);
+  let s = DD(params.sHi, params.sLo);
+  let negS = DD(negf(params.sHi), negf(params.sLo));
+  let stride = num_wg.x * WGS;
+
+  let n_floor = (params.n / stride) * stride;
+  let mainIters = n_floor / stride;
+  for (var iter = 0u; iter < mainIters; iter++) {
+    let id = gid.x + iter * stride;
+    let ix = id * params.x_inc;
+    let iy = id * params.y_inc;
+    let xi = DD(xHi[ix], xLo[ix]);
+    let yi = DD(yHi[iy], yLo[iy]);
+    let xNew = ddAddProtected(ddMulProtected(c, xi, lid.x), ddMulProtected(s, yi, lid.x), lid.x);
+    let yNew = ddAddProtected(ddMulProtected(negS, xi, lid.x), ddMulProtected(c, yi, lid.x), lid.x);
+    xHi[ix] = xNew.hi;
+    xLo[ix] = xNew.lo;
+    yHi[iy] = yNew.hi;
+    yLo[iy] = yNew.lo;
+  }
+
+  // Tail is ragged (0 or 1 extra per thread) — pad to this workgroup's worst
+  // case so every thread in the workgroup still calls ddMulProtected/
+  // ddAddProtected the same number of times (their barriers need that),
+  // masking only the write.
+  let wgBaseGid = wgid.x * WGS;
+  var tailIters = 0u;
+  if (n_floor + wgBaseGid < params.n) {
+    tailIters = (params.n - 1u - n_floor - wgBaseGid) / stride + 1u;
+  }
+  for (var iter = 0u; iter < tailIters; iter++) {
+    let id = n_floor + gid.x + iter * stride;
+    let valid = id < params.n;
+    let ix = select(0u, id * params.x_inc, valid); // index 0 always in-bounds
+    let iy = select(0u, id * params.y_inc, valid);
+    let xi = DD(xHi[ix], xLo[ix]);
+    let yi = DD(yHi[iy], yLo[iy]);
+    let xNew = ddAddProtected(ddMulProtected(c, xi, lid.x), ddMulProtected(s, yi, lid.x), lid.x);
+    let yNew = ddAddProtected(ddMulProtected(negS, xi, lid.x), ddMulProtected(c, yi, lid.x), lid.x);
+    if (valid) {
+      xHi[ix] = xNew.hi;
+      xLo[ix] = xNew.lo;
+      yHi[iy] = yNew.hi;
+      yLo[iy] = yNew.lo;
+    }
+  }
+}

package/src/shaders/drotm.wgsl

@@ -0,0 +1,99 @@
+// drotm: applies a modified Givens rotation H to vectors x and y — double-
+// double (Dekker) f64 emulation of srotm. paramHi/paramLo[0] = flag: -1
+// (full H), 0 (unit diagonal), 1 (unit off-diagonal). param = [ flag, h11,
+// h21, h12, h22 ], each entry an f32 (hi, lo) pair.
+// flag == -2 (identity/no-op) is handled in JS before dispatch reaches here.
+//
+// h11/h12/h21/h22 are resolved once, outside the loop, from the (uniform
+// across every thread) flag — same shape as srot's c/s, so no barrier is
+// needed for that selection itself. Each element then costs four
+// ddMulProtected + two ddAddProtected, same as drot.
+
+@group(0) @binding(0) var<storage, read_write> xHi: array<f32>;
+@group(0) @binding(1) var<storage, read_write> xLo: array<f32>;
+@group(0) @binding(2) var<storage, read_write> yHi: array<f32>;
+@group(0) @binding(3) var<storage, read_write> yLo: array<f32>;
+@group(0) @binding(4) var<storage, read> paramHi: array<f32>;
+@group(0) @binding(5) var<storage, read> paramLo: array<f32>;
+@group(0) @binding(6) var<uniform> params: Params;
+
+struct Params {
+  n:     u32,
+  x_inc: u32,
+  y_inc: u32,
+}
+
+const WGS: u32 = 64;
+const ONE: DD = DD(1.0, 0.0);
+const NEG_ONE: DD = DD(-1.0, 0.0);
+
+@compute @workgroup_size(64)
+fn drotm_main(
+  @builtin(global_invocation_id) gid:    vec3u,
+  @builtin(local_invocation_id)  lid:    vec3u,
+  @builtin(workgroup_id)         wgid:   vec3u,
+  @builtin(num_workgroups)       num_wg: vec3u,
+) {
+  let flag = paramHi[0]; // exact small integer (-1, 0, or 1) — lo is always 0
+
+  var h11: DD; var h12: DD;
+  var h21: DD; var h22: DD;
+
+  if (flag == -1.0) {
+    // full 2x2 matrix
+    h11 = DD(paramHi[1], paramLo[1]); h21 = DD(paramHi[2], paramLo[2]);
+    h12 = DD(paramHi[3], paramLo[3]); h22 = DD(paramHi[4], paramLo[4]);
+  } else if (flag == 0.0) {
+    // diagonal fixed at 1
+    h11 = ONE;                        h21 = DD(paramHi[2], paramLo[2]);
+    h12 = DD(paramHi[3], paramLo[3]); h22 = ONE;
+  } else {
+    // flag == 1.0: off-diagonal fixed at +1 / -1
+    h11 = DD(paramHi[1], paramLo[1]); h21 = NEG_ONE;
+    h12 = ONE;                        h22 = DD(paramHi[4], paramLo[4]);
+  }
+
+  let stride = num_wg.x * WGS;
+
+  let n_floor = (params.n / stride) * stride;
+  let mainIters = n_floor / stride;
+  for (var iter = 0u; iter < mainIters; iter++) {
+    let id = gid.x + iter * stride;
+    let ix = id * params.x_inc;
+    let iy = id * params.y_inc;
+    let xi = DD(xHi[ix], xLo[ix]);
+    let yi = DD(yHi[iy], yLo[iy]);
+    let xNew = ddAddProtected(ddMulProtected(h11, xi, lid.x), ddMulProtected(h12, yi, lid.x), lid.x);
+    let yNew = ddAddProtected(ddMulProtected(h21, xi, lid.x), ddMulProtected(h22, yi, lid.x), lid.x);
+    xHi[ix] = xNew.hi;
+    xLo[ix] = xNew.lo;
+    yHi[iy] = yNew.hi;
+    yLo[iy] = yNew.lo;
+  }
+
+  // Tail is ragged (0 or 1 extra per thread) — pad to this workgroup's worst
+  // case so every thread in the workgroup still calls ddMulProtected/
+  // ddAddProtected the same number of times (their barriers need that),
+  // masking only the write.
+  let wgBaseGid = wgid.x * WGS;
+  var tailIters = 0u;
+  if (n_floor + wgBaseGid < params.n) {
+    tailIters = (params.n - 1u - n_floor - wgBaseGid) / stride + 1u;
+  }
+  for (var iter = 0u; iter < tailIters; iter++) {
+    let id = n_floor + gid.x + iter * stride;
+    let valid = id < params.n;
+    let ix = select(0u, id * params.x_inc, valid); // index 0 always in-bounds
+    let iy = select(0u, id * params.y_inc, valid);
+    let xi = DD(xHi[ix], xLo[ix]);
+    let yi = DD(yHi[iy], yLo[iy]);
+    let xNew = ddAddProtected(ddMulProtected(h11, xi, lid.x), ddMulProtected(h12, yi, lid.x), lid.x);
+    let yNew = ddAddProtected(ddMulProtected(h21, xi, lid.x), ddMulProtected(h22, yi, lid.x), lid.x);
+    if (valid) {
+      xHi[ix] = xNew.hi;
+      xLo[ix] = xNew.lo;
+      yHi[iy] = yNew.hi;
+      yLo[iy] = yNew.lo;
+    }
+  }
+}

package/src/shaders/dscal.wgsl

@@ -0,0 +1,60 @@
+// dscal: x := alpha * x, double-double (Dekker) f64 emulation of sscal.
+// alpha and x are each an f32 (hi, lo) pair. See f64/utils/multiply.wgsl for
+// ddMulProtected and why plain ddMulRaw isn't safe without a renormalizing
+// barrier — that barrier needs a provably uniform loop trip count across
+// every thread in the workgroup, so (like dasum.wgsl's reduction loop) this
+// splits into a uniform main pass plus a ragged, select-masked tail rather
+// than a plain `id < params.n` grid-stride loop.
+
+@group(0) @binding(0) var<storage, read_write> xHi: array<f32>;
+@group(0) @binding(1) var<storage, read_write> xLo: array<f32>;
+@group(0) @binding(2) var<uniform> params: Params;
+
+struct Params {
+  n:       u32,
+  alphaHi: f32,
+  alphaLo: f32,
+  x_inc:   u32,
+}
+
+const WGS: u32 = 64;
+
+@compute @workgroup_size(64)
+fn dscal_main(
+  @builtin(global_invocation_id) gid:    vec3u,
+  @builtin(local_invocation_id)  lid:    vec3u,
+  @builtin(workgroup_id)         wgid:   vec3u,
+  @builtin(num_workgroups)       num_wg: vec3u,
+) {
+  let alpha = DD(params.alphaHi, params.alphaLo);
+  let stride = num_wg.x * WGS;
+
+  let n_floor = (params.n / stride) * stride;
+  let mainIters = n_floor / stride;
+  for (var iter = 0u; iter < mainIters; iter++) {
+    let id = gid.x + iter * stride;
+    let i = id * params.x_inc;
+    let result = ddMulProtected(alpha, DD(xHi[i], xLo[i]), lid.x);
+    xHi[i] = result.hi;
+    xLo[i] = result.lo;
+  }
+
+  // Tail is ragged (0 or 1 extra per thread) — pad to this workgroup's worst
+  // case so every thread in the workgroup still calls ddMulProtected the
+  // same number of times (its barrier needs that), masking only the write.
+  let wgBaseGid = wgid.x * WGS;
+  var tailIters = 0u;
+  if (n_floor + wgBaseGid < params.n) {
+    tailIters = (params.n - 1u - n_floor - wgBaseGid) / stride + 1u;
+  }
+  for (var iter = 0u; iter < tailIters; iter++) {
+    let id = n_floor + gid.x + iter * stride;
+    let valid = id < params.n;
+    let i = select(0u, id * params.x_inc, valid); // index 0 always in-bounds
+    let result = ddMulProtected(alpha, DD(xHi[i], xLo[i]), lid.x);
+    if (valid) {
+      xHi[i] = result.hi;
+      xLo[i] = result.lo;
+    }
+  }
+}

package/src/shaders/dswap.wgsl

@@ -0,0 +1,38 @@
+// dswap: x <-> y, double-double (Dekker) f64 emulation of sswap. A swap is
+// pure data movement — hi and lo are exchanged verbatim, with no arithmetic
+// at all — so (unlike dscal/daxpy/ddot) this needs no
+// ddMulProtected/ddAddProtected renormalizing barrier, and so no
+// ragged-tail-with-barrier split; a plain grid-stride loop is safe, same
+// shape as sswap.wgsl itself.
+
+@group(0) @binding(0) var<storage, read_write> xHi: array<f32>;
+@group(0) @binding(1) var<storage, read_write> xLo: array<f32>;
+@group(0) @binding(2) var<storage, read_write> yHi: array<f32>;
+@group(0) @binding(3) var<storage, read_write> yLo: array<f32>;
+
+struct Params {
+  n:     u32,
+  x_inc: u32,
+  y_inc: u32,
+}
+
+@group(0) @binding(4) var<uniform> params: Params;
+
+const WGS: u32 = 64;
+
+@compute @workgroup_size(64)
+fn main(
+  @builtin(global_invocation_id) gid: vec3u,
+  @builtin(num_workgroups) num_wg: vec3u,
+) {
+  for (var id = gid.x; id < params.n; id += num_wg.x * WGS) {
+    let ix = id * params.x_inc;
+    let iy = id * params.y_inc;
+    let tempHi = xHi[ix];
+    let tempLo = xLo[ix];
+    xHi[ix] = yHi[iy];
+    xLo[ix] = yLo[iy];
+    yHi[iy] = tempHi;
+    yLo[iy] = tempLo;
+  }
+}

package/src/shaders/f64/utils/add.wgsl

@@ -75,3 +75,9 @@ fn ddAddProtected(a: DD, b: DD, threadSlot: u32) -> DD {
   let loSum = a.lo + b.lo;
   return fastTwoSumProtected(s.hi, s.lo + loSum, threadSlot);
 }
+
+// Double-double subtraction — a - b, via exact negation (a sign-bit flip,
+// no rounding) then ddAddProtected. Same protection contract.
+fn ddSubProtected(a: DD, b: DD, threadSlot: u32) -> DD {
+  return ddAddProtected(a, DD(negf(b.hi), negf(b.lo)), threadSlot);
+}

package/src/shaders/f64/utils/divide.wgsl

@@ -0,0 +1,45 @@
+// Requires f64/dekker.wgsl concatenated first for the DD struct, and
+// f64/utils/add.wgsl (ddSubProtected/ddAddProtected/negf) and
+// f64/utils/multiply.wgsl (ddMulProtected).
+//
+// Verified empirically on real hardware (NVIDIA GTX 1650 + Intel Mesa
+// integrated GPU, ~8000 random trials each plus explicit edge cases): no
+// compiler-reassociation-style corruption of the kind that broke twoSum/
+// twoProd (see add.wgsl's/multiply.wgsl's own headers) — every failure
+// found was a genuine algorithm gap, not a driver miscompile, and both are
+// fixed below (the b.hi==0.0 guard). One real, expected-shape difference
+// from every other protected op here: the low-power backend's observed
+// forward-error factor for this op specifically runs noticeably higher
+// (~7-14000x eps, vs ~3-9x on high-performance) than ddSqrtProtected's
+// (~3-4x on both) — division inherently amplifies input imprecision more
+// than a sum/product does, so a real routine built on this needs its own
+// backend-calibrated threshold, same as every other f64 arithmetic routine
+// in this codebase (see e.g. tests/drot/src/test.drot.js's THRESHOLDS).
+//
+// One Newton-style long-division refinement (Bailey/QD-style): q1 = a.hi /
+// b.hi is a plain f32 quotient, accurate to ~24 bits. Computing the residual
+// a - q1*b in DD arithmetic (not f32) recovers the bits q1 lost, and a
+// second plain division of that residual resolves them into a correction
+// term — combining q1 + q2 gives roughly double a lone f32 divide's
+// precision, matching this scheme's ~48-bit double-double target (already
+// short of real f64's 52 bits, so a second refinement step would chase
+// precision this representation has no room for).
+// b.hi == 0.0 makes q1 = a.hi/0.0 already the IEEE-754-correct answer
+// (±Infinity, or NaN for 0/0) via plain float division, but the refinement
+// below would corrupt it: p1 = q1*b multiplies that Infinity by a zero
+// divisor, and Infinity*0 is NaN by definition, poisoning everything after.
+// Substituting a safe non-zero denominator via select() — rather than
+// branching/returning early — keeps every thread calling ddMulProtected/
+// ddSubProtected/ddAddProtected unconditionally, which their internal
+// workgroupBarrier() requires; only the final result is selected between
+// the refined value and q1's own already-correct answer.
+fn ddDivProtected(a: DD, b: DD, threadSlot: u32) -> DD {
+  let bIsZero = b.hi == 0.0;
+  let q1 = a.hi / b.hi;
+  let safeB = DD(select(b.hi, 1.0, bIsZero), select(b.lo, 0.0, bIsZero));
+  let p1 = ddMulProtected(DD(q1, 0.0), safeB, threadSlot);
+  let r1 = ddSubProtected(a, p1, threadSlot);
+  let q2 = r1.hi / safeB.hi;
+  let refined = ddAddProtected(DD(q1, 0.0), DD(q2, 0.0), threadSlot);
+  return DD(select(refined.hi, q1, bIsZero), select(refined.lo, 0.0, bIsZero));
+}

package/src/shaders/f64/utils/multiply.wgsl

@@ -15,7 +15,11 @@ const SPLIT_CONST: f32 = 4097.0;
 
 fn bitSplit(a: f32) -> DD {
   let bits = bitcast<u32>(a);
-  let hiBits = bits & 0xFFFFF800u; // keep sign+exponent+top 12 mantissa bits
+  // Top 11 mantissa bits, so hi carries 12 significant bits with the implicit
+  // leading 1 — the halves are multiplied pairwise and f32 holds 24, so a
+  // wider split rounds those products and the "exact" error term goes wrong.
+  // Matches SPLIT_CONST = 2^12+1 used by the Veltkamp path below.
+  let hiBits = bits & 0xFFFFF000u;
   let hi = bitcast<f32>(hiBits);
   let lo = fsub(a, hi); // exact by Sterbenz's lemma (hi, a share an exponent, are close)
   return DD(hi, lo);
@@ -63,19 +67,24 @@ fn twoProdFma(a: f32, b: f32) -> DD {
 
 // DD × DD product (Dekker/Bailey): twoProdBit(a.hi, b.hi) already captures
 // the dominant term to full DD precision, and the cross terms are below the
-// ~48-bit floor anyway, so folding them in with plain f32 loses nothing —
-// only the final renormalization needs barrier protection. Split into
-// ddMulRaw (unprotected) and ddMulProtected (renormalizes via
-// fastTwoSumProtected) so callers with several products can batch them
-// through one shared barrier. ddMulRaw's result isn't a valid DD pair on
-// its own — it must be renormalized before use.
-fn ddMulRaw(a: DD, b: DD) -> DD {
+// ~48-bit floor anyway, so folding them in with plain f32 loses nothing.
+//
+// Another real compiler bug, distinct from add.wgsl's twoSum one — confirmed
+// on Intel Mesa ANV: when p.lo feeds straight into `crossAndLo` unobserved,
+// the compiler folds it away entirely. Materializing p.lo itself through
+// workgroup memory + workgroupBarrier() (like twoSumProtected does for its
+// sum) is what fixes it, so ddMulRaw now takes threadSlot and always pays
+// that barrier — no longer a plain unprotected batchable helper.
+fn ddMulRaw(a: DD, b: DD, threadSlot: u32) -> DD {
   let p = twoProdBit(a.hi, b.hi);
-  let crossAndLo = p.lo + (a.hi * b.lo + a.lo * b.hi);
+  dekkerScratch[threadSlot] = p.lo;
+  workgroupBarrier();
+  let pLo = dekkerScratch[threadSlot];
+  let crossAndLo = pLo + (a.hi * b.lo + a.lo * b.hi);
   return DD(p.hi, crossAndLo);
 }
 
 fn ddMulProtected(a: DD, b: DD, threadSlot: u32) -> DD {
-  let raw = ddMulRaw(a, b);
+  let raw = ddMulRaw(a, b, threadSlot);
   return fastTwoSumProtected(raw.hi, raw.lo, threadSlot);
 }

package/src/shaders/f64/utils/sqrt.wgsl

@@ -0,0 +1,45 @@
+// Requires f64/dekker.wgsl concatenated first for the DD struct, and
+// f64/utils/add.wgsl (ddSubProtected/ddAddProtected) and
+// f64/utils/multiply.wgsl (twoProdBit — squaring a plain f32 needs no
+// barrier, per multiply.wgsl's own note that twoProdBit is universally safe
+// unprotected).
+//
+// Verified empirically on real hardware (NVIDIA GTX 1650 + Intel Mesa
+// integrated GPU, ~8000 random trials each plus explicit edge cases): no
+// compiler-reassociation-style corruption of the kind that broke twoSum/
+// twoProd (see add.wgsl's/multiply.wgsl's own headers) — every failure
+// found was a genuine algorithm gap (the a.hi==0.0 case below), not a
+// driver miscompile, and is fixed. Observed forward-error factor against a
+// true f64 reference stayed ~3-4x eps on both backends across every random
+// trial — noticeably tighter than ddDivProtected's own low-power spread
+// (see divide.wgsl's header), since sqrt has no denominator to be unlucky
+// about.
+//
+// One Newton refinement step (the classic extended-precision sqrt trick):
+// x0 = sqrt(a.hi) is a plain f32 approximation; the residual a - x0^2,
+// computed in DD arithmetic, recovers what x0 lost, and linearizing sqrt
+// around x0 (dividing that residual by 2*x0) gives a correction term
+// roughly doubling the precision — same ~48-bit target as ddDivProtected,
+// so one step is enough.
+//
+// Undefined for a.hi < 0.0, same as plain sqrt() — callers must guard
+// themselves; this never checks.
+//
+// a.hi == 0.0 (a genuinely zero input, not an underflowed one — zero is
+// exactly representable in f32, unlike this scheme's real range limits;
+// see splitDoubleDouble's own doc comment) makes x0 = sqrt(0) = 0, and the
+// correction step would divide by 2*x0 = 0. Substituting a safe non-zero
+// denominator via select() — rather than branching/returning early — keeps
+// every thread calling ddSubProtected/ddAddProtected unconditionally, which
+// their internal workgroupBarrier() requires; only the final result is
+// selected between the computed value and the exact DD(0,0) answer.
+fn ddSqrtProtected(a: DD, threadSlot: u32) -> DD {
+  let isZero = a.hi == 0.0;
+  let x0 = sqrt(a.hi);
+  let x0sq = twoProdBit(x0, x0);
+  let r = ddSubProtected(a, x0sq, threadSlot);
+  let safeDenom = select(2.0 * x0, 1.0, isZero);
+  let correction = r.hi / safeDenom;
+  let result = ddAddProtected(DD(x0, 0.0), DD(correction, 0.0), threadSlot);
+  return DD(select(result.hi, 0.0, isZero), select(result.lo, 0.0, isZero));
+}