pond-ts 0.24.0 → 0.26.0

package/dist/reducers/percentile.d.ts

@@ -5,36 +5,20 @@ export declare function parsePercentile(op: string): number | undefined;
 /**
  * Shared `reduceColumn` body for percentile-shaped reducers
  * (`median`, `p50`, `p95`, etc.). Walks the validity bitmap to
- * gather defined cells into a dense `Float64Array`, detecting NaN
- * cells in the same pass; sorts and reads the percentile from
- * the sorted view.
+ * gather defined **and finite** cells into a dense `Float64Array`,
+ * sorts with the typed-array intrinsic, and reads the percentile
+ * from the sorted view.
  *
- * **NaN parity with row API.** Two sort behaviors diverge:
+ * Non-finite cells (`NaN` / `±Infinity`) are excluded by the
+ * reducer non-finite policy (docs/notes/reducer-nan-policy.md) —
+ * uniformly, across every path. With non-finite filtered out
+ * before the sort, `Float64Array.prototype.sort()` (the numeric,
+ * NaN-free intrinsic, ~2× faster than `Array.sort` with a
+ * comparator) produces the same total order as the row path's
+ * `Array.sort((a, b) => a - b)` over the same finite values — so
+ * there is no longer any NaN-ordering seam to special-case.
  *
- * - `Array.prototype.sort((a, b) => a - b)` (row-API path) returns
- *   NaN from the comparator on NaN inputs; V8 treats this as
- *   "equal" and leaves NaN cells in undefined order — the
- *   resulting percentile is whatever cell happens to land at the
- *   computed rank, possibly NaN itself.
- * - `Float64Array.prototype.sort()` (typed-array intrinsic) puts
- *   NaN deterministically at the end of the sorted view — the
- *   percentile rank then reads a non-NaN cell unless the rank
- *   lands in the NaN suffix.
- *
- * The first-pass NaN detection lets us use `Float64Array.sort`'s
- * 2× speedup for the common no-NaN case (full parity with row API
- * because both produce identical sorted orders when no NaN
- * present), and fall back to `Array.sort` with comparator only
- * when NaN is present (preserving bug-for-bug row-API parity on
- * the rare contract-violating input).
- *
- * NaN can only reach a `kind: 'number'` column via trusted
- * construction (`fromEvents`); the public `assertCellKind`
- * rejects it at intake. Closed Codex review finding on PR #153 —
- * earlier L2 fix that filtered NaN was correct in spirit but
- * introduced a *different* divergence from the row API. A
- * principled "filter NaN consistently across both paths" fix is
- * tracked in the followup issue.
+ * Empty (no defined+finite values) → `undefined`.
  */
 export declare function reducePercentileColumn(col: Float64Column, q: number): number | undefined;
 export declare function percentileReducer(q: number): ReducerDef;

package/dist/reducers/percentile.js

@@ -18,51 +18,64 @@ export function parsePercentile(op) {
 /**
  * Shared `reduceColumn` body for percentile-shaped reducers
  * (`median`, `p50`, `p95`, etc.). Walks the validity bitmap to
- * gather defined cells into a dense `Float64Array`, detecting NaN
- * cells in the same pass; sorts and reads the percentile from
- * the sorted view.
+ * gather defined **and finite** cells into a dense `Float64Array`,
+ * sorts with the typed-array intrinsic, and reads the percentile
+ * from the sorted view.
  *
- * **NaN parity with row API.** Two sort behaviors diverge:
+ * Non-finite cells (`NaN` / `±Infinity`) are excluded by the
+ * reducer non-finite policy (docs/notes/reducer-nan-policy.md) —
+ * uniformly, across every path. With non-finite filtered out
+ * before the sort, `Float64Array.prototype.sort()` (the numeric,
+ * NaN-free intrinsic, ~2× faster than `Array.sort` with a
+ * comparator) produces the same total order as the row path's
+ * `Array.sort((a, b) => a - b)` over the same finite values — so
+ * there is no longer any NaN-ordering seam to special-case.
  *
- * - `Array.prototype.sort((a, b) => a - b)` (row-API path) returns
- *   NaN from the comparator on NaN inputs; V8 treats this as
- *   "equal" and leaves NaN cells in undefined order — the
- *   resulting percentile is whatever cell happens to land at the
- *   computed rank, possibly NaN itself.
- * - `Float64Array.prototype.sort()` (typed-array intrinsic) puts
- *   NaN deterministically at the end of the sorted view — the
- *   percentile rank then reads a non-NaN cell unless the rank
- *   lands in the NaN suffix.
- *
- * The first-pass NaN detection lets us use `Float64Array.sort`'s
- * 2× speedup for the common no-NaN case (full parity with row API
- * because both produce identical sorted orders when no NaN
- * present), and fall back to `Array.sort` with comparator only
- * when NaN is present (preserving bug-for-bug row-API parity on
- * the rare contract-violating input).
- *
- * NaN can only reach a `kind: 'number'` column via trusted
- * construction (`fromEvents`); the public `assertCellKind`
- * rejects it at intake. Closed Codex review finding on PR #153 —
- * earlier L2 fix that filtered NaN was correct in spirit but
- * introduced a *different* divergence from the row API. A
- * principled "filter NaN consistently across both paths" fix is
- * tracked in the followup issue.
+ * Empty (no defined+finite values) → `undefined`.
  */
 export function reducePercentileColumn(col, q) {
     const validity = col.validity;
     const values = col._values;
     let dense;
     let denseLength = 0;
-    let hasNaN = false;
-    if (validity === undefined) {
+    // Fast path: every defined cell is finite (`Float64Column.allFinite`),
+    // so we gather defined cells with no per-element `Number.isFinite`
+    // filter (reducer non-finite policy, docs/notes/reducer-nan-policy.md).
+    // The subsequent `Float64Array.sort` is the same NaN-free intrinsic
+    // either way → identical order, identical percentile.
+    if (col.allFinite) {
+        if (validity === undefined) {
+            if (col.length === 0)
+                return undefined;
+            dense = new Float64Array(col.length);
+            for (let i = 0; i < col.length; i += 1) {
+                dense[denseLength] = values[i];
+                denseLength += 1;
+            }
+        }
+        else {
+            const definedCount = validity.definedCount;
+            if (definedCount === 0)
+                return undefined;
+            dense = new Float64Array(definedCount);
+            const bits = validity.bits;
+            for (let i = 0; i < col.length; i += 1) {
+                if ((bits[i >> 3] & (1 << (i & 7))) === 0)
+                    continue;
+                dense[denseLength] = values[i];
+                denseLength += 1;
+            }
+        }
+    }
+    else if (validity === undefined) {
+        // Guarded path: filter non-finite before the sort.
         if (col.length === 0)
             return undefined;
         dense = new Float64Array(col.length);
         for (let i = 0; i < col.length; i += 1) {
             const v = values[i];
-            if (Number.isNaN(v))
-                hasNaN = true;
+            if (!Number.isFinite(v))
+                continue;
             dense[denseLength] = v;
             denseLength += 1;
         }
@@ -77,23 +90,16 @@ export function reducePercentileColumn(col, q) {
             if ((bits[i >> 3] & (1 << (i & 7))) === 0)
                 continue;
             const v = values[i];
-            if (Number.isNaN(v))
-                hasNaN = true;
+            if (!Number.isFinite(v))
+                continue;
             dense[denseLength] = v;
             denseLength += 1;
         }
     }
     if (denseLength === 0)
         return undefined;
-    if (hasNaN) {
-        // Match row-API exactly via `Array.sort` with comparator —
-        // diverges from `Float64Array.sort` on NaN ordering.
-        const arr = Array.from(dense.subarray(0, denseLength));
-        arr.sort((a, b) => a - b);
-        return percentileOfSorted(arr, q);
-    }
-    // No NaN: `Float64Array.sort` is parity-correct (same total
-    // order as `Array.sort` with comparator) and ~2× faster.
+    // Non-finite excluded upstream by policy → `Float64Array.sort` (numeric,
+    // NaN-free) gives the same order as the row path's comparator sort.
     const view = dense.subarray(0, denseLength);
     view.sort();
     const rank = (q / 100) * (denseLength - 1);

package/dist/reducers/stdev.js

@@ -20,9 +20,12 @@
  * agree to floating-point noise — bit-for-bit when they see the values in the
  * same order, which the bucketed paths do.
  *
- * `rollingState` is the exception: its sliding window needs `remove`, which
- * Welford can't do stably (windowed removal drifts), so it keeps the one-pass
- * `sq/n − mean²` (with its clamp). A stable rolling stdev is a deferred item.
+ * `rollingState` adds `remove` for its sliding window via Welford's
+ * **order-independent delete** (the reverse recurrence). It works in the same
+ * deviation space — no `sq/n − mean²` cancellation, no shift drift — and
+ * removes *by value*, so it stays correct under the live layer's reorder-mode
+ * eviction (which removes the sorted-prefix, not the oldest-arrived event).
+ * See its inline note for the mechanics.
  */
 // Welford accumulator: `add` each value, then read `result()` for the
 // population stdev (`undefined` if nothing was added). Shared by `reduce` and
@@ -54,20 +57,55 @@ export const stdev = {
     },
     reduceColumn(col) {
         // Inlined Welford (mirrors `welfordStdev`) over the packed array — a single
-        // pass, skipping gaps via the validity bitmask. One division per element vs
-        // the old two-pass's two divisionless scans, but identical to the other
-        // paths' recurrence so the fast path and row path cannot diverge.
+        // pass, skipping gaps via the validity bitmask and skipping non-finite
+        // cells (reducer non-finite policy, docs/notes/reducer-nan-policy.md). One
+        // division per finite element vs the old two-pass's two divisionless scans,
+        // but identical to the other paths' recurrence so the fast path and row
+        // path cannot diverge. `n` counts only finite contributors, so `n === 0`
+        // (no finite cells) → `undefined`.
         const values = col._values;
         const validity = col.validity;
         let n = 0;
         let mean = 0;
         let m2 = 0;
+        // Fast path: every defined cell is finite (`Float64Column.allFinite`),
+        // so the Welford recurrence runs over every defined cell with no
+        // per-element `Number.isFinite` guard (reducer non-finite policy,
+        // docs/notes/reducer-nan-policy.md). Same recurrence as the guarded
+        // path → identical result.
+        if (col.allFinite) {
+            const len = col.length;
+            if (validity === undefined) {
+                for (let i = 0; i < len; i += 1) {
+                    const v = values[i];
+                    n += 1;
+                    const delta = v - mean;
+                    mean += delta / n;
+                    m2 += delta * (v - mean);
+                }
+            }
+            else {
+                const bits = validity.bits;
+                for (let i = 0; i < len; i += 1) {
+                    if ((bits[i >> 3] & (1 << (i & 7))) === 0)
+                        continue;
+                    const v = values[i];
+                    n += 1;
+                    const delta = v - mean;
+                    mean += delta / n;
+                    m2 += delta * (v - mean);
+                }
+            }
+            return n === 0 ? undefined : Math.sqrt(Math.max(0, m2 / n));
+        }
+        // Guarded path: skip non-finite cells; `n` counts only finite
+        // contributors so `n === 0` (no finite cells) → `undefined`.
         if (validity === undefined) {
             const len = col.length;
-            if (len === 0)
-                return undefined;
             for (let i = 0; i < len; i += 1) {
                 const v = values[i];
+                if (!Number.isFinite(v))
+                    continue;
                 n += 1;
                 const delta = v - mean;
                 mean += delta / n;
@@ -75,21 +113,21 @@ export const stdev = {
             }
         }
         else {
-            if (validity.definedCount === 0)
-                return undefined;
             const bits = validity.bits;
             const len = col.length;
             for (let i = 0; i < len; i += 1) {
                 if ((bits[i >> 3] & (1 << (i & 7))) === 0)
                     continue;
                 const v = values[i];
+                if (!Number.isFinite(v))
+                    continue;
                 n += 1;
                 const delta = v - mean;
                 mean += delta / n;
                 m2 += delta * (v - mean);
             }
         }
-        return Math.sqrt(Math.max(0, m2 / n));
+        return n === 0 ? undefined : Math.sqrt(Math.max(0, m2 / n));
     },
     bucketState() {
         const w = welfordStdev();
@@ -104,32 +142,80 @@ export const stdev = {
         };
     },
     rollingState() {
-        // One-pass `sq/n − mean²` with the `Math.max(0, …)` clamp. Unlike the other
-        // paths this has a `remove` for the sliding window, which Welford can't do
-        // stably — a stable rolling stdev is deferred (see the module note above).
-        let s = 0;
-        let sq = 0;
+        // Sliding-window population stdev: Welford's online variance with an
+        // **order-independent delete**. `add(v)` is the standard recurrence;
+        // `remove(v)` reverses it. Working in deviation space, it fixes the old
+        // one-pass `sq/n − mean²` failure modes — catastrophic cancellation on
+        // near-equal large values (`[1e10, 1e10+1, …]` → 0 or a negative variance →
+        // NaN; the audit-§1.1 case, previously still live on this path) and shift
+        // drift on trending data (cumulative distance, elevation).
+        //
+        // Limitation — **outlier eviction** (shared with the old one-pass, so not a
+        // regression): like any *subtractive* sliding variance, evicting a value far
+        // outside the residual spread cancels — `m2 −= huge` loses the small
+        // remainder, and the error persists in the running accumulator (the clamp
+        // floors it to 0 at the extreme). Negligible until the evicted point is
+        // ~1e7–1e8× the residual stdev (e.g. a ~1e6 spike over a ~0.01-σ baseline) —
+        // far beyond realistic monitoring / activity data, and exactly why the
+        // add-only paths use plain Welford. A FIFO-only two-stack merge avoids it
+        // entirely but can't serve the by-value eviction the live layer needs.
+        //
+        // Removal is **by value, not by position**. That is load-bearing for the
+        // live layer: `LiveReduce` shares this state, and a `reorder`-mode source
+        // with retention evicts the sorted-prefix — which may be a later arrival,
+        // not the oldest — so a positional (FIFO) remove would corrupt the window.
+        // A value-based delete is correct regardless of eviction order (the
+        // documented contract; see live-reduce.ts and live-buffer-as-window.test).
+        // The batch rolling driver removes strictly oldest-first — a special case.
+        //
+        // Non-finite / missing cells arrive as `undefined` (the factory wrapper
+        // applies the non-finite policy); `add`/`remove` both skip them symmetrically
+        // so they never enter `n`.
         let n = 0;
+        let mean = 0;
+        let m2 = 0;
         return {
             add(_i, v) {
-                if (typeof v === 'number') {
-                    s += v;
-                    sq += v * v;
-                    n++;
-                }
+                if (typeof v !== 'number')
+                    return;
+                n += 1;
+                const delta = v - mean;
+                mean += delta / n;
+                m2 += delta * (v - mean);
             },
             remove(_i, v) {
-                if (typeof v === 'number') {
-                    s -= v;
-                    sq -= v * v;
-                    n--;
+                if (typeof v !== 'number')
+                    return;
+                if (n <= 1) {
+                    // Removing the final contributor — reset exactly (no 0/0, no drift).
+                    n = 0;
+                    mean = 0;
+                    m2 = 0;
+                    return;
+                }
+                const meanWith = mean;
+                n -= 1;
+                if (n === 1) {
+                    // A single remaining element has population variance *exactly* 0.
+                    // Setting it directly (rather than via the subtraction below) keeps
+                    // the n→1 result exact at any magnitude — the reverse step alone
+                    // leaves rounding residue at large offsets (e.g. ~0.016 on 1e10).
+                    mean = meanWith * 2 - v; // the one survivor: 2·mean₂ − removed
+                    m2 = 0;
+                    return;
                 }
+                // Deviation-space mean update (mean − (v − mean)/n): avoids the large
+                // `n·mean − v` product, staying precise at large magnitudes.
+                mean = meanWith - (v - meanWith) / n;
+                // Reverse Welford: M2 −= (v − meanNew)·(v − meanOld).
+                m2 -= (v - mean) * (v - meanWith);
+                // Normally absorbs FP round-off; on a gross outlier eviction (see the
+                // limitation note above) the subtraction can cancel below 0 — clamp.
+                if (m2 < 0)
+                    m2 = 0;
             },
             snapshot() {
-                if (n === 0)
-                    return undefined;
-                const mean = s / n;
-                return Math.sqrt(Math.max(0, sq / n - mean * mean));
+                return n === 0 ? undefined : Math.sqrt(Math.max(0, m2 / n));
             },
         };
     },

package/dist/reducers/sum.js

@@ -7,18 +7,42 @@ export const sum = {
         const values = col._values;
         const validity = col.validity;
         let s = 0;
+        // Fast path: the column proved every defined cell is finite
+        // (`Float64Column.allFinite`), so we can drop the per-element
+        // `Number.isFinite` guard the reducer non-finite policy
+        // (docs/notes/reducer-nan-policy.md) otherwise requires — plain
+        // accumulate, identical result.
+        if (col.allFinite) {
+            if (validity === undefined) {
+                for (let i = 0; i < col.length; i += 1)
+                    s += values[i];
+                return s;
+            }
+            const bits = validity.bits;
+            for (let i = 0; i < col.length; i += 1) {
+                if ((bits[i >> 3] & (1 << (i & 7))) !== 0)
+                    s += values[i];
+            }
+            return s;
+        }
+        // Guarded path: finiteness not proven, skip non-finite per policy.
         if (validity === undefined) {
-            // Hot path: every cell defined; no per-row branch.
-            for (let i = 0; i < col.length; i += 1)
-                s += values[i];
+            for (let i = 0; i < col.length; i += 1) {
+                const v = values[i];
+                if (Number.isFinite(v))
+                    s += v;
+            }
             return s;
         }
         // Inline bitmap check rather than method dispatch — same pattern
         // the chart-friction-spike notes flagged for hot draw loops.
         const bits = validity.bits;
         for (let i = 0; i < col.length; i += 1) {
-            if ((bits[i >> 3] & (1 << (i & 7))) !== 0)
-                s += values[i];
+            if ((bits[i >> 3] & (1 << (i & 7))) !== 0) {
+                const v = values[i];
+                if (Number.isFinite(v))
+                    s += v;
+            }
         }
         return s;
     },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pond-ts",
-  "version": "0.24.0",
+  "version": "0.26.0",
   "description": "TypeScript-first time series primitives",
   "license": "MIT",
   "repository": {