pond-ts 0.24.0 → 0.25.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -7,7 +7,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
7
7
  file covers both packages. Pre-1.0: minor bumps may include new features and
8
8
  type-level changes; patch bumps are strictly additive.
9
9
 
10
- [Unreleased]: https://github.com/pjm17971/pond-ts/compare/v0.24.0...HEAD
10
+ [Unreleased]: https://github.com/pjm17971/pond-ts/compare/v0.25.0...HEAD
11
+ [0.25.0]: https://github.com/pjm17971/pond-ts/compare/v0.24.0...v0.25.0
11
12
  [0.24.0]: https://github.com/pjm17971/pond-ts/compare/v0.23.0...v0.24.0
12
13
  [0.23.0]: https://github.com/pjm17971/pond-ts/compare/v0.22.0...v0.23.0
13
14
  [0.22.0]: https://github.com/pjm17971/pond-ts/compare/v0.21.0...v0.22.0
@@ -18,6 +19,52 @@ type-level changes; patch bumps are strictly additive.
18
19
 
19
20
  ## [Unreleased]
20
21
 
22
+ ## [0.25.0] — 2026-06-15
23
+
24
+ ### Changed
25
+
26
+ - **Reducers now treat non-finite numerics (`NaN` / `±Infinity`) as missing —
27
+ they are skipped — uniformly across every built-in reducer and all four
28
+ execution paths (`reduce`, the columnar fast path, `aggregate`/bucket, and
29
+ `rolling`/live).** Previously the paths disagreed on non-finite input: e.g.
30
+ `min`/`max` returned a position-dependent wrong extreme on the batch/columnar
31
+ paths but the true extreme on aggregate/rolling; `sum`/`avg` propagated
32
+ `NaN`. Non-finite can't enter via the row API (intake rejects it) — it only
33
+ arises inside computed columns (`cumulative` overflow, `diff`/`rate`
34
+ overflow, `collapse`, trusted construction) — so this only changes results
35
+ for those degenerate values, and makes every path agree. The three-layer
36
+ contract: **intake** stays strict (rejects non-finite), **computed writers**
37
+ stay permissive (pack honest non-finite), **reducers** are robust (skip it).
38
+ A standing parity-matrix test now pins all paths together. See
39
+ `docs/notes/reducer-nan-policy.md`. This also resolves the `aggregate('stdev')`
40
+ divergence class and the `min`/`max` NaN-laundering bug.
41
+ - Internal: `Float64Column` gained an `allFinite` fast-path flag (data-derived
42
+ at construction, conservative-by-default) so reducers skip the per-element
43
+ finite check on provably-finite columns — keeping the policy's cost off the
44
+ hot path (min/max/count stay at their pre-policy speed).
45
+
46
+ ### Fixed
47
+
48
+ - **`rolling(window, { x: 'stdev' })` is now numerically stable.** It was the
49
+ last batch stdev path still on the one-pass `Σx²/n − mean²`, which cancels
50
+ catastrophically on near-equal large values (`[1e10, 1e10+1, …]` → `0`
51
+ instead of ≈1.118, or a negative variance → `NaN`) and drifts on trending
52
+ data (cumulative distance, elevation). It now uses Welford's online variance
53
+ with an order-independent **delete** — deviation-space, so no cancellation,
54
+ and removal **by value**, which keeps it correct under the live layer's
55
+ `reorder`-mode eviction (a positional/FIFO remove would have broken it; the
56
+ documented "stdev is reorder-safe" contract is preserved). Rolling-stdev
57
+ values shift in the last ULPs (now correct); the path stays O(1) and within
58
+ run-noise of the old one-pass, and a single-element window now reports exactly
59
+ `0` at any magnitude. Like any subtractive sliding variance, evicting an
60
+ outlier far outside the residual spread loses precision — negligible until the
61
+ evicted point is ~1e7–1e8× the residual stdev, far beyond realistic data.
62
+ - A standing differential-fuzz parity suite now pins every built-in reducer's
63
+ execution paths (columnar fast path vs `bucket` vs `rolling`, and the FIFO
64
+ sliding window vs a from-scratch recompute) against silent drift across
65
+ randomized magnitudes and window sizes — the class of bug behind the stdev
66
+ and `min`/`max` divergences.
67
+
21
68
  ## [0.24.0] — 2026-06-14
22
69
 
23
70
  ### Changed
@@ -123,26 +123,34 @@ export function tryAggregateColumnarTimeKeyed(begins, getColumn, buckets, column
123
123
  reduced[p] = plan.reduce(plan.column.sliceByRange(start, scan));
124
124
  }
125
125
  else if (plan.which === 'first') {
126
- // First defined cell in [start, scan); scans past missing cells.
126
+ // First defined cell in [start, scan); scans past missing cells and
127
+ // past non-finite numeric cells (reducer non-finite policy,
128
+ // docs/notes/reducer-nan-policy.md — a NaN/±Inf numeric is "not a
129
+ // contributor", matching the row path's `defined` filter).
127
130
  let value;
128
131
  for (let i = start; i < scan; i += 1) {
129
132
  const cell = plan.column.read(i);
130
- if (cell !== undefined) {
131
- value = cell;
132
- break;
133
- }
133
+ if (cell === undefined)
134
+ continue;
135
+ if (typeof cell === 'number' && !Number.isFinite(cell))
136
+ continue;
137
+ value = cell;
138
+ break;
134
139
  }
135
140
  reduced[p] = value;
136
141
  }
137
142
  else {
138
- // Last defined cell in [start, scan); scans backward past missing.
143
+ // Last defined cell in [start, scan); scans backward past missing and
144
+ // past non-finite numeric cells (see the 'first' branch above).
139
145
  let value;
140
146
  for (let i = scan - 1; i >= start; i -= 1) {
141
147
  const cell = plan.column.read(i);
142
- if (cell !== undefined) {
143
- value = cell;
144
- break;
145
- }
148
+ if (cell === undefined)
149
+ continue;
150
+ if (typeof cell === 'number' && !Number.isFinite(cell))
151
+ continue;
152
+ value = cell;
153
+ break;
146
154
  }
147
155
  reduced[p] = value;
148
156
  }
@@ -15,7 +15,7 @@ import { Sequence } from '../sequence/sequence.js';
15
15
  import { IntervalKeyColumn, TimeKeyColumn, TimeRangeKeyColumn, float64ColumnFromArray, stringColumnFromArray, withColumnsRenamed, withColumnsSelected, withKeyColumn, withRowRange, withRowSelection, } from '../columnar/index.js';
16
16
  import { SeriesStore } from '../live/series-store.js';
17
17
  import { parseDuration } from '../core/duration.js';
18
- import { resolveReducer, } from '../reducers/index.js';
18
+ import { resolveReducer, bucketStateFor, rollingStateFor, } from '../reducers/index.js';
19
19
  // JSON ↔ typed-row primitives live in `./json.js`. Both `TimeSeries`
20
20
  // and `LiveSeries` reach for them; extracted to break the import cycle
21
21
  // that would otherwise form (Event needs them, TimeSeries imports
@@ -274,7 +274,11 @@ function bucketOverlapsHalfOpen(bucket, event) {
274
274
  return event.begin() < bucket.end() && bucket.begin() < event.end();
275
275
  }
276
276
  function aggregateValues(operation, values) {
277
- const defined = values.filter((value) => value !== undefined);
277
+ // Non-finite numerics (NaN / ±Inf) are treated as missing — excluded from
278
+ // both `defined` and `numeric` so every built-in reducer skips them exactly
279
+ // as it skips a missing cell. See docs/notes/reducer-nan-policy.md.
280
+ const defined = values.filter((value) => value !== undefined &&
281
+ (typeof value !== 'number' || Number.isFinite(value)));
278
282
  const numeric = defined.filter((value) => typeof value === 'number');
279
283
  return resolveReducer(operation).reduce(defined, numeric);
280
284
  }
@@ -315,7 +319,9 @@ function applyAggregateReducer(reducer, values) {
315
319
  // LivePartitionedSyncRolling) share the same normalisation. See the
316
320
  // import below.
317
321
  function createAggregateBucketState(operation) {
318
- return resolveReducer(operation).bucketState();
322
+ // Delegates to the shared factory so the non-finite skip policy
323
+ // (docs/notes/reducer-nan-policy.md) applies uniformly to batch + live.
324
+ return bucketStateFor(operation);
319
325
  }
320
326
  /**
321
327
  * Resolve the output column kind for an `arrayAggregate` call. Numeric
@@ -336,7 +342,9 @@ function resolveArrayAggregateKind(reducer, explicitKind) {
336
342
  return 'string';
337
343
  }
338
344
  function createRollingReducerState(operation) {
339
- return resolveReducer(operation).rollingState();
345
+ // Delegates to the shared factory so the non-finite skip policy
346
+ // (docs/notes/reducer-nan-policy.md) applies uniformly to batch + live.
347
+ return rollingStateFor(operation);
340
348
  }
341
349
  function duplicateValueColumnNames(schemas) {
342
350
  const counts = new Map();
@@ -416,7 +416,13 @@ export function validateAndNormalizeColumnar(input) {
416
416
  let column;
417
417
  switch (kind) {
418
418
  case 'number':
419
- column = new Float64Column(numberBufs[c], length, validity);
419
+ // Strict intake: `assertCellKind` (kind 'number') already
420
+ // rejected every non-finite cell with a `ValidationError`
421
+ // before it reached `numberBufs`, so a surviving column is
422
+ // provably all-finite → `allFinite: true` (lets reducers skip
423
+ // the per-element finite guard). See the reducer non-finite
424
+ // policy + `Float64Column.allFinite`'s safety contract.
425
+ column = new Float64Column(numberBufs[c], length, validity, true);
420
426
  break;
421
427
  case 'boolean':
422
428
  column = new BooleanColumn(booleanBufs[c], length, validity);
package/dist/column.js CHANGED
@@ -76,33 +76,60 @@ Float64Column.prototype.minMax = function () {
76
76
  const n = this.length;
77
77
  if (n === 0)
78
78
  return undefined;
79
- let i = 0;
80
- let lo;
81
- let hi;
82
- if (!v) {
83
- lo = values[0];
79
+ if (this.allFinite) {
80
+ // Provably finite (the chart axis hot path) → no per-element finite
81
+ // guard; the seeded comparison is unambiguous without NaN.
82
+ let i = 0;
83
+ let lo;
84
+ let hi;
85
+ if (!v) {
86
+ lo = values[0];
87
+ hi = lo;
88
+ for (i = 1; i < n; i += 1) {
89
+ const x = values[i];
90
+ lo = lo <= x ? lo : x;
91
+ hi = hi >= x ? hi : x;
92
+ }
93
+ return [lo, hi];
94
+ }
95
+ while (i < n && !v.isDefined(i))
96
+ i += 1;
97
+ if (i >= n)
98
+ return undefined;
99
+ lo = values[i];
84
100
  hi = lo;
85
- for (i = 1; i < n; i += 1) {
101
+ for (i += 1; i < n; i += 1) {
102
+ if (!v.isDefined(i))
103
+ continue;
86
104
  const x = values[i];
87
105
  lo = lo <= x ? lo : x;
88
106
  hi = hi >= x ? hi : x;
89
107
  }
90
108
  return [lo, hi];
91
109
  }
92
- while (i < n && !v.isDefined(i))
93
- i += 1;
94
- if (i >= n)
95
- return undefined;
96
- lo = values[i];
97
- hi = lo;
98
- for (i += 1; i < n; i += 1) {
99
- if (!v.isDefined(i))
110
+ // Guarded: skip non-finite (and missing) cells, matching the reducer
111
+ // `min`/`max` (reducer non-finite policy — docs/notes/reducer-nan-policy.md).
112
+ // All-non-finite (or empty) → undefined.
113
+ let lo;
114
+ let hi;
115
+ for (let i = 0; i < n; i += 1) {
116
+ if (v && !v.isDefined(i))
100
117
  continue;
101
118
  const x = values[i];
102
- lo = lo <= x ? lo : x;
103
- hi = hi >= x ? hi : x;
119
+ if (!Number.isFinite(x))
120
+ continue;
121
+ if (lo === undefined) {
122
+ lo = x;
123
+ hi = x;
124
+ }
125
+ else {
126
+ if (x < lo)
127
+ lo = x;
128
+ if (x > hi)
129
+ hi = x;
130
+ }
104
131
  }
105
- return [lo, hi];
132
+ return lo === undefined ? undefined : [lo, hi];
106
133
  };
107
134
  Float64Column.prototype.hasMissing = function () {
108
135
  if (!this.validity)
@@ -228,6 +255,44 @@ Float64Column.prototype.bin = function (bins, reducer) {
228
255
  // underlying buffer.
229
256
  const values = this._values;
230
257
  const validity = this.validity;
258
+ if (!this.allFinite) {
259
+ // Guarded path: skip non-finite values (reducer non-finite policy —
260
+ // docs/notes/reducer-nan-policy.md), matching `minMax()` / the reducer
261
+ // min/max. A bin with no finite value (all-missing or all-non-finite)
262
+ // keeps the NaN empty-bin sentinel.
263
+ for (let b = 0; b < bins; b += 1) {
264
+ const start = Math.floor((b * n) / bins);
265
+ const end = Math.floor(((b + 1) * n) / bins);
266
+ let loVal;
267
+ let hiVal;
268
+ for (let i = start; i < end; i += 1) {
269
+ if (validity !== undefined && !validity.isDefined(i))
270
+ continue;
271
+ const x = values[i];
272
+ if (!Number.isFinite(x))
273
+ continue;
274
+ if (loVal === undefined) {
275
+ loVal = x;
276
+ hiVal = x;
277
+ }
278
+ else {
279
+ if (x < loVal)
280
+ loVal = x;
281
+ if (x > hiVal)
282
+ hiVal = x;
283
+ }
284
+ }
285
+ if (loVal === undefined) {
286
+ lo[b] = NaN;
287
+ hi[b] = NaN;
288
+ }
289
+ else {
290
+ lo[b] = loVal;
291
+ hi[b] = hiVal;
292
+ }
293
+ }
294
+ return { lo, hi };
295
+ }
231
296
  if (validity === undefined) {
232
297
  for (let b = 0; b < bins; b += 1) {
233
298
  const start = Math.floor((b * n) / bins);
@@ -568,10 +633,15 @@ ChunkedFloat64Column.prototype.percentile = function (q) {
568
633
  return materializeChunkedFloat64(this).percentile(q);
569
634
  };
570
635
  ChunkedFloat64Column.prototype.count = function () {
571
- // Validity-defined-count is available without materializing.
572
- if (!this.validity)
573
- return this.length;
574
- return this.validity.definedCount;
636
+ // The O(1) defined-count is correct only when the column is provably
637
+ // finite. Non-finite cells are skipped (reducer non-finite policy —
638
+ // docs/notes/reducer-nan-policy.md), so when finiteness isn't proven,
639
+ // delegate to the materialized packed `count()` (which routes through the
640
+ // count reducer and skips non-finite) — matching packed `Float64Column`.
641
+ if (this.allFinite) {
642
+ return this.validity ? this.validity.definedCount : this.length;
643
+ }
644
+ return materializeChunkedFloat64(this).count();
575
645
  };
576
646
  ChunkedFloat64Column.prototype.minMax = function () {
577
647
  return materializeChunkedFloat64(this).minMax();
@@ -72,6 +72,16 @@ export declare class ChunkedFloat64Column {
72
72
  readonly chunks: ReadonlyArray<Float64Column>;
73
73
  readonly chunkOffsets: Int32Array;
74
74
  readonly validity?: ValidityBitmap;
75
+ /**
76
+ * `true` iff **every** chunk is itself `allFinite` — the merged
77
+ * column is all-finite IFF each input is (an AND). Defaults to
78
+ * `false` whenever any chunk's flag is `false` (or absent), which is
79
+ * the safe direction: a chunk that didn't prove finiteness keeps the
80
+ * whole column on the guarded reducer path. Mirrors
81
+ * `Float64Column.allFinite`'s safety contract — see it for why a
82
+ * wrong `true` is unsafe.
83
+ */
84
+ readonly allFinite: boolean;
75
85
  constructor(chunks: ReadonlyArray<Float64Column>);
76
86
  /**
77
87
  * Reads cell `i` by binary-searching `chunkOffsets` to find the
@@ -192,6 +192,16 @@ export class ChunkedFloat64Column {
192
192
  chunks;
193
193
  chunkOffsets;
194
194
  validity;
195
+ /**
196
+ * `true` iff **every** chunk is itself `allFinite` — the merged
197
+ * column is all-finite IFF each input is (an AND). Defaults to
198
+ * `false` whenever any chunk's flag is `false` (or absent), which is
199
+ * the safe direction: a chunk that didn't prove finiteness keeps the
200
+ * whole column on the guarded reducer path. Mirrors
201
+ * `Float64Column.allFinite`'s safety contract — see it for why a
202
+ * wrong `true` is unsafe.
203
+ */
204
+ allFinite;
195
205
  constructor(chunks) {
196
206
  assertChunkKinds(chunks, 'number', 'ChunkedFloat64Column');
197
207
  const { length, chunkOffsets, validity } = buildOffsetsAndAggregateValidity(chunks, 'ChunkedFloat64Column');
@@ -202,6 +212,16 @@ export class ChunkedFloat64Column {
202
212
  this.chunkOffsets = chunkOffsets;
203
213
  if (validity !== undefined)
204
214
  this.validity = validity;
215
+ // AND across chunks: all-finite IFF every chunk is. An empty chunk
216
+ // list is vacuously all-finite (no non-finite cell exists).
217
+ let allFinite = true;
218
+ for (let c = 0; c < chunks.length; c += 1) {
219
+ if (!chunks[c].allFinite) {
220
+ allFinite = false;
221
+ break;
222
+ }
223
+ }
224
+ this.allFinite = allFinite;
205
225
  }
206
226
  /**
207
227
  * Reads cell `i` by binary-searching `chunkOffsets` to find the
@@ -287,10 +307,13 @@ export class ChunkedFloat64Column {
287
307
  out[i] = this.chunks[c]._values[local];
288
308
  validBits[i >> 3] |= 1 << (i & 7);
289
309
  }
310
+ // A gather only reads existing defined cells (out-of-range / invalid
311
+ // slots are skipped → marked invalid below), so the result is finite
312
+ // whenever this chunked column is. Propagate the AND-of-chunks flag.
290
313
  if (!hasInvalid) {
291
- return new Float64Column(out, indices.length);
314
+ return new Float64Column(out, indices.length, undefined, this.allFinite);
292
315
  }
293
- return new Float64Column(out, indices.length, validityFromBits(validBits, indices.length));
316
+ return new Float64Column(out, indices.length, validityFromBits(validBits, indices.length), this.allFinite);
294
317
  }
295
318
  }
296
319
  /* -------------------------------------------------------------------------- */
@@ -631,7 +654,10 @@ export function materializeChunkedFloat64(chunked) {
631
654
  out.set(chunk._values.subarray(0, chunk.length), cursor);
632
655
  cursor += chunk.length;
633
656
  }
634
- return new Float64Column(out, chunked.length, chunked.validity);
657
+ // Compacting doesn't touch values, so finiteness carries over from
658
+ // the chunked column (which is the AND of its chunks). This is what
659
+ // lets a `concatSorted(...).reduce(...)` take the reducer fast path.
660
+ return new Float64Column(out, chunked.length, chunked.validity, chunked.allFinite);
635
661
  }
636
662
  /**
637
663
  * Compacts a `ChunkedBooleanColumn` into a plain `BooleanColumn`.
@@ -128,7 +128,25 @@ export declare class Float64Column implements ColumnBase<number, 'number'> {
128
128
  readonly storage: "packed";
129
129
  readonly length: number;
130
130
  readonly validity?: ValidityBitmap;
131
- constructor(values: Float64Array, length: number, validity?: ValidityBitmap);
131
+ /**
132
+ * **Safety contract.** `true` MUST mean "every *defined* cell is
133
+ * finite — no `NaN`, no `±Infinity`" (missing cells, tracked by
134
+ * `validity`, are irrelevant). It is a promise the producer makes
135
+ * so reducers can skip the per-element `Number.isFinite` guard that
136
+ * the reducer non-finite policy (docs/notes/reducer-nan-policy.md)
137
+ * otherwise requires on every numeric `reduceColumn`.
138
+ *
139
+ * A wrongly-`true` flag makes `reduceColumn` take the unguarded fast
140
+ * path and silently include a non-finite cell → wrong result. So the
141
+ * **default is `false`** (conservative: guarded path, correct-but-
142
+ * slower) and a producer sets `true` ONLY where finiteness is proven
143
+ * — either data-derived by inspecting every cell, validated upstream
144
+ * (strict intake), or propagated from a source column that was
145
+ * itself `allFinite`. When unsure, leave it `false`: a missed `true`
146
+ * is slower, never wrong.
147
+ */
148
+ readonly allFinite: boolean;
149
+ constructor(values: Float64Array, length: number, validity?: ValidityBitmap, allFinite?: boolean);
132
150
  read(i: number): number | undefined;
133
151
  scan(fn: (value: number, i: number) => void, options?: ScanOptions): void;
134
152
  sliceByRange(start: number, end: number): Float64Column;
@@ -41,7 +41,25 @@ export class Float64Column {
41
41
  */
42
42
  _values;
43
43
  validity;
44
- constructor(values, length, validity) {
44
+ /**
45
+ * **Safety contract.** `true` MUST mean "every *defined* cell is
46
+ * finite — no `NaN`, no `±Infinity`" (missing cells, tracked by
47
+ * `validity`, are irrelevant). It is a promise the producer makes
48
+ * so reducers can skip the per-element `Number.isFinite` guard that
49
+ * the reducer non-finite policy (docs/notes/reducer-nan-policy.md)
50
+ * otherwise requires on every numeric `reduceColumn`.
51
+ *
52
+ * A wrongly-`true` flag makes `reduceColumn` take the unguarded fast
53
+ * path and silently include a non-finite cell → wrong result. So the
54
+ * **default is `false`** (conservative: guarded path, correct-but-
55
+ * slower) and a producer sets `true` ONLY where finiteness is proven
56
+ * — either data-derived by inspecting every cell, validated upstream
57
+ * (strict intake), or propagated from a source column that was
58
+ * itself `allFinite`. When unsure, leave it `false`: a missed `true`
59
+ * is slower, never wrong.
60
+ */
61
+ allFinite;
62
+ constructor(values, length, validity, allFinite = false) {
45
63
  validateColumnLength(length, 'Float64Column');
46
64
  if (length > values.length) {
47
65
  throw new RangeError(`Float64Column buffer underflow: length ${length} exceeds values.length ${values.length}`);
@@ -53,6 +71,7 @@ export class Float64Column {
53
71
  this.length = length;
54
72
  if (validity !== undefined)
55
73
  this.validity = validity;
74
+ this.allFinite = allFinite;
56
75
  }
57
76
  read(i) {
58
77
  if (i < 0 || i >= this.length)
@@ -88,7 +107,8 @@ export class Float64Column {
88
107
  }
89
108
  const valuesSlice = this._values.subarray(lo, hi);
90
109
  const validitySlice = validitySliceByRange(this.validity, lo, hi, this.length);
91
- return new Float64Column(valuesSlice, hi - lo, validitySlice);
110
+ // A contiguous slice of an all-finite column is all-finite.
111
+ return new Float64Column(valuesSlice, hi - lo, validitySlice, this.allFinite);
92
112
  }
93
113
  sliceByIndices(indices) {
94
114
  const out = new Float64Array(indices.length);
@@ -99,7 +119,10 @@ export class Float64Column {
99
119
  out[i] = idx >= 0 && idx < this.length ? this._values[idx] : 0;
100
120
  }
101
121
  const validity = validityGatherByIndices(this.validity, indices, this.length);
102
- return new Float64Column(out, indices.length, validity);
122
+ // A gathered subset of finite cells is finite. Out-of-range slots
123
+ // read 0 (finite) and are marked invalid by the validity gather, so
124
+ // they can't violate the contract either way.
125
+ return new Float64Column(out, indices.length, validity, this.allFinite);
103
126
  }
104
127
  }
105
128
  /* -------------------------------------------------------------------------- */
@@ -201,15 +224,27 @@ export function float64ColumnFromArray(source) {
201
224
  const length = source.length;
202
225
  validateColumnLength(length, 'Float64Column');
203
226
  const values = new Float64Array(length);
227
+ // Data-derive `allFinite` in the copy loop: only *defined* cells
228
+ // count (a missing slot doesn't make the column non-finite). Finite
229
+ // data → `true` (reducers take the fast path); an overflow / NaN cell
230
+ // → `false` (guarded path, correct). See the field's safety contract.
231
+ let allFinite = true;
204
232
  for (let i = 0; i < length; i += 1) {
205
233
  const v = source[i];
206
- values[i] = typeof v === 'number' ? v : 0;
234
+ if (typeof v === 'number') {
235
+ values[i] = v;
236
+ if (!Number.isFinite(v))
237
+ allFinite = false;
238
+ }
239
+ else {
240
+ values[i] = 0;
241
+ }
207
242
  }
208
243
  const validity = validityFromPredicate(length, (i) => {
209
244
  const v = source[i];
210
245
  return typeof v === 'number';
211
246
  });
212
- return new Float64Column(values, length, validity);
247
+ return new Float64Column(values, length, validity, allFinite);
213
248
  }
214
249
  /**
215
250
  * Builds a `BooleanColumn` from an array of `boolean | null | undefined`
@@ -8,24 +8,51 @@ export const avg = {
8
8
  reduceColumn(col) {
9
9
  const values = col._values;
10
10
  const validity = col.validity;
11
+ let s = 0;
12
+ let n = 0;
13
+ // Fast path: every defined cell is finite (`Float64Column.allFinite`),
14
+ // so every defined cell is a valid contributor — plain accumulate, and
15
+ // the divisor is `definedCount` (or `col.length`). Drops the
16
+ // per-element finite guard the reducer non-finite policy
17
+ // (docs/notes/reducer-nan-policy.md) otherwise requires.
18
+ if (col.allFinite) {
19
+ if (validity === undefined) {
20
+ for (let i = 0; i < col.length; i += 1)
21
+ s += values[i];
22
+ return col.length === 0 ? undefined : s / col.length;
23
+ }
24
+ const bits = validity.bits;
25
+ for (let i = 0; i < col.length; i += 1) {
26
+ if ((bits[i >> 3] & (1 << (i & 7))) !== 0)
27
+ s += values[i];
28
+ }
29
+ const count = validity.definedCount;
30
+ return count === 0 ? undefined : s / count;
31
+ }
32
+ // Guarded path: divide by the count of *finite* contributors, not
33
+ // `definedCount` — a non-finite cell is skipped per policy, so it must
34
+ // not inflate the divisor.
11
35
  if (validity === undefined) {
12
- if (col.length === 0)
13
- return undefined;
14
- let s = 0;
15
- for (let i = 0; i < col.length; i += 1)
16
- s += values[i];
17
- return s / col.length;
36
+ for (let i = 0; i < col.length; i += 1) {
37
+ const v = values[i];
38
+ if (Number.isFinite(v)) {
39
+ s += v;
40
+ n += 1;
41
+ }
42
+ }
43
+ return n === 0 ? undefined : s / n;
18
44
  }
19
- const definedCount = validity.definedCount;
20
- if (definedCount === 0)
21
- return undefined;
22
45
  const bits = validity.bits;
23
- let s = 0;
24
46
  for (let i = 0; i < col.length; i += 1) {
25
- if ((bits[i >> 3] & (1 << (i & 7))) !== 0)
26
- s += values[i];
47
+ if ((bits[i >> 3] & (1 << (i & 7))) !== 0) {
48
+ const v = values[i];
49
+ if (Number.isFinite(v)) {
50
+ s += v;
51
+ n += 1;
52
+ }
53
+ }
27
54
  }
28
- return s / definedCount;
55
+ return n === 0 ? undefined : s / n;
29
56
  },
30
57
  bucketState() {
31
58
  let s = 0;
@@ -24,10 +24,36 @@ export const count = {
24
24
  return defined.length;
25
25
  },
26
26
  reduceColumn(col) {
27
- // O(1) when validity is precomputed (it always is on Float64Column —
28
- // `validity.definedCount` is cached at construction). Falls back to
29
- // `col.length` when no validity bitmap exists (every cell defined).
30
- return col.validity === undefined ? col.length : col.validity.definedCount;
27
+ const values = col._values;
28
+ const validity = col.validity;
29
+ // Fast path: every defined cell is finite (`Float64Column.allFinite`),
30
+ // so "defined" and "defined AND finite" coincide — the O(1) shortcut
31
+ // is exact again (`definedCount`, or `col.length` when no bitmap).
32
+ // This is the O(N)→O(1) recovery the non-finite policy cost count
33
+ // (docs/notes/reducer-nan-policy.md).
34
+ if (col.allFinite) {
35
+ return validity === undefined ? col.length : validity.definedCount;
36
+ }
37
+ // Guarded path: O(N) scan, NOT the `definedCount` shortcut — the
38
+ // non-finite policy excludes non-finite cells, and `definedCount`
39
+ // counts them as present, so a defined-but-NaN cell would over-count.
40
+ // Walk and count valid AND finite cells.
41
+ let n = 0;
42
+ if (validity === undefined) {
43
+ for (let i = 0; i < col.length; i += 1) {
44
+ if (Number.isFinite(values[i]))
45
+ n += 1;
46
+ }
47
+ return n;
48
+ }
49
+ const bits = validity.bits;
50
+ for (let i = 0; i < col.length; i += 1) {
51
+ if ((bits[i >> 3] & (1 << (i & 7))) !== 0 &&
52
+ Number.isFinite(values[i])) {
53
+ n += 1;
54
+ }
55
+ }
56
+ return n;
31
57
  },
32
58
  bucketState() {
33
59
  let n = 0;
@@ -41,6 +41,38 @@ export function resolveReducer(operation) {
41
41
  return topReducer(n);
42
42
  throw new TypeError(`unsupported aggregate reducer: ${operation}`);
43
43
  }
44
+ /**
45
+ * Non-finite numerics (`NaN` / `±Infinity`) are treated as **missing** by
46
+ * every built-in reducer — the reducer non-finite policy
47
+ * (docs/notes/reducer-nan-policy.md). Row intake keeps user data finite; these
48
+ * values only arise inside computed columns (e.g. `cumulative` overflow) or
49
+ * trusted construction, and skipping them keeps every reducer consistent
50
+ * across all four execution paths. Mapping to `undefined` here lets the
51
+ * existing skip-missing logic in each incremental state do the work — so the
52
+ * policy holds for both the batch and live incremental paths without touching
53
+ * any reducer body. Custom-function reducers are intentionally NOT wrapped:
54
+ * they receive values as-is (the escape hatch decides its own semantics).
55
+ */
56
+ function finiteOrMissing(value) {
57
+ return typeof value === 'number' && !Number.isFinite(value)
58
+ ? undefined
59
+ : value;
60
+ }
61
+ /** Wrap a bucket state so non-finite numerics are skipped. {@link finiteOrMissing} */
62
+ function skipNonFiniteBucket(state) {
63
+ return {
64
+ add: (value) => state.add(finiteOrMissing(value)),
65
+ snapshot: () => state.snapshot(),
66
+ };
67
+ }
68
+ /** Wrap a rolling state so non-finite numerics are skipped. {@link finiteOrMissing} */
69
+ function skipNonFiniteRolling(state) {
70
+ return {
71
+ add: (index, value) => state.add(index, finiteOrMissing(value)),
72
+ remove: (index, value) => state.remove(index, finiteOrMissing(value)),
73
+ snapshot: () => state.snapshot(),
74
+ };
75
+ }
44
76
  /**
45
77
  * Build an `AggregateBucketState` for a reducer that may be either a
46
78
  * built-in name (string) or a custom function. Built-ins use their
@@ -52,7 +84,7 @@ export function resolveReducer(operation) {
52
84
  */
53
85
  export function bucketStateFor(reducer) {
54
86
  if (typeof reducer === 'string') {
55
- return resolveReducer(reducer).bucketState();
87
+ return skipNonFiniteBucket(resolveReducer(reducer).bucketState());
56
88
  }
57
89
  // Custom-function adapter: buffer values, call fn at snapshot time.
58
90
  const items = [];
@@ -90,7 +122,7 @@ export function bucketStateFor(reducer) {
90
122
  */
91
123
  export function rollingStateFor(reducer) {
92
124
  if (typeof reducer === 'string') {
93
- return resolveReducer(reducer).rollingState();
125
+ return skipNonFiniteRolling(resolveReducer(reducer).rollingState());
94
126
  }
95
127
  // Custom-function adapter: Map keyed by event index for O(1) remove.
96
128
  const items = new Map();
@@ -7,20 +7,46 @@ export const max = {
7
7
  : numeric.reduce((a, b) => (a >= b ? a : b));
8
8
  },
9
9
  reduceColumn(col) {
10
- // **NaN parity with row API.** Mirror the row-API expression
11
- // numeric.reduce((a, b) => a >= b ? a : b)
12
- // exactly. See `min.ts` for the full rationale. Closed Codex
13
- // review finding on PR #153.
14
10
  const values = col._values;
15
11
  const validity = col.validity;
16
12
  let hi;
17
- if (validity === undefined) {
18
- if (col.length === 0)
13
+ // Fast path: every defined cell is finite (`Float64Column.allFinite`),
14
+ // so we seed `hi` from the first defined cell and run a plain `v > hi`
15
+ // compare with NO per-element `Number.isFinite` guard and NO in-loop
16
+ // `hi === undefined` check (the seed hoists it out). No NaN to mishandle,
17
+ // also sidesteps the position-dependent `a>=b?a:b` extremum bug the policy
18
+ // fixed (docs/notes/reducer-nan-policy.md). The pre-policy column loop,
19
+ // recovered.
20
+ if (col.allFinite) {
21
+ const len = col.length;
22
+ if (len === 0)
19
23
  return undefined;
20
- hi = values[0];
21
- for (let i = 1; i < col.length; i += 1) {
24
+ if (validity === undefined) {
25
+ hi = values[0];
26
+ for (let i = 1; i < len; i += 1) {
27
+ const v = values[i];
28
+ if (v > hi)
29
+ hi = v;
30
+ }
31
+ return hi;
32
+ }
33
+ const bits = validity.bits;
34
+ for (let i = 0; i < len; i += 1) {
35
+ if ((bits[i >> 3] & (1 << (i & 7))) === 0)
36
+ continue;
37
+ const v = values[i];
38
+ if (hi === undefined || v > hi)
39
+ hi = v;
40
+ }
41
+ return hi;
42
+ }
43
+ // Guarded path: skip non-finite cells (reducer non-finite policy) —
44
+ // matches `bucketState`'s `v > hi`.
45
+ if (validity === undefined) {
46
+ for (let i = 0; i < col.length; i += 1) {
22
47
  const v = values[i];
23
- hi = hi >= v ? hi : v;
48
+ if (Number.isFinite(v) && (hi === undefined || v > hi))
49
+ hi = v;
24
50
  }
25
51
  return hi;
26
52
  }
@@ -29,7 +55,8 @@ export const max = {
29
55
  if ((bits[i >> 3] & (1 << (i & 7))) === 0)
30
56
  continue;
31
57
  const v = values[i];
32
- hi = hi === undefined ? v : hi >= v ? hi : v;
58
+ if (Number.isFinite(v) && (hi === undefined || v > hi))
59
+ hi = v;
33
60
  }
34
61
  return hi;
35
62
  },
@@ -7,33 +7,46 @@ export const min = {
7
7
  : numeric.reduce((a, b) => (a <= b ? a : b));
8
8
  },
9
9
  reduceColumn(col) {
10
- // **NaN parity with row API.** The row-API path uses
11
- // numeric.reduce((a, b) => a <= b ? a : b)
12
- // which has surprising NaN behavior: on `[1, NaN, 2]` it returns
13
- // `2` because the first `1 <= NaN` is false (returning NaN),
14
- // then `NaN <= 2` is also false (returning 2). The "natural"
15
- // column-side loop `if (v < lo) lo = v` would instead return
16
- // `1` (NaN comparisons always false → NaN is skipped). The two
17
- // paths diverge on NaN-bearing input.
18
- //
19
- // We mirror the row-API comparison expression exactly to
20
- // preserve the parity claim, even though both paths exhibit
21
- // surprising results on NaN (which can only reach a `kind:
22
- // 'number'` column via trusted construction — `assertCellKind`
23
- // rejects it at public intake). Closed Codex review finding on
24
- // PR #153. A principled "filter NaN consistently across both
25
- // paths" fix is a separate concern tracked in the followup
26
- // issue.
27
10
  const values = col._values;
28
11
  const validity = col.validity;
29
12
  let lo;
30
- if (validity === undefined) {
31
- if (col.length === 0)
13
+ // Fast path: every defined cell is finite (`Float64Column.allFinite`),
14
+ // so we seed `lo` from the first defined cell and run a plain `v < lo`
15
+ // compare with NO per-element `Number.isFinite` guard and NO in-loop
16
+ // `lo === undefined` check (the seed hoists it out). There is no NaN to
17
+ // mishandle, so this also sidesteps the position-dependent `a<=b?a:b`
18
+ // extremum bug the policy fixed (docs/notes/reducer-nan-policy.md). This
19
+ // is the pre-policy column loop, recovered.
20
+ if (col.allFinite) {
21
+ const len = col.length;
22
+ if (len === 0)
32
23
  return undefined;
33
- lo = values[0];
34
- for (let i = 1; i < col.length; i += 1) {
24
+ if (validity === undefined) {
25
+ lo = values[0];
26
+ for (let i = 1; i < len; i += 1) {
27
+ const v = values[i];
28
+ if (v < lo)
29
+ lo = v;
30
+ }
31
+ return lo;
32
+ }
33
+ const bits = validity.bits;
34
+ for (let i = 0; i < len; i += 1) {
35
+ if ((bits[i >> 3] & (1 << (i & 7))) === 0)
36
+ continue;
37
+ const v = values[i];
38
+ if (lo === undefined || v < lo)
39
+ lo = v;
40
+ }
41
+ return lo;
42
+ }
43
+ // Guarded path: skip non-finite cells (reducer non-finite policy) —
44
+ // matches `bucketState`'s `v < lo`.
45
+ if (validity === undefined) {
46
+ for (let i = 0; i < col.length; i += 1) {
35
47
  const v = values[i];
36
- lo = lo <= v ? lo : v;
48
+ if (Number.isFinite(v) && (lo === undefined || v < lo))
49
+ lo = v;
37
50
  }
38
51
  return lo;
39
52
  }
@@ -42,7 +55,8 @@ export const min = {
42
55
  if ((bits[i >> 3] & (1 << (i & 7))) === 0)
43
56
  continue;
44
57
  const v = values[i];
45
- lo = lo === undefined ? v : lo <= v ? lo : v;
58
+ if (Number.isFinite(v) && (lo === undefined || v < lo))
59
+ lo = v;
46
60
  }
47
61
  return lo;
48
62
  },
@@ -5,36 +5,20 @@ export declare function parsePercentile(op: string): number | undefined;
5
5
  /**
6
6
  * Shared `reduceColumn` body for percentile-shaped reducers
7
7
  * (`median`, `p50`, `p95`, etc.). Walks the validity bitmap to
8
- * gather defined cells into a dense `Float64Array`, detecting NaN
9
- * cells in the same pass; sorts and reads the percentile from
10
- * the sorted view.
8
+ * gather defined **and finite** cells into a dense `Float64Array`,
9
+ * sorts with the typed-array intrinsic, and reads the percentile
10
+ * from the sorted view.
11
11
  *
12
- * **NaN parity with row API.** Two sort behaviors diverge:
12
+ * Non-finite cells (`NaN` / `±Infinity`) are excluded by the
13
+ * reducer non-finite policy (docs/notes/reducer-nan-policy.md) —
14
+ * uniformly, across every path. With non-finite filtered out
15
+ * before the sort, `Float64Array.prototype.sort()` (the numeric,
16
+ * NaN-free intrinsic, ~2× faster than `Array.sort` with a
17
+ * comparator) produces the same total order as the row path's
18
+ * `Array.sort((a, b) => a - b)` over the same finite values — so
19
+ * there is no longer any NaN-ordering seam to special-case.
13
20
  *
14
- * - `Array.prototype.sort((a, b) => a - b)` (row-API path) returns
15
- * NaN from the comparator on NaN inputs; V8 treats this as
16
- * "equal" and leaves NaN cells in undefined order — the
17
- * resulting percentile is whatever cell happens to land at the
18
- * computed rank, possibly NaN itself.
19
- * - `Float64Array.prototype.sort()` (typed-array intrinsic) puts
20
- * NaN deterministically at the end of the sorted view — the
21
- * percentile rank then reads a non-NaN cell unless the rank
22
- * lands in the NaN suffix.
23
- *
24
- * The first-pass NaN detection lets us use `Float64Array.sort`'s
25
- * 2× speedup for the common no-NaN case (full parity with row API
26
- * because both produce identical sorted orders when no NaN
27
- * present), and fall back to `Array.sort` with comparator only
28
- * when NaN is present (preserving bug-for-bug row-API parity on
29
- * the rare contract-violating input).
30
- *
31
- * NaN can only reach a `kind: 'number'` column via trusted
32
- * construction (`fromEvents`); the public `assertCellKind`
33
- * rejects it at intake. Closed Codex review finding on PR #153 —
34
- * earlier L2 fix that filtered NaN was correct in spirit but
35
- * introduced a *different* divergence from the row API. A
36
- * principled "filter NaN consistently across both paths" fix is
37
- * tracked in the followup issue.
21
+ * Empty (no defined+finite values) `undefined`.
38
22
  */
39
23
  export declare function reducePercentileColumn(col: Float64Column, q: number): number | undefined;
40
24
  export declare function percentileReducer(q: number): ReducerDef;
@@ -18,51 +18,64 @@ export function parsePercentile(op) {
18
18
  /**
19
19
  * Shared `reduceColumn` body for percentile-shaped reducers
20
20
  * (`median`, `p50`, `p95`, etc.). Walks the validity bitmap to
21
- * gather defined cells into a dense `Float64Array`, detecting NaN
22
- * cells in the same pass; sorts and reads the percentile from
23
- * the sorted view.
21
+ * gather defined **and finite** cells into a dense `Float64Array`,
22
+ * sorts with the typed-array intrinsic, and reads the percentile
23
+ * from the sorted view.
24
24
  *
25
- * **NaN parity with row API.** Two sort behaviors diverge:
25
+ * Non-finite cells (`NaN` / `±Infinity`) are excluded by the
26
+ * reducer non-finite policy (docs/notes/reducer-nan-policy.md) —
27
+ * uniformly, across every path. With non-finite filtered out
28
+ * before the sort, `Float64Array.prototype.sort()` (the numeric,
29
+ * NaN-free intrinsic, ~2× faster than `Array.sort` with a
30
+ * comparator) produces the same total order as the row path's
31
+ * `Array.sort((a, b) => a - b)` over the same finite values — so
32
+ * there is no longer any NaN-ordering seam to special-case.
26
33
  *
27
- * - `Array.prototype.sort((a, b) => a - b)` (row-API path) returns
28
- * NaN from the comparator on NaN inputs; V8 treats this as
29
- * "equal" and leaves NaN cells in undefined order — the
30
- * resulting percentile is whatever cell happens to land at the
31
- * computed rank, possibly NaN itself.
32
- * - `Float64Array.prototype.sort()` (typed-array intrinsic) puts
33
- * NaN deterministically at the end of the sorted view — the
34
- * percentile rank then reads a non-NaN cell unless the rank
35
- * lands in the NaN suffix.
36
- *
37
- * The first-pass NaN detection lets us use `Float64Array.sort`'s
38
- * 2× speedup for the common no-NaN case (full parity with row API
39
- * because both produce identical sorted orders when no NaN
40
- * present), and fall back to `Array.sort` with comparator only
41
- * when NaN is present (preserving bug-for-bug row-API parity on
42
- * the rare contract-violating input).
43
- *
44
- * NaN can only reach a `kind: 'number'` column via trusted
45
- * construction (`fromEvents`); the public `assertCellKind`
46
- * rejects it at intake. Closed Codex review finding on PR #153 —
47
- * earlier L2 fix that filtered NaN was correct in spirit but
48
- * introduced a *different* divergence from the row API. A
49
- * principled "filter NaN consistently across both paths" fix is
50
- * tracked in the followup issue.
34
+ * Empty (no defined+finite values) `undefined`.
51
35
  */
52
36
  export function reducePercentileColumn(col, q) {
53
37
  const validity = col.validity;
54
38
  const values = col._values;
55
39
  let dense;
56
40
  let denseLength = 0;
57
- let hasNaN = false;
58
- if (validity === undefined) {
41
+ // Fast path: every defined cell is finite (`Float64Column.allFinite`),
42
+ // so we gather defined cells with no per-element `Number.isFinite`
43
+ // filter (reducer non-finite policy, docs/notes/reducer-nan-policy.md).
44
+ // The subsequent `Float64Array.sort` is the same NaN-free intrinsic
45
+ // either way → identical order, identical percentile.
46
+ if (col.allFinite) {
47
+ if (validity === undefined) {
48
+ if (col.length === 0)
49
+ return undefined;
50
+ dense = new Float64Array(col.length);
51
+ for (let i = 0; i < col.length; i += 1) {
52
+ dense[denseLength] = values[i];
53
+ denseLength += 1;
54
+ }
55
+ }
56
+ else {
57
+ const definedCount = validity.definedCount;
58
+ if (definedCount === 0)
59
+ return undefined;
60
+ dense = new Float64Array(definedCount);
61
+ const bits = validity.bits;
62
+ for (let i = 0; i < col.length; i += 1) {
63
+ if ((bits[i >> 3] & (1 << (i & 7))) === 0)
64
+ continue;
65
+ dense[denseLength] = values[i];
66
+ denseLength += 1;
67
+ }
68
+ }
69
+ }
70
+ else if (validity === undefined) {
71
+ // Guarded path: filter non-finite before the sort.
59
72
  if (col.length === 0)
60
73
  return undefined;
61
74
  dense = new Float64Array(col.length);
62
75
  for (let i = 0; i < col.length; i += 1) {
63
76
  const v = values[i];
64
- if (Number.isNaN(v))
65
- hasNaN = true;
77
+ if (!Number.isFinite(v))
78
+ continue;
66
79
  dense[denseLength] = v;
67
80
  denseLength += 1;
68
81
  }
@@ -77,23 +90,16 @@ export function reducePercentileColumn(col, q) {
77
90
  if ((bits[i >> 3] & (1 << (i & 7))) === 0)
78
91
  continue;
79
92
  const v = values[i];
80
- if (Number.isNaN(v))
81
- hasNaN = true;
93
+ if (!Number.isFinite(v))
94
+ continue;
82
95
  dense[denseLength] = v;
83
96
  denseLength += 1;
84
97
  }
85
98
  }
86
99
  if (denseLength === 0)
87
100
  return undefined;
88
- if (hasNaN) {
89
- // Match row-API exactly via `Array.sort` with comparator
90
- // diverges from `Float64Array.sort` on NaN ordering.
91
- const arr = Array.from(dense.subarray(0, denseLength));
92
- arr.sort((a, b) => a - b);
93
- return percentileOfSorted(arr, q);
94
- }
95
- // No NaN: `Float64Array.sort` is parity-correct (same total
96
- // order as `Array.sort` with comparator) and ~2× faster.
101
+ // Non-finite excluded upstream by policy → `Float64Array.sort` (numeric,
102
+ // NaN-free) gives the same order as the row path's comparator sort.
97
103
  const view = dense.subarray(0, denseLength);
98
104
  view.sort();
99
105
  const rank = (q / 100) * (denseLength - 1);
@@ -20,9 +20,12 @@
20
20
  * agree to floating-point noise — bit-for-bit when they see the values in the
21
21
  * same order, which the bucketed paths do.
22
22
  *
23
- * `rollingState` is the exception: its sliding window needs `remove`, which
24
- * Welford can't do stably (windowed removal drifts), so it keeps the one-pass
25
- * `sq/n − mean²` (with its clamp). A stable rolling stdev is a deferred item.
23
+ * `rollingState` adds `remove` for its sliding window via Welford's
24
+ * **order-independent delete** (the reverse recurrence). It works in the same
25
+ * deviation space — no `sq/n − mean²` cancellation, no shift drift and
26
+ * removes *by value*, so it stays correct under the live layer's reorder-mode
27
+ * eviction (which removes the sorted-prefix, not the oldest-arrived event).
28
+ * See its inline note for the mechanics.
26
29
  */
27
30
  // Welford accumulator: `add` each value, then read `result()` for the
28
31
  // population stdev (`undefined` if nothing was added). Shared by `reduce` and
@@ -54,20 +57,55 @@ export const stdev = {
54
57
  },
55
58
  reduceColumn(col) {
56
59
  // Inlined Welford (mirrors `welfordStdev`) over the packed array — a single
57
- // pass, skipping gaps via the validity bitmask. One division per element vs
58
- // the old two-pass's two divisionless scans, but identical to the other
59
- // paths' recurrence so the fast path and row path cannot diverge.
60
+ // pass, skipping gaps via the validity bitmask and skipping non-finite
61
+ // cells (reducer non-finite policy, docs/notes/reducer-nan-policy.md). One
62
+ // division per finite element vs the old two-pass's two divisionless scans,
63
+ // but identical to the other paths' recurrence so the fast path and row
64
+ // path cannot diverge. `n` counts only finite contributors, so `n === 0`
65
+ // (no finite cells) → `undefined`.
60
66
  const values = col._values;
61
67
  const validity = col.validity;
62
68
  let n = 0;
63
69
  let mean = 0;
64
70
  let m2 = 0;
71
+ // Fast path: every defined cell is finite (`Float64Column.allFinite`),
72
+ // so the Welford recurrence runs over every defined cell with no
73
+ // per-element `Number.isFinite` guard (reducer non-finite policy,
74
+ // docs/notes/reducer-nan-policy.md). Same recurrence as the guarded
75
+ // path → identical result.
76
+ if (col.allFinite) {
77
+ const len = col.length;
78
+ if (validity === undefined) {
79
+ for (let i = 0; i < len; i += 1) {
80
+ const v = values[i];
81
+ n += 1;
82
+ const delta = v - mean;
83
+ mean += delta / n;
84
+ m2 += delta * (v - mean);
85
+ }
86
+ }
87
+ else {
88
+ const bits = validity.bits;
89
+ for (let i = 0; i < len; i += 1) {
90
+ if ((bits[i >> 3] & (1 << (i & 7))) === 0)
91
+ continue;
92
+ const v = values[i];
93
+ n += 1;
94
+ const delta = v - mean;
95
+ mean += delta / n;
96
+ m2 += delta * (v - mean);
97
+ }
98
+ }
99
+ return n === 0 ? undefined : Math.sqrt(Math.max(0, m2 / n));
100
+ }
101
+ // Guarded path: skip non-finite cells; `n` counts only finite
102
+ // contributors so `n === 0` (no finite cells) → `undefined`.
65
103
  if (validity === undefined) {
66
104
  const len = col.length;
67
- if (len === 0)
68
- return undefined;
69
105
  for (let i = 0; i < len; i += 1) {
70
106
  const v = values[i];
107
+ if (!Number.isFinite(v))
108
+ continue;
71
109
  n += 1;
72
110
  const delta = v - mean;
73
111
  mean += delta / n;
@@ -75,21 +113,21 @@ export const stdev = {
75
113
  }
76
114
  }
77
115
  else {
78
- if (validity.definedCount === 0)
79
- return undefined;
80
116
  const bits = validity.bits;
81
117
  const len = col.length;
82
118
  for (let i = 0; i < len; i += 1) {
83
119
  if ((bits[i >> 3] & (1 << (i & 7))) === 0)
84
120
  continue;
85
121
  const v = values[i];
122
+ if (!Number.isFinite(v))
123
+ continue;
86
124
  n += 1;
87
125
  const delta = v - mean;
88
126
  mean += delta / n;
89
127
  m2 += delta * (v - mean);
90
128
  }
91
129
  }
92
- return Math.sqrt(Math.max(0, m2 / n));
130
+ return n === 0 ? undefined : Math.sqrt(Math.max(0, m2 / n));
93
131
  },
94
132
  bucketState() {
95
133
  const w = welfordStdev();
@@ -104,32 +142,80 @@ export const stdev = {
104
142
  };
105
143
  },
106
144
  rollingState() {
107
- // One-pass `sq/n mean²` with the `Math.max(0, …)` clamp. Unlike the other
108
- // paths this has a `remove` for the sliding window, which Welford can't do
109
- // stably a stable rolling stdev is deferred (see the module note above).
110
- let s = 0;
111
- let sq = 0;
145
+ // Sliding-window population stdev: Welford's online variance with an
146
+ // **order-independent delete**. `add(v)` is the standard recurrence;
147
+ // `remove(v)` reverses it. Working in deviation space, it fixes the old
148
+ // one-pass `sq/n − mean²` failure modes — catastrophic cancellation on
149
+ // near-equal large values (`[1e10, 1e10+1, …]` → 0 or a negative variance →
150
+ // NaN; the audit-§1.1 case, previously still live on this path) and shift
151
+ // drift on trending data (cumulative distance, elevation).
152
+ //
153
+ // Limitation — **outlier eviction** (shared with the old one-pass, so not a
154
+ // regression): like any *subtractive* sliding variance, evicting a value far
155
+ // outside the residual spread cancels — `m2 −= huge` loses the small
156
+ // remainder, and the error persists in the running accumulator (the clamp
157
+ // floors it to 0 at the extreme). Negligible until the evicted point is
158
+ // ~1e7–1e8× the residual stdev (e.g. a ~1e6 spike over a ~0.01-σ baseline) —
159
+ // far beyond realistic monitoring / activity data, and exactly why the
160
+ // add-only paths use plain Welford. A FIFO-only two-stack merge avoids it
161
+ // entirely but can't serve the by-value eviction the live layer needs.
162
+ //
163
+ // Removal is **by value, not by position**. That is load-bearing for the
164
+ // live layer: `LiveReduce` shares this state, and a `reorder`-mode source
165
+ // with retention evicts the sorted-prefix — which may be a later arrival,
166
+ // not the oldest — so a positional (FIFO) remove would corrupt the window.
167
+ // A value-based delete is correct regardless of eviction order (the
168
+ // documented contract; see live-reduce.ts and live-buffer-as-window.test).
169
+ // The batch rolling driver removes strictly oldest-first — a special case.
170
+ //
171
+ // Non-finite / missing cells arrive as `undefined` (the factory wrapper
172
+ // applies the non-finite policy); `add`/`remove` both skip them symmetrically
173
+ // so they never enter `n`.
112
174
  let n = 0;
175
+ let mean = 0;
176
+ let m2 = 0;
113
177
  return {
114
178
  add(_i, v) {
115
- if (typeof v === 'number') {
116
- s += v;
117
- sq += v * v;
118
- n++;
119
- }
179
+ if (typeof v !== 'number')
180
+ return;
181
+ n += 1;
182
+ const delta = v - mean;
183
+ mean += delta / n;
184
+ m2 += delta * (v - mean);
120
185
  },
121
186
  remove(_i, v) {
122
- if (typeof v === 'number') {
123
- s -= v;
124
- sq -= v * v;
125
- n--;
187
+ if (typeof v !== 'number')
188
+ return;
189
+ if (n <= 1) {
190
+ // Removing the final contributor — reset exactly (no 0/0, no drift).
191
+ n = 0;
192
+ mean = 0;
193
+ m2 = 0;
194
+ return;
195
+ }
196
+ const meanWith = mean;
197
+ n -= 1;
198
+ if (n === 1) {
199
+ // A single remaining element has population variance *exactly* 0.
200
+ // Setting it directly (rather than via the subtraction below) keeps
201
+ // the n→1 result exact at any magnitude — the reverse step alone
202
+ // leaves rounding residue at large offsets (e.g. ~0.016 on 1e10).
203
+ mean = meanWith * 2 - v; // the one survivor: 2·mean₂ − removed
204
+ m2 = 0;
205
+ return;
126
206
  }
207
+ // Deviation-space mean update (mean − (v − mean)/n): avoids the large
208
+ // `n·mean − v` product, staying precise at large magnitudes.
209
+ mean = meanWith - (v - meanWith) / n;
210
+ // Reverse Welford: M2 −= (v − meanNew)·(v − meanOld).
211
+ m2 -= (v - mean) * (v - meanWith);
212
+ // Normally absorbs FP round-off; on a gross outlier eviction (see the
213
+ // limitation note above) the subtraction can cancel below 0 — clamp.
214
+ if (m2 < 0)
215
+ m2 = 0;
127
216
  },
128
217
  snapshot() {
129
- if (n === 0)
130
- return undefined;
131
- const mean = s / n;
132
- return Math.sqrt(Math.max(0, sq / n - mean * mean));
218
+ return n === 0 ? undefined : Math.sqrt(Math.max(0, m2 / n));
133
219
  },
134
220
  };
135
221
  },
@@ -7,18 +7,42 @@ export const sum = {
7
7
  const values = col._values;
8
8
  const validity = col.validity;
9
9
  let s = 0;
10
+ // Fast path: the column proved every defined cell is finite
11
+ // (`Float64Column.allFinite`), so we can drop the per-element
12
+ // `Number.isFinite` guard the reducer non-finite policy
13
+ // (docs/notes/reducer-nan-policy.md) otherwise requires — plain
14
+ // accumulate, identical result.
15
+ if (col.allFinite) {
16
+ if (validity === undefined) {
17
+ for (let i = 0; i < col.length; i += 1)
18
+ s += values[i];
19
+ return s;
20
+ }
21
+ const bits = validity.bits;
22
+ for (let i = 0; i < col.length; i += 1) {
23
+ if ((bits[i >> 3] & (1 << (i & 7))) !== 0)
24
+ s += values[i];
25
+ }
26
+ return s;
27
+ }
28
+ // Guarded path: finiteness not proven, skip non-finite per policy.
10
29
  if (validity === undefined) {
11
- // Hot path: every cell defined; no per-row branch.
12
- for (let i = 0; i < col.length; i += 1)
13
- s += values[i];
30
+ for (let i = 0; i < col.length; i += 1) {
31
+ const v = values[i];
32
+ if (Number.isFinite(v))
33
+ s += v;
34
+ }
14
35
  return s;
15
36
  }
16
37
  // Inline bitmap check rather than method dispatch — same pattern
17
38
  // the chart-friction-spike notes flagged for hot draw loops.
18
39
  const bits = validity.bits;
19
40
  for (let i = 0; i < col.length; i += 1) {
20
- if ((bits[i >> 3] & (1 << (i & 7))) !== 0)
21
- s += values[i];
41
+ if ((bits[i >> 3] & (1 << (i & 7))) !== 0) {
42
+ const v = values[i];
43
+ if (Number.isFinite(v))
44
+ s += v;
45
+ }
22
46
  }
23
47
  return s;
24
48
  },
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "pond-ts",
3
- "version": "0.24.0",
3
+ "version": "0.25.0",
4
4
  "description": "TypeScript-first time series primitives",
5
5
  "license": "MIT",
6
6
  "repository": {