pond-ts 0.23.0 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +96 -1
- package/README.md +18 -14
- package/dist/batch/aggregate-columns.d.ts +0 -24
- package/dist/batch/aggregate-columns.js +55 -32
- package/dist/batch/partitioned-time-series.d.ts +0 -2
- package/dist/batch/partitioned-time-series.js +25 -67
- package/dist/batch/time-series.js +134 -9
- package/dist/batch/validate.js +7 -1
- package/dist/column.js +91 -21
- package/dist/columnar/chunked-column.d.ts +10 -0
- package/dist/columnar/chunked-column.js +29 -3
- package/dist/columnar/column.d.ts +19 -1
- package/dist/columnar/column.js +40 -5
- package/dist/reducers/avg.js +40 -13
- package/dist/reducers/count.js +30 -4
- package/dist/reducers/first.js +1 -0
- package/dist/reducers/index.js +34 -2
- package/dist/reducers/last.js +1 -0
- package/dist/reducers/max.js +37 -10
- package/dist/reducers/min.js +37 -23
- package/dist/reducers/percentile.d.ts +12 -28
- package/dist/reducers/percentile.js +49 -43
- package/dist/reducers/stdev.js +115 -29
- package/dist/reducers/sum.js +29 -5
- package/dist/reducers/types.d.ts +16 -0
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -7,7 +7,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
|
|
7
7
|
file covers both packages. Pre-1.0: minor bumps may include new features and
|
|
8
8
|
type-level changes; patch bumps are strictly additive.
|
|
9
9
|
|
|
10
|
-
[Unreleased]: https://github.com/pjm17971/pond-ts/compare/v0.
|
|
10
|
+
[Unreleased]: https://github.com/pjm17971/pond-ts/compare/v0.25.0...HEAD
|
|
11
|
+
[0.25.0]: https://github.com/pjm17971/pond-ts/compare/v0.24.0...v0.25.0
|
|
12
|
+
[0.24.0]: https://github.com/pjm17971/pond-ts/compare/v0.23.0...v0.24.0
|
|
11
13
|
[0.23.0]: https://github.com/pjm17971/pond-ts/compare/v0.22.0...v0.23.0
|
|
12
14
|
[0.22.0]: https://github.com/pjm17971/pond-ts/compare/v0.21.0...v0.22.0
|
|
13
15
|
[0.21.0]: https://github.com/pjm17971/pond-ts/compare/v0.20.0...v0.21.0
|
|
@@ -17,6 +19,99 @@ type-level changes; patch bumps are strictly additive.
|
|
|
17
19
|
|
|
18
20
|
## [Unreleased]
|
|
19
21
|
|
|
22
|
+
## [0.25.0] — 2026-06-15
|
|
23
|
+
|
|
24
|
+
### Changed
|
|
25
|
+
|
|
26
|
+
- **Reducers now treat non-finite numerics (`NaN` / `±Infinity`) as missing —
|
|
27
|
+
they are skipped — uniformly across every built-in reducer and all four
|
|
28
|
+
execution paths (`reduce`, the columnar fast path, `aggregate`/bucket, and
|
|
29
|
+
`rolling`/live).** Previously the paths disagreed on non-finite input: e.g.
|
|
30
|
+
`min`/`max` returned a position-dependent wrong extreme on the batch/columnar
|
|
31
|
+
paths but the true extreme on aggregate/rolling; `sum`/`avg` propagated
|
|
32
|
+
`NaN`. Non-finite can't enter via the row API (intake rejects it) — it only
|
|
33
|
+
arises inside computed columns (`cumulative` overflow, `diff`/`rate`
|
|
34
|
+
overflow, `collapse`, trusted construction) — so this only changes results
|
|
35
|
+
for those degenerate values, and makes every path agree. The three-layer
|
|
36
|
+
contract: **intake** stays strict (rejects non-finite), **computed writers**
|
|
37
|
+
stay permissive (pack honest non-finite), **reducers** are robust (skip it).
|
|
38
|
+
A standing parity-matrix test now pins all paths together. See
|
|
39
|
+
`docs/notes/reducer-nan-policy.md`. This also resolves the `aggregate('stdev')`
|
|
40
|
+
divergence class and the `min`/`max` NaN-laundering bug.
|
|
41
|
+
- Internal: `Float64Column` gained an `allFinite` fast-path flag (data-derived
|
|
42
|
+
at construction, conservative-by-default) so reducers skip the per-element
|
|
43
|
+
finite check on provably-finite columns — keeping the policy's cost off the
|
|
44
|
+
hot path (min/max/count stay at their pre-policy speed).
|
|
45
|
+
|
|
46
|
+
### Fixed
|
|
47
|
+
|
|
48
|
+
- **`rolling(window, { x: 'stdev' })` is now numerically stable.** It was the
|
|
49
|
+
last batch stdev path still on the one-pass `Σx²/n − mean²`, which cancels
|
|
50
|
+
catastrophically on near-equal large values (`[1e10, 1e10+1, …]` → `0`
|
|
51
|
+
instead of ≈1.118, or a negative variance → `NaN`) and drifts on trending
|
|
52
|
+
data (cumulative distance, elevation). It now uses Welford's online variance
|
|
53
|
+
with an order-independent **delete** — deviation-space, so no cancellation,
|
|
54
|
+
and removal **by value**, which keeps it correct under the live layer's
|
|
55
|
+
`reorder`-mode eviction (a positional/FIFO remove would have broken it; the
|
|
56
|
+
documented "stdev is reorder-safe" contract is preserved). Rolling-stdev
|
|
57
|
+
values shift in the last ULPs (now correct); the path stays O(1) and within
|
|
58
|
+
run-noise of the old one-pass, and a single-element window now reports exactly
|
|
59
|
+
`0` at any magnitude. Like any subtractive sliding variance, evicting an
|
|
60
|
+
outlier far outside the residual spread loses precision — negligible until the
|
|
61
|
+
evicted point is ~1e7–1e8× the residual stdev, far beyond realistic data.
|
|
62
|
+
- A standing differential-fuzz parity suite now pins every built-in reducer's
|
|
63
|
+
execution paths (columnar fast path vs `bucket` vs `rolling`, and the FIFO
|
|
64
|
+
sliding window vs a from-scratch recompute) against silent drift across
|
|
65
|
+
randomized magnitudes and window sizes — the class of bug behind the stdev
|
|
66
|
+
and `min`/`max` divergences.
|
|
67
|
+
|
|
68
|
+
## [0.24.0] — 2026-06-14
|
|
69
|
+
|
|
70
|
+
### Changed
|
|
71
|
+
|
|
72
|
+
- **`TimeSeries.timeRange()` is now a columnar key-axis read instead of a
|
|
73
|
+
reduce over materialized events.** Behavior is unchanged, but the old
|
|
74
|
+
implementation materialized every `Event` on its first call — and because
|
|
75
|
+
`aggregate()` defaults its `range` to `series.timeRange()`, a one-shot
|
|
76
|
+
`aggregate()` paid full event materialization before the columnar fast
|
|
77
|
+
path could run, erasing the win. The new path reads the key column's
|
|
78
|
+
begin/end axis directly: O(1) for time-keyed series, a typed-array scan
|
|
79
|
+
for range/interval-keyed series, with no event materialization. Measured
|
|
80
|
+
on 1M rows: `timeRange()` itself ~407 ms → ~0.002 ms (time-keyed); cold
|
|
81
|
+
`aggregate()` with a defaulted range ~387 ms → ~6 ms (~63×). Every
|
|
82
|
+
`timeRange()` / `overlaps` / `contains` / `intersection` caller benefits.
|
|
83
|
+
(Audit v2 §3.3.)
|
|
84
|
+
- **`aggregate()` now takes the columnar fast path when a mapping mixes
|
|
85
|
+
numeric reducers with `first` / `last`.** Previously a single `first` or
|
|
86
|
+
`last` column (they have no numeric `reduceColumn`) bailed the entire call
|
|
87
|
+
to the row path. They now qualify via a boundary scan — the first/last
|
|
88
|
+
_defined_ cell, on any column kind. Behavior is unchanged. The big
|
|
89
|
+
beneficiary is **partitioned `aggregate`**, which auto-injects a `'first'`
|
|
90
|
+
reducer for the partition column and so was excluded from the fast path on
|
|
91
|
+
every call (audit v2 §3.2/§3.3). Measured on 1M rows, flat
|
|
92
|
+
`{ cpu: 'avg', host: 'first' }`: ~37.7 ms → ~4.8 ms (~7.8×); the
|
|
93
|
+
pure-numeric path is unchanged. (The remaining `partitionBy` materialization
|
|
94
|
+
cost is addressed separately by the columnar `partitionBy` split.)
|
|
95
|
+
- **`partitionBy(...)` now splits the columnar store directly instead of
|
|
96
|
+
materializing events.** `collect()` / the per-partition sugar methods
|
|
97
|
+
(`fill` / `diff` / `rolling` / …) and `toMap()` previously walked
|
|
98
|
+
`this.events` to bucket rows, then rebuilt each partition via `fromEvents`
|
|
99
|
+
(re-validating + re-packing) — silently re-paying the event-materialization
|
|
100
|
+
tax the columnar wave removed, and making `partitionBy(host).fill().collect()`
|
|
101
|
+
the #1 batch hotspot. They now group row indices off the store and gather
|
|
102
|
+
each partition via a zero-materialization columnar selection. Behavior is
|
|
103
|
+
unchanged (partition order, the `' undefined'` missing-key bucket, composite
|
|
104
|
+
keys, and declared `groups` all preserved). Measured on 100k rows / 64
|
|
105
|
+
partitions: `toMap()` ~389 → ~25 ns/row (~15×, no event materialization at
|
|
106
|
+
all); `diff().collect()` ~2×; `fill(hold).collect()` ~1.7× (the residual is
|
|
107
|
+
`TimeSeries.concat` still materializing to re-sort — a separate follow-up).
|
|
108
|
+
Declared-`groups` membership is validated by the same columnar scan, so that
|
|
109
|
+
path is materialization-free too (~331 → ~33 ns/row). **Behavior note:**
|
|
110
|
+
per-partition sub-series from `toMap()` / `apply()` now lazily materialize
|
|
111
|
+
their own `Event` objects rather than reusing the source's instances — cell
|
|
112
|
+
values are identical; only object identity differs (`collect()`, which
|
|
113
|
+
returns the source unchanged, is unaffected). (Audit v2 §3.2.)
|
|
114
|
+
|
|
20
115
|
## [0.23.0] — 2026-06-13
|
|
21
116
|
|
|
22
117
|
### Added
|
package/README.md
CHANGED
|
@@ -152,20 +152,24 @@ const points = series.sample({ reservoir: { size: 500 } }).toRows();
|
|
|
152
152
|
|
|
153
153
|
## Performance
|
|
154
154
|
|
|
155
|
-
pond-ts is
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
|
162
|
-
|
|
|
163
|
-
| **
|
|
164
|
-
| **
|
|
165
|
-
| **
|
|
166
|
-
| **Statistics** |
|
|
167
|
-
| **
|
|
168
|
-
| **
|
|
155
|
+
pond-ts is faster on **every** comparable operation, with no regressions —
|
|
156
|
+
a **~17x** geometric-mean speedup across the measurable ops, plus a handful
|
|
157
|
+
of transforms (`select` / `rename`) that are **effectively instant** (O(1)
|
|
158
|
+
column rebinds, below the timer's resolution). The advantage grows with data
|
|
159
|
+
size.
|
|
160
|
+
|
|
161
|
+
| Category | Speedup (N=16k) | Notes |
|
|
162
|
+
| ----------------- | ---------------------------------------------------- | --------------------------------------------- |
|
|
163
|
+
| **Rate** | ~120x | Single columnar walk vs Pipeline |
|
|
164
|
+
| **Fill** | 77–87x | Single columnar pass vs Pipeline per strategy |
|
|
165
|
+
| **Aggregation** | 57–82x | O(N+B) bucketing vs O(N×B) Pipeline |
|
|
166
|
+
| **Statistics** | 18–80x | Typed-array reduce vs ImmutableJS iteration |
|
|
167
|
+
| **Alignment** | 42x | Forward cursor vs repeated binary search |
|
|
168
|
+
| **Construction** | 13x | Columnar intake vs ImmutableJS wrapping |
|
|
169
|
+
| **Chained** | 8x | Derived constructors vs per-step Pipeline |
|
|
170
|
+
| **Transforms** | `select`/`rename` instant; `collapse` 30x; `map` ~4x | Column reshapes vs Pipeline |
|
|
171
|
+
| **Event access** | 6x | Array indexing vs ImmutableJS `get()` |
|
|
172
|
+
| **Serialization** | 4x | Lightweight columnar representation |
|
|
169
173
|
|
|
170
174
|
See the [full benchmark results](website/docs/reference/benchmarks.mdx)
|
|
171
175
|
for detailed numbers. Run locally:
|
|
@@ -34,30 +34,6 @@ export type AggregateColumnSpec = {
|
|
|
34
34
|
* same `mapping` shape produces the same schema.
|
|
35
35
|
*/
|
|
36
36
|
export declare function normalizeAggregateColumns<S extends SeriesSchema>(schema: S, mapping: AggregateMap<S> | AggregateOutputMap<S>): AggregateColumnSpec[];
|
|
37
|
-
/**
|
|
38
|
-
* **Phase 4.7 step 3B — columnar fast path for time-keyed `aggregate()`.**
|
|
39
|
-
* On sorted time-keyed data each bucket is a contiguous index range, so
|
|
40
|
-
* when every mapped column is a built-in numeric reducer with a
|
|
41
|
-
* `reduceColumn` fast path over a **packed `Float64Column`** source, each
|
|
42
|
-
* bucket reduces straight off the typed-array slice — skipping the
|
|
43
|
-
* `series.events` materialization and the per-cell `state.add` walk the
|
|
44
|
-
* row path pays. Reuses the shipped step-3A `reduceColumn` kernels
|
|
45
|
-
* (sum/min/max/avg 59–73×, stdev 35×, median/p95 3.4×) per bucket.
|
|
46
|
-
*
|
|
47
|
-
* Returns `null` — caller takes the unchanged row path — when any column
|
|
48
|
-
* doesn't qualify: a custom-function reducer, a reducer with no
|
|
49
|
-
* `reduceColumn` (`first` / `last` / `unique` / `top` / `samples`), or a
|
|
50
|
-
* non-numeric / chunked / missing source column. All-or-nothing per call
|
|
51
|
-
* keeps the bucket walk single-pass; mixed mappings fall back wholesale.
|
|
52
|
-
*
|
|
53
|
-
* `begins` is the key column's begin axis (sorted, identical row order to
|
|
54
|
-
* the value columns — both read straight off the store). Bucketing
|
|
55
|
-
* replicates the row path exactly: `cursor` carries across buckets, and a
|
|
56
|
-
* bucket owns `[start, scan)` where `begins[i] ∈ [bucket.begin,
|
|
57
|
-
* bucket.end)`. Empty buckets reduce an empty slice — the reducer's
|
|
58
|
-
* empty-input result (the step-3A parity contract guarantees this matches
|
|
59
|
-
* a zero-`add` bucket snapshot).
|
|
60
|
-
*/
|
|
61
37
|
export declare function tryAggregateColumnarTimeKeyed<B extends {
|
|
62
38
|
begin(): number;
|
|
63
39
|
end(): number;
|
|
@@ -73,45 +73,34 @@ export function normalizeAggregateColumns(schema, mapping) {
|
|
|
73
73
|
}
|
|
74
74
|
return normalized;
|
|
75
75
|
}
|
|
76
|
-
/**
|
|
77
|
-
* **Phase 4.7 step 3B — columnar fast path for time-keyed `aggregate()`.**
|
|
78
|
-
* On sorted time-keyed data each bucket is a contiguous index range, so
|
|
79
|
-
* when every mapped column is a built-in numeric reducer with a
|
|
80
|
-
* `reduceColumn` fast path over a **packed `Float64Column`** source, each
|
|
81
|
-
* bucket reduces straight off the typed-array slice — skipping the
|
|
82
|
-
* `series.events` materialization and the per-cell `state.add` walk the
|
|
83
|
-
* row path pays. Reuses the shipped step-3A `reduceColumn` kernels
|
|
84
|
-
* (sum/min/max/avg 59–73×, stdev 35×, median/p95 3.4×) per bucket.
|
|
85
|
-
*
|
|
86
|
-
* Returns `null` — caller takes the unchanged row path — when any column
|
|
87
|
-
* doesn't qualify: a custom-function reducer, a reducer with no
|
|
88
|
-
* `reduceColumn` (`first` / `last` / `unique` / `top` / `samples`), or a
|
|
89
|
-
* non-numeric / chunked / missing source column. All-or-nothing per call
|
|
90
|
-
* keeps the bucket walk single-pass; mixed mappings fall back wholesale.
|
|
91
|
-
*
|
|
92
|
-
* `begins` is the key column's begin axis (sorted, identical row order to
|
|
93
|
-
* the value columns — both read straight off the store). Bucketing
|
|
94
|
-
* replicates the row path exactly: `cursor` carries across buckets, and a
|
|
95
|
-
* bucket owns `[start, scan)` where `begins[i] ∈ [bucket.begin,
|
|
96
|
-
* bucket.end)`. Empty buckets reduce an empty slice — the reducer's
|
|
97
|
-
* empty-input result (the step-3A parity contract guarantees this matches
|
|
98
|
-
* a zero-`add` bucket snapshot).
|
|
99
|
-
*/
|
|
100
76
|
export function tryAggregateColumnarTimeKeyed(begins, getColumn, buckets, columns) {
|
|
101
77
|
const plans = [];
|
|
102
78
|
for (const spec of columns) {
|
|
103
79
|
if (typeof spec.reducer !== 'string')
|
|
104
80
|
return null; // custom function
|
|
105
81
|
const def = resolveReducer(spec.reducer);
|
|
106
|
-
if (def.reduceColumn === undefined)
|
|
107
|
-
return null; // first/last/unique/...
|
|
108
82
|
const source = getColumn(spec.source);
|
|
109
|
-
if (source === undefined
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
83
|
+
if (source === undefined)
|
|
84
|
+
return null; // missing source
|
|
85
|
+
if (def.definedBoundary !== undefined) {
|
|
86
|
+
// `first` / `last`: pick the first/last *defined* cell in the bucket
|
|
87
|
+
// via a boundary scan over any column kind / storage (`col.read(i)`).
|
|
88
|
+
// This is what lets a partitioned `aggregate` — whose auto-injected
|
|
89
|
+
// partition-column reducer is `'first'` — take the fast path instead
|
|
90
|
+
// of bailing the whole call for lack of a numeric `reduceColumn`.
|
|
91
|
+
plans.push({
|
|
92
|
+
kind: 'boundary',
|
|
93
|
+
column: source,
|
|
94
|
+
which: def.definedBoundary,
|
|
95
|
+
});
|
|
96
|
+
continue;
|
|
113
97
|
}
|
|
114
|
-
|
|
98
|
+
if (def.reduceColumn === undefined)
|
|
99
|
+
return null; // unique / top / samples / keep
|
|
100
|
+
if (source.kind !== 'number' || source.storage !== 'packed') {
|
|
101
|
+
return null; // non-numeric / chunked numeric source
|
|
102
|
+
}
|
|
103
|
+
plans.push({ kind: 'reduce', column: source, reduce: def.reduceColumn });
|
|
115
104
|
}
|
|
116
105
|
const n = begins.length;
|
|
117
106
|
let cursor = 0;
|
|
@@ -130,7 +119,41 @@ export function tryAggregateColumnarTimeKeyed(begins, getColumn, buckets, column
|
|
|
130
119
|
const reduced = new Array(plans.length);
|
|
131
120
|
for (let p = 0; p < plans.length; p += 1) {
|
|
132
121
|
const plan = plans[p];
|
|
133
|
-
|
|
122
|
+
if (plan.kind === 'reduce') {
|
|
123
|
+
reduced[p] = plan.reduce(plan.column.sliceByRange(start, scan));
|
|
124
|
+
}
|
|
125
|
+
else if (plan.which === 'first') {
|
|
126
|
+
// First defined cell in [start, scan); scans past missing cells and
|
|
127
|
+
// past non-finite numeric cells (reducer non-finite policy,
|
|
128
|
+
// docs/notes/reducer-nan-policy.md — a NaN/±Inf numeric is "not a
|
|
129
|
+
// contributor", matching the row path's `defined` filter).
|
|
130
|
+
let value;
|
|
131
|
+
for (let i = start; i < scan; i += 1) {
|
|
132
|
+
const cell = plan.column.read(i);
|
|
133
|
+
if (cell === undefined)
|
|
134
|
+
continue;
|
|
135
|
+
if (typeof cell === 'number' && !Number.isFinite(cell))
|
|
136
|
+
continue;
|
|
137
|
+
value = cell;
|
|
138
|
+
break;
|
|
139
|
+
}
|
|
140
|
+
reduced[p] = value;
|
|
141
|
+
}
|
|
142
|
+
else {
|
|
143
|
+
// Last defined cell in [start, scan); scans backward past missing and
|
|
144
|
+
// past non-finite numeric cells (see the 'first' branch above).
|
|
145
|
+
let value;
|
|
146
|
+
for (let i = scan - 1; i >= start; i -= 1) {
|
|
147
|
+
const cell = plan.column.read(i);
|
|
148
|
+
if (cell === undefined)
|
|
149
|
+
continue;
|
|
150
|
+
if (typeof cell === 'number' && !Number.isFinite(cell))
|
|
151
|
+
continue;
|
|
152
|
+
value = cell;
|
|
153
|
+
break;
|
|
154
|
+
}
|
|
155
|
+
reduced[p] = value;
|
|
156
|
+
}
|
|
134
157
|
}
|
|
135
158
|
rows[b] = Object.freeze([bucket, ...reduced]);
|
|
136
159
|
}
|
|
@@ -196,8 +196,6 @@ export declare class PartitionedTimeSeries<S extends SeriesSchema, K extends str
|
|
|
196
196
|
toMap(): Map<K, TimeSeries<S>>;
|
|
197
197
|
toMap<R extends SeriesSchema>(transform: (group: TimeSeries<S>) => TimeSeries<R>): Map<K, TimeSeries<R>>;
|
|
198
198
|
toMap<R>(transform: (group: TimeSeries<S>) => R): Map<K, R>;
|
|
199
|
-
private static partitionKeyOf;
|
|
200
|
-
private static bucketByPartition;
|
|
201
199
|
private static applyToSource;
|
|
202
200
|
private rewrap;
|
|
203
201
|
/**
|
|
@@ -88,17 +88,17 @@ export class PartitionedTimeSeries {
|
|
|
88
88
|
this.validateGroupMembership();
|
|
89
89
|
}
|
|
90
90
|
}
|
|
91
|
-
// Validate that every
|
|
92
|
-
//
|
|
93
|
-
//
|
|
91
|
+
// Validate that every partition value appears in the declared groups.
|
|
92
|
+
// Scans the partition column columnar-natively via `_distinctPartitionKeys`
|
|
93
|
+
// (no event materialization — same encoding `toMap` produces), so the
|
|
94
|
+
// declared-`groups` path is materialization-free like the rest of the
|
|
95
|
+
// split.
|
|
94
96
|
validateGroupMembership() {
|
|
95
97
|
if (!this.groups)
|
|
96
98
|
return;
|
|
97
99
|
const col = this.by[0];
|
|
98
100
|
const declared = new Set(this.groups);
|
|
99
|
-
const
|
|
100
|
-
for (const event of this.source.events) {
|
|
101
|
-
const key = keyOf(event);
|
|
101
|
+
for (const key of this.source._distinctPartitionKeys(this.by)) {
|
|
102
102
|
if (!declared.has(key)) {
|
|
103
103
|
// Decode the encoder's leading-space sentinel so the message
|
|
104
104
|
// shows the user-facing concept, not the internal encoding.
|
|
@@ -205,96 +205,54 @@ export class PartitionedTimeSeries {
|
|
|
205
205
|
}
|
|
206
206
|
toMap(transform) {
|
|
207
207
|
const result = new Map();
|
|
208
|
-
|
|
208
|
+
// Columnar partition split — each sub-series is gathered straight off
|
|
209
|
+
// the store, no event materialization (the old path rebuilt every
|
|
210
|
+
// partition via `fromEvents`). Map order = first-encountered partition.
|
|
211
|
+
const partitions = this.source.length === 0
|
|
209
212
|
? new Map()
|
|
210
|
-
:
|
|
213
|
+
: this.source._partitionByColumns(this.by);
|
|
211
214
|
if (this.groups) {
|
|
212
215
|
// Declared-order iteration. Empty groups produce empty
|
|
213
216
|
// TimeSeries entries (consistent with pivotByGroup's typed
|
|
214
217
|
// groups behavior, which emits a column for every declared
|
|
215
218
|
// value even when no events match).
|
|
216
219
|
for (const g of this.groups) {
|
|
217
|
-
const
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
220
|
+
const sub = partitions.get(g) ??
|
|
221
|
+
TimeSeries.fromEvents([], {
|
|
222
|
+
schema: this.source.schema,
|
|
223
|
+
name: this.source.name,
|
|
224
|
+
});
|
|
222
225
|
result.set(g, transform ? transform(sub) : sub);
|
|
223
226
|
}
|
|
224
227
|
return result;
|
|
225
228
|
}
|
|
226
229
|
// Insertion-order iteration (matches the order each partition was
|
|
227
230
|
// first encountered in the source events).
|
|
228
|
-
for (const [key,
|
|
229
|
-
const sub = TimeSeries.fromEvents(events, {
|
|
230
|
-
schema: this.source.schema,
|
|
231
|
-
name: this.source.name,
|
|
232
|
-
});
|
|
231
|
+
for (const [key, sub] of partitions) {
|
|
233
232
|
result.set(key, transform ? transform(sub) : sub);
|
|
234
233
|
}
|
|
235
234
|
return result;
|
|
236
235
|
}
|
|
237
|
-
// Build the encoder that produces a string key for an event given
|
|
238
|
-
// the partition columns. Single-column case avoids the JSON encoding
|
|
239
|
-
// overhead. Multi-column uses JSON.stringify to guarantee no key
|
|
240
|
-
// collisions on values containing separators (e.g. region names with
|
|
241
|
-
// spaces) — a naive `parts.join('|')` would collide. `undefined` in a
|
|
242
|
-
// single-column key becomes the literal `' undefined'` (with the
|
|
243
|
-
// leading space ensuring it can never collide with a string column
|
|
244
|
-
// whose value is the literal `'undefined'`).
|
|
245
|
-
static partitionKeyOf(by) {
|
|
246
|
-
if (by.length === 1) {
|
|
247
|
-
const col = by[0];
|
|
248
|
-
return (event) => {
|
|
249
|
-
const v = event.data()[col];
|
|
250
|
-
return v === undefined ? ' undefined' : `${String(v)}`;
|
|
251
|
-
};
|
|
252
|
-
}
|
|
253
|
-
return (event) => {
|
|
254
|
-
const data = event.data();
|
|
255
|
-
const parts = new Array(by.length);
|
|
256
|
-
for (let i = 0; i < by.length; i += 1) {
|
|
257
|
-
parts[i] = data[by[i]] ?? null;
|
|
258
|
-
}
|
|
259
|
-
return JSON.stringify(parts);
|
|
260
|
-
};
|
|
261
|
-
}
|
|
262
|
-
// Group source events into buckets keyed by partition value. Returned
|
|
263
|
-
// Map iteration order = insertion order, which matches the order
|
|
264
|
-
// partitions were first seen in the source events array.
|
|
265
|
-
static bucketByPartition(source, by) {
|
|
266
|
-
const keyOf = PartitionedTimeSeries.partitionKeyOf(by);
|
|
267
|
-
const buckets = new Map();
|
|
268
|
-
for (const event of source.events) {
|
|
269
|
-
const key = keyOf(event);
|
|
270
|
-
let bucket = buckets.get(key);
|
|
271
|
-
if (!bucket) {
|
|
272
|
-
bucket = [];
|
|
273
|
-
buckets.set(key, bucket);
|
|
274
|
-
}
|
|
275
|
-
bucket.push(event);
|
|
276
|
-
}
|
|
277
|
-
return buckets;
|
|
278
|
-
}
|
|
279
236
|
// Internal helper used by both `apply` (terminal) and the sugar
|
|
280
237
|
// methods (which re-wrap the result back into a partitioned view).
|
|
281
238
|
static applyToSource(source, by, fn) {
|
|
282
239
|
// Empty source: apply fn to an empty group so the output schema
|
|
283
240
|
// and name come from fn, not from inferring R structurally.
|
|
284
|
-
if (source.
|
|
241
|
+
if (source.length === 0) {
|
|
285
242
|
const empty = TimeSeries.fromEvents([], {
|
|
286
243
|
schema: source.schema,
|
|
287
244
|
name: source.name,
|
|
288
245
|
});
|
|
289
246
|
return fn(empty);
|
|
290
247
|
}
|
|
291
|
-
|
|
248
|
+
// Columnar partition split — no event materialization. The old path
|
|
249
|
+
// walked `source.events` to bucket, then rebuilt each partition via
|
|
250
|
+
// `fromEvents` (re-validating + re-packing), silently re-paying the
|
|
251
|
+
// 495 ns/row tax the columnar wave removed. `_partitionByColumns`
|
|
252
|
+
// gathers each partition straight off the store (audit v2 §3.2).
|
|
253
|
+
const partitions = source._partitionByColumns(by);
|
|
292
254
|
const transformed = [];
|
|
293
|
-
for (const
|
|
294
|
-
const sub = TimeSeries.fromEvents(events, {
|
|
295
|
-
schema: source.schema,
|
|
296
|
-
name: source.name,
|
|
297
|
-
});
|
|
255
|
+
for (const sub of partitions.values()) {
|
|
298
256
|
transformed.push(fn(sub));
|
|
299
257
|
}
|
|
300
258
|
return TimeSeries.concat(transformed);
|
|
@@ -12,10 +12,10 @@ import { TimeRange } from '../core/time-range.js';
|
|
|
12
12
|
import { compareEventKeys } from '../core/temporal.js';
|
|
13
13
|
import { PartitionedTimeSeries } from './partitioned-time-series.js';
|
|
14
14
|
import { Sequence } from '../sequence/sequence.js';
|
|
15
|
-
import { IntervalKeyColumn, TimeKeyColumn, TimeRangeKeyColumn, float64ColumnFromArray, stringColumnFromArray, withColumnsRenamed, withColumnsSelected, withKeyColumn, withRowRange, } from '../columnar/index.js';
|
|
15
|
+
import { IntervalKeyColumn, TimeKeyColumn, TimeRangeKeyColumn, float64ColumnFromArray, stringColumnFromArray, withColumnsRenamed, withColumnsSelected, withKeyColumn, withRowRange, withRowSelection, } from '../columnar/index.js';
|
|
16
16
|
import { SeriesStore } from '../live/series-store.js';
|
|
17
17
|
import { parseDuration } from '../core/duration.js';
|
|
18
|
-
import { resolveReducer, } from '../reducers/index.js';
|
|
18
|
+
import { resolveReducer, bucketStateFor, rollingStateFor, } from '../reducers/index.js';
|
|
19
19
|
// JSON ↔ typed-row primitives live in `./json.js`. Both `TimeSeries`
|
|
20
20
|
// and `LiveSeries` reach for them; extracted to break the import cycle
|
|
21
21
|
// that would otherwise form (Event needs them, TimeSeries imports
|
|
@@ -274,7 +274,11 @@ function bucketOverlapsHalfOpen(bucket, event) {
|
|
|
274
274
|
return event.begin() < bucket.end() && bucket.begin() < event.end();
|
|
275
275
|
}
|
|
276
276
|
function aggregateValues(operation, values) {
|
|
277
|
-
|
|
277
|
+
// Non-finite numerics (NaN / ±Inf) are treated as missing — excluded from
|
|
278
|
+
// both `defined` and `numeric` so every built-in reducer skips them exactly
|
|
279
|
+
// as it skips a missing cell. See docs/notes/reducer-nan-policy.md.
|
|
280
|
+
const defined = values.filter((value) => value !== undefined &&
|
|
281
|
+
(typeof value !== 'number' || Number.isFinite(value)));
|
|
278
282
|
const numeric = defined.filter((value) => typeof value === 'number');
|
|
279
283
|
return resolveReducer(operation).reduce(defined, numeric);
|
|
280
284
|
}
|
|
@@ -315,7 +319,9 @@ function applyAggregateReducer(reducer, values) {
|
|
|
315
319
|
// LivePartitionedSyncRolling) share the same normalisation. See the
|
|
316
320
|
// import below.
|
|
317
321
|
function createAggregateBucketState(operation) {
|
|
318
|
-
|
|
322
|
+
// Delegates to the shared factory so the non-finite skip policy
|
|
323
|
+
// (docs/notes/reducer-nan-policy.md) applies uniformly to batch + live.
|
|
324
|
+
return bucketStateFor(operation);
|
|
319
325
|
}
|
|
320
326
|
/**
|
|
321
327
|
* Resolve the output column kind for an `arrayAggregate` call. Numeric
|
|
@@ -336,7 +342,9 @@ function resolveArrayAggregateKind(reducer, explicitKind) {
|
|
|
336
342
|
return 'string';
|
|
337
343
|
}
|
|
338
344
|
function createRollingReducerState(operation) {
|
|
339
|
-
|
|
345
|
+
// Delegates to the shared factory so the non-finite skip policy
|
|
346
|
+
// (docs/notes/reducer-nan-policy.md) applies uniformly to batch + live.
|
|
347
|
+
return rollingStateFor(operation);
|
|
340
348
|
}
|
|
341
349
|
function duplicateValueColumnNames(schemas) {
|
|
342
350
|
const counts = new Map();
|
|
@@ -710,6 +718,98 @@ export class TimeSeries {
|
|
|
710
718
|
};
|
|
711
719
|
return new TimeSeries(trustedInput);
|
|
712
720
|
}
|
|
721
|
+
/**
|
|
722
|
+
* @internal Per-row partition-key encoder, mirroring the (removed)
|
|
723
|
+
* event-based `partitionKeyOf` exactly: single column → `String(value)`,
|
|
724
|
+
* or `' undefined'` (leading space, so it can't collide with the literal
|
|
725
|
+
* string `'undefined'`) for a missing cell; composite → `JSON.stringify`
|
|
726
|
+
* of the cells with a `?? null` fallback. Reads cells by index off the
|
|
727
|
+
* store — no event materialization; array cells stringify as the event
|
|
728
|
+
* path did. Shared by `_partitionByColumns` and `_distinctPartitionKeys`
|
|
729
|
+
* so the encoding can't drift between the split and the declared-`groups`
|
|
730
|
+
* membership check.
|
|
731
|
+
*/
|
|
732
|
+
#partitionKeyEncoder(by) {
|
|
733
|
+
const store = this.#store.store;
|
|
734
|
+
if (by.length === 1) {
|
|
735
|
+
// Single-column fast path — no per-row array allocation.
|
|
736
|
+
const col = store.columns.get(by[0]);
|
|
737
|
+
return (i) => {
|
|
738
|
+
const value = col.read(i);
|
|
739
|
+
return value === undefined ? ' undefined' : `${String(value)}`;
|
|
740
|
+
};
|
|
741
|
+
}
|
|
742
|
+
const cols = by.map((name) => store.columns.get(name));
|
|
743
|
+
const parts = new Array(cols.length);
|
|
744
|
+
return (i) => {
|
|
745
|
+
for (let c = 0; c < cols.length; c += 1) {
|
|
746
|
+
parts[c] = cols[c].read(i) ?? null;
|
|
747
|
+
}
|
|
748
|
+
return JSON.stringify(parts);
|
|
749
|
+
};
|
|
750
|
+
}
|
|
751
|
+
/**
|
|
752
|
+
* @internal Columnar partition split — groups row indices by the partition
|
|
753
|
+
* key (see {@link TimeSeries.#partitionKeyEncoder}), gathers each group via
|
|
754
|
+
* `withRowSelection`, and wraps it as a trusted-store `TimeSeries`, with
|
|
755
|
+
* **no event materialization**. The columnar dual of the old
|
|
756
|
+
* `fromEvents`-per-bucket split that `PartitionedTimeSeries` paid through
|
|
757
|
+
* `this.events`. First-encountered partition order is preserved (Map
|
|
758
|
+
* insertion order), matching the previous `bucketByPartition`.
|
|
759
|
+
*
|
|
760
|
+
* **Identity note:** each sub-series lazily materializes its own `Event`
|
|
761
|
+
* objects from the gathered store — it does NOT reuse the source's `Event`
|
|
762
|
+
* instances (the old `fromEvents` path did). Cell values are identical;
|
|
763
|
+
* only object identity differs, and only for the no-transform paths
|
|
764
|
+
* (`toMap()` / `apply(g => g)`). `collect()` returns the source unchanged,
|
|
765
|
+
* so it is unaffected. This is the deliberate cost of skipping
|
|
766
|
+
* materialization.
|
|
767
|
+
*
|
|
768
|
+
* Used by `PartitionedTimeSeries` (`collect` / sugar via `applyToSource`,
|
|
769
|
+
* and `toMap`).
|
|
770
|
+
*/
|
|
771
|
+
_partitionByColumns(by) {
|
|
772
|
+
const columnarStore = this.#store.store;
|
|
773
|
+
const length = columnarStore.length;
|
|
774
|
+
const encode = this.#partitionKeyEncoder(by);
|
|
775
|
+
const groups = new Map();
|
|
776
|
+
for (let i = 0; i < length; i += 1) {
|
|
777
|
+
const key = encode(i);
|
|
778
|
+
let indices = groups.get(key);
|
|
779
|
+
if (indices === undefined) {
|
|
780
|
+
indices = [];
|
|
781
|
+
groups.set(key, indices);
|
|
782
|
+
}
|
|
783
|
+
indices.push(i);
|
|
784
|
+
}
|
|
785
|
+
const result = new Map();
|
|
786
|
+
for (const [key, rowIndices] of groups) {
|
|
787
|
+
const sub = withRowSelection(columnarStore, new Int32Array(rowIndices));
|
|
788
|
+
result.set(key, TimeSeries.#fromTrustedStore(this.name, this.schema, sub));
|
|
789
|
+
}
|
|
790
|
+
return result;
|
|
791
|
+
}
|
|
792
|
+
/**
|
|
793
|
+
* @internal Distinct partition keys in first-encountered order, encoded by
|
|
794
|
+
* {@link TimeSeries.#partitionKeyEncoder} (identical to
|
|
795
|
+
* `_partitionByColumns`). Scans the partition columns off the store — no
|
|
796
|
+
* event materialization. Used by `PartitionedTimeSeries`'s declared-`groups`
|
|
797
|
+
* membership check so that path is materialization-free too.
|
|
798
|
+
*/
|
|
799
|
+
_distinctPartitionKeys(by) {
|
|
800
|
+
const length = this.#store.store.length;
|
|
801
|
+
const encode = this.#partitionKeyEncoder(by);
|
|
802
|
+
const seen = new Set();
|
|
803
|
+
const keys = [];
|
|
804
|
+
for (let i = 0; i < length; i += 1) {
|
|
805
|
+
const key = encode(i);
|
|
806
|
+
if (!seen.has(key)) {
|
|
807
|
+
seen.add(key);
|
|
808
|
+
keys.push(key);
|
|
809
|
+
}
|
|
810
|
+
}
|
|
811
|
+
return keys;
|
|
812
|
+
}
|
|
713
813
|
/**
|
|
714
814
|
* Example: `series.events`. Returns the full event array.
|
|
715
815
|
*
|
|
@@ -2401,12 +2501,37 @@ export class TimeSeries {
|
|
|
2401
2501
|
}
|
|
2402
2502
|
/** Example: `series.timeRange()`. Returns the overall temporal extent of the series, if the series is not empty. */
|
|
2403
2503
|
timeRange() {
|
|
2404
|
-
|
|
2405
|
-
|
|
2504
|
+
// Columnar key-axis read. The old implementation reduced over
|
|
2505
|
+
// `this.events`, materializing every Event (the ~495 ns/row + heap
|
|
2506
|
+
// tax) on every call — including the cold `aggregate()` default
|
|
2507
|
+
// range, which erased the 3B fast-path win for one-shot pipelines.
|
|
2508
|
+
const key = this.keyColumn();
|
|
2509
|
+
const n = key.length;
|
|
2510
|
+
if (n === 0) {
|
|
2406
2511
|
return undefined;
|
|
2407
2512
|
}
|
|
2408
|
-
|
|
2409
|
-
|
|
2513
|
+
// `begin` is sorted non-decreasing (the intake invariant) for every
|
|
2514
|
+
// key kind, so the minimum start is always `begin[0]`.
|
|
2515
|
+
const start = key.begin[0];
|
|
2516
|
+
let end;
|
|
2517
|
+
if (key.kind === 'time') {
|
|
2518
|
+
// Point-in-time keys: `end === begin`, and begin is sorted, so the
|
|
2519
|
+
// maximum end is the final begin. O(1).
|
|
2520
|
+
end = key.begin[n - 1];
|
|
2521
|
+
}
|
|
2522
|
+
else {
|
|
2523
|
+
// Range / interval keys: `begin[i] <= end[i]` per row, but `end` is
|
|
2524
|
+
// NOT monotonic (a long early event can outlast the final row), so
|
|
2525
|
+
// the max end needs a scan — a typed-array scan, not Event
|
|
2526
|
+
// materialization. Seeds with `end[0]` and maxes the rest, matching
|
|
2527
|
+
// the old `reduce` seeded at `first.end()`.
|
|
2528
|
+
const ends = key.end;
|
|
2529
|
+
end = ends[0];
|
|
2530
|
+
for (let i = 1; i < n; i += 1) {
|
|
2531
|
+
if (ends[i] > end)
|
|
2532
|
+
end = ends[i];
|
|
2533
|
+
}
|
|
2534
|
+
}
|
|
2410
2535
|
return new TimeRange({ start, end });
|
|
2411
2536
|
}
|
|
2412
2537
|
/** Example: `series.overlaps(range)`. Returns `true` when the overall series extent overlaps the supplied temporal value. */
|
package/dist/batch/validate.js
CHANGED
|
@@ -416,7 +416,13 @@ export function validateAndNormalizeColumnar(input) {
|
|
|
416
416
|
let column;
|
|
417
417
|
switch (kind) {
|
|
418
418
|
case 'number':
|
|
419
|
-
|
|
419
|
+
// Strict intake: `assertCellKind` (kind 'number') already
|
|
420
|
+
// rejected every non-finite cell with a `ValidationError`
|
|
421
|
+
// before it reached `numberBufs`, so a surviving column is
|
|
422
|
+
// provably all-finite → `allFinite: true` (lets reducers skip
|
|
423
|
+
// the per-element finite guard). See the reducer non-finite
|
|
424
|
+
// policy + `Float64Column.allFinite`'s safety contract.
|
|
425
|
+
column = new Float64Column(numberBufs[c], length, validity, true);
|
|
420
426
|
break;
|
|
421
427
|
case 'boolean':
|
|
422
428
|
column = new BooleanColumn(booleanBufs[c], length, validity);
|