backfold 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,346 @@
1
+ # This file is automatically @generated by Cargo.
2
+ # It is not intended for manual editing.
3
+ version = 4
4
+
5
+ [[package]]
6
+ name = "autocfg"
7
+ version = "1.5.1"
8
+ source = "registry+https://github.com/rust-lang/crates.io-index"
9
+ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
10
+
11
+ [[package]]
12
+ name = "backfold-cli"
13
+ version = "0.2.0"
14
+ dependencies = [
15
+ "backfold-core",
16
+ "serde",
17
+ "serde_json",
18
+ ]
19
+
20
+ [[package]]
21
+ name = "backfold-core"
22
+ version = "0.2.0"
23
+ dependencies = [
24
+ "rand",
25
+ "serde",
26
+ ]
27
+
28
+ [[package]]
29
+ name = "backfold-py"
30
+ version = "0.2.0"
31
+ dependencies = [
32
+ "backfold-core",
33
+ "pyo3",
34
+ ]
35
+
36
+ [[package]]
37
+ name = "cfg-if"
38
+ version = "1.0.5"
39
+ source = "registry+https://github.com/rust-lang/crates.io-index"
40
+ checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
41
+
42
+ [[package]]
43
+ name = "getrandom"
44
+ version = "0.2.17"
45
+ source = "registry+https://github.com/rust-lang/crates.io-index"
46
+ checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"
47
+ dependencies = [
48
+ "cfg-if",
49
+ "libc",
50
+ "wasi",
51
+ ]
52
+
53
+ [[package]]
54
+ name = "heck"
55
+ version = "0.5.0"
56
+ source = "registry+https://github.com/rust-lang/crates.io-index"
57
+ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
58
+
59
+ [[package]]
60
+ name = "indoc"
61
+ version = "2.0.8"
62
+ source = "registry+https://github.com/rust-lang/crates.io-index"
63
+ checksum = "a37b2691796cffeb8a8cd305ac66e65841559f147f4e63231d0eafa4db5384d1"
64
+ dependencies = [
65
+ "rustversion",
66
+ ]
67
+
68
+ [[package]]
69
+ name = "itoa"
70
+ version = "1.0.19"
71
+ source = "registry+https://github.com/rust-lang/crates.io-index"
72
+ checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
73
+
74
+ [[package]]
75
+ name = "libc"
76
+ version = "0.2.191"
77
+ source = "registry+https://github.com/rust-lang/crates.io-index"
78
+ checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a"
79
+
80
+ [[package]]
81
+ name = "memchr"
82
+ version = "2.8.3"
83
+ source = "registry+https://github.com/rust-lang/crates.io-index"
84
+ checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
85
+
86
+ [[package]]
87
+ name = "memoffset"
88
+ version = "0.9.1"
89
+ source = "registry+https://github.com/rust-lang/crates.io-index"
90
+ checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a"
91
+ dependencies = [
92
+ "autocfg",
93
+ ]
94
+
95
+ [[package]]
96
+ name = "once_cell"
97
+ version = "1.21.4"
98
+ source = "registry+https://github.com/rust-lang/crates.io-index"
99
+ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
100
+
101
+ [[package]]
102
+ name = "portable-atomic"
103
+ version = "1.15.0"
104
+ source = "registry+https://github.com/rust-lang/crates.io-index"
105
+ checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
106
+
107
+ [[package]]
108
+ name = "ppv-lite86"
109
+ version = "0.2.21"
110
+ source = "registry+https://github.com/rust-lang/crates.io-index"
111
+ checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
112
+ dependencies = [
113
+ "zerocopy",
114
+ ]
115
+
116
+ [[package]]
117
+ name = "proc-macro2"
118
+ version = "1.0.107"
119
+ source = "registry+https://github.com/rust-lang/crates.io-index"
120
+ checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
121
+ dependencies = [
122
+ "unicode-ident",
123
+ ]
124
+
125
+ [[package]]
126
+ name = "pyo3"
127
+ version = "0.24.2"
128
+ source = "registry+https://github.com/rust-lang/crates.io-index"
129
+ checksum = "e5203598f366b11a02b13aa20cab591229ff0a89fd121a308a5df751d5fc9219"
130
+ dependencies = [
131
+ "cfg-if",
132
+ "indoc",
133
+ "libc",
134
+ "memoffset",
135
+ "once_cell",
136
+ "portable-atomic",
137
+ "pyo3-build-config",
138
+ "pyo3-ffi",
139
+ "pyo3-macros",
140
+ "unindent",
141
+ ]
142
+
143
+ [[package]]
144
+ name = "pyo3-build-config"
145
+ version = "0.24.2"
146
+ source = "registry+https://github.com/rust-lang/crates.io-index"
147
+ checksum = "99636d423fa2ca130fa5acde3059308006d46f98caac629418e53f7ebb1e9999"
148
+ dependencies = [
149
+ "once_cell",
150
+ "target-lexicon",
151
+ ]
152
+
153
+ [[package]]
154
+ name = "pyo3-ffi"
155
+ version = "0.24.2"
156
+ source = "registry+https://github.com/rust-lang/crates.io-index"
157
+ checksum = "78f9cf92ba9c409279bc3305b5409d90db2d2c22392d443a87df3a1adad59e33"
158
+ dependencies = [
159
+ "libc",
160
+ "pyo3-build-config",
161
+ ]
162
+
163
+ [[package]]
164
+ name = "pyo3-macros"
165
+ version = "0.24.2"
166
+ source = "registry+https://github.com/rust-lang/crates.io-index"
167
+ checksum = "0b999cb1a6ce21f9a6b147dcf1be9ffedf02e0043aec74dc390f3007047cecd9"
168
+ dependencies = [
169
+ "proc-macro2",
170
+ "pyo3-macros-backend",
171
+ "quote",
172
+ "syn 2.0.119",
173
+ ]
174
+
175
+ [[package]]
176
+ name = "pyo3-macros-backend"
177
+ version = "0.24.2"
178
+ source = "registry+https://github.com/rust-lang/crates.io-index"
179
+ checksum = "822ece1c7e1012745607d5cf0bcb2874769f0f7cb34c4cde03b9358eb9ef911a"
180
+ dependencies = [
181
+ "heck",
182
+ "proc-macro2",
183
+ "pyo3-build-config",
184
+ "quote",
185
+ "syn 2.0.119",
186
+ ]
187
+
188
+ [[package]]
189
+ name = "quote"
190
+ version = "1.0.47"
191
+ source = "registry+https://github.com/rust-lang/crates.io-index"
192
+ checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
193
+ dependencies = [
194
+ "proc-macro2",
195
+ ]
196
+
197
+ [[package]]
198
+ name = "rand"
199
+ version = "0.8.8"
200
+ source = "registry+https://github.com/rust-lang/crates.io-index"
201
+ checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c"
202
+ dependencies = [
203
+ "libc",
204
+ "rand_chacha",
205
+ "rand_core",
206
+ ]
207
+
208
+ [[package]]
209
+ name = "rand_chacha"
210
+ version = "0.3.1"
211
+ source = "registry+https://github.com/rust-lang/crates.io-index"
212
+ checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
213
+ dependencies = [
214
+ "ppv-lite86",
215
+ "rand_core",
216
+ ]
217
+
218
+ [[package]]
219
+ name = "rand_core"
220
+ version = "0.6.4"
221
+ source = "registry+https://github.com/rust-lang/crates.io-index"
222
+ checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
223
+ dependencies = [
224
+ "getrandom",
225
+ ]
226
+
227
+ [[package]]
228
+ name = "rustversion"
229
+ version = "1.0.23"
230
+ source = "registry+https://github.com/rust-lang/crates.io-index"
231
+ checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
232
+
233
+ [[package]]
234
+ name = "serde"
235
+ version = "1.0.229"
236
+ source = "registry+https://github.com/rust-lang/crates.io-index"
237
+ checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
238
+ dependencies = [
239
+ "serde_core",
240
+ "serde_derive",
241
+ ]
242
+
243
+ [[package]]
244
+ name = "serde_core"
245
+ version = "1.0.229"
246
+ source = "registry+https://github.com/rust-lang/crates.io-index"
247
+ checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
248
+ dependencies = [
249
+ "serde_derive",
250
+ ]
251
+
252
+ [[package]]
253
+ name = "serde_derive"
254
+ version = "1.0.229"
255
+ source = "registry+https://github.com/rust-lang/crates.io-index"
256
+ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
257
+ dependencies = [
258
+ "proc-macro2",
259
+ "quote",
260
+ "syn 3.0.10",
261
+ ]
262
+
263
+ [[package]]
264
+ name = "serde_json"
265
+ version = "1.0.154"
266
+ source = "registry+https://github.com/rust-lang/crates.io-index"
267
+ checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6"
268
+ dependencies = [
269
+ "itoa",
270
+ "memchr",
271
+ "serde",
272
+ "serde_core",
273
+ "zmij",
274
+ ]
275
+
276
+ [[package]]
277
+ name = "syn"
278
+ version = "2.0.119"
279
+ source = "registry+https://github.com/rust-lang/crates.io-index"
280
+ checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
281
+ dependencies = [
282
+ "proc-macro2",
283
+ "quote",
284
+ "unicode-ident",
285
+ ]
286
+
287
+ [[package]]
288
+ name = "syn"
289
+ version = "3.0.10"
290
+ source = "registry+https://github.com/rust-lang/crates.io-index"
291
+ checksum = "4ae9ccbea3a7b0b09443036490c8f34a8b1b760fd345e5c0dae0aac718faea79"
292
+ dependencies = [
293
+ "proc-macro2",
294
+ "quote",
295
+ "unicode-ident",
296
+ ]
297
+
298
+ [[package]]
299
+ name = "target-lexicon"
300
+ version = "0.13.6"
301
+ source = "registry+https://github.com/rust-lang/crates.io-index"
302
+ checksum = "8c447be3a2abfa84b2d91680c56600bd6b2873866de27b36ec911be705e291da"
303
+
304
+ [[package]]
305
+ name = "unicode-ident"
306
+ version = "1.0.27"
307
+ source = "registry+https://github.com/rust-lang/crates.io-index"
308
+ checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"
309
+
310
+ [[package]]
311
+ name = "unindent"
312
+ version = "0.2.4"
313
+ source = "registry+https://github.com/rust-lang/crates.io-index"
314
+ checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3"
315
+
316
+ [[package]]
317
+ name = "wasi"
318
+ version = "0.11.1+wasi-snapshot-preview1"
319
+ source = "registry+https://github.com/rust-lang/crates.io-index"
320
+ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
321
+
322
+ [[package]]
323
+ name = "zerocopy"
324
+ version = "0.8.64"
325
+ source = "registry+https://github.com/rust-lang/crates.io-index"
326
+ checksum = "ed43f3060383c64b000f3c630550aebdfa9d3b76d48de32390483dd2cc7e0865"
327
+ dependencies = [
328
+ "zerocopy-derive",
329
+ ]
330
+
331
+ [[package]]
332
+ name = "zerocopy-derive"
333
+ version = "0.8.64"
334
+ source = "registry+https://github.com/rust-lang/crates.io-index"
335
+ checksum = "a5cbd820fe3e35fb38c83b6dc7e6b1618bc46e0cd10073ae94e8d52dc468d8b9"
336
+ dependencies = [
337
+ "proc-macro2",
338
+ "quote",
339
+ "syn 2.0.119",
340
+ ]
341
+
342
+ [[package]]
343
+ name = "zmij"
344
+ version = "1.0.23"
345
+ source = "registry+https://github.com/rust-lang/crates.io-index"
346
+ checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
@@ -0,0 +1,18 @@
1
+ [workspace]
2
+ resolver = "2"
3
+ members = ["crates/backfold-core", "crates/backfold-py"]
4
+
5
+ [workspace.package]
6
+ version = "0.2.0"
7
+ edition = "2021"
8
+ license = "Apache-2.0"
9
+ description = "Persistent O(log T) associative-monoid engine for out-of-order, retractable, forkable online statistical inference and agent stream compression"
10
+ repository = "https://github.com/pgg898200-ship-it/backfold"
11
+ keywords = ["streaming", "time-series", "kalman", "online-learning", "changepoint"]
12
+ categories = ["algorithms", "science", "mathematics", "data-structures"]
13
+
14
+ [profile.release]
15
+ opt-level = 3
16
+ lto = "thin"
17
+ codegen-units = 1
18
+ panic = "unwind"
backfold-0.2.0/LICENSE ADDED
@@ -0,0 +1,34 @@
1
+ Apache License
2
+ Version 2.0, January 2004
3
+ http://www.apache.org/licenses/
4
+
5
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6
+
7
+ 1. Definitions.
8
+ "License" shall mean the terms and conditions for use, reproduction,
9
+ and distribution as defined by Sections 1 through 9 of this document.
10
+ "Licensor" shall mean the copyright owner or entity authorized by
11
+ the copyright owner that is granting the License.
12
+ "Legal Entity" shall mean the union of the acting entity and all
13
+ other entities that control, are controlled by, or are under common
14
+ control with that entity.
15
+ "You" (or "Your") shall mean an individual or Legal Entity
16
+ exercising permissions granted by this License.
17
+ "Source" form shall mean the preferred form for making modifications.
18
+ "Object" form shall mean any form resulting from mechanical
19
+ transformation or translation of a Source form.
20
+ "Work" shall mean the work of authorship made available under the License.
21
+
22
+ Copyright 2026 Backfold Contributors
23
+
24
+ Licensed under the Apache License, Version 2.0 (the "License");
25
+ you may not use this file except in compliance with the License.
26
+ You may obtain a copy of the License at
27
+
28
+ http://www.apache.org/licenses/LICENSE-2.0
29
+
30
+ Unless required by applicable law or agreed to in writing, software
31
+ distributed under the License is distributed on an "AS IS" BASIS,
32
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
33
+ See the License for the specific language governing permissions and
34
+ limitations under the License.
@@ -0,0 +1,203 @@
1
+ Metadata-Version: 2.4
2
+ Name: backfold
3
+ Version: 0.2.0
4
+ Classifier: Development Status :: 4 - Beta
5
+ Classifier: Intended Audience :: Developers
6
+ Classifier: Intended Audience :: Science/Research
7
+ Classifier: License :: OSI Approved :: Apache Software License
8
+ Classifier: Programming Language :: Rust
9
+ Classifier: Programming Language :: Python :: 3
10
+ Classifier: Programming Language :: Python :: 3.10
11
+ Classifier: Programming Language :: Python :: 3.11
12
+ Classifier: Programming Language :: Python :: 3.12
13
+ Classifier: Programming Language :: Python :: 3.13
14
+ Classifier: Topic :: Scientific/Engineering :: Mathematics
15
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
16
+ Requires-Dist: numpy>=1.23 ; extra == 'bench'
17
+ Requires-Dist: scipy>=1.10 ; extra == 'bench'
18
+ Requires-Dist: pandas>=2.0 ; extra == 'bench'
19
+ Requires-Dist: polars>=1.0 ; extra == 'bench'
20
+ Requires-Dist: river>=0.21 ; extra == 'bench'
21
+ Requires-Dist: pytest>=8.0 ; extra == 'dev'
22
+ Requires-Dist: ruff>=0.6 ; extra == 'dev'
23
+ Requires-Dist: numpy>=1.23 ; extra == 'dev'
24
+ Requires-Dist: numpy>=1.23 ; extra == 'numpy'
25
+ Provides-Extra: bench
26
+ Provides-Extra: dev
27
+ Provides-Extra: numpy
28
+ License-File: LICENSE
29
+ Summary: Persistent O(log T) associative-monoid engine for out-of-order, retractable, forkable online statistical inference and agent stream compression
30
+ Keywords: streaming,time-series,online-learning,kalman-filter,changepoint,e-values,conformal,multi-agent,crdt,rust
31
+ License: Apache-2.0
32
+ Requires-Python: >=3.10
33
+ Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
34
+ Project-URL: Documentation, https://pgg898200-ship-it.github.io/backfold/llms-full.txt
35
+ Project-URL: Homepage, https://pgg898200-ship-it.github.io/backfold/
36
+ Project-URL: Issues, https://github.com/pgg898200-ship-it/backfold/issues
37
+ Project-URL: Repository, https://github.com/pgg898200-ship-it/backfold
38
+
39
+ # backfold
40
+
41
+ [![CI](https://github.com/pgg898200-ship-it/backfold/actions/workflows/ci.yml/badge.svg)](https://github.com/pgg898200-ship-it/backfold/actions/workflows/ci.yml)
42
+ [![License: Apache-2.0](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](LICENSE)
43
+
44
+ **Persistent $O(\log T)$ out-of-order, retractable, forkable online statistical inference, exact RTS Bayesian smoothing & lossless-recoverable AI agent stream compression.**
45
+
46
+ Pure safe Rust (`#![forbid(unsafe_code)]`) · Zero-copy Python bindings · Sub-5ms native CLI · [Docs & `llms.txt`](https://pgg898200-ship-it.github.io/backfold/)
47
+
48
+ ---
49
+
50
+ > **Every streaming library assumes time moves forward, nothing gets retracted, and only one consumer reads the stream.**
51
+ > Real sensor pipelines, financial feeds, telemetry firehoses, and multi-agent LLM swarms break all three assumptions. When an event arrives 50,000 ticks late—or gets retracted, restated, or branched by 64 counterfactual agents—`river`, `pandas.ewm`, and sequential `numpy`/`scipy` Kalman filters/smoothers fall back to **$O(T)$ full forward-backward replay**.
52
+ >
53
+ > `backfold` compiles every operator into an **exact associative monoid** inside a persistent copy-on-write B+ fold tree (`FoldTree<M, B>`): **$O(\log T)$ late edits & retractions, $O(\log T)$ exact Rauch–Tung–Striebel (RTS) Bayesian smoothing & marginal log-likelihood $\log p(y_{1:T})$, $O(\log T)$ arbitrary range queries, $O(1)$ zero-copy `fork()` (~16 ns), echo-proof multi-agent CRDT merges, distribution-free e-BH FDR control, and `<300`-token lossless-recoverable LLM stream digests — with zero approximation (`max |diff| <= 1e-12`).**
54
+
55
+ ---
56
+
57
+ ## 30-Second Quickstart
58
+
59
+ ### Python (`pip install backfold`)
60
+ ```python
61
+ import numpy as np
62
+ import backfold as bf
63
+
64
+ ts = np.arange(100_000, dtype=np.int64)
65
+ dts = np.ones(100_000)
66
+ ys = np.cumsum(np.random.default_rng(0).normal(0.05, 0.1, 100_000))
67
+ m0, p0 = [0.0, 0.0], [10.0, 0.0, 0.0, 10.0]
68
+
69
+ # 1. Exact 2D Constant-Velocity Kalman filter + O(log T) RTS smoother & marginal log-likelihood
70
+ kf = bf.KalmanStream(dim=2)
71
+ kf.ingest_constant_velocity(ts, ys, dts, q_accel=1e-3, r_obs=0.25)
72
+
73
+ # 2. O(1) zero-copy counterfactual fork (~16 ns, 0 bytes copied) + O(log T) late repair + RTS smooth (~12.4 µs)
74
+ branch = kf.fork()
75
+ branch.upsert_constant_velocity(33_333, float(ys[33_333] + 10.0), 1.0, 1e-3, 0.25)
76
+ sm_mean, sm_cov = branch.smooth_at(50_000, m0, p0) # Exact Rauch-Tung-Striebel p(x_t | y_{1:T})
77
+ log_ev = branch.log_marginal_likelihood(m0, p0) # Exact log p(y_{1:T}), O(1) at root!
78
+
79
+ # 3. Exact 106-bit compensated multivariate covariance, correlation & Mahalanobis anomaly scoring
80
+ cov = bf.CovarianceStream(dim=8)
81
+ cov.ingest(ts[:10_000], np.random.default_rng(1).normal(1e12, 1.0, size=(10_000, 8)))
82
+ d2 = cov.mahalanobis_sq([1e12 + 10.0] * 8)
83
+
84
+ # 4. Lossless-recoverable <300-token digest for LLM / coding agents
85
+ digest = bf.digest_stream("sensor_01", ts, ys, top_k=5, threshold=18.0)
86
+ print(digest["markdown"]) # 2,510× smaller than raw CSV, 100% acute-burst window recall
87
+ ```
88
+
89
+ ### Native CLI (sub-5ms cold startup, zero runtime dependencies)
90
+ ```sh
91
+ cargo build --release --bin backfold
92
+
93
+ # Run 6 live verified 100,000-tick demos in 0.56 s (0 approximations, 0 downloads)
94
+ ./target/release/backfold demo
95
+
96
+ # Compress a 100,000-row CSV (~600k tokens) into a ~239-token causal Markdown digest (2,510× compression)
97
+ ./target/release/backfold digest benchmarks/sample_100k.csv
98
+
99
+ # Losslessly drill down to exact raw ticks for any flagged hotspot window
100
+ ./target/release/backfold drill benchmarks/sample_100k.csv --start 22498 --end 22507
101
+
102
+ # Reproduce the live Rust O(log T) vs. handwritten -O3 sequential Rust benchmark
103
+ ./target/release/backfold bench
104
+ ```
105
+
106
+ Runnable recipes in [`examples/`](examples/):
107
+ - [`examples/kalman_rts_smoothing.py`](examples/kalman_rts_smoothing.py) — $O(\log T)$ RTS smoothing, variance tightening & Bayes evidence model selection
108
+ - [`examples/multivariate_anomaly_cov.py`](examples/multivariate_anomaly_cov.py) — `1e12`-offset 8D streaming covariance, regime correlation & batch Mahalanobis scoring
109
+ - [`examples/polars_late_arrivals.py`](examples/polars_late_arrivals.py) — retroactive CUSUM alarms, zero-lookahead bitemporal audit, e-BH FDR & agent digest
110
+
111
+ ---
112
+
113
+ ## Verified Live Benchmarks (`[LIVE-CPU]`, 2-vCPU Linux x86-64, `v0.2.0`)
114
+
115
+ Every number below is measured live on this hardware ([`benchmarks/results.json`](benchmarks/results.json)) and asserts **exact numerical parity** against independent naive sequential oracles ([`benchmarks/oracles.py`](benchmarks/oracles.py)). Python timings conservatively measure a full **edit + query + restore** round-trip (two tree updates + one query).
116
+
117
+ ### 1. Out-of-Order Late Edit Latency — Python API (`T = 100,000`, `max |diff| <= 1e-12`)
118
+
119
+ | Operator | `backfold` (Python) | NumPy | SciPy | Pandas | Polars | River (`T=10k`) | Speedup | Max `\|diff\|` |
120
+ |---|---:|---:|---:|---:|---:|---:|---:|---:|
121
+ | **2D Kalman RTS Smooth + $\log p(y)$** | **12.45 µs** | 6,637,232 µs | — | — | — | — | **533,197× vs NumPy RTS** | `1.16e-14` |
122
+ | **2D Kalman CV Filter** | **9.32 µs** | 4,737,933 µs | — | — | — | — | **508,471× vs NumPy** | `9.09e-13` |
123
+ | **Page CUSUM + 1st alarm** | **5.35 µs** | 134,250 µs *(Py seq)* | — | — | — | — | **25,075× vs seq** | `1.78e-15` |
124
+ | **CT Irregular-$\Delta t$ EWMA** | **4.79 µs** | 1,904.5 µs | — | 4,966.7 µs | — | — | **398× NumPy · 1,037× Pandas** | `4.44e-16` |
125
+ | **8D Covariance Matrix** | **15.51 µs** | 5,367.8 µs | — | — | — | — | **346× vs NumPy `cov`** | `6.66e-16` |
126
+ | **8D Discounted Ridge** | **17.68 µs** | 5,668.3 µs | 6,260.5 µs | — | — | 97,574 µs *(10k)* | **321× NumPy · 354× SciPy · 6,460× River** | `1.33e-15` |
127
+ | **Compensated Moments** *(mean/var/skew/kurt)* | **10.51 µs** | 806.0 µs | — | 1,275.8 µs | 263.4 µs | 8,148 µs *(10k)* | **77× NumPy · 121× Pandas · 25× Polars** | `4.57e-16` |
128
+
129
+ ### 2. Rust-vs-Rust Head-to-Head (`backfold` $O(\log T)$ vs. Handwritten `-O3` Sequential Rust)
130
+
131
+ Not just "Rust vs. Python" — even against tight handwritten `-O3` Rust loops, $O(\log T)$ associative monoids win by **two to four orders of magnitude**:
132
+
133
+ | Operator | `T = 10,000` | `T = 100,000` | `T = 500,000` | `backfold` @ 500k | Seq Rust `-O3` @ 500k | Max `\|diff\|` |
134
+ |---|---:|---:|---:|---:|---:|---:|
135
+ | **CT Irregular-$\Delta t$ EWMA** | 124× | 1,016× | **5,020×** | **1.32 µs** | 6,624.1 µs | `2.95e-17` |
136
+ | **1D Kalman RW** (`KalmanFilter1D`) | 129× | 1,113× | **4,888×** | **1.22 µs** | 5,967.7 µs | `1.67e-16` |
137
+ | **2D Kalman CV** (`KalmanFilter2D`) | 82× | 719× | **3,260×** | **3.34 µs** | 10,898.2 µs | `2.78e-16` |
138
+ | **2D Kalman RTS Smooth + $\log p(y)$** | 79× | 639× | **3,109×** | **6.52 µs** | 20,262.2 µs | `3.06e-14` |
139
+ | **Page CUSUM + 1st alarm** | 32× | 276× | **1,372×** | **1.89 µs** | 2,587.6 µs | `1.78e-15` |
140
+ | **8D Discounted Ridge** | 79× | **640×** | — | *6.22 µs @ 100k* | *3,982.3 µs @ 100k* | `2.00e-15` |
141
+ | **8D Compensated Covariance** | 26× | **234×** | — | *5.69 µs @ 100k* | *1,330.0 µs @ 100k* | `1.93e-14` |
142
+ | **Compensated Moments** | 11× | 88× | **414×** | **3.77 µs** | 1,564.2 µs | `3.12e-14` |
143
+ | **Arbitrary 60% Range Query** | — | 65× | **344×** | **2.76 µs** | 948.8 µs | `9.21e-15` |
144
+
145
+ ### 3. Multi-Agent Coordination, Echo Immunity & 500-Stream FDR Control
146
+
147
+ | Scenario | Naive Baseline | `backfold` | Edge |
148
+ |---|---|---|---|
149
+ | **16-Agent Gossip Swarm** *(2,000 events echoed 3× out-of-order)* | 66.7% overwrite churn (4,000 overwrites); `1.732×` ($\sqrt{3}$) echo z-inflation | **0.0% overwrite churn**; 4,000/4,000 echoes deduplicated; **1.000× exact evidence** | Echo-proof `(event_time, source_id, seq)` CRDT |
150
+ | **500-Stream Firehose** *(10 true shifts + 50 3×-loud traps, $\alpha=0.05$)* | 63 alarms: 10 TP, **53 FP (50/50 loud traps!)**, **FDR = 84.1%** | **10 alarms: 10/10 TP, 0 FP, 0/50 loud traps, FDR = 0.0%** (`259 ms` total) | Causal MAD + composite-null Shiryaev-Roberts e-value + Wang-Ramdas e-BH |
151
+ | **64-Agent Counterfactual Forking** *(`T=100k`, 5 edits/agent)* | 3,937.5 ms (`61.5 ms/agent`) | **1.27 ms** (`0.0198 ms/agent`, `max_diff = 3.55e-15`) | **3,110× faster** ($O(1)$ persistent `Arc` fork) |
152
+
153
+ ### 4. Lossless-Recoverable AI Agent Stream Compression (`50 streams × 20,000 ticks`)
154
+
155
+ | Representation | Avg Tokens | Compression | Acute Burst Exact-Window Recall | Regime Shift Recall | Loud 3×-Distractor FP Rate | Lossless Recovery |
156
+ |---|---:|---:|---:|---:|---:|---|
157
+ | Raw CSV | 120,000 | 1× | — *(overflows context)* | — | — | Yes |
158
+ | `pandas.describe()` | 17 | 7,059× | **0.0%** *(drops timestamps)* | 100.0% | **100.0%** | No |
159
+ | Uniform 200× Downsample | 600 | 200× | **20.0%** *(misses 5-tick spikes)* | 86.7% | **30.0%** | No |
160
+ | **`backfold digest`** | **255.8** | **469.1×** *(2,510× on 100k)* | **100.0%** | **100.0%** | **0.0%** | **Yes (`backfold drill`)** |
161
+
162
+ ---
163
+
164
+ ## What's Inside (7 Exact Associative Operators + Swarm CRDT)
165
+
166
+ 1. **`KalmanStream` / `KalmanFilter1D` / `KalmanFilter2D` / `KalmanMonoid`** — Exact Särkkä & García-Fernández (IEEE TAC 2020) associative Bayesian filter **and exact $O(\log T)$ Rauch–Tung–Striebel (RTS) smoother (`smooth_at`, `smooth_many`) + exact marginal log-likelihood (`log_marginal_likelihood`, `range_log_marginal_likelihood`)**. Hardened with **Anchor-Centered Associative Gaussian Potentials** (eliminating $O(y^2/S)$ cancellation down to `1.14e-15` relative error on 100k drifting streams and `1.04e-12` at `1e6` offset), Joseph-form PSD leaves, zero-alloc stack `1D`/`2D` fast paths, and x86-64 subnormal flush (`|x| < 1e-290 => 0`).
167
+ 2. **`CovarianceStream` / `CovarianceMonoid`** — Exact multivariate streaming covariance, Pearson correlation, arbitrary `[t1, t2]` regime correlation, and Jacobi-equilibrated Cholesky **Mahalanobis anomaly scoring** (`mahalanobis_sq`, factored batch `mahalanobis_many`), using 106-bit Knuth TwoSum compensated mean vectors and zero-alloc inline stack storage for `dim <= 8`.
168
+ 3. **`MomentsStream` / `MomentsMonoid`** — Centered Chan–Pébay–Terriberry higher-order moments (`mean`, `variance`, `skewness`, `excess_kurtosis`, `min`/`max` + exact timestamps) with **Knuth TwoSum 106-bit mean compensation** (`<1e-12` variance accuracy even on `1e12 + N(0, 1)` streams).
169
+ 4. **`EwmaStream` / `EwmaMonoid`** — Continuous-time irregular-$\Delta t$ centered exponentially weighted mean & variance with subnormal-safe decay clamping (`lambda * dt > 700 => 0`).
170
+ 5. **`CusumStream` / `CusumMonoid`** — Two-sided tropical-semiring Page CUSUM with non-empty `max_pref` (identity = $-\infty$) supporting exact **$O(\log T)$ earliest-alarm binary descent** (`first_alarm(threshold)`).
171
+ 6. **`EProcessStream` / `EProcessMonoid` + `ebh_reject`** — Simultaneously tracks Waudby-Smith–Ramdas 8-point symmetric betting martingale (`wealth`), Shin–Ramdas–Rinaldo Page e-CUSUM (`e_detector`), and exact log-sum-exp **Shiryaev–Roberts (`sr_evalue`) + all-subinterval scan (`scan_evalue`)** yielding a provably valid composite-null e-value ($\mathbb{E}_0[e] \le 1$, even under heavy-tailed Cauchy/Student-$t$ noise) for Wang–Ramdas arbitrary-dependence **e-BH FDR control**.
172
+ 7. **`RidgeStream` / `RidgeMonoid`** — Continuous-time exponentially discounted Ridge / RLS regression with zero-alloc inline stack storage for `dim <= 8`, in-place `combine_assign`, **Jacobi-equilibrated Cholesky** ($D^{-1/2}(G + \alpha I)D^{-1/2}$) with pivoted LU & relative-Tikhonov fallback, and exact predictive variance.
173
+ 8. **`SwarmEProcessLedger` & `BitemporalMoments`** — Echo-proof multi-agent CRDT ledger (`Arc::ptr_eq`-accelerated idempotent merge) and zero-lookahead bitemporal `as_of(knowledge_time)` snapshots.
174
+
175
+ ---
176
+
177
+ ## Honest Crossovers & Limitations
178
+
179
+ `backfold` is **not** magic for every workload. Know where it wins and where a simpler tool wins:
180
+
181
+ - **Pure append-only, single-pass, in-order aggregation:** If you never receive late arrivals, never retract/edit history, never query arbitrary sub-intervals `[t1, t2]`, and never fork branches, a plain sequential loop is faster on initial ingest. On `MomentsMonoid` at `T = 100k–500k`, bulk tree construction takes `10.2 ms` (`T=100k`) / `47.9 ms` (`T=500k`) vs. `0.30 ms` / `1.56 ms` for one sequential Rust pass — breaking even after **31–34 late updates or range queries**.
182
+ - **High-dimensional dense matrices (`d > 64`):** `RidgeMonoid` and `CovarianceMonoid` maintain packed upper-triangular $O(d^2)$ matrices per tree node. They shine for streaming telemetry, control, sensor fusion, and factor models (`d <= 32`, zero-alloc inline for `d <= 8`), not 10,000-feature dense NLP embeddings.
183
+ - **Exact quantile / median queries:** Order statistics (arbitrary quantiles) do not admit a fixed-size exact associative monoid without sketch approximation. `backfold` sticks strictly to **exact** operators (`causal_robust_scale` computes exact median/MAD on causal warm-up prefixes).
184
+
185
+ ---
186
+
187
+ ## For AI Coding Agents
188
+
189
+ - **Agent discovery index:** [`llms.txt`](llms.txt) and complete API reference [`llms-full.txt`](llms-full.txt) (pure Markdown).
190
+ - **Drop-in coding-agent skill:** [`skills/backfold/SKILL.md`](skills/backfold/SKILL.md) (works with Claude Code, Cursor, Codex, OpenCode, and MCP-compatible agents).
191
+ - **Technical deep-dive:** [`docs/SUBNORMAL_KALMAN.md`](docs/SUBNORMAL_KALMAN.md) (x86-64 subnormal traps, Anchor-Centered Gaussian Potentials for $O(\log T)$ RTS smoothing, 106-bit TwoSum moments, and Shiryaev–Roberts e-values).
192
+ - **Reproduce all tests & benchmarks:**
193
+ ```sh
194
+ cargo test --package backfold-core --package backfold-cli --release
195
+ cargo clippy --workspace --all-targets -- -D warnings
196
+ pytest -v tests/
197
+ python -m benchmarks.run_benchmarks
198
+ ```
199
+
200
+ ## License
201
+
202
+ Licensed under the [Apache License, Version 2.0](LICENSE).
203
+