backfold 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {backfold-0.2.0 → backfold-0.3.0}/Cargo.lock +12 -3
- {backfold-0.2.0 → backfold-0.3.0}/Cargo.toml +1 -1
- {backfold-0.2.0 → backfold-0.3.0}/PKG-INFO +79 -43
- backfold-0.3.0/README.md +200 -0
- backfold-0.3.0/crates/backfold-core/README.md +200 -0
- {backfold-0.2.0 → backfold-0.3.0}/crates/backfold-core/src/ops/covariance.rs +203 -0
- {backfold-0.2.0 → backfold-0.3.0}/crates/backfold-core/src/ops/eprocess.rs +125 -0
- {backfold-0.2.0 → backfold-0.3.0}/crates/backfold-core/src/ops/kalman.rs +497 -75
- backfold-0.3.0/crates/backfold-core/src/ops/mod.rs +22 -0
- {backfold-0.2.0 → backfold-0.3.0}/crates/backfold-core/src/ops/moments.rs +75 -50
- {backfold-0.2.0 → backfold-0.3.0}/crates/backfold-core/src/tree.rs +163 -9
- {backfold-0.2.0 → backfold-0.3.0}/crates/backfold-py/src/lib.rs +504 -162
- {backfold-0.2.0 → backfold-0.3.0}/pyproject.toml +5 -3
- {backfold-0.2.0 → backfold-0.3.0}/python/backfold/__init__.py +2 -0
- backfold-0.2.0/README.md +0 -164
- backfold-0.2.0/crates/backfold-core/README.md +0 -164
- backfold-0.2.0/crates/backfold-core/src/ops/mod.rs +0 -19
- {backfold-0.2.0 → backfold-0.3.0}/LICENSE +0 -0
- {backfold-0.2.0 → backfold-0.3.0}/crates/backfold-core/Cargo.toml +0 -0
- {backfold-0.2.0 → backfold-0.3.0}/crates/backfold-core/src/digest.rs +0 -0
- {backfold-0.2.0 → backfold-0.3.0}/crates/backfold-core/src/key.rs +0 -0
- {backfold-0.2.0 → backfold-0.3.0}/crates/backfold-core/src/lib.rs +0 -0
- {backfold-0.2.0 → backfold-0.3.0}/crates/backfold-core/src/monoid.rs +0 -0
- {backfold-0.2.0 → backfold-0.3.0}/crates/backfold-core/src/ops/cusum.rs +0 -0
- {backfold-0.2.0 → backfold-0.3.0}/crates/backfold-core/src/ops/ewma.rs +0 -0
- {backfold-0.2.0 → backfold-0.3.0}/crates/backfold-core/src/ops/ridge.rs +0 -0
- {backfold-0.2.0 → backfold-0.3.0}/crates/backfold-core/src/swarm.rs +0 -0
- {backfold-0.2.0 → backfold-0.3.0}/crates/backfold-core/tests/breaker_audit.rs +0 -0
- {backfold-0.2.0 → backfold-0.3.0}/crates/backfold-py/Cargo.toml +0 -0
|
@@ -10,7 +10,7 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
|
|
|
10
10
|
|
|
11
11
|
[[package]]
|
|
12
12
|
name = "backfold-cli"
|
|
13
|
-
version = "0.
|
|
13
|
+
version = "0.3.0"
|
|
14
14
|
dependencies = [
|
|
15
15
|
"backfold-core",
|
|
16
16
|
"serde",
|
|
@@ -19,7 +19,7 @@ dependencies = [
|
|
|
19
19
|
|
|
20
20
|
[[package]]
|
|
21
21
|
name = "backfold-core"
|
|
22
|
-
version = "0.
|
|
22
|
+
version = "0.3.0"
|
|
23
23
|
dependencies = [
|
|
24
24
|
"rand",
|
|
25
25
|
"serde",
|
|
@@ -27,12 +27,21 @@ dependencies = [
|
|
|
27
27
|
|
|
28
28
|
[[package]]
|
|
29
29
|
name = "backfold-py"
|
|
30
|
-
version = "0.
|
|
30
|
+
version = "0.3.0"
|
|
31
31
|
dependencies = [
|
|
32
32
|
"backfold-core",
|
|
33
33
|
"pyo3",
|
|
34
34
|
]
|
|
35
35
|
|
|
36
|
+
[[package]]
|
|
37
|
+
name = "backfold-wasm"
|
|
38
|
+
version = "0.3.0"
|
|
39
|
+
dependencies = [
|
|
40
|
+
"backfold-core",
|
|
41
|
+
"serde",
|
|
42
|
+
"serde_json",
|
|
43
|
+
]
|
|
44
|
+
|
|
36
45
|
[[package]]
|
|
37
46
|
name = "cfg-if"
|
|
38
47
|
version = "1.0.5"
|
|
@@ -3,7 +3,7 @@ resolver = "2"
|
|
|
3
3
|
members = ["crates/backfold-core", "crates/backfold-py"]
|
|
4
4
|
|
|
5
5
|
[workspace.package]
|
|
6
|
-
version = "0.
|
|
6
|
+
version = "0.3.0"
|
|
7
7
|
edition = "2021"
|
|
8
8
|
license = "Apache-2.0"
|
|
9
9
|
description = "Persistent O(log T) associative-monoid engine for out-of-order, retractable, forkable online statistical inference and agent stream compression"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: backfold
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Classifier: Development Status :: 4 - Beta
|
|
5
5
|
Classifier: Intended Audience :: Developers
|
|
6
6
|
Classifier: Intended Audience :: Science/Research
|
|
@@ -26,8 +26,8 @@ Provides-Extra: bench
|
|
|
26
26
|
Provides-Extra: dev
|
|
27
27
|
Provides-Extra: numpy
|
|
28
28
|
License-File: LICENSE
|
|
29
|
-
Summary: Persistent O(log T) associative-monoid engine for out-of-order, retractable, forkable online statistical inference and agent stream compression
|
|
30
|
-
Keywords: streaming,time-series,online-learning,kalman-filter,changepoint,e-values,
|
|
29
|
+
Summary: Persistent O(log T) associative-monoid engine for out-of-order, retractable, forkable online statistical inference, exact RTS smoothing, and agent stream compression
|
|
30
|
+
Keywords: streaming,time-series,online-learning,kalman-filter,rts-smoother,changepoint,pca,e-values,mcp,multi-agent,crdt,rust
|
|
31
31
|
License: Apache-2.0
|
|
32
32
|
Requires-Python: >=3.10
|
|
33
33
|
Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
|
|
@@ -39,24 +39,33 @@ Project-URL: Repository, https://github.com/pgg898200-ship-it/backfold
|
|
|
39
39
|
# backfold
|
|
40
40
|
|
|
41
41
|
[](https://github.com/pgg898200-ship-it/backfold/actions/workflows/ci.yml)
|
|
42
|
+
[](https://pypi.org/project/backfold/)
|
|
42
43
|
[](LICENSE)
|
|
43
44
|
|
|
44
|
-
**Persistent $O(\log T)$ out-of-order, retractable, forkable online statistical inference, exact RTS Bayesian smoothing & lossless-recoverable AI agent stream compression.**
|
|
45
|
+
**Persistent $O(\log T)$ out-of-order, retractable, forkable online statistical inference, exact RTS Bayesian smoothing, streaming PCA, native MCP server & lossless-recoverable AI agent stream compression.**
|
|
45
46
|
|
|
46
|
-
Pure safe Rust (`#![forbid(unsafe_code)]`) · Zero-copy Python
|
|
47
|
+
Pure safe Rust (`#![forbid(unsafe_code)]`) · Zero-copy GIL-free Python · Sub-5ms native CLI & MCP server · **[Live WebAssembly Playground & `llms.txt`](https://pgg898200-ship-it.github.io/backfold/)**
|
|
47
48
|
|
|
48
49
|
---
|
|
49
50
|
|
|
50
51
|
> **Every streaming library assumes time moves forward, nothing gets retracted, and only one consumer reads the stream.**
|
|
51
52
|
> Real sensor pipelines, financial feeds, telemetry firehoses, and multi-agent LLM swarms break all three assumptions. When an event arrives 50,000 ticks late—or gets retracted, restated, or branched by 64 counterfactual agents—`river`, `pandas.ewm`, and sequential `numpy`/`scipy` Kalman filters/smoothers fall back to **$O(T)$ full forward-backward replay**.
|
|
52
53
|
>
|
|
53
|
-
> `backfold` compiles every operator into an **exact associative monoid** inside a persistent copy-on-write B+ fold tree (`FoldTree<M, B>`): **$O(\log T)$ late edits & retractions, $O(\log T)$ exact Rauch–Tung–Striebel (RTS) Bayesian smoothing & marginal log-likelihood $\log p(y_{1:T})$,
|
|
54
|
+
> `backfold` compiles every operator into an **exact associative monoid** inside a persistent copy-on-write B+ fold tree (`FoldTree<M, B>`): **$O(\log T)$ late edits & retractions, $O(\log T)$ exact Rauch–Tung–Striebel (RTS) Bayesian smoothing & marginal log-likelihood $\log p(y_{1:T})$, zero-blind-spot Bayesian changepoint shock localization, 106-bit streaming PCA & Mahalanobis anomaly scoring, $O(1)$ zero-copy `fork()` (~20 ns), echo-proof multi-agent CRDT merges, distribution-free e-BH FDR control (`sift_streams`), a zero-dependency stdio MCP server (`backfold mcp`), and `<300`-token lossless-recoverable LLM stream digests — with zero approximation (`max |diff| <= 1e-12`).**
|
|
55
|
+
|
|
56
|
+
---
|
|
57
|
+
|
|
58
|
+
## Live Interactive WebAssembly Playground
|
|
59
|
+
|
|
60
|
+
Try real compiled Rust `backfold-core` (`backfold_wasm.wasm`, zero backend server) directly in your browser on 20,000 ticks — click to inject late glitches, Shift+Click to retract ticks, localize off-grid shocks, and benchmark **WASM $O(\log T)$ (`~8 µs`) vs. JavaScript sequential RTS (`~28 ms`, ~3,600× faster, `diff < 1e-14`)** live on your own machine:
|
|
61
|
+
|
|
62
|
+
**[https://pgg898200-ship-it.github.io/backfold/](https://pgg898200-ship-it.github.io/backfold/)**
|
|
54
63
|
|
|
55
64
|
---
|
|
56
65
|
|
|
57
66
|
## 30-Second Quickstart
|
|
58
67
|
|
|
59
|
-
### Python (`pip install backfold`)
|
|
68
|
+
### Python (`pip install backfold` — prebuilt wheels for Linux, macOS, Windows, Py 3.10–3.13)
|
|
60
69
|
```python
|
|
61
70
|
import numpy as np
|
|
62
71
|
import backfold as bf
|
|
@@ -66,31 +75,55 @@ dts = np.ones(100_000)
|
|
|
66
75
|
ys = np.cumsum(np.random.default_rng(0).normal(0.05, 0.1, 100_000))
|
|
67
76
|
m0, p0 = [0.0, 0.0], [10.0, 0.0, 0.0, 10.0]
|
|
68
77
|
|
|
69
|
-
# 1. Exact 2D Constant-Velocity Kalman filter + O(log T) RTS smoother
|
|
78
|
+
# 1. Exact 2D Constant-Velocity Kalman filter + O(log T) RTS smoother + Bayesian changepoint locator
|
|
70
79
|
kf = bf.KalmanStream(dim=2)
|
|
71
80
|
kf.ingest_constant_velocity(ts, ys, dts, q_accel=1e-3, r_obs=0.25)
|
|
72
81
|
|
|
73
|
-
# 2. O(1) zero-copy counterfactual fork (~
|
|
82
|
+
# 2. O(1) zero-copy counterfactual fork (~20 ns) + O(log T) late repair + RTS smooth (~11.1 µs)
|
|
74
83
|
branch = kf.fork()
|
|
75
84
|
branch.upsert_constant_velocity(33_333, float(ys[33_333] + 10.0), 1.0, 1e-3, 0.25)
|
|
76
|
-
sm_mean, sm_cov = branch.smooth_at(50_000, m0, p0)
|
|
77
|
-
log_ev = branch.log_marginal_likelihood(m0, p0)
|
|
85
|
+
sm_mean, sm_cov = branch.smooth_at(50_000, m0, p0) # Exact RTS p(x_t | y_{1:T})
|
|
86
|
+
log_ev = branch.log_marginal_likelihood(m0, p0) # Exact log p(y_{1:T}), O(1) at root!
|
|
87
|
+
cp = branch.locate_changepoint(m0, p0, [200.0, 0.0, 0.0, 1.0]) # Zero-blind-spot shock locator (<0.6 ms!)
|
|
78
88
|
|
|
79
|
-
# 3. Exact 106-bit compensated multivariate covariance,
|
|
89
|
+
# 3. Exact 106-bit compensated multivariate covariance, cyclic-Jacobi streaming PCA & Mahalanobis
|
|
80
90
|
cov = bf.CovarianceStream(dim=8)
|
|
81
91
|
cov.ingest(ts[:10_000], np.random.default_rng(1).normal(1e12, 1.0, size=(10_000, 8)))
|
|
92
|
+
pca = cov.range_pca(1_000, 9_000, ddof=1, top_k=3) # Zero BLAS, exact even at 1e12 offset!
|
|
82
93
|
d2 = cov.mahalanobis_sq([1e12 + 10.0] * 8)
|
|
83
94
|
|
|
84
|
-
# 4.
|
|
95
|
+
# 4. Single-call GIL-free 500-channel Wang-Ramdas e-BH FDR sifter (provable FDR <= alpha)
|
|
96
|
+
sift = bf.sift_streams(np.random.default_rng(2).normal(size=(500, 400)), warmup=120, alpha=0.05)
|
|
97
|
+
|
|
98
|
+
# 5. Lossless-recoverable <300-token digest for LLM / coding agents
|
|
85
99
|
digest = bf.digest_stream("sensor_01", ts, ys, top_k=5, threshold=18.0)
|
|
86
100
|
print(digest["markdown"]) # 2,510× smaller than raw CSV, 100% acute-burst window recall
|
|
87
101
|
```
|
|
88
102
|
|
|
103
|
+
### Zero-Dependency Native MCP Server (`backfold mcp`)
|
|
104
|
+
Drop the precompiled sub-1 MB `backfold` binary straight into **Claude Desktop**, **Cursor**, **Windsurf**, or **Claude Code** (zero Python, Node, or Docker runtime required):
|
|
105
|
+
```json
|
|
106
|
+
{
|
|
107
|
+
"mcpServers": {
|
|
108
|
+
"backfold": {
|
|
109
|
+
"command": "backfold",
|
|
110
|
+
"args": ["mcp"]
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
```
|
|
115
|
+
Exposes 5 production streaming tools over newline-delimited stdio JSON-RPC 2.0 (`MCP 2024-11-05`):
|
|
116
|
+
- `backfold_digest_csv` — `<300`-token lossless-recoverable Markdown statistical digest
|
|
117
|
+
- `backfold_drill_csv` — exact 106-bit range moments + raw ticks for `[start, end]`
|
|
118
|
+
- `backfold_kalman_rts_csv` — $O(\log T)$ RTS smoothing + $\log p(y_{1:T})$ + zero-blind-spot shock localization
|
|
119
|
+
- `backfold_covariance_pca_csv` — 106-bit streaming covariance, correlation, cyclic-Jacobi PCA & top Mahalanobis outliers
|
|
120
|
+
- `backfold_ebh_sift_csv` — Wang–Ramdas arbitrary-dependence e-BH FDR-controlled multi-channel alarm sifter
|
|
121
|
+
|
|
89
122
|
### Native CLI (sub-5ms cold startup, zero runtime dependencies)
|
|
90
123
|
```sh
|
|
91
124
|
cargo build --release --bin backfold
|
|
92
125
|
|
|
93
|
-
# Run
|
|
126
|
+
# Run 7 live verified 100,000-tick demos in 0.53 s (0 approximations, 0 downloads)
|
|
94
127
|
./target/release/backfold demo
|
|
95
128
|
|
|
96
129
|
# Compress a 100,000-row CSV (~600k tokens) into a ~239-token causal Markdown digest (2,510× compression)
|
|
@@ -105,12 +138,13 @@ cargo build --release --bin backfold
|
|
|
105
138
|
|
|
106
139
|
Runnable recipes in [`examples/`](examples/):
|
|
107
140
|
- [`examples/kalman_rts_smoothing.py`](examples/kalman_rts_smoothing.py) — $O(\log T)$ RTS smoothing, variance tightening & Bayes evidence model selection
|
|
141
|
+
- [`examples/streaming_pca_and_changepoint_bf.py`](examples/streaming_pca_and_changepoint_bf.py) — zero-blind-spot Bayesian shock localization (`+1e6` offset in `0.39 ms`), `1e12` range PCA & 500-channel `sift_streams`
|
|
108
142
|
- [`examples/multivariate_anomaly_cov.py`](examples/multivariate_anomaly_cov.py) — `1e12`-offset 8D streaming covariance, regime correlation & batch Mahalanobis scoring
|
|
109
143
|
- [`examples/polars_late_arrivals.py`](examples/polars_late_arrivals.py) — retroactive CUSUM alarms, zero-lookahead bitemporal audit, e-BH FDR & agent digest
|
|
110
144
|
|
|
111
145
|
---
|
|
112
146
|
|
|
113
|
-
## Verified Live Benchmarks (`[LIVE-CPU]`, 2-vCPU Linux x86-64, `v0.
|
|
147
|
+
## Verified Live Benchmarks (`[LIVE-CPU]`, 2-vCPU Linux x86-64, `v0.3.0`)
|
|
114
148
|
|
|
115
149
|
Every number below is measured live on this hardware ([`benchmarks/results.json`](benchmarks/results.json)) and asserts **exact numerical parity** against independent naive sequential oracles ([`benchmarks/oracles.py`](benchmarks/oracles.py)). Python timings conservatively measure a full **edit + query + restore** round-trip (two tree updates + one query).
|
|
116
150
|
|
|
@@ -118,37 +152,37 @@ Every number below is measured live on this hardware ([`benchmarks/results.json`
|
|
|
118
152
|
|
|
119
153
|
| Operator | `backfold` (Python) | NumPy | SciPy | Pandas | Polars | River (`T=10k`) | Speedup | Max `\|diff\|` |
|
|
120
154
|
|---|---:|---:|---:|---:|---:|---:|---:|---:|
|
|
121
|
-
| **2D Kalman RTS Smooth + $\log p(y)$** | **12
|
|
122
|
-
| **2D Kalman CV Filter** | **
|
|
123
|
-
| **Page CUSUM + 1st alarm** | **
|
|
124
|
-
| **CT Irregular-$\Delta t$ EWMA** | **
|
|
125
|
-
| **8D Covariance Matrix** | **15.
|
|
126
|
-
| **8D Discounted Ridge** | **
|
|
127
|
-
| **Compensated Moments** *(mean/var/skew/kurt)* | **
|
|
155
|
+
| **2D Kalman RTS Smooth + $\log p(y)$** | **11.12 µs** | 6,437,010 µs | — | — | — | — | **578,868× vs NumPy RTS** | `1.47e-14` |
|
|
156
|
+
| **2D Kalman CV Filter** | **8.82 µs** | 4,563,614 µs | — | — | — | — | **517,710× vs NumPy** | `9.09e-13` |
|
|
157
|
+
| **Page CUSUM + 1st alarm** | **3.83 µs** | 137,450 µs *(Py seq)* | — | — | — | — | **35,897× vs seq** | `1.78e-15` |
|
|
158
|
+
| **CT Irregular-$\Delta t$ EWMA** | **3.92 µs** | 1,940.7 µs | — | 4,831.8 µs | — | — | **495× NumPy · 1,231× Pandas** | `1.78e-15` |
|
|
159
|
+
| **8D Covariance Matrix** | **15.60 µs** | 5,589.1 µs | — | — | — | — | **358× vs NumPy `cov`** | `6.66e-16` |
|
|
160
|
+
| **8D Discounted Ridge** | **16.95 µs** | 5,567.1 µs | 5,920.6 µs | — | — | 103,981 µs *(10k)* | **328× NumPy · 349× SciPy · 7,092× River** | `1.33e-15` |
|
|
161
|
+
| **Compensated Moments** *(mean/var/skew/kurt)* | **6.06 µs** | 730.4 µs | — | 1,293.5 µs | 273.0 µs | 8,215 µs *(10k)* | **121× NumPy · 213× Pandas · 45× Polars** | `4.54e-16` |
|
|
128
162
|
|
|
129
163
|
### 2. Rust-vs-Rust Head-to-Head (`backfold` $O(\log T)$ vs. Handwritten `-O3` Sequential Rust)
|
|
130
164
|
|
|
131
|
-
Not just "Rust vs. Python" — even against tight handwritten `-O3` Rust loops,
|
|
165
|
+
Not just "Rust vs. Python" — even against tight handwritten `-O3` Rust loops, Pareto-tuned (`B = 4`, single-pass `split_summary`, in-place `combine_assign`) associative monoids win by **up to 8,152×** (sub-microsecond at half a million ticks!):
|
|
132
166
|
|
|
133
167
|
| Operator | `T = 10,000` | `T = 100,000` | `T = 500,000` | `backfold` @ 500k | Seq Rust `-O3` @ 500k | Max `\|diff\|` |
|
|
134
168
|
|---|---:|---:|---:|---:|---:|---:|
|
|
135
|
-
| **CT Irregular-$\Delta t$ EWMA** |
|
|
136
|
-
| **1D Kalman RW** (`KalmanFilter1D`) |
|
|
137
|
-
| **2D Kalman
|
|
138
|
-
| **2D Kalman
|
|
139
|
-
| **Page CUSUM + 1st alarm** |
|
|
140
|
-
| **
|
|
141
|
-
| **
|
|
142
|
-
| **
|
|
143
|
-
| **
|
|
169
|
+
| **CT Irregular-$\Delta t$ EWMA** | 207× | 1,736× | **8,152×** | **0.806 µs** | 6,570.7 µs | `2.26e-17` |
|
|
170
|
+
| **1D Kalman RW** (`KalmanFilter1D`) | 160× | 1,338× | **6,185×** | **0.979 µs** | 6,052.4 µs | `1.39e-16` |
|
|
171
|
+
| **2D Kalman RTS Smooth + $\log p(y)$** | 89× | 760× | **3,694×** | **5.367 µs** | 19,823.4 µs | `3.06e-14` |
|
|
172
|
+
| **2D Kalman CV** (`KalmanFilter2D`) | 89× | 760× | **3,538×** | **3.110 µs** | 11,000.2 µs | `2.78e-16` |
|
|
173
|
+
| **Page CUSUM + 1st alarm** | 53× | 454× | **2,177×** | **1.181 µs** | 2,571.3 µs | `1.78e-15` |
|
|
174
|
+
| **Compensated Moments** | 32× | 259× | **1,288×** | **1.278 µs** | 1,646.3 µs | `3.12e-14` |
|
|
175
|
+
| **Arbitrary 60% Range Query** | — | 154× | **664×** | **1.404 µs** | 932.2 µs | `8.77e-15` |
|
|
176
|
+
| **8D Discounted Ridge** | 79× | **659×** | — | *6.108 µs @ 100k* | *4,022.8 µs @ 100k* | `2.00e-15` |
|
|
177
|
+
| **8D Compensated Covariance** | 26× | **262×** | — | *5.677 µs @ 100k* | *1,484.7 µs @ 100k* | `1.93e-14` |
|
|
144
178
|
|
|
145
179
|
### 3. Multi-Agent Coordination, Echo Immunity & 500-Stream FDR Control
|
|
146
180
|
|
|
147
181
|
| Scenario | Naive Baseline | `backfold` | Edge |
|
|
148
182
|
|---|---|---|---|
|
|
149
183
|
| **16-Agent Gossip Swarm** *(2,000 events echoed 3× out-of-order)* | 66.7% overwrite churn (4,000 overwrites); `1.732×` ($\sqrt{3}$) echo z-inflation | **0.0% overwrite churn**; 4,000/4,000 echoes deduplicated; **1.000× exact evidence** | Echo-proof `(event_time, source_id, seq)` CRDT |
|
|
150
|
-
| **500-Stream Firehose** *(10 true shifts + 50 3×-loud traps, $\alpha=0.05$)* | 63 alarms: 10 TP, **53 FP (50/50 loud traps!)**, **FDR = 84.1%** | **10 alarms: 10/10 TP, 0 FP, 0/50 loud traps, FDR = 0.0%** (`
|
|
151
|
-
| **64-Agent Counterfactual Forking** *(`T=100k`, 5 edits/agent)* |
|
|
184
|
+
| **500-Stream Firehose** *(10 true shifts + 50 3×-loud traps, $\alpha=0.05$)* | 63 alarms: 10 TP, **53 FP (50/50 loud traps!)**, **FDR = 84.1%** | **10 alarms: 10/10 TP, 0 FP, 0/50 loud traps, FDR = 0.0%** (`107.4 ms` via `sift_streams`) | Pure-Rust GIL-free causal MAD + Shiryaev–Roberts e-value + Wang–Ramdas e-BH |
|
|
185
|
+
| **64-Agent Counterfactual Forking** *(`T=100k`, 5 edits/agent)* | 4,688.1 ms (`73.3 ms/agent`) | **1.41 ms** (`0.022 ms/agent`, `max_diff = 3.55e-15`) | **3,326× faster** ($O(1)$ persistent `Arc` fork) |
|
|
152
186
|
|
|
153
187
|
### 4. Lossless-Recoverable AI Agent Stream Compression (`50 streams × 20,000 ticks`)
|
|
154
188
|
|
|
@@ -161,16 +195,17 @@ Not just "Rust vs. Python" — even against tight handwritten `-O3` Rust loops,
|
|
|
161
195
|
|
|
162
196
|
---
|
|
163
197
|
|
|
164
|
-
## What's Inside (7 Exact Associative Operators + Swarm CRDT)
|
|
198
|
+
## What's Inside (7 Exact Associative Operators + Swarm CRDT + Native MCP + WASM)
|
|
165
199
|
|
|
166
|
-
1. **`KalmanStream` / `KalmanFilter1D` / `KalmanFilter2D` / `KalmanMonoid`** — Exact Särkkä & García-Fernández (IEEE TAC 2020) associative Bayesian filter **
|
|
167
|
-
2. **`CovarianceStream` / `CovarianceMonoid`** — Exact multivariate streaming covariance, Pearson correlation, arbitrary `[t1, t2]` regime correlation, and Jacobi-equilibrated Cholesky **Mahalanobis anomaly scoring** (`mahalanobis_sq`, factored batch `mahalanobis_many`),
|
|
168
|
-
3. **`MomentsStream` / `MomentsMonoid`** — Centered Chan–Pébay–Terriberry higher-order moments (`mean`, `variance`, `skewness`, `excess_kurtosis`, `min`/`max` + exact timestamps) with **Knuth TwoSum 106-bit mean compensation** (`<1e-12` variance accuracy even on `1e12 + N(0, 1)` streams)
|
|
200
|
+
1. **`KalmanStream` / `KalmanFilter1D` / `KalmanFilter2D` / `KalmanMonoid`** — Exact Särkkä & García-Fernández (IEEE TAC 2020) associative Bayesian filter, **single-pass $O(\log T)$ Rauch–Tung–Striebel (RTS) smoother (`smooth_at`, `smooth_many`)**, exact marginal log-likelihood (`log_marginal_likelihood`, `range_log_marginal_likelihood`), **zero-cancellation Bayesian changepoint Bayes factors (`changepoint_bayes_factor`, `scan_changepoints`)**, and **zero-blind-spot cached-subtree Bayesian shock localization (`locate_changepoint`)**. Hardened with **Anchor-Centered Associative Gaussian Potentials** (`1.14e-15` rel err @ 100k; `1.04e-12` at `1e6` offset), Joseph-form PSD leaves, zero-alloc stack `1D`/`2D` paths, and x86-64 subnormal flush (`|x| < 1e-290 => 0`).
|
|
201
|
+
2. **`CovarianceStream` / `CovarianceMonoid`** — Exact multivariate streaming covariance, Pearson correlation, arbitrary `[t1, t2]` regime correlation, **exact pure-safe cyclic-Jacobi streaming PCA (`pca`, `range_pca` — zero BLAS/LAPACK, `<1e-11` accuracy even at `1e12` offset!)**, and Jacobi-equilibrated Cholesky **Mahalanobis anomaly scoring** (`mahalanobis_sq`, factored batch `mahalanobis_many`), with 106-bit Knuth TwoSum compensated mean vectors and zero-alloc inline stack storage for `dim <= 8`.
|
|
202
|
+
3. **`MomentsStream` / `MomentsMonoid`** — Centered Chan–Pébay–Terriberry higher-order moments (`mean`, `variance`, `skewness`, `excess_kurtosis`, `min`/`max` + exact timestamps) with **Knuth TwoSum 106-bit mean compensation** (`<1e-12` variance accuracy even on `1e12 + N(0, 1)` streams) and singleton-leaf fast-path `combine_assign`.
|
|
169
203
|
4. **`EwmaStream` / `EwmaMonoid`** — Continuous-time irregular-$\Delta t$ centered exponentially weighted mean & variance with subnormal-safe decay clamping (`lambda * dt > 700 => 0`).
|
|
170
204
|
5. **`CusumStream` / `CusumMonoid`** — Two-sided tropical-semiring Page CUSUM with non-empty `max_pref` (identity = $-\infty$) supporting exact **$O(\log T)$ earliest-alarm binary descent** (`first_alarm(threshold)`).
|
|
171
|
-
6. **`EProcessStream` / `EProcessMonoid` + `ebh_reject`** — Simultaneously tracks Waudby-Smith–Ramdas 8-point symmetric betting martingale (`wealth`), Shin–Ramdas–Rinaldo Page e-CUSUM (`e_detector`), and exact log-sum-exp **Shiryaev–Roberts (`sr_evalue`) + all-subinterval scan (`scan_evalue`)** yielding a provably valid composite-null e-value ($\mathbb{E}_0[e] \le 1$, even under heavy-tailed Cauchy/Student-$t$ noise) for Wang–Ramdas arbitrary-dependence **e-BH FDR control
|
|
205
|
+
6. **`EProcessStream` / `EProcessMonoid` + `ebh_reject` + `sift_streams`** — Simultaneously tracks Waudby-Smith–Ramdas 8-point symmetric betting martingale (`wealth`), Shin–Ramdas–Rinaldo Page e-CUSUM (`e_detector`), and exact log-sum-exp **Shiryaev–Roberts (`sr_evalue`) + all-subinterval scan (`scan_evalue`)** yielding a provably valid composite-null e-value ($\mathbb{E}_0[e] \le 1$, even under heavy-tailed Cauchy/Student-$t$ noise) for Wang–Ramdas arbitrary-dependence **e-BH FDR control**, plus single-call GIL-free batch `sift_streams`.
|
|
172
206
|
7. **`RidgeStream` / `RidgeMonoid`** — Continuous-time exponentially discounted Ridge / RLS regression with zero-alloc inline stack storage for `dim <= 8`, in-place `combine_assign`, **Jacobi-equilibrated Cholesky** ($D^{-1/2}(G + \alpha I)D^{-1/2}$) with pivoted LU & relative-Tikhonov fallback, and exact predictive variance.
|
|
173
207
|
8. **`SwarmEProcessLedger` & `BitemporalMoments`** — Echo-proof multi-agent CRDT ledger (`Arc::ptr_eq`-accelerated idempotent merge) and zero-lookahead bitemporal `as_of(knowledge_time)` snapshots.
|
|
208
|
+
9. **GIL-free Python scaling** — Bulk `ingest*`, `smooth_many`, `scan_changepoints`, `locate_changepoint`, `mahalanobis_many`, `digest_stream`, and `sift_streams` release the Python GIL (`py.allow_threads`) for true multi-core scaling in Python agent swarms.
|
|
174
209
|
|
|
175
210
|
---
|
|
176
211
|
|
|
@@ -178,7 +213,7 @@ Not just "Rust vs. Python" — even against tight handwritten `-O3` Rust loops,
|
|
|
178
213
|
|
|
179
214
|
`backfold` is **not** magic for every workload. Know where it wins and where a simpler tool wins:
|
|
180
215
|
|
|
181
|
-
- **Pure append-only, single-pass, in-order aggregation:** If you never receive late arrivals, never retract/edit history, never query arbitrary sub-intervals `[t1, t2]`, and never fork branches, a plain sequential loop is faster on initial ingest. On `MomentsMonoid` at `T = 100k–500k`, bulk tree construction takes `
|
|
216
|
+
- **Pure append-only, single-pass, in-order aggregation:** If you never receive late arrivals, never retract/edit history, never query arbitrary sub-intervals `[t1, t2]`, and never fork branches, a plain sequential loop is faster on initial ingest. On `MomentsMonoid` at `T = 100k–500k`, bulk tree construction takes `9.99 ms` (`T=100k`) / `51.07 ms` (`T=500k`) vs. `0.30 ms` / `1.65 ms` for one sequential Rust pass — breaking even after **31–33 late updates or range queries**.
|
|
182
217
|
- **High-dimensional dense matrices (`d > 64`):** `RidgeMonoid` and `CovarianceMonoid` maintain packed upper-triangular $O(d^2)$ matrices per tree node. They shine for streaming telemetry, control, sensor fusion, and factor models (`d <= 32`, zero-alloc inline for `d <= 8`), not 10,000-feature dense NLP embeddings.
|
|
183
218
|
- **Exact quantile / median queries:** Order statistics (arbitrary quantiles) do not admit a fixed-size exact associative monoid without sketch approximation. `backfold` sticks strictly to **exact** operators (`causal_robust_scale` computes exact median/MAD on causal warm-up prefixes).
|
|
184
219
|
|
|
@@ -186,12 +221,13 @@ Not just "Rust vs. Python" — even against tight handwritten `-O3` Rust loops,
|
|
|
186
221
|
|
|
187
222
|
## For AI Coding Agents
|
|
188
223
|
|
|
189
|
-
- **
|
|
224
|
+
- **Native stdio MCP server:** `backfold mcp` (5 built-in CSV streaming tools, zero runtime dependencies).
|
|
225
|
+
- **Agent discovery index:** [`llms.txt`](llms.txt) and complete API reference [`llms-full.txt`](llms-full.txt) (pure Markdown, also hosted at `https://pgg898200-ship-it.github.io/backfold/llms.txt`).
|
|
190
226
|
- **Drop-in coding-agent skill:** [`skills/backfold/SKILL.md`](skills/backfold/SKILL.md) (works with Claude Code, Cursor, Codex, OpenCode, and MCP-compatible agents).
|
|
191
|
-
- **Technical deep-dive:** [`docs/SUBNORMAL_KALMAN.md`](docs/SUBNORMAL_KALMAN.md) (x86-64 subnormal traps, Anchor-Centered Gaussian Potentials
|
|
227
|
+
- **Technical deep-dive:** [`docs/SUBNORMAL_KALMAN.md`](docs/SUBNORMAL_KALMAN.md) (x86-64 subnormal traps, Anchor-Centered Gaussian Potentials, zero-cancellation changepoint Bayes factors, 106-bit TwoSum moments, and Shiryaev–Roberts e-values).
|
|
192
228
|
- **Reproduce all tests & benchmarks:**
|
|
193
229
|
```sh
|
|
194
|
-
cargo test --package backfold-core --package backfold-cli --release
|
|
230
|
+
cargo test --package backfold-core --package backfold-cli --package backfold-wasm --release
|
|
195
231
|
cargo clippy --workspace --all-targets -- -D warnings
|
|
196
232
|
pytest -v tests/
|
|
197
233
|
python -m benchmarks.run_benchmarks
|
backfold-0.3.0/README.md
ADDED
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
# backfold
|
|
2
|
+
|
|
3
|
+
[](https://github.com/pgg898200-ship-it/backfold/actions/workflows/ci.yml)
|
|
4
|
+
[](https://pypi.org/project/backfold/)
|
|
5
|
+
[](LICENSE)
|
|
6
|
+
|
|
7
|
+
**Persistent $O(\log T)$ out-of-order, retractable, forkable online statistical inference, exact RTS Bayesian smoothing, streaming PCA, native MCP server & lossless-recoverable AI agent stream compression.**
|
|
8
|
+
|
|
9
|
+
Pure safe Rust (`#![forbid(unsafe_code)]`) · Zero-copy GIL-free Python · Sub-5ms native CLI & MCP server · **[Live WebAssembly Playground & `llms.txt`](https://pgg898200-ship-it.github.io/backfold/)**
|
|
10
|
+
|
|
11
|
+
---
|
|
12
|
+
|
|
13
|
+
> **Every streaming library assumes time moves forward, nothing gets retracted, and only one consumer reads the stream.**
|
|
14
|
+
> Real sensor pipelines, financial feeds, telemetry firehoses, and multi-agent LLM swarms break all three assumptions. When an event arrives 50,000 ticks late—or gets retracted, restated, or branched by 64 counterfactual agents—`river`, `pandas.ewm`, and sequential `numpy`/`scipy` Kalman filters/smoothers fall back to **$O(T)$ full forward-backward replay**.
|
|
15
|
+
>
|
|
16
|
+
> `backfold` compiles every operator into an **exact associative monoid** inside a persistent copy-on-write B+ fold tree (`FoldTree<M, B>`): **$O(\log T)$ late edits & retractions, $O(\log T)$ exact Rauch–Tung–Striebel (RTS) Bayesian smoothing & marginal log-likelihood $\log p(y_{1:T})$, zero-blind-spot Bayesian changepoint shock localization, 106-bit streaming PCA & Mahalanobis anomaly scoring, $O(1)$ zero-copy `fork()` (~20 ns), echo-proof multi-agent CRDT merges, distribution-free e-BH FDR control (`sift_streams`), a zero-dependency stdio MCP server (`backfold mcp`), and `<300`-token lossless-recoverable LLM stream digests — with zero approximation (`max |diff| <= 1e-12`).**
|
|
17
|
+
|
|
18
|
+
---
|
|
19
|
+
|
|
20
|
+
## Live Interactive WebAssembly Playground
|
|
21
|
+
|
|
22
|
+
Try real compiled Rust `backfold-core` (`backfold_wasm.wasm`, zero backend server) directly in your browser on 20,000 ticks — click to inject late glitches, Shift+Click to retract ticks, localize off-grid shocks, and benchmark **WASM $O(\log T)$ (`~8 µs`) vs. JavaScript sequential RTS (`~28 ms`, ~3,600× faster, `diff < 1e-14`)** live on your own machine:
|
|
23
|
+
|
|
24
|
+
**[https://pgg898200-ship-it.github.io/backfold/](https://pgg898200-ship-it.github.io/backfold/)**
|
|
25
|
+
|
|
26
|
+
---
|
|
27
|
+
|
|
28
|
+
## 30-Second Quickstart
|
|
29
|
+
|
|
30
|
+
### Python (`pip install backfold` — prebuilt wheels for Linux, macOS, Windows, Py 3.10–3.13)
|
|
31
|
+
```python
|
|
32
|
+
import numpy as np
|
|
33
|
+
import backfold as bf
|
|
34
|
+
|
|
35
|
+
ts = np.arange(100_000, dtype=np.int64)
|
|
36
|
+
dts = np.ones(100_000)
|
|
37
|
+
ys = np.cumsum(np.random.default_rng(0).normal(0.05, 0.1, 100_000))
|
|
38
|
+
m0, p0 = [0.0, 0.0], [10.0, 0.0, 0.0, 10.0]
|
|
39
|
+
|
|
40
|
+
# 1. Exact 2D Constant-Velocity Kalman filter + O(log T) RTS smoother + Bayesian changepoint locator
|
|
41
|
+
kf = bf.KalmanStream(dim=2)
|
|
42
|
+
kf.ingest_constant_velocity(ts, ys, dts, q_accel=1e-3, r_obs=0.25)
|
|
43
|
+
|
|
44
|
+
# 2. O(1) zero-copy counterfactual fork (~20 ns) + O(log T) late repair + RTS smooth (~11.1 µs)
|
|
45
|
+
branch = kf.fork()
|
|
46
|
+
branch.upsert_constant_velocity(33_333, float(ys[33_333] + 10.0), 1.0, 1e-3, 0.25)
|
|
47
|
+
sm_mean, sm_cov = branch.smooth_at(50_000, m0, p0) # Exact RTS p(x_t | y_{1:T})
|
|
48
|
+
log_ev = branch.log_marginal_likelihood(m0, p0) # Exact log p(y_{1:T}), O(1) at root!
|
|
49
|
+
cp = branch.locate_changepoint(m0, p0, [200.0, 0.0, 0.0, 1.0]) # Zero-blind-spot shock locator (<0.6 ms!)
|
|
50
|
+
|
|
51
|
+
# 3. Exact 106-bit compensated multivariate covariance, cyclic-Jacobi streaming PCA & Mahalanobis
|
|
52
|
+
cov = bf.CovarianceStream(dim=8)
|
|
53
|
+
cov.ingest(ts[:10_000], np.random.default_rng(1).normal(1e12, 1.0, size=(10_000, 8)))
|
|
54
|
+
pca = cov.range_pca(1_000, 9_000, ddof=1, top_k=3) # Zero BLAS, exact even at 1e12 offset!
|
|
55
|
+
d2 = cov.mahalanobis_sq([1e12 + 10.0] * 8)
|
|
56
|
+
|
|
57
|
+
# 4. Single-call GIL-free 500-channel Wang-Ramdas e-BH FDR sifter (provable FDR <= alpha)
|
|
58
|
+
sift = bf.sift_streams(np.random.default_rng(2).normal(size=(500, 400)), warmup=120, alpha=0.05)
|
|
59
|
+
|
|
60
|
+
# 5. Lossless-recoverable <300-token digest for LLM / coding agents
|
|
61
|
+
digest = bf.digest_stream("sensor_01", ts, ys, top_k=5, threshold=18.0)
|
|
62
|
+
print(digest["markdown"]) # 2,510× smaller than raw CSV, 100% acute-burst window recall
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
### Zero-Dependency Native MCP Server (`backfold mcp`)
|
|
66
|
+
Drop the precompiled sub-1 MB `backfold` binary straight into **Claude Desktop**, **Cursor**, **Windsurf**, or **Claude Code** (zero Python, Node, or Docker runtime required):
|
|
67
|
+
```json
|
|
68
|
+
{
|
|
69
|
+
"mcpServers": {
|
|
70
|
+
"backfold": {
|
|
71
|
+
"command": "backfold",
|
|
72
|
+
"args": ["mcp"]
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
```
|
|
77
|
+
Exposes 5 production streaming tools over newline-delimited stdio JSON-RPC 2.0 (`MCP 2024-11-05`):
|
|
78
|
+
- `backfold_digest_csv` — `<300`-token lossless-recoverable Markdown statistical digest
|
|
79
|
+
- `backfold_drill_csv` — exact 106-bit range moments + raw ticks for `[start, end]`
|
|
80
|
+
- `backfold_kalman_rts_csv` — $O(\log T)$ RTS smoothing + $\log p(y_{1:T})$ + zero-blind-spot shock localization
|
|
81
|
+
- `backfold_covariance_pca_csv` — 106-bit streaming covariance, correlation, cyclic-Jacobi PCA & top Mahalanobis outliers
|
|
82
|
+
- `backfold_ebh_sift_csv` — Wang–Ramdas arbitrary-dependence e-BH FDR-controlled multi-channel alarm sifter
|
|
83
|
+
|
|
84
|
+
### Native CLI (sub-5ms cold startup, zero runtime dependencies)
|
|
85
|
+
```sh
|
|
86
|
+
cargo build --release --bin backfold
|
|
87
|
+
|
|
88
|
+
# Run 7 live verified 100,000-tick demos in 0.53 s (0 approximations, 0 downloads)
|
|
89
|
+
./target/release/backfold demo
|
|
90
|
+
|
|
91
|
+
# Compress a 100,000-row CSV (~600k tokens) into a ~239-token causal Markdown digest (2,510× compression)
|
|
92
|
+
./target/release/backfold digest benchmarks/sample_100k.csv
|
|
93
|
+
|
|
94
|
+
# Losslessly drill down to exact raw ticks for any flagged hotspot window
|
|
95
|
+
./target/release/backfold drill benchmarks/sample_100k.csv --start 22498 --end 22507
|
|
96
|
+
|
|
97
|
+
# Reproduce the live Rust O(log T) vs. handwritten -O3 sequential Rust benchmark
|
|
98
|
+
./target/release/backfold bench
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
Runnable recipes in [`examples/`](examples/):
|
|
102
|
+
- [`examples/kalman_rts_smoothing.py`](examples/kalman_rts_smoothing.py) — $O(\log T)$ RTS smoothing, variance tightening & Bayes evidence model selection
|
|
103
|
+
- [`examples/streaming_pca_and_changepoint_bf.py`](examples/streaming_pca_and_changepoint_bf.py) — zero-blind-spot Bayesian shock localization (`+1e6` offset in `0.39 ms`), `1e12` range PCA & 500-channel `sift_streams`
|
|
104
|
+
- [`examples/multivariate_anomaly_cov.py`](examples/multivariate_anomaly_cov.py) — `1e12`-offset 8D streaming covariance, regime correlation & batch Mahalanobis scoring
|
|
105
|
+
- [`examples/polars_late_arrivals.py`](examples/polars_late_arrivals.py) — retroactive CUSUM alarms, zero-lookahead bitemporal audit, e-BH FDR & agent digest
|
|
106
|
+
|
|
107
|
+
---
|
|
108
|
+
|
|
109
|
+
## Verified Live Benchmarks (`[LIVE-CPU]`, 2-vCPU Linux x86-64, `v0.3.0`)
|
|
110
|
+
|
|
111
|
+
Every number below is measured live on this hardware ([`benchmarks/results.json`](benchmarks/results.json)) and asserts **exact numerical parity** against independent naive sequential oracles ([`benchmarks/oracles.py`](benchmarks/oracles.py)). Python timings conservatively measure a full **edit + query + restore** round-trip (two tree updates + one query).
|
|
112
|
+
|
|
113
|
+
### 1. Out-of-Order Late Edit Latency — Python API (`T = 100,000`, `max |diff| <= 1e-12`)
|
|
114
|
+
|
|
115
|
+
| Operator | `backfold` (Python) | NumPy | SciPy | Pandas | Polars | River (`T=10k`) | Speedup | Max `\|diff\|` |
|
|
116
|
+
|---|---:|---:|---:|---:|---:|---:|---:|---:|
|
|
117
|
+
| **2D Kalman RTS Smooth + $\log p(y)$** | **11.12 µs** | 6,437,010 µs | — | — | — | — | **578,868× vs NumPy RTS** | `1.47e-14` |
|
|
118
|
+
| **2D Kalman CV Filter** | **8.82 µs** | 4,563,614 µs | — | — | — | — | **517,710× vs NumPy** | `9.09e-13` |
|
|
119
|
+
| **Page CUSUM + 1st alarm** | **3.83 µs** | 137,450 µs *(Py seq)* | — | — | — | — | **35,897× vs seq** | `1.78e-15` |
|
|
120
|
+
| **CT Irregular-$\Delta t$ EWMA** | **3.92 µs** | 1,940.7 µs | — | 4,831.8 µs | — | — | **495× NumPy · 1,231× Pandas** | `1.78e-15` |
|
|
121
|
+
| **8D Covariance Matrix** | **15.60 µs** | 5,589.1 µs | — | — | — | — | **358× vs NumPy `cov`** | `6.66e-16` |
|
|
122
|
+
| **8D Discounted Ridge** | **16.95 µs** | 5,567.1 µs | 5,920.6 µs | — | — | 103,981 µs *(10k)* | **328× NumPy · 349× SciPy · 7,092× River** | `1.33e-15` |
|
|
123
|
+
| **Compensated Moments** *(mean/var/skew/kurt)* | **6.06 µs** | 730.4 µs | — | 1,293.5 µs | 273.0 µs | 8,215 µs *(10k)* | **121× NumPy · 213× Pandas · 45× Polars** | `4.54e-16` |
|
|
124
|
+
|
|
125
|
+
### 2. Rust-vs-Rust Head-to-Head (`backfold` $O(\log T)$ vs. Handwritten `-O3` Sequential Rust)
|
|
126
|
+
|
|
127
|
+
Not just "Rust vs. Python" — even against tight handwritten `-O3` Rust loops, Pareto-tuned (`B = 4`, single-pass `split_summary`, in-place `combine_assign`) associative monoids win by **up to 8,152×** (sub-microsecond at half a million ticks!):
|
|
128
|
+
|
|
129
|
+
| Operator | `T = 10,000` | `T = 100,000` | `T = 500,000` | `backfold` @ 500k | Seq Rust `-O3` @ 500k | Max `\|diff\|` |
|
|
130
|
+
|---|---:|---:|---:|---:|---:|---:|
|
|
131
|
+
| **CT Irregular-$\Delta t$ EWMA** | 207× | 1,736× | **8,152×** | **0.806 µs** | 6,570.7 µs | `2.26e-17` |
|
|
132
|
+
| **1D Kalman RW** (`KalmanFilter1D`) | 160× | 1,338× | **6,185×** | **0.979 µs** | 6,052.4 µs | `1.39e-16` |
|
|
133
|
+
| **2D Kalman RTS Smooth + $\log p(y)$** | 89× | 760× | **3,694×** | **5.367 µs** | 19,823.4 µs | `3.06e-14` |
|
|
134
|
+
| **2D Kalman CV** (`KalmanFilter2D`) | 89× | 760× | **3,538×** | **3.110 µs** | 11,000.2 µs | `2.78e-16` |
|
|
135
|
+
| **Page CUSUM + 1st alarm** | 53× | 454× | **2,177×** | **1.181 µs** | 2,571.3 µs | `1.78e-15` |
|
|
136
|
+
| **Compensated Moments** | 32× | 259× | **1,288×** | **1.278 µs** | 1,646.3 µs | `3.12e-14` |
|
|
137
|
+
| **Arbitrary 60% Range Query** | — | 154× | **664×** | **1.404 µs** | 932.2 µs | `8.77e-15` |
|
|
138
|
+
| **8D Discounted Ridge** | 79× | **659×** | — | *6.108 µs @ 100k* | *4,022.8 µs @ 100k* | `2.00e-15` |
|
|
139
|
+
| **8D Compensated Covariance** | 26× | **262×** | — | *5.677 µs @ 100k* | *1,484.7 µs @ 100k* | `1.93e-14` |
|
|
140
|
+
|
|
141
|
+
### 3. Multi-Agent Coordination, Echo Immunity & 500-Stream FDR Control
|
|
142
|
+
|
|
143
|
+
| Scenario | Naive Baseline | `backfold` | Edge |
|
|
144
|
+
|---|---|---|---|
|
|
145
|
+
| **16-Agent Gossip Swarm** *(2,000 events echoed 3× out-of-order)* | 66.7% overwrite churn (4,000 overwrites); `1.732×` ($\sqrt{3}$) echo z-inflation | **0.0% overwrite churn**; 4,000/4,000 echoes deduplicated; **1.000× exact evidence** | Echo-proof `(event_time, source_id, seq)` CRDT |
|
|
146
|
+
| **500-Stream Firehose** *(10 true shifts + 50 3×-loud traps, $\alpha=0.05$)* | 63 alarms: 10 TP, **53 FP (50/50 loud traps!)**, **FDR = 84.1%** | **10 alarms: 10/10 TP, 0 FP, 0/50 loud traps, FDR = 0.0%** (`107.4 ms` via `sift_streams`) | Pure-Rust GIL-free causal MAD + Shiryaev–Roberts e-value + Wang–Ramdas e-BH |
|
|
147
|
+
| **64-Agent Counterfactual Forking** *(`T=100k`, 5 edits/agent)* | 4,688.1 ms (`73.3 ms/agent`) | **1.41 ms** (`0.022 ms/agent`, `max_diff = 3.55e-15`) | **3,326× faster** ($O(1)$ persistent `Arc` fork) |
|
|
148
|
+
|
|
149
|
+
### 4. Lossless-Recoverable AI Agent Stream Compression (`50 streams × 20,000 ticks`)
|
|
150
|
+
|
|
151
|
+
| Representation | Avg Tokens | Compression | Acute Burst Exact-Window Recall | Regime Shift Recall | Loud 3×-Distractor FP Rate | Lossless Recovery |
|
|
152
|
+
|---|---:|---:|---:|---:|---:|---|
|
|
153
|
+
| Raw CSV | 120,000 | 1× | — *(overflows context)* | — | — | Yes |
|
|
154
|
+
| `pandas.describe()` | 17 | 7,059× | **0.0%** *(drops timestamps)* | 100.0% | **100.0%** | No |
|
|
155
|
+
| Uniform 200× Downsample | 600 | 200× | **20.0%** *(misses 5-tick spikes)* | 86.7% | **30.0%** | No |
|
|
156
|
+
| **`backfold digest`** | **255.8** | **469.1×** *(2,510× on 100k)* | **100.0%** | **100.0%** | **0.0%** | **Yes (`backfold drill`)** |
|
|
157
|
+
|
|
158
|
+
---
|
|
159
|
+
|
|
160
|
+
## What's Inside (7 Exact Associative Operators + Swarm CRDT + Native MCP + WASM)
|
|
161
|
+
|
|
162
|
+
1. **`KalmanStream` / `KalmanFilter1D` / `KalmanFilter2D` / `KalmanMonoid`** — Exact Särkkä & García-Fernández (IEEE TAC 2020) associative Bayesian filter, **single-pass $O(\log T)$ Rauch–Tung–Striebel (RTS) smoother (`smooth_at`, `smooth_many`)**, exact marginal log-likelihood (`log_marginal_likelihood`, `range_log_marginal_likelihood`), **zero-cancellation Bayesian changepoint Bayes factors (`changepoint_bayes_factor`, `scan_changepoints`)**, and **zero-blind-spot cached-subtree Bayesian shock localization (`locate_changepoint`)**. Hardened with **Anchor-Centered Associative Gaussian Potentials** (`1.14e-15` rel err @ 100k; `1.04e-12` at `1e6` offset), Joseph-form PSD leaves, zero-alloc stack `1D`/`2D` paths, and x86-64 subnormal flush (`|x| < 1e-290 => 0`).
|
|
163
|
+
2. **`CovarianceStream` / `CovarianceMonoid`** — Exact multivariate streaming covariance, Pearson correlation, arbitrary `[t1, t2]` regime correlation, **exact pure-safe cyclic-Jacobi streaming PCA (`pca`, `range_pca` — zero BLAS/LAPACK, `<1e-11` accuracy even at `1e12` offset!)**, and Jacobi-equilibrated Cholesky **Mahalanobis anomaly scoring** (`mahalanobis_sq`, factored batch `mahalanobis_many`), with 106-bit Knuth TwoSum compensated mean vectors and zero-alloc inline stack storage for `dim <= 8`.
|
|
164
|
+
3. **`MomentsStream` / `MomentsMonoid`** — Centered Chan–Pébay–Terriberry higher-order moments (`mean`, `variance`, `skewness`, `excess_kurtosis`, `min`/`max` + exact timestamps) with **Knuth TwoSum 106-bit mean compensation** (`<1e-12` variance accuracy even on `1e12 + N(0, 1)` streams) and singleton-leaf fast-path `combine_assign`.
|
|
165
|
+
4. **`EwmaStream` / `EwmaMonoid`** — Continuous-time irregular-$\Delta t$ centered exponentially weighted mean & variance with subnormal-safe decay clamping (`lambda * dt > 700 => 0`).
|
|
166
|
+
5. **`CusumStream` / `CusumMonoid`** — Two-sided tropical-semiring Page CUSUM with non-empty `max_pref` (identity = $-\infty$) supporting exact **$O(\log T)$ earliest-alarm binary descent** (`first_alarm(threshold)`).
|
|
167
|
+
6. **`EProcessStream` / `EProcessMonoid` + `ebh_reject` + `sift_streams`** — Simultaneously tracks Waudby-Smith–Ramdas 8-point symmetric betting martingale (`wealth`), Shin–Ramdas–Rinaldo Page e-CUSUM (`e_detector`), and exact log-sum-exp **Shiryaev–Roberts (`sr_evalue`) + all-subinterval scan (`scan_evalue`)** yielding a provably valid composite-null e-value ($\mathbb{E}_0[e] \le 1$, even under heavy-tailed Cauchy/Student-$t$ noise) for Wang–Ramdas arbitrary-dependence **e-BH FDR control**, plus single-call GIL-free batch `sift_streams`.
|
|
168
|
+
7. **`RidgeStream` / `RidgeMonoid`** — Continuous-time exponentially discounted Ridge / RLS regression with zero-alloc inline stack storage for `dim <= 8`, in-place `combine_assign`, **Jacobi-equilibrated Cholesky** ($D^{-1/2}(G + \alpha I)D^{-1/2}$) with pivoted LU & relative-Tikhonov fallback, and exact predictive variance.
|
|
169
|
+
8. **`SwarmEProcessLedger` & `BitemporalMoments`** — Echo-proof multi-agent CRDT ledger (`Arc::ptr_eq`-accelerated idempotent merge) and zero-lookahead bitemporal `as_of(knowledge_time)` snapshots.
|
|
170
|
+
9. **GIL-free Python scaling** — Bulk `ingest*`, `smooth_many`, `scan_changepoints`, `locate_changepoint`, `mahalanobis_many`, `digest_stream`, and `sift_streams` release the Python GIL (`py.allow_threads`) for true multi-core scaling in Python agent swarms.
|
|
171
|
+
|
|
172
|
+
---
|
|
173
|
+
|
|
174
|
+
## Honest Crossovers & Limitations
|
|
175
|
+
|
|
176
|
+
`backfold` is **not** magic for every workload. Know where it wins and where a simpler tool wins:
|
|
177
|
+
|
|
178
|
+
- **Pure append-only, single-pass, in-order aggregation:** If you never receive late arrivals, never retract/edit history, never query arbitrary sub-intervals `[t1, t2]`, and never fork branches, a plain sequential loop is faster on initial ingest. On `MomentsMonoid` at `T = 100k–500k`, bulk tree construction takes `9.99 ms` (`T=100k`) / `51.07 ms` (`T=500k`) vs. `0.30 ms` / `1.65 ms` for one sequential Rust pass — breaking even after **31–33 late updates or range queries**.
|
|
179
|
+
- **High-dimensional dense matrices (`d > 64`):** `RidgeMonoid` and `CovarianceMonoid` maintain packed upper-triangular $O(d^2)$ matrices per tree node. They shine for streaming telemetry, control, sensor fusion, and factor models (`d <= 32`, zero-alloc inline for `d <= 8`), not 10,000-feature dense NLP embeddings.
|
|
180
|
+
- **Exact quantile / median queries:** Order statistics (arbitrary quantiles) do not admit a fixed-size exact associative monoid without sketch approximation. `backfold` sticks strictly to **exact** operators (`causal_robust_scale` computes exact median/MAD on causal warm-up prefixes).
|
|
181
|
+
|
|
182
|
+
---
|
|
183
|
+
|
|
184
|
+
## For AI Coding Agents
|
|
185
|
+
|
|
186
|
+
- **Native stdio MCP server:** `backfold mcp` (5 built-in CSV streaming tools, zero runtime dependencies).
|
|
187
|
+
- **Agent discovery index:** [`llms.txt`](llms.txt) and complete API reference [`llms-full.txt`](llms-full.txt) (pure Markdown, also hosted at `https://pgg898200-ship-it.github.io/backfold/llms.txt`).
|
|
188
|
+
- **Drop-in coding-agent skill:** [`skills/backfold/SKILL.md`](skills/backfold/SKILL.md) (works with Claude Code, Cursor, Codex, OpenCode, and MCP-compatible agents).
|
|
189
|
+
- **Technical deep-dive:** [`docs/SUBNORMAL_KALMAN.md`](docs/SUBNORMAL_KALMAN.md) (x86-64 subnormal traps, Anchor-Centered Gaussian Potentials, zero-cancellation changepoint Bayes factors, 106-bit TwoSum moments, and Shiryaev–Roberts e-values).
|
|
190
|
+
- **Reproduce all tests & benchmarks:**
|
|
191
|
+
```sh
|
|
192
|
+
cargo test --package backfold-core --package backfold-cli --package backfold-wasm --release
|
|
193
|
+
cargo clippy --workspace --all-targets -- -D warnings
|
|
194
|
+
pytest -v tests/
|
|
195
|
+
python -m benchmarks.run_benchmarks
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
## License
|
|
199
|
+
|
|
200
|
+
Licensed under the [Apache License, Version 2.0](LICENSE).
|