telemetry-anomdet 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/LICENSE +1 -1
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/PKG-INFO +18 -14
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/README.md +11 -10
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/pyproject.toml +14 -5
- telemetry_anomdet-0.3.0/src/telemetry_anomdet/ingest/__init__.py +41 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/src/telemetry_anomdet/ingest/dataset.py +3 -6
- telemetry_anomdet-0.3.0/src/telemetry_anomdet/ingest/esa.py +347 -0
- telemetry_anomdet-0.3.0/src/telemetry_anomdet/models/deep/_ar.py +186 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/src/telemetry_anomdet/models/deep/_kan.py +137 -1
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/src/telemetry_anomdet/models/deep/_net.py +9 -1
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/src/telemetry_anomdet/models/deep/codegen.py +155 -26
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/src/telemetry_anomdet/models/deep/distill.py +52 -8
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/src/telemetry_anomdet/models/deep/gdn.py +243 -13
- telemetry_anomdet-0.3.0/src/telemetry_anomdet/models/deep/kan_gdn.py +164 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/src/telemetry_anomdet/thresholding.py +17 -18
- telemetry_anomdet-0.3.0/src/telemetry_anomdet/version.py +12 -0
- telemetry_anomdet-0.3.0/src/telemetry_anomdet/visualization.py +425 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/src/telemetry_anomdet.egg-info/PKG-INFO +18 -14
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/src/telemetry_anomdet.egg-info/SOURCES.txt +8 -1
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/src/telemetry_anomdet.egg-info/requires.txt +6 -1
- telemetry_anomdet-0.3.0/tests/test_deep_ar.py +429 -0
- telemetry_anomdet-0.3.0/tests/test_deep_attribution.py +246 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/tests/test_deep_codegen.py +57 -20
- telemetry_anomdet-0.3.0/tests/test_ingest_esa.py +349 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/tests/test_threshold_budget.py +25 -13
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/tests/test_thresholding.py +10 -0
- telemetry_anomdet-0.3.0/tests/test_visualization.py +257 -0
- telemetry_anomdet-0.2.0/src/telemetry_anomdet/ingest/__init__.py +0 -21
- telemetry_anomdet-0.2.0/src/telemetry_anomdet/models/deep/kan_gdn.py +0 -99
- telemetry_anomdet-0.2.0/src/telemetry_anomdet/version.py +0 -15
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/setup.cfg +0 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/src/telemetry_anomdet/__init__.py +0 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/src/telemetry_anomdet/evaluation.py +0 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/src/telemetry_anomdet/feature_extraction/features.py +0 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/src/telemetry_anomdet/ingest/ccsds_loader.py +0 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/src/telemetry_anomdet/ingest/csv_loader.py +0 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/src/telemetry_anomdet/ingest/smap.py +0 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/src/telemetry_anomdet/models/__init__.py +0 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/src/telemetry_anomdet/models/base.py +0 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/src/telemetry_anomdet/models/deep/__init__.py +0 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/src/telemetry_anomdet/models/ensemble.py +0 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/src/telemetry_anomdet/models/supervised/__init__.py +0 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/src/telemetry_anomdet/models/unsupervised/__init__.py +0 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/src/telemetry_anomdet/models/unsupervised/gaussian_nb.py +0 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/src/telemetry_anomdet/models/unsupervised/isolation_forest.py +0 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/src/telemetry_anomdet/models/unsupervised/kmeans.py +0 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/src/telemetry_anomdet/models/unsupervised/pca.py +0 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/src/telemetry_anomdet/preprocessing/__init__.py +0 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/src/telemetry_anomdet/preprocessing/preprocessing.py +0 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/src/telemetry_anomdet/py.typed +0 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/src/telemetry_anomdet.egg-info/dependency_links.txt +0 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/src/telemetry_anomdet.egg-info/top_level.txt +0 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/tests/test_base_detector.py +0 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/tests/test_csv_loader.py +0 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/tests/test_deep_distill.py +0 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/tests/test_deep_gdn.py +0 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/tests/test_deep_kan.py +0 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/tests/test_deep_kangdn.py +0 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/tests/test_deep_smoothing.py +0 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/tests/test_ensemble.py +0 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/tests/test_evaluation.py +0 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/tests/test_evaluation_prauc.py +0 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/tests/test_evaluation_sequences.py +0 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/tests/test_import.py +0 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/tests/test_preprocessing.py +0 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/tests/test_score_channels.py +0 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/tests/test_smap.py +0 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/tests/test_unsup_kmeans.py +0 -0
- {telemetry_anomdet-0.2.0 → telemetry_anomdet-0.3.0}/tests/test_unsup_pca.py +0 -0
|
@@ -186,7 +186,7 @@
|
|
|
186
186
|
same "printed page" as the copyright notice for easier
|
|
187
187
|
identification within third-party archives.
|
|
188
188
|
|
|
189
|
-
Copyright 2026
|
|
189
|
+
Copyright 2025-2026 Julio Anzaldo
|
|
190
190
|
|
|
191
191
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
192
192
|
you may not use this file except in compliance with the License.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: telemetry-anomdet
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Spacecraft telemetry anomaly detection: classical and graph deep detectors, label-free thresholding, and distillation to C for flight hardware.
|
|
5
5
|
Author-email: Julio Anzaldo <anzaldojuliojesus@gmail.com>
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -14,10 +14,10 @@ Classifier: Natural Language :: English
|
|
|
14
14
|
Classifier: Operating System :: OS Independent
|
|
15
15
|
Classifier: Programming Language :: Python :: 3
|
|
16
16
|
Classifier: Programming Language :: Python :: 3 :: Only
|
|
17
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
18
17
|
Classifier: Programming Language :: Python :: 3.11
|
|
19
18
|
Classifier: Programming Language :: Python :: 3.12
|
|
20
19
|
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
21
21
|
Classifier: Programming Language :: C
|
|
22
22
|
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
23
23
|
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
@@ -27,17 +27,20 @@ Classifier: Topic :: Software Development :: Embedded Systems
|
|
|
27
27
|
Classifier: Topic :: Software Development :: Code Generators
|
|
28
28
|
Classifier: Topic :: System :: Monitoring
|
|
29
29
|
Classifier: Typing :: Typed
|
|
30
|
-
Requires-Python: >=3.
|
|
30
|
+
Requires-Python: >=3.11
|
|
31
31
|
Description-Content-Type: text/markdown
|
|
32
32
|
License-File: LICENSE
|
|
33
33
|
Requires-Dist: pandas<3.0.0,>=2.2.2
|
|
34
34
|
Requires-Dist: numpy<3.0,>=2.1
|
|
35
35
|
Requires-Dist: scikit-learn<2.0,>=1.3
|
|
36
|
-
Requires-Dist: typing-extensions>=4.16.0
|
|
37
36
|
Provides-Extra: ccsds
|
|
38
37
|
Requires-Dist: ccsdspy<3.0,>=2.0.0; extra == "ccsds"
|
|
39
38
|
Provides-Extra: deep
|
|
40
39
|
Requires-Dist: torch>=2.0; extra == "deep"
|
|
40
|
+
Provides-Extra: viz
|
|
41
|
+
Requires-Dist: matplotlib>=3.7; extra == "viz"
|
|
42
|
+
Provides-Extra: track
|
|
43
|
+
Requires-Dist: wandb>=0.17; extra == "track"
|
|
41
44
|
Dynamic: license-file
|
|
42
45
|
|
|
43
46
|
<h2 align="center">telemetry-anomdet</h2>
|
|
@@ -50,9 +53,9 @@ Dynamic: license-file
|
|
|
50
53
|
<a href="https://github.com/JulioAnzaldo/telemetry-anomdet/blob/main/LICENSE"><img alt="License" src="https://img.shields.io/badge/License-Apache%202.0-blue"></a>
|
|
51
54
|
</p>
|
|
52
55
|
|
|
53
|
-
**telemetry-anomdet** is an open-source anomaly detection toolkit for spacecraft telemetry. It runs classical and graph-based deep detectors behind one interface, selects alarm thresholds without labels, and distills a trained detector down to Power of Ten conformant C that runs on flight hardware.
|
|
56
|
+
**telemetry-anomdet** is an open-source anomaly detection toolkit for spacecraft telemetry. It runs classical and graph-based deep detectors behind one interface, selects alarm thresholds without labels, points each alarm at the channels that raised it, and distills a trained detector down to Power of Ten conformant C that runs on flight hardware. SHAP attribution and LLM-generated diagnostic reports are on the roadmap.
|
|
54
57
|
|
|
55
|
-
Benchmarked on SMAP
|
|
58
|
+
Benchmarked on SMAP and MSL (NASA) and ESA-ADB (ESA).
|
|
56
59
|
|
|
57
60
|
---
|
|
58
61
|
|
|
@@ -85,6 +88,8 @@ The base install is torch-free. The deep detectors (`GDN`, `KANGDN`) need the
|
|
|
85
88
|
uv add "telemetry-anomdet[deep]"
|
|
86
89
|
```
|
|
87
90
|
|
|
91
|
+
Plotting, including the learned sensor graph, needs the `viz` extra.
|
|
92
|
+
|
|
88
93
|
## Example Usage
|
|
89
94
|
|
|
90
95
|
### Fit the Ensemble on Nominal Telemetry
|
|
@@ -184,13 +189,14 @@ Datasets are never committed; `data/` is gitignored.
|
|
|
184
189
|
**Available now**
|
|
185
190
|
|
|
186
191
|
- **One interface for every detector.** Classical and deep detectors share the same `fit` / `decision_function` / `predict` / `is_anomaly` API, so stacking or swapping models needs no per-model glue.
|
|
187
|
-
- **Graph deviation detectors.** `GDN` and `KANGDN` forecast each channel from its learned top-k neighbours and score the deviation, catching readings that are individually plausible but wrong in relation to each other.
|
|
192
|
+
- **Graph deviation detectors.** `GDN` and `KANGDN` forecast each channel from its learned top-k neighbours and score the deviation, catching readings that are individually plausible but wrong in relation to each other. `feat_mode` selects how `KANGDN` encodes each channel's history (`linear`, `kan`, `ar_kan`, `ar_kan_residual`).
|
|
193
|
+
- **Which sensor, not just when.** `channel_deviations` and `dominant_channels` attribute each alarm to the channels driving it, and `plot_sensor_graph` draws the graph the detector learned.
|
|
188
194
|
- **Alarms without labels.** `threshold_for_budget` and `dynamic_threshold` pick an operating point from the score distribution alone. Operations can state an alarm rate; they cannot state a recall they have no way to observe.
|
|
189
|
-
- **Honest evaluation.** Event-level scoring alongside the conventional point-adjusted F1, plus a random baseline row in the benchmark
|
|
190
|
-
- **Runs on flight hardware.** A fitted `KANGDN` distills to a torch-free NumPy evaluator, then to Power of Ten conformant C with golden vectors
|
|
195
|
+
- **Honest evaluation.** Event-level scoring alongside the conventional point-adjusted F1, plus a random baseline row in the benchmark. ESA-ADB is scored by its own protocol, with F0.5 as the headline metric.
|
|
196
|
+
- **Runs on flight hardware.** A fitted `KANGDN` distills to a torch-free NumPy evaluator, then to Power of Ten conformant C with golden vectors, for every `feat_mode`. Host and ESP32-S3 targets are in [`targets/`](targets/).
|
|
191
197
|
- **Stacking ensemble** with robust (median + IQR) score normalization, built for anomaly scores that are extreme by definition.
|
|
192
198
|
- **Per-model score decomposition** via `score_components()`, the hook that enables per-channel attribution without retraining.
|
|
193
|
-
- **Spacecraft-native ingestion.** SMAP and CSV load straight into a long-form `TelemetryDataset`, no schema wrangling.
|
|
199
|
+
- **Spacecraft-native ingestion.** SMAP, MSL, ESA-ADB and CSV load straight into a long-form `TelemetryDataset`, no schema wrangling. The ESA-ADB loader reads the official archive directly, with no preprocessing step; its payloads are pickled, so load only that archive, never untrusted files.
|
|
194
200
|
- **Runtime sensitivity control.** `is_anomaly()` accepts a percentile or threshold override, so operators re-tune without retraining.
|
|
195
201
|
|
|
196
202
|
**On the roadmap**
|
|
@@ -199,16 +205,14 @@ Datasets are never committed; `data/` is gitignored.
|
|
|
199
205
|
- `SHAPExplainer`: per-channel attribution over `score_components()` (Phase 3)
|
|
200
206
|
- LLM diagnostic reports, SHAP chart supplied as an image (Phase 4)
|
|
201
207
|
- Human-in-the-loop threshold feedback (Phase 5)
|
|
202
|
-
- MSL results, using the existing SMAP loader (`spacecraft = "MSL"`) (v0.3.0)
|
|
203
|
-
- ESA-ADB evaluation: the multivariate benchmark that exercises the graph (v0.3.0)
|
|
204
208
|
|
|
205
209
|
## AI Transparency
|
|
206
210
|
|
|
207
|
-
AI assistants are used while writing and maintaining this project's documentation
|
|
211
|
+
AI assistants are used while writing and maintaining this project's documentation.
|
|
208
212
|
|
|
209
213
|
What that does not change:
|
|
210
214
|
|
|
211
|
-
- **Numbers come from runs.** Benchmark figures are produced by [`examples/smap_benchmark.py`](examples/smap_benchmark.py), and the hardware results in the deployment docs are measured on an ESP32-S3 by the conformance harness in [`targets/`](targets/). Nothing quoted in these docs is estimated or inferred.
|
|
215
|
+
- **Numbers come from runs.** Benchmark figures are produced by [`examples/smap_benchmark.py`](examples/smap_benchmark.py) and [`examples/esa_benchmark.py`](examples/esa_benchmark.py), and the hardware results in the deployment docs are measured on an ESP32-S3 by the conformance harness in [`targets/`](targets/). Nothing quoted in these docs is estimated or inferred.
|
|
212
216
|
- **Behavior claims are backed by tests.** The suite gates CI, and the documented API is covered by it.
|
|
213
217
|
|
|
214
218
|
Everything published here is reviewed before it lands.
|
|
@@ -8,9 +8,9 @@
|
|
|
8
8
|
<a href="https://github.com/JulioAnzaldo/telemetry-anomdet/blob/main/LICENSE"><img alt="License" src="https://img.shields.io/badge/License-Apache%202.0-blue"></a>
|
|
9
9
|
</p>
|
|
10
10
|
|
|
11
|
-
**telemetry-anomdet** is an open-source anomaly detection toolkit for spacecraft telemetry. It runs classical and graph-based deep detectors behind one interface, selects alarm thresholds without labels, and distills a trained detector down to Power of Ten conformant C that runs on flight hardware.
|
|
11
|
+
**telemetry-anomdet** is an open-source anomaly detection toolkit for spacecraft telemetry. It runs classical and graph-based deep detectors behind one interface, selects alarm thresholds without labels, points each alarm at the channels that raised it, and distills a trained detector down to Power of Ten conformant C that runs on flight hardware. SHAP attribution and LLM-generated diagnostic reports are on the roadmap.
|
|
12
12
|
|
|
13
|
-
Benchmarked on SMAP
|
|
13
|
+
Benchmarked on SMAP and MSL (NASA) and ESA-ADB (ESA).
|
|
14
14
|
|
|
15
15
|
---
|
|
16
16
|
|
|
@@ -43,6 +43,8 @@ The base install is torch-free. The deep detectors (`GDN`, `KANGDN`) need the
|
|
|
43
43
|
uv add "telemetry-anomdet[deep]"
|
|
44
44
|
```
|
|
45
45
|
|
|
46
|
+
Plotting, including the learned sensor graph, needs the `viz` extra.
|
|
47
|
+
|
|
46
48
|
## Example Usage
|
|
47
49
|
|
|
48
50
|
### Fit the Ensemble on Nominal Telemetry
|
|
@@ -142,13 +144,14 @@ Datasets are never committed; `data/` is gitignored.
|
|
|
142
144
|
**Available now**
|
|
143
145
|
|
|
144
146
|
- **One interface for every detector.** Classical and deep detectors share the same `fit` / `decision_function` / `predict` / `is_anomaly` API, so stacking or swapping models needs no per-model glue.
|
|
145
|
-
- **Graph deviation detectors.** `GDN` and `KANGDN` forecast each channel from its learned top-k neighbours and score the deviation, catching readings that are individually plausible but wrong in relation to each other.
|
|
147
|
+
- **Graph deviation detectors.** `GDN` and `KANGDN` forecast each channel from its learned top-k neighbours and score the deviation, catching readings that are individually plausible but wrong in relation to each other. `feat_mode` selects how `KANGDN` encodes each channel's history (`linear`, `kan`, `ar_kan`, `ar_kan_residual`).
|
|
148
|
+
- **Which sensor, not just when.** `channel_deviations` and `dominant_channels` attribute each alarm to the channels driving it, and `plot_sensor_graph` draws the graph the detector learned.
|
|
146
149
|
- **Alarms without labels.** `threshold_for_budget` and `dynamic_threshold` pick an operating point from the score distribution alone. Operations can state an alarm rate; they cannot state a recall they have no way to observe.
|
|
147
|
-
- **Honest evaluation.** Event-level scoring alongside the conventional point-adjusted F1, plus a random baseline row in the benchmark
|
|
148
|
-
- **Runs on flight hardware.** A fitted `KANGDN` distills to a torch-free NumPy evaluator, then to Power of Ten conformant C with golden vectors
|
|
150
|
+
- **Honest evaluation.** Event-level scoring alongside the conventional point-adjusted F1, plus a random baseline row in the benchmark. ESA-ADB is scored by its own protocol, with F0.5 as the headline metric.
|
|
151
|
+
- **Runs on flight hardware.** A fitted `KANGDN` distills to a torch-free NumPy evaluator, then to Power of Ten conformant C with golden vectors, for every `feat_mode`. Host and ESP32-S3 targets are in [`targets/`](targets/).
|
|
149
152
|
- **Stacking ensemble** with robust (median + IQR) score normalization, built for anomaly scores that are extreme by definition.
|
|
150
153
|
- **Per-model score decomposition** via `score_components()`, the hook that enables per-channel attribution without retraining.
|
|
151
|
-
- **Spacecraft-native ingestion.** SMAP and CSV load straight into a long-form `TelemetryDataset`, no schema wrangling.
|
|
154
|
+
- **Spacecraft-native ingestion.** SMAP, MSL, ESA-ADB and CSV load straight into a long-form `TelemetryDataset`, no schema wrangling. The ESA-ADB loader reads the official archive directly, with no preprocessing step; its payloads are pickled, so load only that archive, never untrusted files.
|
|
152
155
|
- **Runtime sensitivity control.** `is_anomaly()` accepts a percentile or threshold override, so operators re-tune without retraining.
|
|
153
156
|
|
|
154
157
|
**On the roadmap**
|
|
@@ -157,16 +160,14 @@ Datasets are never committed; `data/` is gitignored.
|
|
|
157
160
|
- `SHAPExplainer`: per-channel attribution over `score_components()` (Phase 3)
|
|
158
161
|
- LLM diagnostic reports, SHAP chart supplied as an image (Phase 4)
|
|
159
162
|
- Human-in-the-loop threshold feedback (Phase 5)
|
|
160
|
-
- MSL results, using the existing SMAP loader (`spacecraft = "MSL"`) (v0.3.0)
|
|
161
|
-
- ESA-ADB evaluation: the multivariate benchmark that exercises the graph (v0.3.0)
|
|
162
163
|
|
|
163
164
|
## AI Transparency
|
|
164
165
|
|
|
165
|
-
AI assistants are used while writing and maintaining this project's documentation
|
|
166
|
+
AI assistants are used while writing and maintaining this project's documentation.
|
|
166
167
|
|
|
167
168
|
What that does not change:
|
|
168
169
|
|
|
169
|
-
- **Numbers come from runs.** Benchmark figures are produced by [`examples/smap_benchmark.py`](examples/smap_benchmark.py), and the hardware results in the deployment docs are measured on an ESP32-S3 by the conformance harness in [`targets/`](targets/). Nothing quoted in these docs is estimated or inferred.
|
|
170
|
+
- **Numbers come from runs.** Benchmark figures are produced by [`examples/smap_benchmark.py`](examples/smap_benchmark.py) and [`examples/esa_benchmark.py`](examples/esa_benchmark.py), and the hardware results in the deployment docs are measured on an ESP32-S3 by the conformance harness in [`targets/`](targets/). Nothing quoted in these docs is estimated or inferred.
|
|
170
171
|
- **Behavior claims are backed by tests.** The suite gates CI, and the documented API is covered by it.
|
|
171
172
|
|
|
172
173
|
Everything published here is reviewed before it lands.
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "telemetry-anomdet"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.3.0"
|
|
4
4
|
description = "Spacecraft telemetry anomaly detection: classical and graph deep detectors, label-free thresholding, and distillation to C for flight hardware."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "Apache-2.0"
|
|
7
7
|
license-files = ["LICENSE"]
|
|
8
|
-
requires-python = ">=3.
|
|
8
|
+
requires-python = ">=3.11"
|
|
9
9
|
authors = [
|
|
10
10
|
{ name = "Julio Anzaldo", email = "anzaldojuliojesus@gmail.com" }
|
|
11
11
|
]
|
|
@@ -13,7 +13,6 @@ dependencies = [
|
|
|
13
13
|
"pandas>=2.2.2,<3.0.0",
|
|
14
14
|
"numpy>=2.1,<3.0",
|
|
15
15
|
"scikit-learn>=1.3,<2.0",
|
|
16
|
-
"typing-extensions>=4.16.0",
|
|
17
16
|
]
|
|
18
17
|
keywords = ["telemetry", "anomaly-detection", "machine-learning", "python"]
|
|
19
18
|
# License is declared as an SPDX expression above, not a classifier: license
|
|
@@ -26,10 +25,10 @@ classifiers = [
|
|
|
26
25
|
"Operating System :: OS Independent",
|
|
27
26
|
"Programming Language :: Python :: 3",
|
|
28
27
|
"Programming Language :: Python :: 3 :: Only",
|
|
29
|
-
"Programming Language :: Python :: 3.10",
|
|
30
28
|
"Programming Language :: Python :: 3.11",
|
|
31
29
|
"Programming Language :: Python :: 3.12",
|
|
32
30
|
"Programming Language :: Python :: 3.13",
|
|
31
|
+
"Programming Language :: Python :: 3.14",
|
|
33
32
|
# The toolkit is pure Python; the artifact it generates for flight targets is C.
|
|
34
33
|
"Programming Language :: C",
|
|
35
34
|
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
@@ -51,6 +50,16 @@ ccsds = [
|
|
|
51
50
|
deep = [
|
|
52
51
|
"torch>=2.0",
|
|
53
52
|
]
|
|
53
|
+
# Plotting. Optional so the base install stays lean; matplotlib is imported
|
|
54
|
+
# lazily inside each figure function.
|
|
55
|
+
viz = [
|
|
56
|
+
"matplotlib>=3.7",
|
|
57
|
+
]
|
|
58
|
+
# Experiment tracking for the benchmark scripts. Nothing in the library imports
|
|
59
|
+
# it; examples/smap_benchmark.py logs to it only when TAD_WANDB_PROJECT is set.
|
|
60
|
+
track = [
|
|
61
|
+
"wandb>=0.17",
|
|
62
|
+
]
|
|
54
63
|
|
|
55
64
|
[dependency-groups]
|
|
56
65
|
dev = [
|
|
@@ -90,7 +99,7 @@ build-backend = "setuptools.build_meta"
|
|
|
90
99
|
|
|
91
100
|
[tool.ruff]
|
|
92
101
|
line-length = 100
|
|
93
|
-
target-version = "
|
|
102
|
+
target-version = "py311"
|
|
94
103
|
src = ["src", "tests"]
|
|
95
104
|
|
|
96
105
|
[tool.ruff.lint]
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# src/telemetry_anomdet/ingest/__init__.py
|
|
2
|
+
|
|
3
|
+
"""
|
|
4
|
+
Ingest subpackage: dataset loaders and parsers.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from .csv_loader import load_from_csv
|
|
8
|
+
from .dataset import TelemetryDataset
|
|
9
|
+
from .esa import (
|
|
10
|
+
MISSION1_LIGHTWEIGHT,
|
|
11
|
+
MISSION1_SPAN,
|
|
12
|
+
esa_splits,
|
|
13
|
+
load_esa,
|
|
14
|
+
load_esa_channels,
|
|
15
|
+
load_esa_labels,
|
|
16
|
+
rescope_anomaly_types,
|
|
17
|
+
)
|
|
18
|
+
from .smap import (
|
|
19
|
+
anomaly_point_mask,
|
|
20
|
+
load_smap,
|
|
21
|
+
load_smap_channel,
|
|
22
|
+
load_smap_labels,
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
# ccsds_loader is deliberately absent: it sits behind the optional ccsds extra,
|
|
26
|
+
# and importing it here would make the base install fail without ccsdspy.
|
|
27
|
+
__all__ = [
|
|
28
|
+
"TelemetryDataset",
|
|
29
|
+
"load_from_csv",
|
|
30
|
+
"load_smap",
|
|
31
|
+
"load_smap_channel",
|
|
32
|
+
"load_smap_labels",
|
|
33
|
+
"anomaly_point_mask",
|
|
34
|
+
"MISSION1_LIGHTWEIGHT",
|
|
35
|
+
"MISSION1_SPAN",
|
|
36
|
+
"esa_splits",
|
|
37
|
+
"load_esa",
|
|
38
|
+
"load_esa_channels",
|
|
39
|
+
"load_esa_labels",
|
|
40
|
+
"rescope_anomaly_types",
|
|
41
|
+
]
|
|
@@ -3,20 +3,17 @@
|
|
|
3
3
|
"""
|
|
4
4
|
Telemetry dataset loader.
|
|
5
5
|
|
|
6
|
-
This module a lightweight class
|
|
6
|
+
This module provides TelemetryDataset, a lightweight class that wraps telemetry
|
|
7
|
+
data in a consistent form for downstream processing.
|
|
7
8
|
"""
|
|
8
9
|
|
|
9
10
|
from __future__ import annotations
|
|
10
11
|
|
|
11
12
|
from dataclasses import dataclass
|
|
13
|
+
from typing import Self
|
|
12
14
|
|
|
13
15
|
import pandas as pd
|
|
14
16
|
|
|
15
|
-
try:
|
|
16
|
-
from typing import Self
|
|
17
|
-
except ImportError:
|
|
18
|
-
from typing_extensions import Self
|
|
19
|
-
|
|
20
17
|
|
|
21
18
|
@dataclass
|
|
22
19
|
class TelemetryDataset:
|
|
@@ -0,0 +1,347 @@
|
|
|
1
|
+
# src/telemetry_anomdet/ingest/esa.py
|
|
2
|
+
|
|
3
|
+
"""
|
|
4
|
+
Loader for the ESA Anomaly Dataset / ESA-ADB (Zenodo DOI 10.5281/zenodo.12528696).
|
|
5
|
+
|
|
6
|
+
A mission archive holds one ``channels/channel_N.zip`` per telemetry channel and
|
|
7
|
+
one ``telecommands/telecommand_N.zip`` per command, alongside four metadata CSVs:
|
|
8
|
+
|
|
9
|
+
``channels.csv``
|
|
10
|
+
``Channel, Subsystem, Physical Unit, Group, Target, Categorical``. Only
|
|
11
|
+
``Target = YES`` channels are monitored for anomalies (58 of 76 in Mission1);
|
|
12
|
+
the rest exist to support detection.
|
|
13
|
+
``labels.csv``
|
|
14
|
+
``ID, Channel, StartTime, EndTime``. One row per *segment*, so a single event
|
|
15
|
+
``ID`` appears once per affected channel and may appear several times for one
|
|
16
|
+
channel when the event is a burst of separate disturbances.
|
|
17
|
+
``anomaly_types.csv``
|
|
18
|
+
``ID, Class, Subclass, Category, Dimensionality, Locality, Length``, where
|
|
19
|
+
``Category`` is Anomaly, Rare Event, or Communication Gap.
|
|
20
|
+
``telecommands.csv``
|
|
21
|
+
``Telecommand, Priority``.
|
|
22
|
+
|
|
23
|
+
Unlike SMAP/MSL this dataset carries real timestamps, so nothing is synthesised.
|
|
24
|
+
Output is the canonical long form ``[timestamp, variable, value]`` wrapped in a
|
|
25
|
+
:class:`TelemetryDataset`.
|
|
26
|
+
|
|
27
|
+
.. warning::
|
|
28
|
+
Channel payloads are **pickled pandas objects**, so loading executes the
|
|
29
|
+
pickle stream. That is safe for the official Zenodo archive and is not safe
|
|
30
|
+
for arbitrary user-supplied paths. Point this loader at data you trust.
|
|
31
|
+
|
|
32
|
+
Two facts about the annotations drive how results must be scored, and both
|
|
33
|
+
differ from SMAP:
|
|
34
|
+
|
|
35
|
+
* An event is identified by its ``ID``, not by contiguity. Roughly a fifth of
|
|
36
|
+
the (event, channel) pairs in the Mission1 lightweight subset are split across
|
|
37
|
+
two to five segments, deliberately, so that a burst of related disturbances
|
|
38
|
+
counts once. Scoring segments as separate events inflates false positives and
|
|
39
|
+
deflates recall at the same time.
|
|
40
|
+
* ``Rare Event`` rows are atypical but expected behaviour (commanded manoeuvres,
|
|
41
|
+
resets, calibrations). ESA-ADB's headline results score every category except
|
|
42
|
+
``Communication Gap``; :func:`load_esa_labels` keeps the column so either
|
|
43
|
+
convention can be applied explicitly rather than by accident.
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
from __future__ import annotations
|
|
47
|
+
|
|
48
|
+
import io
|
|
49
|
+
import zipfile
|
|
50
|
+
from collections.abc import Sequence
|
|
51
|
+
from pathlib import Path
|
|
52
|
+
|
|
53
|
+
import numpy as np
|
|
54
|
+
import pandas as pd
|
|
55
|
+
|
|
56
|
+
from telemetry_anomdet.ingest.dataset import TelemetryDataset
|
|
57
|
+
|
|
58
|
+
# Canonical columns
|
|
59
|
+
_TS, _VAR, _VAL = "timestamp", "variable", "value"
|
|
60
|
+
|
|
61
|
+
#: Mission1's lightweight subset: the six channels ESA-ADB reports its headline
|
|
62
|
+
#: benchmark on, all of subsystem_5 / physical_unit_4 / group 8. The benchmark's
|
|
63
|
+
#: full 58-channel set defeats every algorithm it published, so this subset is
|
|
64
|
+
#: where a comparable result can be had.
|
|
65
|
+
MISSION1_LIGHTWEIGHT = tuple(f"channel_{i}" for i in range(41, 47))
|
|
66
|
+
|
|
67
|
+
#: Categories excluded from ESA-ADB's headline scoring.
|
|
68
|
+
NON_EVENT_CATEGORIES = ("Communication Gap",)
|
|
69
|
+
|
|
70
|
+
#: Mission1's span, end exclusive. Used to derive the benchmark splits without
|
|
71
|
+
#: opening a 15-million-row archive first.
|
|
72
|
+
MISSION1_SPAN = (pd.Timestamp("2000-01-01"), pd.Timestamp("2014-01-01"))
|
|
73
|
+
|
|
74
|
+
#: Mission1's dominant sampling frequency is 0.033 Hz. This is the grid the
|
|
75
|
+
#: benchmark's own definitions are stated against (a point anomaly is up to
|
|
76
|
+
#: three samples *after resampling to the dominant frequency*), so it is the
|
|
77
|
+
#: canonical cadence rather than merely a convenient one.
|
|
78
|
+
MISSION1_CADENCE_S = 30
|
|
79
|
+
|
|
80
|
+
#: Longest event, in samples at the dominant cadence, still counted as a point
|
|
81
|
+
#: anomaly rather than a subsequence.
|
|
82
|
+
POINT_MAX_SAMPLES = 3
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def esa_splits(
|
|
86
|
+
span: tuple[pd.Timestamp, pd.Timestamp] = MISSION1_SPAN,
|
|
87
|
+
*,
|
|
88
|
+
validation_months: int = 3,
|
|
89
|
+
) -> dict[str, tuple[pd.Timestamp, pd.Timestamp]]:
|
|
90
|
+
"""
|
|
91
|
+
The benchmark's train / validation / test division.
|
|
92
|
+
|
|
93
|
+
ESA-ADB halves each mission chronologically, trains on the first half, and
|
|
94
|
+
reserves the last three months of that half for validation, so every
|
|
95
|
+
validation and test sample is later than every training one.
|
|
96
|
+
|
|
97
|
+
Arguments:
|
|
98
|
+
span: ``(start, end)`` of the mission, end exclusive.
|
|
99
|
+
validation_months: Months at the end of the training half held out.
|
|
100
|
+
Returns:
|
|
101
|
+
dict: ``{"train": (start, end), "validation": ..., "test": ...}``, each
|
|
102
|
+
half-open ``[start, end)``.
|
|
103
|
+
|
|
104
|
+
Notes:
|
|
105
|
+
Anomalies appear in **all three** splits, training included. That is
|
|
106
|
+
deliberate on the benchmark's part, and it breaks an assumption
|
|
107
|
+
inherited from SMAP, whose training split is anomaly-free: the
|
|
108
|
+
per-channel error median and IQR that calibrate the deviation score are
|
|
109
|
+
fitted on data that contains faults. Both statistics are robust, which
|
|
110
|
+
is what makes the arrangement workable, but a detector calibrated here
|
|
111
|
+
is not calibrated on clean data.
|
|
112
|
+
"""
|
|
113
|
+
start, end = pd.Timestamp(span[0]), pd.Timestamp(span[1])
|
|
114
|
+
if end <= start:
|
|
115
|
+
raise ValueError(f"span must be increasing, got {start} to {end}")
|
|
116
|
+
|
|
117
|
+
midpoint = start + (end - start) / 2
|
|
118
|
+
validation_start = midpoint - pd.DateOffset(months=validation_months)
|
|
119
|
+
if validation_start <= start:
|
|
120
|
+
raise ValueError(
|
|
121
|
+
f"validation_months={validation_months} leaves no training data in {start}..{end}"
|
|
122
|
+
)
|
|
123
|
+
return {
|
|
124
|
+
"train": (start, validation_start),
|
|
125
|
+
"validation": (validation_start, midpoint),
|
|
126
|
+
"test": (midpoint, end),
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _read_pickled_zip(path: Path, member: str) -> pd.DataFrame | pd.Series:
|
|
131
|
+
"""Read a single pickled member out of one of the dataset's zip archives."""
|
|
132
|
+
with zipfile.ZipFile(path) as archive:
|
|
133
|
+
return pd.read_pickle(io.BytesIO(archive.read(member)))
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def load_esa_channels(root: str | Path) -> pd.DataFrame:
|
|
137
|
+
"""
|
|
138
|
+
Load ``channels.csv``, the channel metadata table.
|
|
139
|
+
|
|
140
|
+
Arguments:
|
|
141
|
+
root: Mission directory, e.g. ``.../ESA-Mission1``.
|
|
142
|
+
Returns:
|
|
143
|
+
pd.DataFrame: One row per channel, with a boolean ``is_target`` column
|
|
144
|
+
added alongside the file's own ``YES``/``NO`` strings.
|
|
145
|
+
"""
|
|
146
|
+
channels = pd.read_csv(Path(root) / "channels.csv")
|
|
147
|
+
channels["is_target"] = channels["Target"].str.upper().eq("YES")
|
|
148
|
+
return channels
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def rescope_anomaly_types(
|
|
152
|
+
labels: pd.DataFrame,
|
|
153
|
+
*,
|
|
154
|
+
cadence_s: int = MISSION1_CADENCE_S,
|
|
155
|
+
point_max_samples: int = POINT_MAX_SAMPLES,
|
|
156
|
+
) -> pd.DataFrame:
|
|
157
|
+
"""
|
|
158
|
+
Recompute the type attributes that depend on which channels are in scope.
|
|
159
|
+
|
|
160
|
+
``anomaly_types.csv`` describes each event across the *whole* mission, so an
|
|
161
|
+
event listed as Multivariate may touch only one channel of a subset, and one
|
|
162
|
+
listed as Subsequence may be a point anomaly within it. ESA-ADB says as much
|
|
163
|
+
and ships ``infer_anomaly_types.py`` for the purpose; this is the same rule
|
|
164
|
+
applied in-process.
|
|
165
|
+
|
|
166
|
+
Arguments:
|
|
167
|
+
labels: Segment rows, already filtered to the channels of interest.
|
|
168
|
+
cadence_s: Dominant sampling period in seconds.
|
|
169
|
+
point_max_samples: Longest event still counted as a point anomaly.
|
|
170
|
+
Returns:
|
|
171
|
+
pd.DataFrame: A copy with ``Dimensionality`` and ``Length`` recomputed
|
|
172
|
+
for the channels present, and the originals preserved as
|
|
173
|
+
``Dimensionality_mission`` and ``Length_mission``.
|
|
174
|
+
|
|
175
|
+
Notes:
|
|
176
|
+
``Locality`` is **not** recomputed and remains mission-scope. Deciding it
|
|
177
|
+
requires the min/max of every nominal sample per channel, which means
|
|
178
|
+
reading the full signal; inferring it from annotations alone is not
|
|
179
|
+
possible. Treat that column as mission-wide wherever it is used.
|
|
180
|
+
|
|
181
|
+
Length follows the benchmark's rule that a fragmented event counts as a
|
|
182
|
+
point anomaly only if *every* one of its regions does.
|
|
183
|
+
"""
|
|
184
|
+
out = labels.copy()
|
|
185
|
+
out["Dimensionality_mission"] = out["Dimensionality"]
|
|
186
|
+
out["Length_mission"] = out["Length"]
|
|
187
|
+
|
|
188
|
+
affected = out.groupby("ID")["Channel"].transform("nunique")
|
|
189
|
+
out["Dimensionality"] = np.where(affected > 1, "Multivariate", "Univariate")
|
|
190
|
+
|
|
191
|
+
span_s = (out["EndTime"] - out["StartTime"]).dt.total_seconds()
|
|
192
|
+
segment_is_point = span_s <= point_max_samples * cadence_s
|
|
193
|
+
event_is_point = segment_is_point.groupby(out["ID"]).transform("all")
|
|
194
|
+
out["Length"] = np.where(event_is_point, "Point", "Subsequence")
|
|
195
|
+
return out
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def load_esa_labels(
|
|
199
|
+
root: str | Path,
|
|
200
|
+
channels: Sequence[str] | None = None,
|
|
201
|
+
*,
|
|
202
|
+
drop_categories: Sequence[str] | None = NON_EVENT_CATEGORIES,
|
|
203
|
+
rescope_types: bool = True,
|
|
204
|
+
cadence_s: int = MISSION1_CADENCE_S,
|
|
205
|
+
) -> pd.DataFrame:
|
|
206
|
+
"""
|
|
207
|
+
Load the annotations, joined to their event types.
|
|
208
|
+
|
|
209
|
+
Arguments:
|
|
210
|
+
root: Mission directory, e.g. ``.../ESA-Mission1``.
|
|
211
|
+
channels: Keep only segments affecting these channels. None keeps all.
|
|
212
|
+
drop_categories: Categories to exclude. Defaults to communication gaps,
|
|
213
|
+
matching ESA-ADB's headline scoring, which reports every other
|
|
214
|
+
category together. Pass None to keep everything, or
|
|
215
|
+
``("Communication Gap", "Rare Event")`` to score true anomalies only.
|
|
216
|
+
rescope_types: Recompute ``Dimensionality`` and ``Length`` for the
|
|
217
|
+
selected channels via :func:`rescope_anomaly_types`. Ignored when
|
|
218
|
+
``channels`` is None, where mission scope is already correct.
|
|
219
|
+
cadence_s: Dominant sampling period, used only by the rescoping.
|
|
220
|
+
Returns:
|
|
221
|
+
pd.DataFrame: One row per *segment*, with columns ``ID``, ``Channel``,
|
|
222
|
+
``StartTime``, ``EndTime`` (tz-naive datetimes) and the event's
|
|
223
|
+
``Class``, ``Subclass``, ``Category``, ``Dimensionality``,
|
|
224
|
+
``Locality`` and ``Length``.
|
|
225
|
+
|
|
226
|
+
Notes:
|
|
227
|
+
Group by ``ID`` to recover events. A count of rows is a count of
|
|
228
|
+
segments, which is a different and larger number: one event spans every
|
|
229
|
+
channel it affects, and may be split into several disjoint regions per
|
|
230
|
+
channel so that a burst of related disturbances still counts once.
|
|
231
|
+
"""
|
|
232
|
+
root = Path(root)
|
|
233
|
+
labels = pd.read_csv(root / "labels.csv", parse_dates=["StartTime", "EndTime"])
|
|
234
|
+
# Timestamps are stored with a Z suffix; the channel index is tz-naive UTC,
|
|
235
|
+
# so drop the tz rather than leave the two incomparable.
|
|
236
|
+
for column in ("StartTime", "EndTime"):
|
|
237
|
+
if isinstance(labels[column].dtype, pd.DatetimeTZDtype):
|
|
238
|
+
labels[column] = labels[column].dt.tz_convert(None)
|
|
239
|
+
|
|
240
|
+
labels = labels.merge(pd.read_csv(root / "anomaly_types.csv"), on="ID", how="left")
|
|
241
|
+
if channels is not None:
|
|
242
|
+
labels = labels[labels["Channel"].isin(list(channels))]
|
|
243
|
+
if drop_categories:
|
|
244
|
+
labels = labels[~labels["Category"].isin(list(drop_categories))]
|
|
245
|
+
|
|
246
|
+
labels = labels.sort_values(["StartTime", "Channel"], ignore_index=True)
|
|
247
|
+
if rescope_types and channels is not None:
|
|
248
|
+
labels = rescope_anomaly_types(labels, cadence_s=cadence_s)
|
|
249
|
+
return labels
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def load_esa(
|
|
253
|
+
root: str | Path,
|
|
254
|
+
channels: Sequence[str] | None = None,
|
|
255
|
+
*,
|
|
256
|
+
split: str | None = None,
|
|
257
|
+
span: tuple[pd.Timestamp, pd.Timestamp] = MISSION1_SPAN,
|
|
258
|
+
validation_months: int = 3,
|
|
259
|
+
start: str | pd.Timestamp | None = None,
|
|
260
|
+
end: str | pd.Timestamp | None = None,
|
|
261
|
+
resample_rule: str | None = None,
|
|
262
|
+
) -> TelemetryDataset:
|
|
263
|
+
"""
|
|
264
|
+
Load telemetry channels into the canonical long form.
|
|
265
|
+
|
|
266
|
+
Arguments:
|
|
267
|
+
root: Mission directory, e.g. ``.../ESA-Mission1``.
|
|
268
|
+
channels: Channel names to load, e.g. ``("channel_41", "channel_42")``.
|
|
269
|
+
Defaults to :data:`MISSION1_LIGHTWEIGHT`.
|
|
270
|
+
split: ``"train"``, ``"validation"`` or ``"test"``, resolved through
|
|
271
|
+
:func:`esa_splits`. Use this rather than hand-written dates so
|
|
272
|
+
results stay comparable with the published benchmark. Cannot be
|
|
273
|
+
combined with ``start`` or ``end``.
|
|
274
|
+
span: Mission span passed to :func:`esa_splits` when ``split`` is given.
|
|
275
|
+
validation_months: Months of the training half held out for validation.
|
|
276
|
+
The benchmark's default is 3; the paper also studies shorter
|
|
277
|
+
training sets to find the earliest mission phase that supports a
|
|
278
|
+
reliable detector.
|
|
279
|
+
start: Keep samples at or after this timestamp. None for no lower bound.
|
|
280
|
+
end: Keep samples strictly before this timestamp. None for no upper
|
|
281
|
+
bound. Slicing happens per channel, before the frames are combined,
|
|
282
|
+
so memory scales with the slice rather than the archive.
|
|
283
|
+
resample_rule: Optional pandas offset alias (lowercase, e.g. ``"5min"``)
|
|
284
|
+
to downsample onto a regular grid, taking the mean of each bin.
|
|
285
|
+
None keeps the native cadence.
|
|
286
|
+
Returns:
|
|
287
|
+
TelemetryDataset: Long-form ``[timestamp, variable, value]``.
|
|
288
|
+
|
|
289
|
+
Notes:
|
|
290
|
+
Mission1 runs 14 years at a nominal 30 s cadence, so one channel is
|
|
291
|
+
about 15.4 million samples. Six of them at full resolution is roughly a
|
|
292
|
+
gigabyte once windowed, and a whole split is far more than that: prefer
|
|
293
|
+
a slice within the split while iterating.
|
|
294
|
+
|
|
295
|
+
Downsampling is not free, and on this dataset it is barely affordable
|
|
296
|
+
at all: annotated events are short enough that a coarse rule erases
|
|
297
|
+
them rather than blurring them. See :doc:`/user_guide/anomaly_scoring`
|
|
298
|
+
for the measured cost at each rule, and prefer a slice at the native
|
|
299
|
+
cadence over a resampled span.
|
|
300
|
+
"""
|
|
301
|
+
root = Path(root)
|
|
302
|
+
names = list(MISSION1_LIGHTWEIGHT if channels is None else channels)
|
|
303
|
+
if not names:
|
|
304
|
+
raise ValueError("channels must not be empty")
|
|
305
|
+
|
|
306
|
+
if split is not None:
|
|
307
|
+
if start is not None or end is not None:
|
|
308
|
+
raise ValueError("Pass either split or start/end, not both.")
|
|
309
|
+
splits = esa_splits(span, validation_months=validation_months)
|
|
310
|
+
if split not in splits:
|
|
311
|
+
raise ValueError(f"split must be one of {sorted(splits)}, got {split!r}")
|
|
312
|
+
start, end = splits[split]
|
|
313
|
+
|
|
314
|
+
start = None if start is None else pd.Timestamp(start)
|
|
315
|
+
end = None if end is None else pd.Timestamp(end)
|
|
316
|
+
|
|
317
|
+
frames = []
|
|
318
|
+
for name in names:
|
|
319
|
+
archive = root / "channels" / f"{name}.zip"
|
|
320
|
+
if not archive.exists():
|
|
321
|
+
raise FileNotFoundError(f"No archive for {name!r} at {archive}")
|
|
322
|
+
|
|
323
|
+
series = _read_pickled_zip(archive, name)
|
|
324
|
+
if isinstance(series, pd.DataFrame):
|
|
325
|
+
series = series.iloc[:, 0]
|
|
326
|
+
# Slice before anything else: the archives are large and the caller is
|
|
327
|
+
# usually after a fraction of the 14-year span.
|
|
328
|
+
if start is not None or end is not None:
|
|
329
|
+
series = series.loc[start:end]
|
|
330
|
+
if end is not None and len(series) and series.index[-1] == end:
|
|
331
|
+
series = series.iloc[:-1] # `end` is exclusive
|
|
332
|
+
if resample_rule is not None:
|
|
333
|
+
series = series.resample(resample_rule).mean().dropna()
|
|
334
|
+
|
|
335
|
+
frames.append(
|
|
336
|
+
pd.DataFrame(
|
|
337
|
+
{
|
|
338
|
+
_TS: series.index,
|
|
339
|
+
_VAR: name,
|
|
340
|
+
_VAL: series.to_numpy(dtype="float32"),
|
|
341
|
+
}
|
|
342
|
+
)
|
|
343
|
+
)
|
|
344
|
+
|
|
345
|
+
long = pd.concat(frames, ignore_index=True)
|
|
346
|
+
long = long.sort_values([_TS, _VAR], ignore_index=True)
|
|
347
|
+
return TelemetryDataset(long)
|