splittrace 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- splittrace-0.1.0/CONTRIBUTING.md +44 -0
- splittrace-0.1.0/LICENSE +21 -0
- splittrace-0.1.0/MANIFEST.in +3 -0
- splittrace-0.1.0/PKG-INFO +247 -0
- splittrace-0.1.0/README.md +229 -0
- splittrace-0.1.0/SECURITY.md +16 -0
- splittrace-0.1.0/examples/derived-assignments.json +12 -0
- splittrace-0.1.0/examples/derived-samples.json +23 -0
- splittrace-0.1.0/pyproject.toml +30 -0
- splittrace-0.1.0/setup.cfg +4 -0
- splittrace-0.1.0/src/splittrace/__init__.py +4 -0
- splittrace-0.1.0/src/splittrace/cli.py +70 -0
- splittrace-0.1.0/src/splittrace/core.py +194 -0
- splittrace-0.1.0/src/splittrace.egg-info/PKG-INFO +247 -0
- splittrace-0.1.0/src/splittrace.egg-info/SOURCES.txt +17 -0
- splittrace-0.1.0/src/splittrace.egg-info/dependency_links.txt +1 -0
- splittrace-0.1.0/src/splittrace.egg-info/entry_points.txt +2 -0
- splittrace-0.1.0/src/splittrace.egg-info/top_level.txt +1 -0
- splittrace-0.1.0/tests/test_audit.py +170 -0
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# Contributing
|
|
2
|
+
|
|
3
|
+
SplitTrace is an experimental, metadata-only auditor. Small, reproducible examples
|
|
4
|
+
are more useful at this stage than a large new feature surface.
|
|
5
|
+
|
|
6
|
+
## Report a problem or try the prototype
|
|
7
|
+
|
|
8
|
+
Open an issue with:
|
|
9
|
+
|
|
10
|
+
- Python version and the commit you tested;
|
|
11
|
+
- intended evaluation policy (new subjects or future windows in a recording);
|
|
12
|
+
- a **synthetic** sample manifest and split assignments;
|
|
13
|
+
- expected outcome and actual report;
|
|
14
|
+
- how you currently handle this with another tool or a short script.
|
|
15
|
+
|
|
16
|
+
Please do not upload patient records, private datasets, credentials, or reports
|
|
17
|
+
containing real subject identifiers. Reduce examples to invented IDs and intervals.
|
|
18
|
+
Private security issues should use the process in [SECURITY.md](SECURITY.md).
|
|
19
|
+
|
|
20
|
+
## Development
|
|
21
|
+
|
|
22
|
+
```sh
|
|
23
|
+
python3 -m venv .venv
|
|
24
|
+
.venv/bin/python -m pip install -e .
|
|
25
|
+
.venv/bin/python -m unittest discover -s tests -v
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
Keep core runtime dependencies empty. Add behavior tests for new rules, including
|
|
29
|
+
a legitimate split that should not be flagged and a missing-metadata case.
|
|
30
|
+
Do not silently turn UNKNOWN into PASS or remove evaluation examples automatically.
|
|
31
|
+
Document changes to the data contract and report schema.
|
|
32
|
+
|
|
33
|
+
## Optional comparison
|
|
34
|
+
|
|
35
|
+
Use a separate environment and `requirements-comparison.txt` to run
|
|
36
|
+
`scripts/compare.py`. It regenerates synthetic fixtures and reports. Do not infer
|
|
37
|
+
real-world accuracy, adoption or speed from this small functional comparison.
|
|
38
|
+
When comparing tools, show unsupported inputs and manual adapters separately.
|
|
39
|
+
|
|
40
|
+
## Pull requests
|
|
41
|
+
|
|
42
|
+
Explain the concrete scenario, expected behavior and tests run. Contributions
|
|
43
|
+
are provided under the repository's MIT license. Do not submit code or data you
|
|
44
|
+
do not have permission to share.
|
splittrace-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Ryu Koda
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: splittrace
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Metadata-first audit of derived samples across existing ML splits
|
|
5
|
+
Author: Ryu Koda
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Repository, https://github.com/R-Koda/splittrace
|
|
8
|
+
Project-URL: Issues, https://github.com/R-Koda/splittrace/issues
|
|
9
|
+
Keywords: data-leakage,cross-validation,eeg,provenance,machine-learning
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Requires-Python: >=3.10
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
License-File: LICENSE
|
|
17
|
+
Dynamic: license-file
|
|
18
|
+
|
|
19
|
+
# SplitTrace — trace data leakage back to its source
|
|
20
|
+
|
|
21
|
+
[](https://github.com/R-Koda/splittrace/actions/workflows/tests.yml)
|
|
22
|
+
[](LICENSE)
|
|
23
|
+
|
|
24
|
+
**Experimental Python library for auditing existing EEG and sensor-data splits.**
|
|
25
|
+
|
|
26
|
+
Audit existing train/validation/test assignments for EEG and sensor experiments.
|
|
27
|
+
Trace derived examples back through declared parent IDs, inspect intervals in
|
|
28
|
+
original-recording coordinates, and return evidence for each violation.
|
|
29
|
+
|
|
30
|
+
Different filenames and different pixel values can still come from the same
|
|
31
|
+
source epoch. SplitTrace accepts declared parent relationships and reports the
|
|
32
|
+
path that connects a training example to an evaluation example:
|
|
33
|
+
|
|
34
|
+
```text
|
|
35
|
+
train: a → image-01 → epoch-01 ← b :test
|
|
36
|
+
^ shared source example
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
It also checks subject mixing, overlapping source intervals and, under an explicit
|
|
40
|
+
policy, chronological ordering. Missing metadata is **UNKNOWN**, not a clean bill
|
|
41
|
+
of health. All bundled examples are synthetic; no patient data is included.
|
|
42
|
+
|
|
43
|
+
**Early prototype:** small manifests only, with no real-user validation yet.
|
|
44
|
+
The distribution name and import name are both `splittrace`.
|
|
45
|
+
|
|
46
|
+
日本語の試用手順は [PILOT.md](https://github.com/R-Koda/splittrace/blob/main/PILOT.md)。実行済みの比較結果は
|
|
47
|
+
[results/comparison.html](https://github.com/R-Koda/splittrace/blob/main/results/comparison.html)、派生元追跡の出力例は
|
|
48
|
+
[results/derived-audit.html](https://github.com/R-Koda/splittrace/blob/main/results/derived-audit.html) を開いてください。
|
|
49
|
+
|
|
50
|
+
## Install from PyPI / pipでインストール
|
|
51
|
+
|
|
52
|
+
Python 3.10 or later. Create and activate a virtual environment first:
|
|
53
|
+
|
|
54
|
+
```sh
|
|
55
|
+
python3 -m venv .venv
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
macOS / Linux: `source .venv/bin/activate`
|
|
59
|
+
|
|
60
|
+
Windows PowerShell: `.venv\Scripts\Activate.ps1`
|
|
61
|
+
|
|
62
|
+
```sh
|
|
63
|
+
python -m pip install splittrace
|
|
64
|
+
splittrace --help
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
The help screen confirms that the command is available. Install only the tools you need.
|
|
68
|
+
[PyPI package](https://pypi.org/project/splittrace/) · [Source and examples](https://github.com/R-Koda/splittrace)
|
|
69
|
+
|
|
70
|
+
サンプルCSV・Notebookなどはpipでは配置されません。公開デモはブラウザで開けます。自分でサンプルを実行する場合はGitHubの「Code → Download ZIP」で取得・展開し、READMEの相対パスをそのフォルダー内から指定してください。
|
|
71
|
+
|
|
72
|
+
## Article & live demo / 紹介記事と公開デモ
|
|
73
|
+
|
|
74
|
+
- [個別ガイド / SplitTrace:データ分割の混入を派生元から点検する](https://ryu-koda.com/blog/splittrace-data-leakage-check/)
|
|
75
|
+
- [紹介記事:データを分ける・評価する・公開する。Pythonの小さな確認ツールを3つ作りました](https://ryu-koda.com/blog/python-tools-splittrace-eventscore-notebook/) — Japanese overview of SplitTrace, EventScore Explorer, and Notebook Publish Check.
|
|
76
|
+
- [公開デモをブラウザで開く / Open the live demo](https://ryu-koda.com/articles/python-tools/splittrace-demo.html) — Synthetic data only; no installation required.
|
|
77
|
+
|
|
78
|
+
公開デモは合成データの出力例です。自分のファイルをアップロードする画面ではありません。
|
|
79
|
+
The hosted demo is a sample report, not a file-upload service.
|
|
80
|
+
|
|
81
|
+
## Run without installing anything
|
|
82
|
+
|
|
83
|
+
Python 3.10+; the core and CLI use only the standard library.
|
|
84
|
+
|
|
85
|
+
```sh
|
|
86
|
+
git clone https://github.com/R-Koda/splittrace.git
|
|
87
|
+
cd splittrace
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
From the repository directory:
|
|
91
|
+
|
|
92
|
+
```sh
|
|
93
|
+
PYTHONPATH=src python3 -m splittrace.cli \
|
|
94
|
+
examples/derived-samples.json examples/derived-assignments.json \
|
|
95
|
+
--policy cross_subject --lineage-complete \
|
|
96
|
+
--output results/local-audit.json --html results/local-audit.html
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
This intentionally leaking example exits **1**; that is the expected result,
|
|
100
|
+
not an installation error. The report contains the paths
|
|
101
|
+
`a → image-01 → epoch-01` and `b → epoch-01`. No signal values are read.
|
|
102
|
+
CLI exit codes: `0` = all applicable checks PASS; `1` = a declared policy violation;
|
|
103
|
+
`2` = UNKNOWN coverage or invalid input (inspect `status` to distinguish them).
|
|
104
|
+
|
|
105
|
+
For an editable install in your own virtual environment:
|
|
106
|
+
|
|
107
|
+
```sh
|
|
108
|
+
python3 -m venv .venv
|
|
109
|
+
.venv/bin/python -m pip install -e .
|
|
110
|
+
.venv/bin/splittrace --help
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
## Python API
|
|
114
|
+
|
|
115
|
+
```python
|
|
116
|
+
from splittrace import audit, map_interval
|
|
117
|
+
|
|
118
|
+
samples = [
|
|
119
|
+
{"sample_id": "epoch-01"},
|
|
120
|
+
{"sample_id": "image-01", "parent_ids": ["epoch-01"]},
|
|
121
|
+
{"sample_id": "a", "parent_ids": ["image-01"]},
|
|
122
|
+
{"sample_id": "b", "parent_ids": ["epoch-01"]},
|
|
123
|
+
]
|
|
124
|
+
assignments = [
|
|
125
|
+
{"sample_id": "a", "split": "train", "fold": "0"},
|
|
126
|
+
{"sample_id": "b", "split": "test", "fold": "0"},
|
|
127
|
+
]
|
|
128
|
+
report = audit(samples, assignments, policy="cross_subject", lineage_complete=True)
|
|
129
|
+
assert report["status"] == "FAIL"
|
|
130
|
+
# Other checks remain UNKNOWN when their required metadata is absent.
|
|
131
|
+
|
|
132
|
+
# Convert a local, resampled interval to original sample coordinates explicitly:
|
|
133
|
+
start, end = map_interval(0, 128, origin_offset=256, original_units_per_local_unit=2)
|
|
134
|
+
assert (start, end) == (256, 512)
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
## Data contract
|
|
138
|
+
|
|
139
|
+
Samples are dictionaries / JSON objects. CSV is also accepted; its `parent_ids`
|
|
140
|
+
cell must be a JSON array, quoted following normal CSV rules. Blank optional CSV
|
|
141
|
+
cells are missing metadata. Numeric interval cells are parsed as floats.
|
|
142
|
+
|
|
143
|
+
- `sample_id`: nonempty, globally unique string. Reference nodes may exist without
|
|
144
|
+
assignment to a split. Parent references must resolve; cycles are rejected.
|
|
145
|
+
- `parent_ids`: list of directly contributing source-example IDs. Declare all
|
|
146
|
+
parents for mixtures. Examples sharing a sample ancestor are flagged across
|
|
147
|
+
partitions. This is a conservative **source-example separation policy**, not a
|
|
148
|
+
statistical proof that all shared ancestry invalidates every evaluation.
|
|
149
|
+
- `kind`: `sample` (default) or `recording`. A whole raw recording used as a graph
|
|
150
|
+
parent should be `recording`; merely sharing that parent does not fail lineage.
|
|
151
|
+
Interval or subject checks still apply. Do not label an actual shared training
|
|
152
|
+
example as a recording to bypass the check.
|
|
153
|
+
- `subject_id`: canonical ID; namespace IDs that are only unique within a site.
|
|
154
|
+
- `recording_id`: canonical original-recording ID, not a derived filename.
|
|
155
|
+
- `coordinate_system`: explicit common coordinate label, e.g.
|
|
156
|
+
`original-samples-256hz` or `original-seconds`.
|
|
157
|
+
- `start`, `end`: finite values in that original coordinate system, `[start,end)`.
|
|
158
|
+
Touching intervals are not overlapping. Values are not rounded implicitly.
|
|
159
|
+
|
|
160
|
+
The library does not automatically inherit time intervals or subject metadata
|
|
161
|
+
through arbitrary transformations. Attach canonical metadata to assigned leaves.
|
|
162
|
+
`map_interval` only implements a supplied positive affine mapping. Nonlinear
|
|
163
|
+
warps, filter support, padding and unknown clock shifts require user-provided
|
|
164
|
+
provenance or must remain unverified.
|
|
165
|
+
|
|
166
|
+
Assignments contain `sample_id`, `split` (`train`, `validation`, `test`) and `fold`
|
|
167
|
+
(string, default `"0"`). Each fold needs train and an evaluation partition. All
|
|
168
|
+
partition pairs within a fold are checked. Ordinary reuse across independent
|
|
169
|
+
folds is allowed; nested-CV semantics are not implemented.
|
|
170
|
+
|
|
171
|
+
## Policies and coverage
|
|
172
|
+
|
|
173
|
+
- `cross_subject`: subject IDs must not cross partitions. No global time ordering
|
|
174
|
+
is required. Shared source examples and raw interval overlap are also checked.
|
|
175
|
+
- `within_recording_future`: subject sharing is allowed; within each common
|
|
176
|
+
recording/coordinate system, train must end no later than validation/test
|
|
177
|
+
starts, and validation must precede test. Cross-recording chronological claims
|
|
178
|
+
are UNKNOWN because clocks are not assumed comparable. This narrowly scoped
|
|
179
|
+
policy does not audit future-target availability or preprocessing fit history.
|
|
180
|
+
|
|
181
|
+
`lineage_complete=True` is a declaration by the caller, not something the library
|
|
182
|
+
can prove. Omitting it makes lineage coverage UNKNOWN even if no shared ancestor
|
|
183
|
+
was found. Missing subject or interval metadata makes affected checks UNKNOWN.
|
|
184
|
+
A proven violation makes the overall result FAIL even if other checks are UNKNOWN.
|
|
185
|
+
PASS means only that the supplied metadata satisfies the applicable rules.
|
|
186
|
+
|
|
187
|
+
Results include per-fold checks, unknown-pair counts, violation counts and evidence
|
|
188
|
+
paths. `max_findings` limits returned examples, not failure counts. HTML/JSON
|
|
189
|
+
reports include identifiers and paths; keep them local or redact before sharing.
|
|
190
|
+
The tool performs no network requests, uploads, model training or automatic edits.
|
|
191
|
+
|
|
192
|
+
## Validation performed
|
|
193
|
+
|
|
194
|
+
On 2026-11-09, Python 3.14.7:
|
|
195
|
+
|
|
196
|
+
- 23 standard-library tests passed, including malformed manifests, parent cycles,
|
|
197
|
+
multiple parents, interval boundaries, coordinate mapping, missing metadata,
|
|
198
|
+
ordinary K-fold reuse, validation/test contamination and HTML escaping.
|
|
199
|
+
- 7 synthetic scenarios matched their expected PASS/FAIL/UNKNOWN outcomes.
|
|
200
|
+
- Compared against **leakfence 0.5.0**, NumPy 2.5.4 in an isolated environment.
|
|
201
|
+
Subject mixing and interval overlap were detected by both tools. Clean splits
|
|
202
|
+
were not flagged. The full machine-readable output is `results/comparison.json`.
|
|
203
|
+
|
|
204
|
+
In the derived-source case, leakfence's native input has no parent-graph field.
|
|
205
|
+
Calling it with only the available native metadata produced no flag; that is not
|
|
206
|
+
a safety verdict. **Manually grouping leaves by source also made leakfence detect
|
|
207
|
+
the problem.** Our demonstrated distinction is accepting the parent graph and
|
|
208
|
+
returning the chain of evidence, not a uniquely powerful detection algorithm.
|
|
209
|
+
|
|
210
|
+
The comparison uses the same available supported metadata, with the manual
|
|
211
|
+
source-group adapter shown separately. It is a small functional comparison, not
|
|
212
|
+
an independent benchmark, speed test, accuracy improvement, or proof of demand.
|
|
213
|
+
Deepchecks and other competitors were not executed in this prototype comparison.
|
|
214
|
+
|
|
215
|
+
Reproduce tests and the nearest-competitor comparison:
|
|
216
|
+
|
|
217
|
+
```sh
|
|
218
|
+
PYTHONPATH=src python3 -m unittest discover -s tests -v
|
|
219
|
+
python3 -m venv /tmp/splittrace-comparison
|
|
220
|
+
/tmp/splittrace-comparison/bin/python -m pip install -r requirements-comparison.txt
|
|
221
|
+
/tmp/splittrace-comparison/bin/python scripts/compare.py
|
|
222
|
+
```
|
|
223
|
+
|
|
224
|
+
## Deliberate limits
|
|
225
|
+
|
|
226
|
+
Pairwise scanning is quadratic in partition sizes; ancestor-path storage also
|
|
227
|
+
grows with graph size/depth. Use small manifests. There is no performance claim.
|
|
228
|
+
No approximate content matching, nested-CV audit, persistent exception management,
|
|
229
|
+
automatic clock recovery, preprocessing tracing, causal/label leakage analysis or
|
|
230
|
+
proof of independence is implemented. Synthetic tests are not real-user validation.
|
|
231
|
+
|
|
232
|
+
Feedback is welcome from EEG, wearable-sensor and other windowed-signal projects:
|
|
233
|
+
can you supply source IDs/coordinates, and does a provenance path help you fix a
|
|
234
|
+
split? See [CONTRIBUTING.md](https://github.com/R-Koda/splittrace/blob/main/CONTRIBUTING.md) and the Japanese [trial guide](https://github.com/R-Koda/splittrace/blob/main/PILOT.md).
|
|
235
|
+
|
|
236
|
+
## References
|
|
237
|
+
|
|
238
|
+
- [leakfence](https://github.com/ptapal/leakfence)
|
|
239
|
+
- [Deepchecks train-test validation](https://docs.deepchecks.com/stable/api/generated/deepchecks.tabular.suites.train_test_validation.html)
|
|
240
|
+
- [bioLeak](https://github.com/selcukorkmaz/bioLeak)
|
|
241
|
+
|
|
242
|
+
This prototype contains no copied competitor implementation. Optional comparison
|
|
243
|
+
dependencies retain their own licenses and are installed separately.
|
|
244
|
+
|
|
245
|
+
## License
|
|
246
|
+
|
|
247
|
+
[MIT](https://github.com/R-Koda/splittrace/blob/main/LICENSE). Copyright (c) 2026 Ryu Koda.
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
# SplitTrace — trace data leakage back to its source
|
|
2
|
+
|
|
3
|
+
[](https://github.com/R-Koda/splittrace/actions/workflows/tests.yml)
|
|
4
|
+
[](LICENSE)
|
|
5
|
+
|
|
6
|
+
**Experimental Python library for auditing existing EEG and sensor-data splits.**
|
|
7
|
+
|
|
8
|
+
Audit existing train/validation/test assignments for EEG and sensor experiments.
|
|
9
|
+
Trace derived examples back through declared parent IDs, inspect intervals in
|
|
10
|
+
original-recording coordinates, and return evidence for each violation.
|
|
11
|
+
|
|
12
|
+
Different filenames and different pixel values can still come from the same
|
|
13
|
+
source epoch. SplitTrace accepts declared parent relationships and reports the
|
|
14
|
+
path that connects a training example to an evaluation example:
|
|
15
|
+
|
|
16
|
+
```text
|
|
17
|
+
train: a → image-01 → epoch-01 ← b :test
|
|
18
|
+
^ shared source example
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
It also checks subject mixing, overlapping source intervals and, under an explicit
|
|
22
|
+
policy, chronological ordering. Missing metadata is **UNKNOWN**, not a clean bill
|
|
23
|
+
of health. All bundled examples are synthetic; no patient data is included.
|
|
24
|
+
|
|
25
|
+
**Early prototype:** small manifests only, with no real-user validation yet.
|
|
26
|
+
The distribution name and import name are both `splittrace`.
|
|
27
|
+
|
|
28
|
+
日本語の試用手順は [PILOT.md](https://github.com/R-Koda/splittrace/blob/main/PILOT.md)。実行済みの比較結果は
|
|
29
|
+
[results/comparison.html](https://github.com/R-Koda/splittrace/blob/main/results/comparison.html)、派生元追跡の出力例は
|
|
30
|
+
[results/derived-audit.html](https://github.com/R-Koda/splittrace/blob/main/results/derived-audit.html) を開いてください。
|
|
31
|
+
|
|
32
|
+
## Install from PyPI / pipでインストール
|
|
33
|
+
|
|
34
|
+
Python 3.10 or later. Create and activate a virtual environment first:
|
|
35
|
+
|
|
36
|
+
```sh
|
|
37
|
+
python3 -m venv .venv
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
macOS / Linux: `source .venv/bin/activate`
|
|
41
|
+
|
|
42
|
+
Windows PowerShell: `.venv\Scripts\Activate.ps1`
|
|
43
|
+
|
|
44
|
+
```sh
|
|
45
|
+
python -m pip install splittrace
|
|
46
|
+
splittrace --help
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
The help screen confirms that the command is available. Install only the tools you need.
|
|
50
|
+
[PyPI package](https://pypi.org/project/splittrace/) · [Source and examples](https://github.com/R-Koda/splittrace)
|
|
51
|
+
|
|
52
|
+
サンプルCSV・Notebookなどはpipでは配置されません。公開デモはブラウザで開けます。自分でサンプルを実行する場合はGitHubの「Code → Download ZIP」で取得・展開し、READMEの相対パスをそのフォルダー内から指定してください。
|
|
53
|
+
|
|
54
|
+
## Article & live demo / 紹介記事と公開デモ
|
|
55
|
+
|
|
56
|
+
- [個別ガイド / SplitTrace:データ分割の混入を派生元から点検する](https://ryu-koda.com/blog/splittrace-data-leakage-check/)
|
|
57
|
+
- [紹介記事:データを分ける・評価する・公開する。Pythonの小さな確認ツールを3つ作りました](https://ryu-koda.com/blog/python-tools-splittrace-eventscore-notebook/) — Japanese overview of SplitTrace, EventScore Explorer, and Notebook Publish Check.
|
|
58
|
+
- [公開デモをブラウザで開く / Open the live demo](https://ryu-koda.com/articles/python-tools/splittrace-demo.html) — Synthetic data only; no installation required.
|
|
59
|
+
|
|
60
|
+
公開デモは合成データの出力例です。自分のファイルをアップロードする画面ではありません。
|
|
61
|
+
The hosted demo is a sample report, not a file-upload service.
|
|
62
|
+
|
|
63
|
+
## Run without installing anything
|
|
64
|
+
|
|
65
|
+
Python 3.10+; the core and CLI use only the standard library.
|
|
66
|
+
|
|
67
|
+
```sh
|
|
68
|
+
git clone https://github.com/R-Koda/splittrace.git
|
|
69
|
+
cd splittrace
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
From the repository directory:
|
|
73
|
+
|
|
74
|
+
```sh
|
|
75
|
+
PYTHONPATH=src python3 -m splittrace.cli \
|
|
76
|
+
examples/derived-samples.json examples/derived-assignments.json \
|
|
77
|
+
--policy cross_subject --lineage-complete \
|
|
78
|
+
--output results/local-audit.json --html results/local-audit.html
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
This intentionally leaking example exits **1**; that is the expected result,
|
|
82
|
+
not an installation error. The report contains the paths
|
|
83
|
+
`a → image-01 → epoch-01` and `b → epoch-01`. No signal values are read.
|
|
84
|
+
CLI exit codes: `0` = all applicable checks PASS; `1` = a declared policy violation;
|
|
85
|
+
`2` = UNKNOWN coverage or invalid input (inspect `status` to distinguish them).
|
|
86
|
+
|
|
87
|
+
For an editable install in your own virtual environment:
|
|
88
|
+
|
|
89
|
+
```sh
|
|
90
|
+
python3 -m venv .venv
|
|
91
|
+
.venv/bin/python -m pip install -e .
|
|
92
|
+
.venv/bin/splittrace --help
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
## Python API
|
|
96
|
+
|
|
97
|
+
```python
|
|
98
|
+
from splittrace import audit, map_interval
|
|
99
|
+
|
|
100
|
+
samples = [
|
|
101
|
+
{"sample_id": "epoch-01"},
|
|
102
|
+
{"sample_id": "image-01", "parent_ids": ["epoch-01"]},
|
|
103
|
+
{"sample_id": "a", "parent_ids": ["image-01"]},
|
|
104
|
+
{"sample_id": "b", "parent_ids": ["epoch-01"]},
|
|
105
|
+
]
|
|
106
|
+
assignments = [
|
|
107
|
+
{"sample_id": "a", "split": "train", "fold": "0"},
|
|
108
|
+
{"sample_id": "b", "split": "test", "fold": "0"},
|
|
109
|
+
]
|
|
110
|
+
report = audit(samples, assignments, policy="cross_subject", lineage_complete=True)
|
|
111
|
+
assert report["status"] == "FAIL"
|
|
112
|
+
# Other checks remain UNKNOWN when their required metadata is absent.
|
|
113
|
+
|
|
114
|
+
# Convert a local, resampled interval to original sample coordinates explicitly:
|
|
115
|
+
start, end = map_interval(0, 128, origin_offset=256, original_units_per_local_unit=2)
|
|
116
|
+
assert (start, end) == (256, 512)
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
## Data contract
|
|
120
|
+
|
|
121
|
+
Samples are dictionaries / JSON objects. CSV is also accepted; its `parent_ids`
|
|
122
|
+
cell must be a JSON array, quoted following normal CSV rules. Blank optional CSV
|
|
123
|
+
cells are missing metadata. Numeric interval cells are parsed as floats.
|
|
124
|
+
|
|
125
|
+
- `sample_id`: nonempty, globally unique string. Reference nodes may exist without
|
|
126
|
+
assignment to a split. Parent references must resolve; cycles are rejected.
|
|
127
|
+
- `parent_ids`: list of directly contributing source-example IDs. Declare all
|
|
128
|
+
parents for mixtures. Examples sharing a sample ancestor are flagged across
|
|
129
|
+
partitions. This is a conservative **source-example separation policy**, not a
|
|
130
|
+
statistical proof that all shared ancestry invalidates every evaluation.
|
|
131
|
+
- `kind`: `sample` (default) or `recording`. A whole raw recording used as a graph
|
|
132
|
+
parent should be `recording`; merely sharing that parent does not fail lineage.
|
|
133
|
+
Interval or subject checks still apply. Do not label an actual shared training
|
|
134
|
+
example as a recording to bypass the check.
|
|
135
|
+
- `subject_id`: canonical ID; namespace IDs that are only unique within a site.
|
|
136
|
+
- `recording_id`: canonical original-recording ID, not a derived filename.
|
|
137
|
+
- `coordinate_system`: explicit common coordinate label, e.g.
|
|
138
|
+
`original-samples-256hz` or `original-seconds`.
|
|
139
|
+
- `start`, `end`: finite values in that original coordinate system, `[start,end)`.
|
|
140
|
+
Touching intervals are not overlapping. Values are not rounded implicitly.
|
|
141
|
+
|
|
142
|
+
The library does not automatically inherit time intervals or subject metadata
|
|
143
|
+
through arbitrary transformations. Attach canonical metadata to assigned leaves.
|
|
144
|
+
`map_interval` only implements a supplied positive affine mapping. Nonlinear
|
|
145
|
+
warps, filter support, padding and unknown clock shifts require user-provided
|
|
146
|
+
provenance or must remain unverified.
|
|
147
|
+
|
|
148
|
+
Assignments contain `sample_id`, `split` (`train`, `validation`, `test`) and `fold`
|
|
149
|
+
(string, default `"0"`). Each fold needs train and an evaluation partition. All
|
|
150
|
+
partition pairs within a fold are checked. Ordinary reuse across independent
|
|
151
|
+
folds is allowed; nested-CV semantics are not implemented.
|
|
152
|
+
|
|
153
|
+
## Policies and coverage
|
|
154
|
+
|
|
155
|
+
- `cross_subject`: subject IDs must not cross partitions. No global time ordering
|
|
156
|
+
is required. Shared source examples and raw interval overlap are also checked.
|
|
157
|
+
- `within_recording_future`: subject sharing is allowed; within each common
|
|
158
|
+
recording/coordinate system, train must end no later than validation/test
|
|
159
|
+
starts, and validation must precede test. Cross-recording chronological claims
|
|
160
|
+
are UNKNOWN because clocks are not assumed comparable. This narrowly scoped
|
|
161
|
+
policy does not audit future-target availability or preprocessing fit history.
|
|
162
|
+
|
|
163
|
+
`lineage_complete=True` is a declaration by the caller, not something the library
|
|
164
|
+
can prove. Omitting it makes lineage coverage UNKNOWN even if no shared ancestor
|
|
165
|
+
was found. Missing subject or interval metadata makes affected checks UNKNOWN.
|
|
166
|
+
A proven violation makes the overall result FAIL even if other checks are UNKNOWN.
|
|
167
|
+
PASS means only that the supplied metadata satisfies the applicable rules.
|
|
168
|
+
|
|
169
|
+
Results include per-fold checks, unknown-pair counts, violation counts and evidence
|
|
170
|
+
paths. `max_findings` limits returned examples, not failure counts. HTML/JSON
|
|
171
|
+
reports include identifiers and paths; keep them local or redact before sharing.
|
|
172
|
+
The tool performs no network requests, uploads, model training or automatic edits.
|
|
173
|
+
|
|
174
|
+
## Validation performed
|
|
175
|
+
|
|
176
|
+
On 2026-11-09, Python 3.14.7:
|
|
177
|
+
|
|
178
|
+
- 23 standard-library tests passed, including malformed manifests, parent cycles,
|
|
179
|
+
multiple parents, interval boundaries, coordinate mapping, missing metadata,
|
|
180
|
+
ordinary K-fold reuse, validation/test contamination and HTML escaping.
|
|
181
|
+
- 7 synthetic scenarios matched their expected PASS/FAIL/UNKNOWN outcomes.
|
|
182
|
+
- Compared against **leakfence 0.5.0**, NumPy 2.5.4 in an isolated environment.
|
|
183
|
+
Subject mixing and interval overlap were detected by both tools. Clean splits
|
|
184
|
+
were not flagged. The full machine-readable output is `results/comparison.json`.
|
|
185
|
+
|
|
186
|
+
In the derived-source case, leakfence's native input has no parent-graph field.
|
|
187
|
+
Calling it with only the available native metadata produced no flag; that is not
|
|
188
|
+
a safety verdict. **Manually grouping leaves by source also made leakfence detect
|
|
189
|
+
the problem.** Our demonstrated distinction is accepting the parent graph and
|
|
190
|
+
returning the chain of evidence, not a uniquely powerful detection algorithm.
|
|
191
|
+
|
|
192
|
+
The comparison uses the same available supported metadata, with the manual
|
|
193
|
+
source-group adapter shown separately. It is a small functional comparison, not
|
|
194
|
+
an independent benchmark, speed test, accuracy improvement, or proof of demand.
|
|
195
|
+
Deepchecks and other competitors were not executed in this prototype comparison.
|
|
196
|
+
|
|
197
|
+
Reproduce tests and the nearest-competitor comparison:
|
|
198
|
+
|
|
199
|
+
```sh
|
|
200
|
+
PYTHONPATH=src python3 -m unittest discover -s tests -v
|
|
201
|
+
python3 -m venv /tmp/splittrace-comparison
|
|
202
|
+
/tmp/splittrace-comparison/bin/python -m pip install -r requirements-comparison.txt
|
|
203
|
+
/tmp/splittrace-comparison/bin/python scripts/compare.py
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
## Deliberate limits
|
|
207
|
+
|
|
208
|
+
Pairwise scanning is quadratic in partition sizes; ancestor-path storage also
|
|
209
|
+
grows with graph size/depth. Use small manifests. There is no performance claim.
|
|
210
|
+
No approximate content matching, nested-CV audit, persistent exception management,
|
|
211
|
+
automatic clock recovery, preprocessing tracing, causal/label leakage analysis or
|
|
212
|
+
proof of independence is implemented. Synthetic tests are not real-user validation.
|
|
213
|
+
|
|
214
|
+
Feedback is welcome from EEG, wearable-sensor and other windowed-signal projects:
|
|
215
|
+
can you supply source IDs/coordinates, and does a provenance path help you fix a
|
|
216
|
+
split? See [CONTRIBUTING.md](https://github.com/R-Koda/splittrace/blob/main/CONTRIBUTING.md) and the Japanese [trial guide](https://github.com/R-Koda/splittrace/blob/main/PILOT.md).
|
|
217
|
+
|
|
218
|
+
## References
|
|
219
|
+
|
|
220
|
+
- [leakfence](https://github.com/ptapal/leakfence)
|
|
221
|
+
- [Deepchecks train-test validation](https://docs.deepchecks.com/stable/api/generated/deepchecks.tabular.suites.train_test_validation.html)
|
|
222
|
+
- [bioLeak](https://github.com/selcukorkmaz/bioLeak)
|
|
223
|
+
|
|
224
|
+
This prototype contains no copied competitor implementation. Optional comparison
|
|
225
|
+
dependencies retain their own licenses and are installed separately.
|
|
226
|
+
|
|
227
|
+
## License
|
|
228
|
+
|
|
229
|
+
[MIT](https://github.com/R-Koda/splittrace/blob/main/LICENSE). Copyright (c) 2026 Ryu Koda.
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# Security
|
|
2
|
+
|
|
3
|
+
This is an early-stage local tool. Only the current main branch is maintained;
|
|
4
|
+
there is no security-response SLA.
|
|
5
|
+
|
|
6
|
+
For a sensitive vulnerability, use GitHub's **Report a vulnerability** feature:
|
|
7
|
+
https://github.com/R-Koda/splittrace/security/advisories/new
|
|
8
|
+
|
|
9
|
+
Do not put credentials, patient information or sensitive sample manifests in
|
|
10
|
+
public issues. Include a synthetic reproducer and the affected commit where
|
|
11
|
+
possible. Non-sensitive correctness bugs can be reported in ordinary issues.
|
|
12
|
+
|
|
13
|
+
The core does not upload data or execute manifest content. Reports may contain
|
|
14
|
+
identifiers supplied by the caller. Review them before sharing. Large manifests
|
|
15
|
+
or deeply connected provenance graphs can consume substantial CPU and memory;
|
|
16
|
+
the prototype is intended for small, locally trusted inputs.
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
[
|
|
2
|
+
{
|
|
3
|
+
"sample_id": "epoch-01"
|
|
4
|
+
},
|
|
5
|
+
{
|
|
6
|
+
"sample_id": "image-01",
|
|
7
|
+
"parent_ids": [
|
|
8
|
+
"epoch-01"
|
|
9
|
+
]
|
|
10
|
+
},
|
|
11
|
+
{
|
|
12
|
+
"sample_id": "a",
|
|
13
|
+
"parent_ids": [
|
|
14
|
+
"image-01"
|
|
15
|
+
]
|
|
16
|
+
},
|
|
17
|
+
{
|
|
18
|
+
"sample_id": "b",
|
|
19
|
+
"parent_ids": [
|
|
20
|
+
"epoch-01"
|
|
21
|
+
]
|
|
22
|
+
}
|
|
23
|
+
]
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77.0.3"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "splittrace"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Metadata-first audit of derived samples across existing ML splits"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
authors = [{name = "Ryu Koda"}]
|
|
12
|
+
requires-python = ">=3.10"
|
|
13
|
+
dependencies = []
|
|
14
|
+
keywords = ["data-leakage", "cross-validation", "eeg", "provenance", "machine-learning"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 3 - Alpha",
|
|
17
|
+
"Intended Audience :: Science/Research",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"Operating System :: OS Independent",
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
[project.urls]
|
|
23
|
+
Repository = "https://github.com/R-Koda/splittrace"
|
|
24
|
+
Issues = "https://github.com/R-Koda/splittrace/issues"
|
|
25
|
+
|
|
26
|
+
[project.scripts]
|
|
27
|
+
splittrace = "splittrace.cli:main"
|
|
28
|
+
|
|
29
|
+
[tool.setuptools.packages.find]
|
|
30
|
+
where = ["src"]
|