falsification-ledger 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- falsification_ledger-0.1.0/LICENSE +21 -0
- falsification_ledger-0.1.0/PKG-INFO +208 -0
- falsification_ledger-0.1.0/README.md +187 -0
- falsification_ledger-0.1.0/pyproject.toml +34 -0
- falsification_ledger-0.1.0/setup.cfg +4 -0
- falsification_ledger-0.1.0/src/falsification_ledger/__init__.py +40 -0
- falsification_ledger-0.1.0/src/falsification_ledger/__main__.py +4 -0
- falsification_ledger-0.1.0/src/falsification_ledger/cli.py +146 -0
- falsification_ledger-0.1.0/src/falsification_ledger/contracts.py +107 -0
- falsification_ledger-0.1.0/src/falsification_ledger/ledger.py +363 -0
- falsification_ledger-0.1.0/src/falsification_ledger.egg-info/PKG-INFO +208 -0
- falsification_ledger-0.1.0/src/falsification_ledger.egg-info/SOURCES.txt +17 -0
- falsification_ledger-0.1.0/src/falsification_ledger.egg-info/dependency_links.txt +1 -0
- falsification_ledger-0.1.0/src/falsification_ledger.egg-info/entry_points.txt +3 -0
- falsification_ledger-0.1.0/src/falsification_ledger.egg-info/requires.txt +1 -0
- falsification_ledger-0.1.0/src/falsification_ledger.egg-info/top_level.txt +1 -0
- falsification_ledger-0.1.0/tests/test_cli.py +133 -0
- falsification_ledger-0.1.0/tests/test_contracts.py +110 -0
- falsification_ledger-0.1.0/tests/test_ledger.py +125 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Falsification Ledger contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: falsification-ledger
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Pre-registration and falsification ledger for research: hash-chained, append-only, with Wilson-CI hit-rate reporting and fail-closed evidence contracts.
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Keywords: preregistration,falsification,reproducibility,ledger,research,quant
|
|
7
|
+
Classifier: Development Status :: 3 - Alpha
|
|
8
|
+
Classifier: Environment :: Console
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: Intended Audience :: Science/Research
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Topic :: Scientific/Engineering
|
|
16
|
+
Requires-Python: >=3.11
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Requires-Dist: jsonschema>=4.18
|
|
20
|
+
Dynamic: license-file
|
|
21
|
+
|
|
22
|
+
# falsification-ledger
|
|
23
|
+
|
|
24
|
+
A hash-chained, append-only ledger for research claims: **pre-register the
|
|
25
|
+
hypothesis and the evidence that would kill it *before* the study runs**,
|
|
26
|
+
then adjudicate honestly and measure your hit rate against a random
|
|
27
|
+
baseline. Python 3.11+, one dependency (`jsonschema`), Windows / Linux /
|
|
28
|
+
macOS.
|
|
29
|
+
|
|
30
|
+
**Status:** v0.1 — alpha. The ledger semantics are distilled from a
|
|
31
|
+
production research pipeline, but this standalone package is new: expect the
|
|
32
|
+
CLI and schemas to shift before v1.0.
|
|
33
|
+
|
|
34
|
+
## Why this exists
|
|
35
|
+
|
|
36
|
+
Quantitative research has a self-deception problem: you test 500 factor
|
|
37
|
+
ideas, remember the 3 that worked, and forget the 497 that died. By the time
|
|
38
|
+
you "validate" the lucky survivors, the evidence is already contaminated by
|
|
39
|
+
what you saw. Every backtest-hygiene tool on the market attacks the
|
|
40
|
+
*statistics* of this problem (deflated Sharpe, PBO, multiple-testing
|
|
41
|
+
corrections). `falsification-ledger` attacks the *process*: it makes you
|
|
42
|
+
write down, before seeing evidence:
|
|
43
|
+
|
|
44
|
+
- what you expect (`support` / `against` / `uncertain`), and
|
|
45
|
+
- what evidence would **kill** your claim (the falsification contract).
|
|
46
|
+
|
|
47
|
+
Then it keeps the receipts. Every event lands in an append-only JSONL
|
|
48
|
+
**hash chain** — any edit after the fact is detected by `fl verify` — and
|
|
49
|
+
the report answers the only question that matters: *do your pre-registered
|
|
50
|
+
beliefs actually hit, or is your hit rate indistinguishable from a random
|
|
51
|
+
baseline?* (Wilson 95% CI vs the most common actual verdict.)
|
|
52
|
+
|
|
53
|
+
## Philosophy
|
|
54
|
+
|
|
55
|
+
**Research is a promise; the ledger keeps it.**
|
|
56
|
+
|
|
57
|
+
- **Falsifiability is the default, not the exception.** Popper's criterion
|
|
58
|
+
— a claim is scientific only if something could count against it — is
|
|
59
|
+
usually invoked as a lecture. Here it is a required JSON field
|
|
60
|
+
(`falsification_contract` on `preregister`).
|
|
61
|
+
- **Pre-analysis plans have known costs and benefits.** [Olken (2015),
|
|
62
|
+
"Promises and Perils of Pre-Analysis Plans"](https://www.aeaweb.org/articles?id=10.1257/jep.29.3.61)
|
|
63
|
+
(JEP 29(3)) documents both; this tool implements the benefits (frozen
|
|
64
|
+
expectations, audit trail) while keeping the costs explicit (`uncertain`
|
|
65
|
+
verdicts and exploratory source types are first-class, so you can register
|
|
66
|
+
what you genuinely do not know).
|
|
67
|
+
- **Moderation beats total freezing.** [Banerjee & Duflo, "In Praise of
|
|
68
|
+
Moderation"](https://www.semanticscholar.org/paper/05ecf99a05419f0a268fe885be11a2cf4a8dbd46)
|
|
69
|
+
argue for layered pre-registration; `source_type` (paper / business /
|
|
70
|
+
cross_domain / pipeline / other) exists so confirmatory and exploratory
|
|
71
|
+
claims are never mixed in the same bucket.
|
|
72
|
+
- **Finance can become scientific.** [López de Prado (2023), *Causal Factor
|
|
73
|
+
Investing*](https://www.cambridge.org/core/elements/causal-factor-investing/9AFE270D7099B787B8FD4F4CBADE0C6E)
|
|
74
|
+
asks whether factor investing can become a science; this ledger is one
|
|
75
|
+
concrete answer — evidence with a chain of custody, adjudicated against a
|
|
76
|
+
pre-registered expectation.
|
|
77
|
+
- **Automated research needs machine-checkable evidence.** [EviBound
|
|
78
|
+
(arXiv:2511.05524)](https://ar5iv.labs.arxiv.org/html/2511.05524) and
|
|
79
|
+
[ECLIPSE v2.0](https://ideas.repec.org/p/osf/metaar/z3fke_v1.html) argue
|
|
80
|
+
that agentic research pipelines must eliminate false claims through
|
|
81
|
+
verifiable evidence; `fl submit` validates falsification reports against a
|
|
82
|
+
JSON Schema and computes content IDs, so gates can trust the evidence
|
|
83
|
+
without trusting the messenger.
|
|
84
|
+
|
|
85
|
+
## Quick start
|
|
86
|
+
|
|
87
|
+
```bash
|
|
88
|
+
# install from PyPI (once published)
|
|
89
|
+
pip install falsification-ledger
|
|
90
|
+
|
|
91
|
+
# or run without installing anything:
|
|
92
|
+
# PYTHONPATH=src python -m falsification_ledger --help
|
|
93
|
+
|
|
94
|
+
# try the full loop on a scratch ledger (creates files under a temp dir)
|
|
95
|
+
python examples/demo.py
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
The manual loop:
|
|
99
|
+
|
|
100
|
+
```bash
|
|
101
|
+
fl init --state-dir ~/.research-ledger
|
|
102
|
+
|
|
103
|
+
# 1. BEFORE running the study: register what you expect,
|
|
104
|
+
# and what evidence would kill the claim.
|
|
105
|
+
fl preregister --state-dir ~/.research-ledger \
|
|
106
|
+
--case-id MOMENTUM-OOS-2026Q3 \
|
|
107
|
+
--verdict support \
|
|
108
|
+
--reason "momentum rank IC stays positive OOS" \
|
|
109
|
+
--source-type paper \
|
|
110
|
+
--contract kill-criteria.json
|
|
111
|
+
|
|
112
|
+
# 2. When an independent check produces evidence, submit it:
|
|
113
|
+
fl submit --report falsification-report.json
|
|
114
|
+
# -> {"content_id": "sha256:...", "evidence_status": "valid", ...}
|
|
115
|
+
|
|
116
|
+
# 3. AFTER the study: adjudicate honestly.
|
|
117
|
+
fl adjudicate --state-dir ~/.research-ledger \
|
|
118
|
+
--case-id MOMENTUM-OOS-2026Q3 --verdict support
|
|
119
|
+
|
|
120
|
+
# 4. Measure whether you are better than a coin flip.
|
|
121
|
+
fl report --state-dir ~/.research-ledger --min-cases 20
|
|
122
|
+
|
|
123
|
+
# 5. Any time: prove nobody rewrote history.
|
|
124
|
+
fl verify --state-dir ~/.research-ledger
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
## Commands
|
|
128
|
+
|
|
129
|
+
| Command | What it does |
|
|
130
|
+
| --- | --- |
|
|
131
|
+
| `init` | Create the ledger state directory |
|
|
132
|
+
| `preregister` | Register a claim: `--case-id`, `--verdict` (support/against/uncertain), `--reason`, optional `--source-type`, optional `--contract` (falsification contract JSON). Duplicate registration for the same case is rejected |
|
|
133
|
+
| `submit` | Validate a falsification report against the contract schema; print its content ID (`sha256:...`) and evidence status (`valid` / `invalid` / `missing`). Read-only; exits non-zero on blockers |
|
|
134
|
+
| `adjudicate` | Backfill the actual verdict for a registered case (register required; once per case) |
|
|
135
|
+
| `report` | Hit-rate report: resolved cases, completeness, participation, hit rate with **Wilson 95% CI**, random baseline, per-source-type breakdown, `verdict_ready` gate |
|
|
136
|
+
| `verify` | Recompute the hash chain of the whole ledger; detects any edit, insertion, or reordering |
|
|
137
|
+
| `version` | Print version |
|
|
138
|
+
|
|
139
|
+
Global flag: `--state-dir` on every stateful command (default: none — the
|
|
140
|
+
ledger path is always explicit, so a `git add .` can never sweep it into
|
|
141
|
+
version control).
|
|
142
|
+
|
|
143
|
+
## Ledger format
|
|
144
|
+
|
|
145
|
+
The ledger is a JSONL file at `<state-dir>/ledger.jsonl`. Every line is one
|
|
146
|
+
event:
|
|
147
|
+
|
|
148
|
+
```json
|
|
149
|
+
{"schema_version": "falsification_ledger.prediction_event.v1",
|
|
150
|
+
"event": "register", "record_id": "...", "case_id": "CASE-1",
|
|
151
|
+
"expected_verdict": "support", "expected_reason": "...",
|
|
152
|
+
"source_type": "paper", "falsification_contract": {...},
|
|
153
|
+
"actual_verdict": null, "recorded_at": "...", "concluded_at": null,
|
|
154
|
+
"prev_hash": null,
|
|
155
|
+
"event_hash": "sha256(prev_hash || 0x00 || canonical payload)"}
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
`verify` recomputes every `event_hash` and checks each `prev_hash` link.
|
|
159
|
+
**Any tampering — editing a reason, deleting a line, reordering events —
|
|
160
|
+
breaks the chain at a specific line number.**
|
|
161
|
+
|
|
162
|
+
## Falsification reports
|
|
163
|
+
|
|
164
|
+
A falsification report is the machine-readable evidence produced by an
|
|
165
|
+
independent check (null-model randomization, OOS rank IC, FDR correction,
|
|
166
|
+
protocol deviation, effect CI, cost sensitivity, ...). The contract:
|
|
167
|
+
|
|
168
|
+
- schema: [`schema/falsification-report.schema.json`](schema/falsification-report.schema.json)
|
|
169
|
+
(draft 2020-12, `additionalProperties: false`, fail-closed);
|
|
170
|
+
- content ID: `sha256:` over `domain-prefix || 0x00 || canonical JSON` —
|
|
171
|
+
the same report always yields the same ID, a one-field change yields a
|
|
172
|
+
different ID;
|
|
173
|
+
- evidence status (fail-closed for gates):
|
|
174
|
+
- `valid` — conformant, conclusion `not_falsified`, consistency intact;
|
|
175
|
+
- `invalid` — non-conformant, or conclusion `falsified`, or explicitly
|
|
176
|
+
inconsistent;
|
|
177
|
+
- `missing` — conclusion `inconclusive`: treated as *absent* evidence.
|
|
178
|
+
|
|
179
|
+
## Verification model
|
|
180
|
+
|
|
181
|
+
`fl verify` is the tamper-evidence layer: it re-derives the entire chain
|
|
182
|
+
from the file bytes and reports the first bad line. Combined with
|
|
183
|
+
`preregister` (frozen expectations) and `submit` (content-addressed
|
|
184
|
+
evidence), a research pipeline can prove to itself — and to reviewers —
|
|
185
|
+
that the expectation existed before the evidence did. Nothing here trades,
|
|
186
|
+
prices, or decides.
|
|
187
|
+
|
|
188
|
+
## Development
|
|
189
|
+
|
|
190
|
+
```bash
|
|
191
|
+
python -m pip install -e . pytest
|
|
192
|
+
python -m pytest
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
CI runs the full test suite on Ubuntu, Windows and macOS with Python 3.11
|
|
196
|
+
and 3.12. Issues are handled on weekends; pull requests are welcome.
|
|
197
|
+
|
|
198
|
+
## Related work
|
|
199
|
+
|
|
200
|
+
- [Olken (2015), Promises and Perils of Pre-Analysis Plans](https://www.aeaweb.org/articles?id=10.1257/jep.29.3.61) — the economics of freezing expectations
|
|
201
|
+
- [Banerjee & Duflo, In Praise of Moderation](https://www.semanticscholar.org/paper/05ecf99a05419f0a268fe885be11a2cf4a8dbd46) — layered pre-registration
|
|
202
|
+
- [López de Prado (2023), Causal Factor Investing](https://www.cambridge.org/core/elements/causal-factor-investing/9AFE270D7099B787B8FD4F4CBADE0C6E) — can factor investing become scientific?
|
|
203
|
+
- [EviBound: Evidence-Bound Autonomous Research (arXiv:2511.05524)](https://ar5iv.labs.arxiv.org/html/2511.05524) — governance for agentic research
|
|
204
|
+
- [ECLIPSE v2.0: A Systematic Falsification Framework](https://ideas.repec.org/p/osf/metaar/z3fke_v1.html) — enforce falsifiability integrity
|
|
205
|
+
|
|
206
|
+
## License
|
|
207
|
+
|
|
208
|
+
MIT
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
# falsification-ledger
|
|
2
|
+
|
|
3
|
+
A hash-chained, append-only ledger for research claims: **pre-register the
|
|
4
|
+
hypothesis and the evidence that would kill it *before* the study runs**,
|
|
5
|
+
then adjudicate honestly and measure your hit rate against a random
|
|
6
|
+
baseline. Python 3.11+, one dependency (`jsonschema`), Windows / Linux /
|
|
7
|
+
macOS.
|
|
8
|
+
|
|
9
|
+
**Status:** v0.1 — alpha. The ledger semantics are distilled from a
|
|
10
|
+
production research pipeline, but this standalone package is new: expect the
|
|
11
|
+
CLI and schemas to shift before v1.0.
|
|
12
|
+
|
|
13
|
+
## Why this exists
|
|
14
|
+
|
|
15
|
+
Quantitative research has a self-deception problem: you test 500 factor
|
|
16
|
+
ideas, remember the 3 that worked, and forget the 497 that died. By the time
|
|
17
|
+
you "validate" the lucky survivors, the evidence is already contaminated by
|
|
18
|
+
what you saw. Every backtest-hygiene tool on the market attacks the
|
|
19
|
+
*statistics* of this problem (deflated Sharpe, PBO, multiple-testing
|
|
20
|
+
corrections). `falsification-ledger` attacks the *process*: it makes you
|
|
21
|
+
write down, before seeing evidence:
|
|
22
|
+
|
|
23
|
+
- what you expect (`support` / `against` / `uncertain`), and
|
|
24
|
+
- what evidence would **kill** your claim (the falsification contract).
|
|
25
|
+
|
|
26
|
+
Then it keeps the receipts. Every event lands in an append-only JSONL
|
|
27
|
+
**hash chain** — any edit after the fact is detected by `fl verify` — and
|
|
28
|
+
the report answers the only question that matters: *do your pre-registered
|
|
29
|
+
beliefs actually hit, or is your hit rate indistinguishable from a random
|
|
30
|
+
baseline?* (Wilson 95% CI vs the most common actual verdict.)
|
|
31
|
+
|
|
32
|
+
## Philosophy
|
|
33
|
+
|
|
34
|
+
**Research is a promise; the ledger keeps it.**
|
|
35
|
+
|
|
36
|
+
- **Falsifiability is the default, not the exception.** Popper's criterion
|
|
37
|
+
— a claim is scientific only if something could count against it — is
|
|
38
|
+
usually invoked as a lecture. Here it is a required JSON field
|
|
39
|
+
(`falsification_contract` on `preregister`).
|
|
40
|
+
- **Pre-analysis plans have known costs and benefits.** [Olken (2015),
|
|
41
|
+
"Promises and Perils of Pre-Analysis Plans"](https://www.aeaweb.org/articles?id=10.1257/jep.29.3.61)
|
|
42
|
+
(JEP 29(3)) documents both; this tool implements the benefits (frozen
|
|
43
|
+
expectations, audit trail) while keeping the costs explicit (`uncertain`
|
|
44
|
+
verdicts and exploratory source types are first-class, so you can register
|
|
45
|
+
what you genuinely do not know).
|
|
46
|
+
- **Moderation beats total freezing.** [Banerjee & Duflo, "In Praise of
|
|
47
|
+
Moderation"](https://www.semanticscholar.org/paper/05ecf99a05419f0a268fe885be11a2cf4a8dbd46)
|
|
48
|
+
argue for layered pre-registration; `source_type` (paper / business /
|
|
49
|
+
cross_domain / pipeline / other) exists so confirmatory and exploratory
|
|
50
|
+
claims are never mixed in the same bucket.
|
|
51
|
+
- **Finance can become scientific.** [López de Prado (2023), *Causal Factor
|
|
52
|
+
Investing*](https://www.cambridge.org/core/elements/causal-factor-investing/9AFE270D7099B787B8FD4F4CBADE0C6E)
|
|
53
|
+
asks whether factor investing can become a science; this ledger is one
|
|
54
|
+
concrete answer — evidence with a chain of custody, adjudicated against a
|
|
55
|
+
pre-registered expectation.
|
|
56
|
+
- **Automated research needs machine-checkable evidence.** [EviBound
|
|
57
|
+
(arXiv:2511.05524)](https://ar5iv.labs.arxiv.org/html/2511.05524) and
|
|
58
|
+
[ECLIPSE v2.0](https://ideas.repec.org/p/osf/metaar/z3fke_v1.html) argue
|
|
59
|
+
that agentic research pipelines must eliminate false claims through
|
|
60
|
+
verifiable evidence; `fl submit` validates falsification reports against a
|
|
61
|
+
JSON Schema and computes content IDs, so gates can trust the evidence
|
|
62
|
+
without trusting the messenger.
|
|
63
|
+
|
|
64
|
+
## Quick start
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
# install from PyPI (once published)
|
|
68
|
+
pip install falsification-ledger
|
|
69
|
+
|
|
70
|
+
# or run without installing anything:
|
|
71
|
+
# PYTHONPATH=src python -m falsification_ledger --help
|
|
72
|
+
|
|
73
|
+
# try the full loop on a scratch ledger (creates files under a temp dir)
|
|
74
|
+
python examples/demo.py
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
The manual loop:
|
|
78
|
+
|
|
79
|
+
```bash
|
|
80
|
+
fl init --state-dir ~/.research-ledger
|
|
81
|
+
|
|
82
|
+
# 1. BEFORE running the study: register what you expect,
|
|
83
|
+
# and what evidence would kill the claim.
|
|
84
|
+
fl preregister --state-dir ~/.research-ledger \
|
|
85
|
+
--case-id MOMENTUM-OOS-2026Q3 \
|
|
86
|
+
--verdict support \
|
|
87
|
+
--reason "momentum rank IC stays positive OOS" \
|
|
88
|
+
--source-type paper \
|
|
89
|
+
--contract kill-criteria.json
|
|
90
|
+
|
|
91
|
+
# 2. When an independent check produces evidence, submit it:
|
|
92
|
+
fl submit --report falsification-report.json
|
|
93
|
+
# -> {"content_id": "sha256:...", "evidence_status": "valid", ...}
|
|
94
|
+
|
|
95
|
+
# 3. AFTER the study: adjudicate honestly.
|
|
96
|
+
fl adjudicate --state-dir ~/.research-ledger \
|
|
97
|
+
--case-id MOMENTUM-OOS-2026Q3 --verdict support
|
|
98
|
+
|
|
99
|
+
# 4. Measure whether you are better than a coin flip.
|
|
100
|
+
fl report --state-dir ~/.research-ledger --min-cases 20
|
|
101
|
+
|
|
102
|
+
# 5. Any time: prove nobody rewrote history.
|
|
103
|
+
fl verify --state-dir ~/.research-ledger
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
## Commands
|
|
107
|
+
|
|
108
|
+
| Command | What it does |
|
|
109
|
+
| --- | --- |
|
|
110
|
+
| `init` | Create the ledger state directory |
|
|
111
|
+
| `preregister` | Register a claim: `--case-id`, `--verdict` (support/against/uncertain), `--reason`, optional `--source-type`, optional `--contract` (falsification contract JSON). Duplicate registration for the same case is rejected |
|
|
112
|
+
| `submit` | Validate a falsification report against the contract schema; print its content ID (`sha256:...`) and evidence status (`valid` / `invalid` / `missing`). Read-only; exits non-zero on blockers |
|
|
113
|
+
| `adjudicate` | Backfill the actual verdict for a registered case (register required; once per case) |
|
|
114
|
+
| `report` | Hit-rate report: resolved cases, completeness, participation, hit rate with **Wilson 95% CI**, random baseline, per-source-type breakdown, `verdict_ready` gate |
|
|
115
|
+
| `verify` | Recompute the hash chain of the whole ledger; detects any edit, insertion, or reordering |
|
|
116
|
+
| `version` | Print version |
|
|
117
|
+
|
|
118
|
+
Global flag: `--state-dir` on every stateful command (default: none — the
|
|
119
|
+
ledger path is always explicit, so a `git add .` can never sweep it into
|
|
120
|
+
version control).
|
|
121
|
+
|
|
122
|
+
## Ledger format
|
|
123
|
+
|
|
124
|
+
The ledger is a JSONL file at `<state-dir>/ledger.jsonl`. Every line is one
|
|
125
|
+
event:
|
|
126
|
+
|
|
127
|
+
```json
|
|
128
|
+
{"schema_version": "falsification_ledger.prediction_event.v1",
|
|
129
|
+
"event": "register", "record_id": "...", "case_id": "CASE-1",
|
|
130
|
+
"expected_verdict": "support", "expected_reason": "...",
|
|
131
|
+
"source_type": "paper", "falsification_contract": {...},
|
|
132
|
+
"actual_verdict": null, "recorded_at": "...", "concluded_at": null,
|
|
133
|
+
"prev_hash": null,
|
|
134
|
+
"event_hash": "sha256(prev_hash || 0x00 || canonical payload)"}
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
`verify` recomputes every `event_hash` and checks each `prev_hash` link.
|
|
138
|
+
**Any tampering — editing a reason, deleting a line, reordering events —
|
|
139
|
+
breaks the chain at a specific line number.**
|
|
140
|
+
|
|
141
|
+
## Falsification reports
|
|
142
|
+
|
|
143
|
+
A falsification report is the machine-readable evidence produced by an
|
|
144
|
+
independent check (null-model randomization, OOS rank IC, FDR correction,
|
|
145
|
+
protocol deviation, effect CI, cost sensitivity, ...). The contract:
|
|
146
|
+
|
|
147
|
+
- schema: [`schema/falsification-report.schema.json`](schema/falsification-report.schema.json)
|
|
148
|
+
(draft 2020-12, `additionalProperties: false`, fail-closed);
|
|
149
|
+
- content ID: `sha256:` over `domain-prefix || 0x00 || canonical JSON` —
|
|
150
|
+
the same report always yields the same ID, a one-field change yields a
|
|
151
|
+
different ID;
|
|
152
|
+
- evidence status (fail-closed for gates):
|
|
153
|
+
- `valid` — conformant, conclusion `not_falsified`, consistency intact;
|
|
154
|
+
- `invalid` — non-conformant, or conclusion `falsified`, or explicitly
|
|
155
|
+
inconsistent;
|
|
156
|
+
- `missing` — conclusion `inconclusive`: treated as *absent* evidence.
|
|
157
|
+
|
|
158
|
+
## Verification model
|
|
159
|
+
|
|
160
|
+
`fl verify` is the tamper-evidence layer: it re-derives the entire chain
|
|
161
|
+
from the file bytes and reports the first bad line. Combined with
|
|
162
|
+
`preregister` (frozen expectations) and `submit` (content-addressed
|
|
163
|
+
evidence), a research pipeline can prove to itself — and to reviewers —
|
|
164
|
+
that the expectation existed before the evidence did. Nothing here trades,
|
|
165
|
+
prices, or decides.
|
|
166
|
+
|
|
167
|
+
## Development
|
|
168
|
+
|
|
169
|
+
```bash
|
|
170
|
+
python -m pip install -e . pytest
|
|
171
|
+
python -m pytest
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
CI runs the full test suite on Ubuntu, Windows and macOS with Python 3.11
|
|
175
|
+
and 3.12. Issues are handled on weekends; pull requests are welcome.
|
|
176
|
+
|
|
177
|
+
## Related work
|
|
178
|
+
|
|
179
|
+
- [Olken (2015), Promises and Perils of Pre-Analysis Plans](https://www.aeaweb.org/articles?id=10.1257/jep.29.3.61) — the economics of freezing expectations
|
|
180
|
+
- [Banerjee & Duflo, In Praise of Moderation](https://www.semanticscholar.org/paper/05ecf99a05419f0a268fe885be11a2cf4a8dbd46) — layered pre-registration
|
|
181
|
+
- [López de Prado (2023), Causal Factor Investing](https://www.cambridge.org/core/elements/causal-factor-investing/9AFE270D7099B787B8FD4F4CBADE0C6E) — can factor investing become scientific?
|
|
182
|
+
- [EviBound: Evidence-Bound Autonomous Research (arXiv:2511.05524)](https://ar5iv.labs.arxiv.org/html/2511.05524) — governance for agentic research
|
|
183
|
+
- [ECLIPSE v2.0: A Systematic Falsification Framework](https://ideas.repec.org/p/osf/metaar/z3fke_v1.html) — enforce falsifiability integrity
|
|
184
|
+
|
|
185
|
+
## License
|
|
186
|
+
|
|
187
|
+
MIT
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=64"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "falsification-ledger"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Pre-registration and falsification ledger for research: hash-chained, append-only, with Wilson-CI hit-rate reporting and fail-closed evidence contracts."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
dependencies = ["jsonschema>=4.18"]
|
|
13
|
+
keywords = ["preregistration", "falsification", "reproducibility", "ledger", "research", "quant"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 3 - Alpha",
|
|
16
|
+
"Environment :: Console",
|
|
17
|
+
"Intended Audience :: Developers",
|
|
18
|
+
"Intended Audience :: Science/Research",
|
|
19
|
+
"Operating System :: OS Independent",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3.11",
|
|
22
|
+
"Programming Language :: Python :: 3.12",
|
|
23
|
+
"Topic :: Scientific/Engineering",
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
[project.scripts]
|
|
27
|
+
fl = "falsification_ledger.cli:main"
|
|
28
|
+
falsification-ledger = "falsification_ledger.cli:main"
|
|
29
|
+
|
|
30
|
+
[tool.setuptools.packages.find]
|
|
31
|
+
where = ["src"]
|
|
32
|
+
|
|
33
|
+
[tool.pytest.ini_options]
|
|
34
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""falsification-ledger: pre-registration and falsification ledger.
|
|
2
|
+
|
|
3
|
+
A hash-chained, append-only JSONL ledger for research claims: register the
|
|
4
|
+
hypothesis and the evidence that would kill it *before* running the study,
|
|
5
|
+
submit evidence, adjudicate, and report hit-rate with a Wilson confidence
|
|
6
|
+
interval against a random baseline. Every event is content-addressed and
|
|
7
|
+
chain-verified; nothing here trades, prices, or decides.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from .contracts import (
|
|
11
|
+
evidence_status,
|
|
12
|
+
falsification_object_id,
|
|
13
|
+
load_falsification_schema,
|
|
14
|
+
validate_falsification_report,
|
|
15
|
+
)
|
|
16
|
+
from .ledger import (
|
|
17
|
+
conclude_prediction,
|
|
18
|
+
ensure_prediction_registered,
|
|
19
|
+
has_prediction_registered,
|
|
20
|
+
ledger_path,
|
|
21
|
+
register_prediction,
|
|
22
|
+
report_prediction_hitrate,
|
|
23
|
+
verify_chain,
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
__version__ = "0.1.0"
|
|
27
|
+
|
|
28
|
+
__all__ = [
|
|
29
|
+
"conclude_prediction",
|
|
30
|
+
"ensure_prediction_registered",
|
|
31
|
+
"evidence_status",
|
|
32
|
+
"falsification_object_id",
|
|
33
|
+
"has_prediction_registered",
|
|
34
|
+
"ledger_path",
|
|
35
|
+
"load_falsification_schema",
|
|
36
|
+
"register_prediction",
|
|
37
|
+
"report_prediction_hitrate",
|
|
38
|
+
"validate_falsification_report",
|
|
39
|
+
"verify_chain",
|
|
40
|
+
]
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
"""Command-line interface for falsification-ledger.
|
|
2
|
+
|
|
3
|
+
Subcommands:
|
|
4
|
+
|
|
5
|
+
- ``init`` create the ledger state directory
|
|
6
|
+
- ``preregister`` register a claim (verdict + reason) before evidence;
|
|
7
|
+
optionally attach a falsification contract JSON
|
|
8
|
+
- ``submit`` validate a falsification report and print its
|
|
9
|
+
content ID and evidence status (read-only)
|
|
10
|
+
- ``adjudicate`` backfill the actual verdict for a case
|
|
11
|
+
- ``report`` hit-rate report (Wilson 95% CI vs random baseline)
|
|
12
|
+
- ``verify`` hash-chain integrity check of the whole ledger
|
|
13
|
+
- ``version`` print version
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import argparse
|
|
19
|
+
import json
|
|
20
|
+
import sys
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
from typing import Any
|
|
23
|
+
|
|
24
|
+
from . import __version__
|
|
25
|
+
from .contracts import evidence_status, falsification_object_id, validate_falsification_report
|
|
26
|
+
from .ledger import (
|
|
27
|
+
conclude_prediction,
|
|
28
|
+
ledger_path,
|
|
29
|
+
register_prediction,
|
|
30
|
+
report_prediction_hitrate,
|
|
31
|
+
verify_chain,
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _print_json(body: dict[str, Any]) -> None:
|
|
36
|
+
print(json.dumps(body, ensure_ascii=False, indent=2))
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _load_json_file(path: str) -> dict[str, Any]:
|
|
40
|
+
value = json.loads(Path(path).read_text(encoding="utf-8"))
|
|
41
|
+
if not isinstance(value, dict):
|
|
42
|
+
raise ValueError(f"expected a JSON object: {path}")
|
|
43
|
+
return value
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
47
|
+
parser = argparse.ArgumentParser(
|
|
48
|
+
prog="fl",
|
|
49
|
+
description="Pre-registration and falsification ledger for research.",
|
|
50
|
+
)
|
|
51
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
52
|
+
|
|
53
|
+
init = sub.add_parser("init", help="create the ledger state directory")
|
|
54
|
+
init.add_argument("--state-dir", required=True, help="ledger state directory")
|
|
55
|
+
|
|
56
|
+
prereg = sub.add_parser("preregister", help="register a claim before evidence")
|
|
57
|
+
prereg.add_argument("--state-dir", required=True)
|
|
58
|
+
prereg.add_argument("--case-id", required=True)
|
|
59
|
+
prereg.add_argument(
|
|
60
|
+
"--verdict", required=True, choices=["support", "against", "uncertain"]
|
|
61
|
+
)
|
|
62
|
+
prereg.add_argument("--reason", required=True)
|
|
63
|
+
prereg.add_argument("--source-type", default="other",
|
|
64
|
+
choices=["paper", "business", "cross_domain", "pipeline", "other"])
|
|
65
|
+
prereg.add_argument("--contract", default=None,
|
|
66
|
+
help="falsification contract JSON: what evidence would kill the claim")
|
|
67
|
+
|
|
68
|
+
submit = sub.add_parser("submit", help="validate a falsification report (read-only)")
|
|
69
|
+
submit.add_argument("--report", required=True, help="falsification report JSON path")
|
|
70
|
+
submit.add_argument("--schema", default=None, help="override schema JSON path")
|
|
71
|
+
|
|
72
|
+
adjudicate = sub.add_parser("adjudicate", help="backfill the actual verdict")
|
|
73
|
+
adjudicate.add_argument("--state-dir", required=True)
|
|
74
|
+
adjudicate.add_argument("--case-id", required=True)
|
|
75
|
+
adjudicate.add_argument(
|
|
76
|
+
"--verdict", required=True, choices=["support", "against", "uncertain"]
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
report = sub.add_parser("report", help="hit-rate report")
|
|
80
|
+
report.add_argument("--state-dir", required=True)
|
|
81
|
+
report.add_argument("--min-cases", type=int, default=20)
|
|
82
|
+
|
|
83
|
+
verify = sub.add_parser("verify", help="hash-chain integrity check")
|
|
84
|
+
verify.add_argument("--state-dir", required=True)
|
|
85
|
+
|
|
86
|
+
sub.add_parser("version", help="print version")
|
|
87
|
+
return parser
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def main(argv: list[str] | None = None) -> int:
|
|
91
|
+
parser = build_parser()
|
|
92
|
+
args = parser.parse_args(argv)
|
|
93
|
+
|
|
94
|
+
if args.command == "version":
|
|
95
|
+
print(__version__)
|
|
96
|
+
return 0
|
|
97
|
+
|
|
98
|
+
if args.command == "init":
|
|
99
|
+
path = ledger_path(args.state_dir)
|
|
100
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
101
|
+
print(f"init: ledger ready at {path}")
|
|
102
|
+
return 0
|
|
103
|
+
|
|
104
|
+
if args.command == "preregister":
|
|
105
|
+
contract = _load_json_file(args.contract) if args.contract else None
|
|
106
|
+
result = register_prediction(
|
|
107
|
+
args.state_dir,
|
|
108
|
+
args.case_id,
|
|
109
|
+
args.verdict,
|
|
110
|
+
args.reason,
|
|
111
|
+
args.source_type,
|
|
112
|
+
falsification_contract=contract,
|
|
113
|
+
)
|
|
114
|
+
_print_json(result)
|
|
115
|
+
return 0
|
|
116
|
+
|
|
117
|
+
if args.command == "submit":
|
|
118
|
+
report = _load_json_file(args.report)
|
|
119
|
+
blockers = validate_falsification_report(report, args.schema)
|
|
120
|
+
body = {
|
|
121
|
+
"content_id": falsification_object_id(report),
|
|
122
|
+
"evidence_status": evidence_status(report, args.schema),
|
|
123
|
+
"blockers": blockers,
|
|
124
|
+
}
|
|
125
|
+
_print_json(body)
|
|
126
|
+
return 0 if not blockers else 1
|
|
127
|
+
|
|
128
|
+
if args.command == "adjudicate":
|
|
129
|
+
_print_json(conclude_prediction(args.state_dir, args.case_id, args.verdict))
|
|
130
|
+
return 0
|
|
131
|
+
|
|
132
|
+
if args.command == "report":
|
|
133
|
+
_print_json(report_prediction_hitrate(args.state_dir, min_cases=args.min_cases))
|
|
134
|
+
return 0
|
|
135
|
+
|
|
136
|
+
if args.command == "verify":
|
|
137
|
+
body = verify_chain(args.state_dir)
|
|
138
|
+
_print_json(body)
|
|
139
|
+
return 0 if body.get("ok") else 1
|
|
140
|
+
|
|
141
|
+
parser.error(f"unknown command: {args.command}")
|
|
142
|
+
return 2
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
if __name__ == "__main__":
|
|
146
|
+
sys.exit(main())
|