mizan-ledger 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mizan_ledger-0.1.0/LICENSE +21 -0
- mizan_ledger-0.1.0/MANIFEST.in +7 -0
- mizan_ledger-0.1.0/PKG-INFO +109 -0
- mizan_ledger-0.1.0/README.md +90 -0
- mizan_ledger-0.1.0/js/mizan-ledger.mjs +45 -0
- mizan_ledger-0.1.0/js/test.mjs +27 -0
- mizan_ledger-0.1.0/pyproject.toml +41 -0
- mizan_ledger-0.1.0/setup.cfg +4 -0
- mizan_ledger-0.1.0/src/mizan_ledger/__init__.py +13 -0
- mizan_ledger-0.1.0/src/mizan_ledger/adapters/__init__.py +1 -0
- mizan_ledger-0.1.0/src/mizan_ledger/adapters/trace_eval.py +41 -0
- mizan_ledger-0.1.0/src/mizan_ledger/cli.py +22 -0
- mizan_ledger-0.1.0/src/mizan_ledger/core.py +247 -0
- mizan_ledger-0.1.0/src/mizan_ledger/metrics.py +111 -0
- mizan_ledger-0.1.0/src/mizan_ledger/pytest_plugin.py +81 -0
- mizan_ledger-0.1.0/src/mizan_ledger.egg-info/PKG-INFO +109 -0
- mizan_ledger-0.1.0/src/mizan_ledger.egg-info/SOURCES.txt +23 -0
- mizan_ledger-0.1.0/src/mizan_ledger.egg-info/dependency_links.txt +1 -0
- mizan_ledger-0.1.0/src/mizan_ledger.egg-info/entry_points.txt +5 -0
- mizan_ledger-0.1.0/src/mizan_ledger.egg-info/requires.txt +3 -0
- mizan_ledger-0.1.0/src/mizan_ledger.egg-info/top_level.txt +1 -0
- mizan_ledger-0.1.0/tests/conftest.py +3 -0
- mizan_ledger-0.1.0/tests/test_adapters.py +75 -0
- mizan_ledger-0.1.0/tests/test_core.py +139 -0
- mizan_ledger-0.1.0/vectors/commitments.json +38 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 QANTS
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: mizan-ledger
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Declare your agent's confidence before the task, commit a salted hash, score honesty against real outcomes. Local-first, hash-only.
|
|
5
|
+
Author: QANTS
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://qants-site.qants.workers.dev
|
|
8
|
+
Project-URL: SeedAgents, https://qants-site.qants.workers.dev/seed-agents
|
|
9
|
+
Keywords: calibration,agents,brier,commitment,pytest-plugin
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Framework :: Pytest
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Requires-Python: >=3.9
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
License-File: LICENSE
|
|
16
|
+
Provides-Extra: dev
|
|
17
|
+
Requires-Dist: pytest>=7; extra == "dev"
|
|
18
|
+
Dynamic: license-file
|
|
19
|
+
|
|
20
|
+
# mizan-ledger
|
|
21
|
+
|
|
22
|
+
Declare how confident your agent is **before** a task, commit a salted hash of that declaration, then score honesty against the **real outcome** (your own tests, evals or checks). Local-first, hash-only, verifiable offline.
|
|
23
|
+
|
|
24
|
+
> Status: **v0.1.0, early alpha** by QANTS. On PyPI (not on npm). MIT licensed (see `LICENSE`). No network access, no telemetry, no dependencies in the core library.
|
|
25
|
+
|
|
26
|
+
## Why
|
|
27
|
+
|
|
28
|
+
Agents often report success with the same unearned confidence whether they are right or wrong. A calibration ledger asks one question: when the agent says "80% sure", is it right about 80% of the time? Brier score and ECE answer that. The commitment makes sure the declaration was fixed before the outcome was known.
|
|
29
|
+
|
|
30
|
+
## Install
|
|
31
|
+
|
|
32
|
+
```
|
|
33
|
+
pip install mizan-ledger
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
Or from a local checkout:
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
pip install -e . # Python >= 3.9, pytest only for development
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## 30-second example
|
|
43
|
+
|
|
44
|
+
```python
|
|
45
|
+
from mizan_ledger import Ledger
|
|
46
|
+
|
|
47
|
+
led = Ledger("ledger/", private_path="private/") # public dir is safe to share; private dir is not
|
|
48
|
+
led.declare("fix-bug-421", 0.80, agent_id="my-agent", version="1.2.0") # hash-only receipt, timestamped
|
|
49
|
+
# ... run the task, run the tests ...
|
|
50
|
+
led.resolve("fix-bug-421", outcome=True) # reveals the opening, scores it
|
|
51
|
+
|
|
52
|
+
print(led.summary("my-agent")) # n, Brier + 95% interval, ECE, log loss, accuracy (Wilson), coverage
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
mizan-ledger verify ledger/ # offline check of hash chains, openings and scores
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
## What gets recorded
|
|
60
|
+
|
|
61
|
+
| File | Visibility | Contents |
|
|
62
|
+
|---|---|---|
|
|
63
|
+
| `receipts.jsonl` | public | `agent_id`, `agent_version`, `commitment` (SHA-256), `declared_at`, hash-chain link |
|
|
64
|
+
| `resolutions.jsonl` | public after resolution | outcome, opening (`task_id`, `p`, `salt`), squared error, log loss, hash-chain link |
|
|
65
|
+
| `private/openings.jsonl` | **private** (mode 600) | the same openings, kept until the task is resolved |
|
|
66
|
+
|
|
67
|
+
Commitment = `SHA-256(canonical_json({agent_id, agent_version, p, salt, schema, task_id}))` with `p` as a fixed 6-decimal string, a fresh 128-bit random salt per declaration, sorted keys and no whitespace. Shared test vectors (`vectors/commitments.json`) are checked by both the Python and the JavaScript implementation.
|
|
68
|
+
|
|
69
|
+
## Guarantees and honest limits
|
|
70
|
+
|
|
71
|
+
- A declaration cannot be replaced or re-resolved. Deleting or editing a record breaks the hash chain and `verify` fails.
|
|
72
|
+
- **Coverage is reported.** Declared, resolved and pending are all counted, so you cannot quietly drop the bad ones. Anything unresolved shows as `pending`.
|
|
73
|
+
- **Timestamps are your local clock.** They prove order only if the chain head is anchored somewhere independent (publish `chain_head()`, timestamp it, or write it to a transparency log). Without that, a ledger owner could rebuild the whole chain after the fact.
|
|
74
|
+
- The library cannot tell whether the outcome you pass to `resolve` is honest. Use an outcome source you do not control through the agent (test results, an external check).
|
|
75
|
+
- Small samples give wide intervals; ECE is biased upward for small n. The summary prints the interval and `n`; read them.
|
|
76
|
+
- Scores are about calibration, not about whether the agent is good, safe or secure.
|
|
77
|
+
|
|
78
|
+
## Scores (`mizan_ledger.metrics`)
|
|
79
|
+
|
|
80
|
+
Brier (mean squared error), log loss (p clipped to [0.01, 0.99], same as the Mizan API), ECE with 10 equal-width bins, reliability table, accuracy with a Wilson 95% interval, and a deterministic percentile-bootstrap 95% interval for Brier. The always-0.5 baseline Brier is 0.25.
|
|
81
|
+
|
|
82
|
+
## Adapters
|
|
83
|
+
|
|
84
|
+
| Adapter | Status |
|
|
85
|
+
|---|---|
|
|
86
|
+
| **pytest plugin** (`mizan_ledger.pytest_plugin`): `@pytest.mark.confidence(0.9)` declares before the test body, pass/fail resolves it. Run with `pytest --mizan-ledger-dir .mizan --mizan-agent-id my-agent`. | Tested here (pytester). |
|
|
87
|
+
| **Generic trace evaluator** (`mizan_ledger.adapters.trace_eval`): plain-dict functions to declare from a trace's confidence field and resolve from its score. | Unit-tested with synthetic dicts only. **Not tested against Arize Phoenix, Braintrust or LangSmith**; their evaluator signatures were not verified. Treat as a stub. |
|
|
88
|
+
| **JavaScript shim** (`js/mizan-ledger.mjs`): commitment, declare, verify opening. Node >= 18, `node:crypto` only. | Tested against the shared vectors (`node --test js/test.mjs`). It does not implement the file ledger. |
|
|
89
|
+
|
|
90
|
+
No CI workflow file is included in this repository.
|
|
91
|
+
|
|
92
|
+
## Tests
|
|
93
|
+
|
|
94
|
+
```bash
|
|
95
|
+
python3 -m pytest -q # Python suite
|
|
96
|
+
node --test js/test.mjs # JS shim vs shared vectors
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
## Seed agents
|
|
100
|
+
|
|
101
|
+
`seeds/` holds three demonstration agents operated by QANTS (ids start with `qants-`). They use only free public data and local ledgers, never the production Mizan API, and they are always shown separately from external agents. Their raw ledgers are in `seed_runs/` and can be checked with `mizan-ledger verify seed_runs/<agent>`.
|
|
102
|
+
|
|
103
|
+
## Security notes
|
|
104
|
+
|
|
105
|
+
Keep `private/` out of version control and backups you share. The salt is what stops guessing of low-entropy declarations.
|
|
106
|
+
|
|
107
|
+
## License
|
|
108
|
+
|
|
109
|
+
MIT, see `LICENSE`. Copyright (c) 2026 QANTS.
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
# mizan-ledger
|
|
2
|
+
|
|
3
|
+
Declare how confident your agent is **before** a task, commit a salted hash of that declaration, then score honesty against the **real outcome** (your own tests, evals or checks). Local-first, hash-only, verifiable offline.
|
|
4
|
+
|
|
5
|
+
> Status: **v0.1.0, early alpha** by QANTS. On PyPI (not on npm). MIT licensed (see `LICENSE`). No network access, no telemetry, no dependencies in the core library.
|
|
6
|
+
|
|
7
|
+
## Why
|
|
8
|
+
|
|
9
|
+
Agents often report success with the same unearned confidence whether they are right or wrong. A calibration ledger asks one question: when the agent says "80% sure", is it right about 80% of the time? Brier score and ECE answer that. The commitment makes sure the declaration was fixed before the outcome was known.
|
|
10
|
+
|
|
11
|
+
## Install
|
|
12
|
+
|
|
13
|
+
```
|
|
14
|
+
pip install mizan-ledger
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
Or from a local checkout:
|
|
18
|
+
|
|
19
|
+
```bash
|
|
20
|
+
pip install -e . # Python >= 3.9, pytest only for development
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
## 30-second example
|
|
24
|
+
|
|
25
|
+
```python
|
|
26
|
+
from mizan_ledger import Ledger
|
|
27
|
+
|
|
28
|
+
led = Ledger("ledger/", private_path="private/") # public dir is safe to share; private dir is not
|
|
29
|
+
led.declare("fix-bug-421", 0.80, agent_id="my-agent", version="1.2.0") # hash-only receipt, timestamped
|
|
30
|
+
# ... run the task, run the tests ...
|
|
31
|
+
led.resolve("fix-bug-421", outcome=True) # reveals the opening, scores it
|
|
32
|
+
|
|
33
|
+
print(led.summary("my-agent")) # n, Brier + 95% interval, ECE, log loss, accuracy (Wilson), coverage
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
mizan-ledger verify ledger/ # offline check of hash chains, openings and scores
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
## What gets recorded
|
|
41
|
+
|
|
42
|
+
| File | Visibility | Contents |
|
|
43
|
+
|---|---|---|
|
|
44
|
+
| `receipts.jsonl` | public | `agent_id`, `agent_version`, `commitment` (SHA-256), `declared_at`, hash-chain link |
|
|
45
|
+
| `resolutions.jsonl` | public after resolution | outcome, opening (`task_id`, `p`, `salt`), squared error, log loss, hash-chain link |
|
|
46
|
+
| `private/openings.jsonl` | **private** (mode 600) | the same openings, kept until the task is resolved |
|
|
47
|
+
|
|
48
|
+
Commitment = `SHA-256(canonical_json({agent_id, agent_version, p, salt, schema, task_id}))` with `p` as a fixed 6-decimal string, a fresh 128-bit random salt per declaration, sorted keys and no whitespace. Shared test vectors (`vectors/commitments.json`) are checked by both the Python and the JavaScript implementation.
|
|
49
|
+
|
|
50
|
+
## Guarantees and honest limits
|
|
51
|
+
|
|
52
|
+
- A declaration cannot be replaced or re-resolved. Deleting or editing a record breaks the hash chain and `verify` fails.
|
|
53
|
+
- **Coverage is reported.** Declared, resolved and pending are all counted, so you cannot quietly drop the bad ones. Anything unresolved shows as `pending`.
|
|
54
|
+
- **Timestamps are your local clock.** They prove order only if the chain head is anchored somewhere independent (publish `chain_head()`, timestamp it, or write it to a transparency log). Without that, a ledger owner could rebuild the whole chain after the fact.
|
|
55
|
+
- The library cannot tell whether the outcome you pass to `resolve` is honest. Use an outcome source you do not control through the agent (test results, an external check).
|
|
56
|
+
- Small samples give wide intervals; ECE is biased upward for small n. The summary prints the interval and `n`; read them.
|
|
57
|
+
- Scores are about calibration, not about whether the agent is good, safe or secure.
|
|
58
|
+
|
|
59
|
+
## Scores (`mizan_ledger.metrics`)
|
|
60
|
+
|
|
61
|
+
Brier (mean squared error), log loss (p clipped to [0.01, 0.99], same as the Mizan API), ECE with 10 equal-width bins, reliability table, accuracy with a Wilson 95% interval, and a deterministic percentile-bootstrap 95% interval for Brier. The always-0.5 baseline Brier is 0.25.
|
|
62
|
+
|
|
63
|
+
## Adapters
|
|
64
|
+
|
|
65
|
+
| Adapter | Status |
|
|
66
|
+
|---|---|
|
|
67
|
+
| **pytest plugin** (`mizan_ledger.pytest_plugin`): `@pytest.mark.confidence(0.9)` declares before the test body, pass/fail resolves it. Run with `pytest --mizan-ledger-dir .mizan --mizan-agent-id my-agent`. | Tested here (pytester). |
|
|
68
|
+
| **Generic trace evaluator** (`mizan_ledger.adapters.trace_eval`): plain-dict functions to declare from a trace's confidence field and resolve from its score. | Unit-tested with synthetic dicts only. **Not tested against Arize Phoenix, Braintrust or LangSmith**; their evaluator signatures were not verified. Treat as a stub. |
|
|
69
|
+
| **JavaScript shim** (`js/mizan-ledger.mjs`): commitment, declare, verify opening. Node >= 18, `node:crypto` only. | Tested against the shared vectors (`node --test js/test.mjs`). It does not implement the file ledger. |
|
|
70
|
+
|
|
71
|
+
No CI workflow file is included in this repository.
|
|
72
|
+
|
|
73
|
+
## Tests
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
python3 -m pytest -q # Python suite
|
|
77
|
+
node --test js/test.mjs # JS shim vs shared vectors
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
## Seed agents
|
|
81
|
+
|
|
82
|
+
`seeds/` holds three demonstration agents operated by QANTS (ids start with `qants-`). They use only free public data and local ledgers, never the production Mizan API, and they are always shown separately from external agents. Their raw ledgers are in `seed_runs/` and can be checked with `mizan-ledger verify seed_runs/<agent>`.
|
|
83
|
+
|
|
84
|
+
## Security notes
|
|
85
|
+
|
|
86
|
+
Keep `private/` out of version control and backups you share. The salt is what stops guessing of low-entropy declarations.
|
|
87
|
+
|
|
88
|
+
## License
|
|
89
|
+
|
|
90
|
+
MIT, see `LICENSE`. Copyright (c) 2026 QANTS.
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
// Tiny JS shim: commitment hashing + offline opening verification. Uses node:crypto only.
|
|
2
|
+
// Mirrors the Python implementation byte for byte (shared vectors in ../vectors/commitments.json).
|
|
3
|
+
import { createHash, randomBytes } from "node:crypto";
|
|
4
|
+
|
|
5
|
+
export const SCHEMA = "mizan-ledger/1";
|
|
6
|
+
|
|
7
|
+
export function fmtP(p) {
|
|
8
|
+
const x = Number(p);
|
|
9
|
+
if (!Number.isFinite(x) || x < 0 || x > 1) throw new Error("p_success must be a number in [0, 1]");
|
|
10
|
+
return x.toFixed(6);
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
// Strings only, flat object: sorted keys, no whitespace.
|
|
14
|
+
export function canonicalJson(obj) {
|
|
15
|
+
const out = {};
|
|
16
|
+
for (const k of Object.keys(obj).sort()) out[k] = obj[k];
|
|
17
|
+
return JSON.stringify(out);
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export const sha256Hex = (s) => createHash("sha256").update(s, "utf8").digest("hex");
|
|
21
|
+
|
|
22
|
+
export function commitmentHash({ taskId, p, agentId, agentVersion = "unspecified", salt }) {
|
|
23
|
+
return sha256Hex(canonicalJson({
|
|
24
|
+
agent_id: agentId, agent_version: agentVersion, p: fmtP(p), salt, schema: SCHEMA, task_id: taskId,
|
|
25
|
+
}));
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
// Create a commitment. Keep the returned `opening` private until the outcome is known.
|
|
29
|
+
export function declare({ taskId, p, agentId, agentVersion = "unspecified" }) {
|
|
30
|
+
const salt = randomBytes(16).toString("hex");
|
|
31
|
+
const commitment = commitmentHash({ taskId, p, agentId, agentVersion, salt });
|
|
32
|
+
return {
|
|
33
|
+
receipt: { schema: SCHEMA, agent_id: agentId, agent_version: agentVersion, commitment, declared_at: new Date().toISOString() },
|
|
34
|
+
opening: { task_id: taskId, p: fmtP(p), salt },
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export function verifyOpening(receipt, opening) {
|
|
39
|
+
try {
|
|
40
|
+
return commitmentHash({ taskId: opening.task_id, p: Number(opening.p), agentId: receipt.agent_id,
|
|
41
|
+
agentVersion: receipt.agent_version, salt: opening.salt }) === receipt.commitment;
|
|
42
|
+
} catch { return false; }
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export const squaredError = (p, outcome) => (Number(p) - (outcome ? 1 : 0)) ** 2;
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import test from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
import { readFileSync } from "node:fs";
|
|
4
|
+
import { commitmentHash, declare, verifyOpening, fmtP, squaredError } from "./mizan-ledger.mjs";
|
|
5
|
+
|
|
6
|
+
const vectors = JSON.parse(readFileSync(new URL("../vectors/commitments.json", import.meta.url), "utf8"));
|
|
7
|
+
|
|
8
|
+
test("shared vectors match the Python implementation", () => {
|
|
9
|
+
for (const v of vectors) {
|
|
10
|
+
assert.equal(fmtP(v.p), v.p_fmt);
|
|
11
|
+
assert.equal(commitmentHash({ taskId: v.task_id, p: v.p, agentId: v.agent_id, agentVersion: v.agent_version, salt: v.salt }), v.commitment);
|
|
12
|
+
}
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
test("declare then verify, and tampering is detected", () => {
|
|
16
|
+
const { receipt, opening } = declare({ taskId: "t1", p: 0.8, agentId: "a", agentVersion: "1" });
|
|
17
|
+
assert.ok(verifyOpening(receipt, opening));
|
|
18
|
+
assert.ok(!verifyOpening(receipt, { ...opening, p: "0.900000" }));
|
|
19
|
+
assert.ok(!verifyOpening(receipt, { ...opening, task_id: "t2" }));
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
test("p outside [0,1] is rejected", () => {
|
|
23
|
+
assert.throws(() => fmtP(1.2));
|
|
24
|
+
assert.throws(() => fmtP(NaN));
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
test("squared error", () => { assert.equal(squaredError(0.8, true).toFixed(6), "0.040000"); });
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "mizan-ledger"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Declare your agent's confidence before the task, commit a salted hash, score honesty against real outcomes. Local-first, hash-only."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
requires-python = ">=3.9"
|
|
13
|
+
authors = [{ name = "QANTS" }]
|
|
14
|
+
keywords = ["calibration", "agents", "brier", "commitment", "pytest-plugin"]
|
|
15
|
+
dependencies = []
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Programming Language :: Python :: 3",
|
|
18
|
+
"Framework :: Pytest",
|
|
19
|
+
"Development Status :: 3 - Alpha",
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
[project.urls]
|
|
23
|
+
Homepage = "https://qants-site.qants.workers.dev"
|
|
24
|
+
SeedAgents = "https://qants-site.qants.workers.dev/seed-agents"
|
|
25
|
+
|
|
26
|
+
[project.optional-dependencies]
|
|
27
|
+
dev = ["pytest>=7"]
|
|
28
|
+
|
|
29
|
+
[project.scripts]
|
|
30
|
+
mizan-ledger = "mizan_ledger.cli:main"
|
|
31
|
+
|
|
32
|
+
[project.entry-points.pytest11]
|
|
33
|
+
mizan_ledger = "mizan_ledger.pytest_plugin"
|
|
34
|
+
|
|
35
|
+
[tool.setuptools.packages.find]
|
|
36
|
+
where = ["src"]
|
|
37
|
+
|
|
38
|
+
[tool.pytest.ini_options]
|
|
39
|
+
testpaths = ["tests"]
|
|
40
|
+
pythonpath = ["src"]
|
|
41
|
+
addopts = "-p no:cacheprovider"
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""mizan-ledger: confidence commitments and honesty scoring. Hash-only, local-first, offline-verifiable."""
|
|
2
|
+
from .core import (
|
|
3
|
+
SCHEMA, Ledger, LedgerError, canonical_json, commitment_hash, fmt_p,
|
|
4
|
+
verify_opening, verify_chain, verify_ledger_dir,
|
|
5
|
+
)
|
|
6
|
+
from .metrics import brier, log_loss, ece, accuracy_wilson, brier_interval, summarize
|
|
7
|
+
|
|
8
|
+
__version__ = "0.1.0"
|
|
9
|
+
__all__ = [
|
|
10
|
+
"SCHEMA", "Ledger", "LedgerError", "canonical_json", "commitment_hash", "fmt_p",
|
|
11
|
+
"verify_opening", "verify_chain", "verify_ledger_dir",
|
|
12
|
+
"brier", "log_loss", "ece", "accuracy_wilson", "brier_interval", "summarize",
|
|
13
|
+
]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Adapters. See each module's docstring for what has and has not been tested."""
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""Generic trace-evaluator adapter (STUB, UNTESTED against real vendors).
|
|
2
|
+
|
|
3
|
+
Purpose: let an observability/eval tool (Arize Phoenix, Braintrust, LangSmith or similar) feed a confidence
|
|
4
|
+
ledger from traces it already holds. The adapter works on plain dicts only and imports no vendor SDK.
|
|
5
|
+
|
|
6
|
+
declare_from_trace(ledger, trace, agent_id, version) # at the start of a run: reads confidence
|
|
7
|
+
resolve_from_trace(ledger, trace, agent_id, version) # after evaluation: reads pass/score
|
|
8
|
+
|
|
9
|
+
Status: unit-tested with synthetic dict traces in this repository ONLY. It has NOT been run inside Phoenix,
|
|
10
|
+
Braintrust or LangSmith, and their evaluator signatures were not verified. A vendor wrapper is expected to be
|
|
11
|
+
a few lines that map their run/example objects to the dict shape below.
|
|
12
|
+
|
|
13
|
+
Expected dict shape (keys are configurable):
|
|
14
|
+
{"id": "run-123", "confidence": 0.8} # at declaration
|
|
15
|
+
{"id": "run-123", "score": 1.0} # at resolution; score >= threshold means success
|
|
16
|
+
"""
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
from typing import Callable, Optional
|
|
19
|
+
from ..core import Ledger, LedgerError
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def declare_from_trace(ledger: Ledger, trace: dict, agent_id: str, version: str = "unspecified",
|
|
23
|
+
id_key: str = "id", confidence_key: str = "confidence") -> dict:
|
|
24
|
+
if confidence_key not in trace or trace[confidence_key] is None:
|
|
25
|
+
raise LedgerError(f"trace has no {confidence_key!r}; nothing to declare")
|
|
26
|
+
return ledger.declare(str(trace[id_key]), float(trace[confidence_key]), agent_id, version)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def resolve_from_trace(ledger: Ledger, trace: dict, agent_id: str, version: str = "unspecified",
|
|
30
|
+
id_key: str = "id", score_key: str = "score", threshold: float = 0.5) -> dict:
|
|
31
|
+
if score_key not in trace or trace[score_key] is None:
|
|
32
|
+
raise LedgerError(f"trace has no {score_key!r}; leave it pending instead of guessing")
|
|
33
|
+
return ledger.resolve(str(trace[id_key]), float(trace[score_key]) >= threshold, agent_id, version)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def make_evaluator(ledger: Ledger, agent_id: str, version: str = "unspecified", threshold: float = 0.5) -> Callable[[dict], Optional[dict]]:
|
|
37
|
+
"""Return fn(trace) -> {'brier_term': ...} usable as a custom evaluator body once wrapped for a vendor."""
|
|
38
|
+
def evaluator(trace: dict):
|
|
39
|
+
rec = resolve_from_trace(ledger, trace, agent_id, version, threshold=threshold)
|
|
40
|
+
return {"key": "mizan_squared_error", "score": rec["score"]["squared_error"]}
|
|
41
|
+
return evaluator
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""Command line: mizan-ledger verify <dir> | summary <dir> [--agent ID]"""
|
|
2
|
+
import argparse, json, sys
|
|
3
|
+
from .core import Ledger, verify_ledger_dir
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def main(argv=None) -> int:
|
|
7
|
+
ap = argparse.ArgumentParser(prog="mizan-ledger", description="Offline verification and scoring of a local confidence ledger")
|
|
8
|
+
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
9
|
+
v = sub.add_parser("verify", help="verify hash chains, openings and scores (no network, no private files)"); v.add_argument("dir")
|
|
10
|
+
s = sub.add_parser("summary", help="print scores for a ledger directory"); s.add_argument("dir"); s.add_argument("--agent"); s.add_argument("--version")
|
|
11
|
+
a = ap.parse_args(argv)
|
|
12
|
+
if a.cmd == "verify":
|
|
13
|
+
rep = verify_ledger_dir(a.dir)
|
|
14
|
+
print(json.dumps(rep, indent=2, sort_keys=True))
|
|
15
|
+
return 0 if rep["ok"] else 1
|
|
16
|
+
led = Ledger(a.dir, private_path=a.dir + "/.noprivate")
|
|
17
|
+
print(json.dumps(led.summary(a.agent, a.version), indent=2, sort_keys=True))
|
|
18
|
+
return 0
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
if __name__ == "__main__":
|
|
22
|
+
sys.exit(main())
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
"""Core ledger: salted commitments, hash chain, resolution, offline verification.
|
|
2
|
+
|
|
3
|
+
Design rules
|
|
4
|
+
* The public receipt contains only a salted hash, the agent id/version and a timestamp.
|
|
5
|
+
* The opening (task_id, p, salt) stays in a private file until the task is resolved.
|
|
6
|
+
* Timestamps come from the local clock. They are self-reported until the chain head
|
|
7
|
+
hash is anchored somewhere independent (see README, "Anchoring").
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
import datetime as _dt
|
|
11
|
+
import hashlib, json, math, os, secrets
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Callable, Optional
|
|
14
|
+
|
|
15
|
+
from . import metrics
|
|
16
|
+
|
|
17
|
+
SCHEMA = "mizan-ledger/1"
|
|
18
|
+
GENESIS = "0" * 64
|
|
19
|
+
LOG_LOSS_CLIP = metrics.LOG_LOSS_CLIP
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class LedgerError(Exception):
|
|
23
|
+
pass
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def canonical_json(obj) -> str:
|
|
27
|
+
"""Sorted keys, no whitespace, UTF-8 characters kept (matches JSON.stringify on sorted keys)."""
|
|
28
|
+
return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def sha256_hex(s: str) -> str:
|
|
32
|
+
return hashlib.sha256(s.encode("utf-8")).hexdigest()
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def fmt_p(p: float) -> str:
|
|
36
|
+
"""Fixed 6-decimal string so every language serialises the same probability identically."""
|
|
37
|
+
p = float(p)
|
|
38
|
+
if not (0.0 <= p <= 1.0) or math.isnan(p):
|
|
39
|
+
raise LedgerError("p_success must be a number in [0, 1]")
|
|
40
|
+
return f"{p:.6f}"
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def commitment_hash(task_id: str, p: float, agent_id: str, version: str, salt: str) -> str:
|
|
44
|
+
return sha256_hex(canonical_json({
|
|
45
|
+
"agent_id": agent_id, "agent_version": version, "p": fmt_p(p),
|
|
46
|
+
"salt": salt, "schema": SCHEMA, "task_id": task_id,
|
|
47
|
+
}))
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _now() -> str:
|
|
51
|
+
return _dt.datetime.now(_dt.timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z")
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _parse(ts: str) -> _dt.datetime:
|
|
55
|
+
return _dt.datetime.fromisoformat(ts.replace("Z", "+00:00"))
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _chain_hash(rec: dict, key: str) -> str:
|
|
59
|
+
body = {k: v for k, v in rec.items() if k != key}
|
|
60
|
+
return sha256_hex(canonical_json(body))
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _read_jsonl(path: Path) -> list:
|
|
64
|
+
if not path.exists():
|
|
65
|
+
return []
|
|
66
|
+
out = []
|
|
67
|
+
with path.open("r", encoding="utf-8") as f:
|
|
68
|
+
for i, line in enumerate(f, 1):
|
|
69
|
+
line = line.strip()
|
|
70
|
+
if line:
|
|
71
|
+
try:
|
|
72
|
+
out.append(json.loads(line))
|
|
73
|
+
except json.JSONDecodeError as e:
|
|
74
|
+
raise LedgerError(f"{path.name}: line {i} is not valid JSON ({e})")
|
|
75
|
+
return out
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _append_jsonl(path: Path, rec: dict, private: bool = False) -> None:
|
|
79
|
+
new = not path.exists()
|
|
80
|
+
with path.open("a", encoding="utf-8") as f:
|
|
81
|
+
f.write(canonical_json(rec) + "\n")
|
|
82
|
+
f.flush(); os.fsync(f.fileno())
|
|
83
|
+
if new and private:
|
|
84
|
+
os.chmod(path, 0o600)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
class Ledger:
|
|
88
|
+
"""Append-only local ledger.
|
|
89
|
+
|
|
90
|
+
public dir: receipts.jsonl, resolutions.jsonl (safe to share; receipts are hash-only)
|
|
91
|
+
private dir: openings.jsonl (task_id, p, salt; keep secret until resolved)
|
|
92
|
+
"""
|
|
93
|
+
|
|
94
|
+
def __init__(self, path, private_path=None, clock: Optional[Callable[[], str]] = None):
|
|
95
|
+
self.dir = Path(path)
|
|
96
|
+
self.private_dir = Path(private_path) if private_path else self.dir / "private"
|
|
97
|
+
self.dir.mkdir(parents=True, exist_ok=True)
|
|
98
|
+
self._clock = clock or _now
|
|
99
|
+
self.receipts_path = self.dir / "receipts.jsonl"
|
|
100
|
+
self.resolutions_path = self.dir / "resolutions.jsonl"
|
|
101
|
+
self.openings_path = self.private_dir / "openings.jsonl"
|
|
102
|
+
|
|
103
|
+
# -- reading ---------------------------------------------------------------
|
|
104
|
+
def receipts(self) -> list: return _read_jsonl(self.receipts_path)
|
|
105
|
+
def resolutions(self) -> list: return _read_jsonl(self.resolutions_path)
|
|
106
|
+
def _openings(self) -> list: return _read_jsonl(self.openings_path)
|
|
107
|
+
|
|
108
|
+
def _find_opening(self, task_id, agent_id, version):
|
|
109
|
+
for o in self._openings():
|
|
110
|
+
if o["task_id"] == task_id and o["agent_id"] == agent_id and o["agent_version"] == version:
|
|
111
|
+
return o
|
|
112
|
+
return None
|
|
113
|
+
|
|
114
|
+
# -- declare ---------------------------------------------------------------
|
|
115
|
+
def declare(self, task_id: str, p_success: float, agent_id: str, version: str = "unspecified") -> dict:
|
|
116
|
+
"""Commit to a probability of success BEFORE the outcome is known. Returns the public receipt."""
|
|
117
|
+
if not task_id or not agent_id:
|
|
118
|
+
raise LedgerError("task_id and agent_id are required")
|
|
119
|
+
p = float(p_success)
|
|
120
|
+
fmt_p(p) # validates
|
|
121
|
+
if self._find_opening(task_id, agent_id, version):
|
|
122
|
+
raise LedgerError(f"task {task_id!r} was already declared for {agent_id}@{version}; a declaration cannot be replaced")
|
|
123
|
+
self.private_dir.mkdir(parents=True, exist_ok=True)
|
|
124
|
+
try: os.chmod(self.private_dir, 0o700)
|
|
125
|
+
except OSError: pass
|
|
126
|
+
salt = secrets.token_hex(16)
|
|
127
|
+
commitment = commitment_hash(task_id, p, agent_id, version, salt)
|
|
128
|
+
recs = self.receipts()
|
|
129
|
+
prev = recs[-1]["receipt_hash"] if recs else GENESIS
|
|
130
|
+
rec = {"schema": SCHEMA, "seq": len(recs) + 1, "agent_id": agent_id, "agent_version": version,
|
|
131
|
+
"commitment": commitment, "declared_at": self._clock(), "prev": prev}
|
|
132
|
+
rec["receipt_hash"] = _chain_hash(rec, "receipt_hash")
|
|
133
|
+
# private opening first: if the process dies between the two writes we never publish a hash we cannot open
|
|
134
|
+
_append_jsonl(self.openings_path, {"agent_id": agent_id, "agent_version": version, "task_id": task_id,
|
|
135
|
+
"p": fmt_p(p), "salt": salt, "commitment": commitment}, private=True)
|
|
136
|
+
_append_jsonl(self.receipts_path, rec)
|
|
137
|
+
return rec
|
|
138
|
+
|
|
139
|
+
# -- resolve ---------------------------------------------------------------
|
|
140
|
+
def resolve(self, task_id: str, outcome: bool, agent_id: Optional[str] = None, version: Optional[str] = None) -> dict:
|
|
141
|
+
"""Record the real outcome (True = the task succeeded). Reveals the opening so anyone can verify it."""
|
|
142
|
+
cands = [o for o in self._openings() if o["task_id"] == task_id
|
|
143
|
+
and (agent_id is None or o["agent_id"] == agent_id)
|
|
144
|
+
and (version is None or o["agent_version"] == version)]
|
|
145
|
+
if not cands:
|
|
146
|
+
raise LedgerError(f"no declaration found for task {task_id!r}")
|
|
147
|
+
if len(cands) > 1:
|
|
148
|
+
raise LedgerError("task_id is ambiguous; pass agent_id and version")
|
|
149
|
+
o = cands[0]
|
|
150
|
+
res = self.resolutions()
|
|
151
|
+
if any(r["commitment"] == o["commitment"] for r in res):
|
|
152
|
+
raise LedgerError(f"task {task_id!r} is already resolved; outcomes cannot be changed")
|
|
153
|
+
rcpt = next(r for r in self.receipts() if r["commitment"] == o["commitment"])
|
|
154
|
+
now = self._clock()
|
|
155
|
+
if _parse(now) < _parse(rcpt["declared_at"]):
|
|
156
|
+
raise LedgerError("resolution time is earlier than declaration time (clock moved backwards)")
|
|
157
|
+
y = 1 if outcome else 0
|
|
158
|
+
p = float(o["p"])
|
|
159
|
+
q = min(LOG_LOSS_CLIP[1], max(LOG_LOSS_CLIP[0], p))
|
|
160
|
+
rec = {"schema": SCHEMA, "seq": len(res) + 1, "commitment": o["commitment"],
|
|
161
|
+
"agent_id": o["agent_id"], "agent_version": o["agent_version"],
|
|
162
|
+
"outcome": y, "resolved_at": now,
|
|
163
|
+
"opening": {"task_id": o["task_id"], "p": o["p"], "salt": o["salt"]},
|
|
164
|
+
"score": {"squared_error": round((p - y) ** 2, 12),
|
|
165
|
+
"log_loss": round(-(math.log(q) if y else math.log(1 - q)), 12)},
|
|
166
|
+
"prev": res[-1]["record_hash"] if res else GENESIS}
|
|
167
|
+
rec["record_hash"] = _chain_hash(rec, "record_hash")
|
|
168
|
+
_append_jsonl(self.resolutions_path, rec)
|
|
169
|
+
return rec
|
|
170
|
+
|
|
171
|
+
# -- scoring ---------------------------------------------------------------
|
|
172
|
+
def pairs(self, agent_id=None, version=None) -> list:
|
|
173
|
+
return [(float(r["opening"]["p"]), r["outcome"]) for r in self.resolutions()
|
|
174
|
+
if (agent_id is None or r["agent_id"] == agent_id) and (version is None or r["agent_version"] == version)]
|
|
175
|
+
|
|
176
|
+
def summary(self, agent_id=None, version=None) -> dict:
|
|
177
|
+
rcpts = [r for r in self.receipts() if (agent_id is None or r["agent_id"] == agent_id)
|
|
178
|
+
and (version is None or r["agent_version"] == version)]
|
|
179
|
+
pr = self.pairs(agent_id, version)
|
|
180
|
+
out = metrics.summarize(pr) if pr else {"n": 0}
|
|
181
|
+
out.update({"declared": len(rcpts), "resolved": len(pr), "pending": len(rcpts) - len(pr),
|
|
182
|
+
"coverage": (len(pr) / len(rcpts)) if rcpts else None})
|
|
183
|
+
return out
|
|
184
|
+
|
|
185
|
+
def chain_head(self) -> dict:
|
|
186
|
+
r = self.receipts(); s = self.resolutions()
|
|
187
|
+
return {"receipts": len(r), "receipt_head": r[-1]["receipt_hash"] if r else GENESIS,
|
|
188
|
+
"resolutions": len(s), "resolution_head": s[-1]["record_hash"] if s else GENESIS}
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
# ---------------------------------------------------------------------------
|
|
192
|
+
# Offline verification (needs no network and no private files)
|
|
193
|
+
# ---------------------------------------------------------------------------
|
|
194
|
+
def verify_opening(receipt: dict, opening: dict) -> bool:
|
|
195
|
+
"""Does (task_id, p, salt) hash to the receipt's commitment?"""
|
|
196
|
+
try:
|
|
197
|
+
return commitment_hash(opening["task_id"], float(opening["p"]), receipt["agent_id"],
|
|
198
|
+
receipt["agent_version"], opening["salt"]) == receipt["commitment"]
|
|
199
|
+
except (KeyError, ValueError, LedgerError):
|
|
200
|
+
return False
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def verify_chain(records: list, hash_key: str) -> list:
|
|
204
|
+
errs, prev = [], GENESIS
|
|
205
|
+
for i, r in enumerate(records, 1):
|
|
206
|
+
if r.get("seq") != i: errs.append(f"{hash_key}: seq {r.get('seq')} at position {i}")
|
|
207
|
+
if r.get("prev") != prev: errs.append(f"{hash_key}: broken link at seq {i}")
|
|
208
|
+
if r.get(hash_key) != _chain_hash(r, hash_key): errs.append(f"{hash_key}: hash mismatch at seq {i}")
|
|
209
|
+
prev = r.get(hash_key)
|
|
210
|
+
return errs
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def verify_ledger_dir(path) -> dict:
|
|
214
|
+
"""Verify receipts.jsonl and resolutions.jsonl in a directory. Returns {ok, errors, summary}."""
|
|
215
|
+
d = Path(path)
|
|
216
|
+
errs = []
|
|
217
|
+
try:
|
|
218
|
+
receipts = _read_jsonl(d / "receipts.jsonl")
|
|
219
|
+
resolutions = _read_jsonl(d / "resolutions.jsonl")
|
|
220
|
+
except LedgerError as e:
|
|
221
|
+
return {"ok": False, "errors": [str(e)], "summary": None}
|
|
222
|
+
errs += verify_chain(receipts, "receipt_hash")
|
|
223
|
+
errs += verify_chain(resolutions, "record_hash")
|
|
224
|
+
by_c = {r["commitment"]: r for r in receipts}
|
|
225
|
+
seen = set()
|
|
226
|
+
for r in resolutions:
|
|
227
|
+
rc = by_c.get(r.get("commitment"))
|
|
228
|
+
if not rc: errs.append(f"resolution seq {r.get('seq')}: no matching receipt"); continue
|
|
229
|
+
if r["commitment"] in seen: errs.append(f"resolution seq {r['seq']}: duplicate resolution")
|
|
230
|
+
seen.add(r["commitment"])
|
|
231
|
+
if rc["agent_id"] != r.get("agent_id") or rc["agent_version"] != r.get("agent_version"):
|
|
232
|
+
errs.append(f"resolution seq {r['seq']}: agent/version differs from receipt")
|
|
233
|
+
if not verify_opening(rc, r.get("opening", {})): errs.append(f"resolution seq {r['seq']}: opening does not match commitment")
|
|
234
|
+
try:
|
|
235
|
+
if _parse(r["resolved_at"]) < _parse(rc["declared_at"]): errs.append(f"resolution seq {r['seq']}: resolved before declared")
|
|
236
|
+
except Exception: errs.append(f"resolution seq {r['seq']}: bad timestamp")
|
|
237
|
+
try:
|
|
238
|
+
p = float(r["opening"]["p"]); y = r["outcome"]
|
|
239
|
+
if y not in (0, 1): errs.append(f"resolution seq {r['seq']}: outcome must be 0 or 1")
|
|
240
|
+
elif abs(r["score"]["squared_error"] - (p - y) ** 2) > 1e-9: errs.append(f"resolution seq {r['seq']}: score does not match p and outcome")
|
|
241
|
+
except Exception: errs.append(f"resolution seq {r['seq']}: malformed record")
|
|
242
|
+
pairs = [(float(r["opening"]["p"]), r["outcome"]) for r in resolutions if r.get("outcome") in (0, 1)]
|
|
243
|
+
summ = metrics.summarize(pairs) if pairs else {"n": 0}
|
|
244
|
+
summ.update({"declared": len(receipts), "resolved": len(resolutions), "pending": len(receipts) - len(resolutions)})
|
|
245
|
+
return {"ok": not errs, "errors": errs, "summary": summ,
|
|
246
|
+
"chain_heads": {"receipt_head": receipts[-1]["receipt_hash"] if receipts else GENESIS,
|
|
247
|
+
"resolution_head": resolutions[-1]["record_hash"] if resolutions else GENESIS}}
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
"""Scoring helpers. Pure functions on lists of (p, outcome) with outcome in {0, 1}.
|
|
2
|
+
|
|
3
|
+
Conventions match the Mizan API: log loss clips p to [0.01, 0.99], ECE uses 10 equal-width bins.
|
|
4
|
+
"""
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
import math, random
|
|
7
|
+
from typing import Iterable, Sequence
|
|
8
|
+
|
|
9
|
+
LOG_LOSS_CLIP = (0.01, 0.99)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _pairs(pairs: Iterable) -> list:
|
|
13
|
+
out = []
|
|
14
|
+
for p, y in pairs:
|
|
15
|
+
p = float(p); y = 1 if y else 0
|
|
16
|
+
if not (0.0 <= p <= 1.0):
|
|
17
|
+
raise ValueError("p must be in [0, 1]")
|
|
18
|
+
out.append((p, y))
|
|
19
|
+
return out
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def brier(pairs) -> float:
|
|
23
|
+
xs = _pairs(pairs)
|
|
24
|
+
if not xs: raise ValueError("no data")
|
|
25
|
+
return sum((p - y) ** 2 for p, y in xs) / len(xs)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def log_loss(pairs) -> float:
|
|
29
|
+
xs = _pairs(pairs)
|
|
30
|
+
if not xs: raise ValueError("no data")
|
|
31
|
+
lo, hi = LOG_LOSS_CLIP
|
|
32
|
+
tot = 0.0
|
|
33
|
+
for p, y in xs:
|
|
34
|
+
q = min(hi, max(lo, p))
|
|
35
|
+
tot += -(math.log(q) if y else math.log(1 - q))
|
|
36
|
+
return tot / len(xs)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def ece(pairs, bins: int = 10) -> float:
|
|
40
|
+
"""Expected calibration error with equal-width bins; p == 1.0 falls in the last bin."""
|
|
41
|
+
xs = _pairs(pairs)
|
|
42
|
+
if not xs: raise ValueError("no data")
|
|
43
|
+
buckets = [[] for _ in range(bins)]
|
|
44
|
+
for p, y in xs:
|
|
45
|
+
buckets[min(bins - 1, int(p * bins))].append((p, y))
|
|
46
|
+
n = len(xs)
|
|
47
|
+
return sum(len(b) / n * abs(sum(p for p, _ in b) / len(b) - sum(y for _, y in b) / len(b)) for b in buckets if b)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def reliability_table(pairs, bins: int = 10) -> list:
|
|
51
|
+
xs = _pairs(pairs)
|
|
52
|
+
buckets = [[] for _ in range(bins)]
|
|
53
|
+
for p, y in xs:
|
|
54
|
+
buckets[min(bins - 1, int(p * bins))].append((p, y))
|
|
55
|
+
rows = []
|
|
56
|
+
for i, b in enumerate(buckets):
|
|
57
|
+
if b:
|
|
58
|
+
rows.append({"bin": f"{i/bins:.1f}-{(i+1)/bins:.1f}", "n": len(b),
|
|
59
|
+
"mean_p": round(sum(p for p, _ in b) / len(b), 4),
|
|
60
|
+
"freq": round(sum(y for _, y in b) / len(b), 4)})
|
|
61
|
+
return rows
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def accuracy_wilson(pairs, z: float = 1.959964) -> dict:
|
|
65
|
+
"""Accuracy of thresholded predictions (p >= 0.5 -> 1) with a Wilson 95% interval."""
|
|
66
|
+
xs = _pairs(pairs)
|
|
67
|
+
n = len(xs)
|
|
68
|
+
if not n: raise ValueError("no data")
|
|
69
|
+
k = sum(1 for p, y in xs if (p >= 0.5) == bool(y))
|
|
70
|
+
ph = k / n
|
|
71
|
+
d = 1 + z * z / n
|
|
72
|
+
c = (ph + z * z / (2 * n)) / d
|
|
73
|
+
h = z * math.sqrt(ph * (1 - ph) / n + z * z / (4 * n * n)) / d
|
|
74
|
+
return {"accuracy": ph, "ci95": [max(0.0, c - h), min(1.0, c + h)], "n": n}
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def brier_interval(pairs, reps: int = 2000, seed: int = 20261116) -> dict:
|
|
78
|
+
"""Percentile bootstrap 95% interval for the mean Brier score (deterministic seed)."""
|
|
79
|
+
xs = _pairs(pairs)
|
|
80
|
+
n = len(xs)
|
|
81
|
+
if not n: raise ValueError("no data")
|
|
82
|
+
sq = [(p - y) ** 2 for p, y in xs]
|
|
83
|
+
mean = sum(sq) / n
|
|
84
|
+
if n < 2:
|
|
85
|
+
return {"brier": mean, "ci95": [None, None], "method": "bootstrap", "n": n}
|
|
86
|
+
rng = random.Random(seed)
|
|
87
|
+
means = sorted(sum(sq[rng.randrange(n)] for _ in range(n)) / n for _ in range(reps))
|
|
88
|
+
return {"brier": mean, "ci95": [means[int(0.025 * reps)], means[int(0.975 * reps) - 1]], "method": f"percentile bootstrap, {reps} reps, seed {seed}", "n": n}
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def summarize(pairs) -> dict:
|
|
92
|
+
xs = _pairs(pairs)
|
|
93
|
+
n = len(xs)
|
|
94
|
+
if not n:
|
|
95
|
+
return {"n": 0}
|
|
96
|
+
base = sum(y for _, y in xs) / n
|
|
97
|
+
bi = brier_interval(xs)
|
|
98
|
+
return {
|
|
99
|
+
"n": n,
|
|
100
|
+
"base_rate": base,
|
|
101
|
+
"brier": bi["brier"],
|
|
102
|
+
"brier_ci95": bi["ci95"],
|
|
103
|
+
"brier_ci_method": bi["method"],
|
|
104
|
+
"brier_always_0.5": 0.25,
|
|
105
|
+
"brier_base_rate_hindsight": base * (1 - base), # in-sample best constant; a ceiling for "no skill" claims
|
|
106
|
+
"log_loss": log_loss(xs),
|
|
107
|
+
"ece": ece(xs),
|
|
108
|
+
"accuracy": accuracy_wilson(xs),
|
|
109
|
+
"reliability": reliability_table(xs),
|
|
110
|
+
"note": "With small n the intervals are wide; ECE is biased upward for small n.",
|
|
111
|
+
}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""pytest plugin: declare confidence before a test runs, resolve with its pass/fail.
|
|
2
|
+
|
|
3
|
+
@pytest.mark.confidence(0.9)
|
|
4
|
+
def test_parser(): ...
|
|
5
|
+
|
|
6
|
+
pytest --mizan-ledger-dir .mizan --mizan-agent-id my-agent --mizan-agent-version 1.2.0
|
|
7
|
+
|
|
8
|
+
Only tests carrying the marker are recorded. The declaration is committed after skip marks and fixture setup, right
|
|
9
|
+
before the test body. A test that is declared but then xfails stays "pending" (counted in coverage, never silently
|
|
10
|
+
dropped). Skipped tests and fixture errors are never declared.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
import datetime as _dt
|
|
14
|
+
import pytest
|
|
15
|
+
from .core import Ledger, LedgerError
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def pytest_addoption(parser):
|
|
19
|
+
g = parser.getgroup("mizan-ledger")
|
|
20
|
+
g.addoption("--mizan-ledger-dir", default=None, help="enable mizan-ledger and store the ledger in this directory")
|
|
21
|
+
g.addoption("--mizan-private-dir", default=None, help="directory for private openings (default <ledger>/private)")
|
|
22
|
+
g.addoption("--mizan-agent-id", default="pytest-agent")
|
|
23
|
+
g.addoption("--mizan-agent-version", default="unspecified")
|
|
24
|
+
g.addoption("--mizan-run-id", default=None, help="suffix that makes task ids unique per run (default: UTC start time)")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def pytest_configure(config):
|
|
28
|
+
config.addinivalue_line("markers", "confidence(p): declared probability (0..1) that this test passes, committed before it runs")
|
|
29
|
+
d = config.getoption("--mizan-ledger-dir")
|
|
30
|
+
if d:
|
|
31
|
+
config._mizan = {
|
|
32
|
+
"ledger": Ledger(d, private_path=config.getoption("--mizan-private-dir")),
|
|
33
|
+
"agent": config.getoption("--mizan-agent-id"),
|
|
34
|
+
"version": config.getoption("--mizan-agent-version"),
|
|
35
|
+
"run": config.getoption("--mizan-run-id") or _dt.datetime.now(_dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ"),
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _task_id(cfg, item):
|
|
40
|
+
return f"{item.nodeid}@{cfg['run']}"
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@pytest.hookimpl(trylast=True) # after skip marks and fixture setup: declared right before the test body
|
|
44
|
+
def pytest_runtest_setup(item):
|
|
45
|
+
cfg = getattr(item.config, "_mizan", None)
|
|
46
|
+
m = item.get_closest_marker("confidence")
|
|
47
|
+
if not cfg or m is None:
|
|
48
|
+
return
|
|
49
|
+
if not m.args:
|
|
50
|
+
raise pytest.UsageError("@pytest.mark.confidence needs a probability, e.g. @pytest.mark.confidence(0.9)")
|
|
51
|
+
try:
|
|
52
|
+
rec = cfg["ledger"].declare(_task_id(cfg, item), float(m.args[0]), cfg["agent"], cfg["version"])
|
|
53
|
+
except LedgerError as e:
|
|
54
|
+
raise pytest.UsageError(str(e))
|
|
55
|
+
item._mizan_declared = rec["commitment"]
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@pytest.hookimpl(hookwrapper=True)
|
|
59
|
+
def pytest_runtest_makereport(item, call):
|
|
60
|
+
out = yield
|
|
61
|
+
rep = out.get_result()
|
|
62
|
+
cfg = getattr(item.config, "_mizan", None)
|
|
63
|
+
if not cfg or not hasattr(item, "_mizan_declared") or call.when != "call":
|
|
64
|
+
return
|
|
65
|
+
if hasattr(rep, "wasxfail") or rep.skipped:
|
|
66
|
+
return # not a real outcome: stays pending
|
|
67
|
+
cfg["ledger"].resolve(_task_id(cfg, item), rep.passed, cfg["agent"], cfg["version"])
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def pytest_terminal_summary(terminalreporter, config):
|
|
71
|
+
cfg = getattr(config, "_mizan", None)
|
|
72
|
+
if not cfg:
|
|
73
|
+
return
|
|
74
|
+
s = cfg["ledger"].summary(cfg["agent"], cfg["version"])
|
|
75
|
+
tr = terminalreporter
|
|
76
|
+
tr.section("mizan-ledger")
|
|
77
|
+
if s.get("resolved"):
|
|
78
|
+
tr.line(f"agent {cfg['agent']}@{cfg['version']}: declared {s['declared']}, resolved {s['resolved']}, pending {s['pending']}; "
|
|
79
|
+
f"Brier {s['brier']:.4f} (always-0.5 = 0.25), ECE {s['ece']:.4f}, n={s['n']}")
|
|
80
|
+
else:
|
|
81
|
+
tr.line("no resolved confidence declarations in this ledger yet")
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: mizan-ledger
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Declare your agent's confidence before the task, commit a salted hash, score honesty against real outcomes. Local-first, hash-only.
|
|
5
|
+
Author: QANTS
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://qants-site.qants.workers.dev
|
|
8
|
+
Project-URL: SeedAgents, https://qants-site.qants.workers.dev/seed-agents
|
|
9
|
+
Keywords: calibration,agents,brier,commitment,pytest-plugin
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Framework :: Pytest
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Requires-Python: >=3.9
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
License-File: LICENSE
|
|
16
|
+
Provides-Extra: dev
|
|
17
|
+
Requires-Dist: pytest>=7; extra == "dev"
|
|
18
|
+
Dynamic: license-file
|
|
19
|
+
|
|
20
|
+
# mizan-ledger
|
|
21
|
+
|
|
22
|
+
Declare how confident your agent is **before** a task, commit a salted hash of that declaration, then score honesty against the **real outcome** (your own tests, evals or checks). Local-first, hash-only, verifiable offline.
|
|
23
|
+
|
|
24
|
+
> Status: **v0.1.0, early alpha** by QANTS. On PyPI (not on npm). MIT licensed (see `LICENSE`). No network access, no telemetry, no dependencies in the core library.
|
|
25
|
+
|
|
26
|
+
## Why
|
|
27
|
+
|
|
28
|
+
Agents often report success with the same unearned confidence whether they are right or wrong. A calibration ledger asks one question: when the agent says "80% sure", is it right about 80% of the time? Brier score and ECE answer that. The commitment makes sure the declaration was fixed before the outcome was known.
|
|
29
|
+
|
|
30
|
+
## Install
|
|
31
|
+
|
|
32
|
+
```
|
|
33
|
+
pip install mizan-ledger
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
Or from a local checkout:
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
pip install -e . # Python >= 3.9, pytest only for development
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## 30-second example
|
|
43
|
+
|
|
44
|
+
```python
|
|
45
|
+
from mizan_ledger import Ledger
|
|
46
|
+
|
|
47
|
+
led = Ledger("ledger/", private_path="private/") # public dir is safe to share; private dir is not
|
|
48
|
+
led.declare("fix-bug-421", 0.80, agent_id="my-agent", version="1.2.0") # hash-only receipt, timestamped
|
|
49
|
+
# ... run the task, run the tests ...
|
|
50
|
+
led.resolve("fix-bug-421", outcome=True) # reveals the opening, scores it
|
|
51
|
+
|
|
52
|
+
print(led.summary("my-agent")) # n, Brier + 95% interval, ECE, log loss, accuracy (Wilson), coverage
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
mizan-ledger verify ledger/ # offline check of hash chains, openings and scores
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
## What gets recorded
|
|
60
|
+
|
|
61
|
+
| File | Visibility | Contents |
|
|
62
|
+
|---|---|---|
|
|
63
|
+
| `receipts.jsonl` | public | `agent_id`, `agent_version`, `commitment` (SHA-256), `declared_at`, hash-chain link |
|
|
64
|
+
| `resolutions.jsonl` | public after resolution | outcome, opening (`task_id`, `p`, `salt`), squared error, log loss, hash-chain link |
|
|
65
|
+
| `private/openings.jsonl` | **private** (mode 600) | the same openings, kept until the task is resolved |
|
|
66
|
+
|
|
67
|
+
Commitment = `SHA-256(canonical_json({agent_id, agent_version, p, salt, schema, task_id}))` with `p` as a fixed 6-decimal string, a fresh 128-bit random salt per declaration, sorted keys and no whitespace. Shared test vectors (`vectors/commitments.json`) are checked by both the Python and the JavaScript implementation.
|
|
68
|
+
|
|
69
|
+
## Guarantees and honest limits
|
|
70
|
+
|
|
71
|
+
- A declaration cannot be replaced or re-resolved. Deleting or editing a record breaks the hash chain and `verify` fails.
|
|
72
|
+
- **Coverage is reported.** Declared, resolved and pending are all counted, so you cannot quietly drop the bad ones. Anything unresolved shows as `pending`.
|
|
73
|
+
- **Timestamps are your local clock.** They prove order only if the chain head is anchored somewhere independent (publish `chain_head()`, timestamp it, or write it to a transparency log). Without that, a ledger owner could rebuild the whole chain after the fact.
|
|
74
|
+
- The library cannot tell whether the outcome you pass to `resolve` is honest. Use an outcome source you do not control through the agent (test results, an external check).
|
|
75
|
+
- Small samples give wide intervals; ECE is biased upward for small n. The summary prints the interval and `n`; read them.
|
|
76
|
+
- Scores are about calibration, not about whether the agent is good, safe or secure.
|
|
77
|
+
|
|
78
|
+
## Scores (`mizan_ledger.metrics`)
|
|
79
|
+
|
|
80
|
+
Brier (mean squared error), log loss (p clipped to [0.01, 0.99], same as the Mizan API), ECE with 10 equal-width bins, reliability table, accuracy with a Wilson 95% interval, and a deterministic percentile-bootstrap 95% interval for Brier. The always-0.5 baseline Brier is 0.25.
|
|
81
|
+
|
|
82
|
+
## Adapters
|
|
83
|
+
|
|
84
|
+
| Adapter | Status |
|
|
85
|
+
|---|---|
|
|
86
|
+
| **pytest plugin** (`mizan_ledger.pytest_plugin`): `@pytest.mark.confidence(0.9)` declares before the test body, pass/fail resolves it. Run with `pytest --mizan-ledger-dir .mizan --mizan-agent-id my-agent`. | Tested here (pytester). |
|
|
87
|
+
| **Generic trace evaluator** (`mizan_ledger.adapters.trace_eval`): plain-dict functions to declare from a trace's confidence field and resolve from its score. | Unit-tested with synthetic dicts only. **Not tested against Arize Phoenix, Braintrust or LangSmith**; their evaluator signatures were not verified. Treat as a stub. |
|
|
88
|
+
| **JavaScript shim** (`js/mizan-ledger.mjs`): commitment, declare, verify opening. Node >= 18, `node:crypto` only. | Tested against the shared vectors (`node --test js/test.mjs`). It does not implement the file ledger. |
|
|
89
|
+
|
|
90
|
+
No CI workflow file is included in this repository.
|
|
91
|
+
|
|
92
|
+
## Tests
|
|
93
|
+
|
|
94
|
+
```bash
|
|
95
|
+
python3 -m pytest -q # Python suite
|
|
96
|
+
node --test js/test.mjs # JS shim vs shared vectors
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
## Seed agents
|
|
100
|
+
|
|
101
|
+
`seeds/` holds three demonstration agents operated by QANTS (ids start with `qants-`). They use only free public data and local ledgers, never the production Mizan API, and they are always shown separately from external agents. Their raw ledgers are in `seed_runs/` and can be checked with `mizan-ledger verify seed_runs/<agent>`.
|
|
102
|
+
|
|
103
|
+
## Security notes
|
|
104
|
+
|
|
105
|
+
Keep `private/` out of version control and backups you share. The salt is what stops guessing of low-entropy declarations.
|
|
106
|
+
|
|
107
|
+
## License
|
|
108
|
+
|
|
109
|
+
MIT, see `LICENSE`. Copyright (c) 2026 QANTS.
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
MANIFEST.in
|
|
3
|
+
README.md
|
|
4
|
+
pyproject.toml
|
|
5
|
+
js/mizan-ledger.mjs
|
|
6
|
+
js/test.mjs
|
|
7
|
+
src/mizan_ledger/__init__.py
|
|
8
|
+
src/mizan_ledger/cli.py
|
|
9
|
+
src/mizan_ledger/core.py
|
|
10
|
+
src/mizan_ledger/metrics.py
|
|
11
|
+
src/mizan_ledger/pytest_plugin.py
|
|
12
|
+
src/mizan_ledger.egg-info/PKG-INFO
|
|
13
|
+
src/mizan_ledger.egg-info/SOURCES.txt
|
|
14
|
+
src/mizan_ledger.egg-info/dependency_links.txt
|
|
15
|
+
src/mizan_ledger.egg-info/entry_points.txt
|
|
16
|
+
src/mizan_ledger.egg-info/requires.txt
|
|
17
|
+
src/mizan_ledger.egg-info/top_level.txt
|
|
18
|
+
src/mizan_ledger/adapters/__init__.py
|
|
19
|
+
src/mizan_ledger/adapters/trace_eval.py
|
|
20
|
+
tests/conftest.py
|
|
21
|
+
tests/test_adapters.py
|
|
22
|
+
tests/test_core.py
|
|
23
|
+
vectors/commitments.json
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
mizan_ledger
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
from mizan_ledger import Ledger, LedgerError, verify_ledger_dir
|
|
3
|
+
from mizan_ledger.adapters import trace_eval as te
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def test_trace_adapter_roundtrip(tmp_path):
|
|
7
|
+
led = Ledger(tmp_path / "pub", tmp_path / "priv")
|
|
8
|
+
te.declare_from_trace(led, {"id": "run-1", "confidence": 0.9}, "agent-t", "1")
|
|
9
|
+
rec = te.resolve_from_trace(led, {"id": "run-1", "score": 1.0}, "agent-t", "1")
|
|
10
|
+
assert rec["outcome"] == 1
|
|
11
|
+
te.declare_from_trace(led, {"id": "run-2", "confidence": 0.9}, "agent-t", "1")
|
|
12
|
+
assert te.resolve_from_trace(led, {"id": "run-2", "score": 0.2}, "agent-t", "1")["outcome"] == 0
|
|
13
|
+
assert verify_ledger_dir(tmp_path / "pub")["ok"]
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def test_trace_adapter_refuses_to_guess(tmp_path):
|
|
17
|
+
led = Ledger(tmp_path / "pub", tmp_path / "priv")
|
|
18
|
+
with pytest.raises(LedgerError): te.declare_from_trace(led, {"id": "r"}, "a")
|
|
19
|
+
led.declare("r", 0.5, "a")
|
|
20
|
+
with pytest.raises(LedgerError): te.resolve_from_trace(led, {"id": "r"}, "a")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def test_evaluator_callable(tmp_path):
|
|
24
|
+
led = Ledger(tmp_path / "pub", tmp_path / "priv")
|
|
25
|
+
led.declare("r", 0.8, "a", "1")
|
|
26
|
+
out = te.make_evaluator(led, "a", "1")({"id": "r", "score": 1})
|
|
27
|
+
assert out["score"] == pytest.approx(0.04)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
# --- pytest plugin, tested through pytester ---------------------------------
|
|
31
|
+
def test_pytest_plugin(pytester, tmp_path):
|
|
32
|
+
pytester.makepyfile("""
|
|
33
|
+
import pytest
|
|
34
|
+
@pytest.mark.confidence(0.9)
|
|
35
|
+
def test_pass(): assert True
|
|
36
|
+
@pytest.mark.confidence(0.8)
|
|
37
|
+
def test_fail(): assert False
|
|
38
|
+
@pytest.mark.confidence(0.5)
|
|
39
|
+
@pytest.mark.skip(reason="x")
|
|
40
|
+
def test_skip(): pass
|
|
41
|
+
def test_unmarked(): assert True
|
|
42
|
+
""")
|
|
43
|
+
ld = tmp_path / "ledger"
|
|
44
|
+
r = pytester.runpytest("-p", "mizan_ledger.pytest_plugin", "--mizan-ledger-dir", str(ld),
|
|
45
|
+
"--mizan-agent-id", "plug-agent", "--mizan-agent-version", "9", "--mizan-run-id", "r1")
|
|
46
|
+
r.assert_outcomes(passed=2, failed=1, skipped=1)
|
|
47
|
+
led = Ledger(ld)
|
|
48
|
+
s = led.summary("plug-agent", "9")
|
|
49
|
+
assert s["declared"] == 2 and s["resolved"] == 2 and s["pending"] == 0 # skipped test is never declared
|
|
50
|
+
assert s["brier"] == pytest.approx(((0.9 - 1) ** 2 + (0.8 - 0) ** 2) / 2)
|
|
51
|
+
assert verify_ledger_dir(ld)["ok"]
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def test_xfail_stays_pending(pytester, tmp_path):
|
|
55
|
+
pytester.makepyfile("""
|
|
56
|
+
import pytest
|
|
57
|
+
@pytest.mark.confidence(0.7)
|
|
58
|
+
@pytest.mark.xfail
|
|
59
|
+
def test_x(): assert False
|
|
60
|
+
""")
|
|
61
|
+
ld = tmp_path / "ledger"
|
|
62
|
+
pytester.runpytest("-p", "mizan_ledger.pytest_plugin", "--mizan-ledger-dir", str(ld), "--mizan-agent-id", "x", "--mizan-run-id", "r")
|
|
63
|
+
s = Ledger(ld).summary("x")
|
|
64
|
+
assert s["declared"] == 1 and s["pending"] == 1
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def test_cli_verify(tmp_path, capsys):
|
|
68
|
+
from mizan_ledger.cli import main
|
|
69
|
+
led = Ledger(tmp_path / "pub", tmp_path / "priv")
|
|
70
|
+
led.declare("t", 0.5, "a"); led.resolve("t", True)
|
|
71
|
+
assert main(["verify", str(tmp_path / "pub")]) == 0
|
|
72
|
+
assert not (tmp_path / "pub" / ".noprivate").exists()
|
|
73
|
+
main(["summary", str(tmp_path / "pub")])
|
|
74
|
+
(tmp_path / "pub" / "resolutions.jsonl").write_text("")
|
|
75
|
+
assert main(["verify", str(tmp_path / "pub")]) == 0
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
import json, math, pathlib
|
|
2
|
+
import pytest
|
|
3
|
+
from mizan_ledger import Ledger, LedgerError, commitment_hash, verify_opening, verify_ledger_dir, fmt_p, metrics
|
|
4
|
+
|
|
5
|
+
VEC = json.loads((pathlib.Path(__file__).resolve().parents[1] / "vectors" / "commitments.json").read_text("utf8"))
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def make(tmp_path, times=None):
|
|
9
|
+
it = iter(times or [f"2026-11-16T10:00:{i:02d}.000Z" for i in range(60)])
|
|
10
|
+
return Ledger(tmp_path / "pub", tmp_path / "priv", clock=lambda: next(it))
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def test_shared_vectors():
|
|
14
|
+
for v in VEC:
|
|
15
|
+
assert fmt_p(v["p"]) == v["p_fmt"]
|
|
16
|
+
assert commitment_hash(v["task_id"], v["p"], v["agent_id"], v["agent_version"], v["salt"]) == v["commitment"]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def test_declare_is_hash_only(tmp_path):
|
|
20
|
+
led = make(tmp_path)
|
|
21
|
+
r = led.declare("secret-task-name", 0.8123, "agent-a", "1")
|
|
22
|
+
pub = (tmp_path / "pub" / "receipts.jsonl").read_text()
|
|
23
|
+
assert "secret-task-name" not in pub and "0.8123" not in pub and "0.812300" not in pub
|
|
24
|
+
assert len(r["commitment"]) == 64 and r["declared_at"].endswith("Z")
|
|
25
|
+
assert oct((tmp_path / "priv" / "openings.jsonl").stat().st_mode & 0o777) == "0o600"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def test_salt_makes_same_input_differ(tmp_path):
|
|
29
|
+
led = make(tmp_path)
|
|
30
|
+
a = led.declare("t1", 0.5, "a", "1")
|
|
31
|
+
b = led.declare("t1", 0.5, "a", "2")
|
|
32
|
+
assert a["commitment"] != b["commitment"]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def test_resolve_scores_and_reveals(tmp_path):
|
|
36
|
+
led = make(tmp_path)
|
|
37
|
+
led.declare("t1", 0.8, "a", "1")
|
|
38
|
+
rec = led.resolve("t1", True)
|
|
39
|
+
assert rec["score"]["squared_error"] == pytest.approx(0.04)
|
|
40
|
+
assert rec["score"]["log_loss"] == pytest.approx(-math.log(0.8))
|
|
41
|
+
rc = led.receipts()[0]
|
|
42
|
+
assert verify_opening(rc, rec["opening"])
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def test_cannot_redeclare_or_re_resolve(tmp_path):
|
|
46
|
+
led = make(tmp_path)
|
|
47
|
+
led.declare("t1", 0.8, "a", "1")
|
|
48
|
+
with pytest.raises(LedgerError): led.declare("t1", 0.2, "a", "1")
|
|
49
|
+
led.resolve("t1", False)
|
|
50
|
+
with pytest.raises(LedgerError): led.resolve("t1", True)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def test_resolve_unknown_and_bad_p(tmp_path):
|
|
54
|
+
led = make(tmp_path)
|
|
55
|
+
with pytest.raises(LedgerError): led.resolve("nope", True)
|
|
56
|
+
for bad in (-0.1, 1.1, float("nan")):
|
|
57
|
+
with pytest.raises(LedgerError): led.declare("x", bad, "a")
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def test_clock_backwards_rejected(tmp_path):
|
|
61
|
+
led = make(tmp_path, ["2026-11-16T10:00:10.000Z", "2026-11-16T10:00:05.000Z"])
|
|
62
|
+
led.declare("t1", 0.5, "a", "1")
|
|
63
|
+
with pytest.raises(LedgerError): led.resolve("t1", True)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def test_verify_ok_and_tamper_detection(tmp_path):
|
|
67
|
+
led = make(tmp_path)
|
|
68
|
+
for i, (p, y) in enumerate([(0.9, 1), (0.7, 1), (0.2, 0), (0.6, 0)]):
|
|
69
|
+
led.declare(f"t{i}", p, "a", "1"); led.resolve(f"t{i}", bool(y))
|
|
70
|
+
led.declare("pending", 0.5, "a", "1")
|
|
71
|
+
rep = verify_ledger_dir(tmp_path / "pub")
|
|
72
|
+
assert rep["ok"], rep["errors"]
|
|
73
|
+
assert rep["summary"]["declared"] == 5 and rep["summary"]["pending"] == 1 and rep["summary"]["n"] == 4
|
|
74
|
+
# 1) change an outcome in resolutions
|
|
75
|
+
rp = tmp_path / "pub" / "resolutions.jsonl"
|
|
76
|
+
orig = rp.read_text()
|
|
77
|
+
rp.write_text(orig.replace('"outcome":1', '"outcome":0', 1))
|
|
78
|
+
assert not verify_ledger_dir(tmp_path / "pub")["ok"]
|
|
79
|
+
rp.write_text(orig)
|
|
80
|
+
# 2) delete a receipt in the middle
|
|
81
|
+
cp = tmp_path / "pub" / "receipts.jsonl"
|
|
82
|
+
lines = cp.read_text().splitlines(True)
|
|
83
|
+
cp.write_text("".join(lines[:1] + lines[2:]))
|
|
84
|
+
assert not verify_ledger_dir(tmp_path / "pub")["ok"]
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def test_verify_detects_opening_swap(tmp_path):
|
|
88
|
+
led = make(tmp_path)
|
|
89
|
+
led.declare("t1", 0.9, "a", "1"); led.resolve("t1", False)
|
|
90
|
+
rp = tmp_path / "pub" / "resolutions.jsonl"
|
|
91
|
+
d = json.loads(rp.read_text()); d["opening"]["p"] = "0.100000"
|
|
92
|
+
# recompute record hash so only the opening check can catch it
|
|
93
|
+
from mizan_ledger.core import _chain_hash, canonical_json
|
|
94
|
+
d["record_hash"] = _chain_hash(d, "record_hash")
|
|
95
|
+
rp.write_text(canonical_json(d) + "\n")
|
|
96
|
+
rep = verify_ledger_dir(tmp_path / "pub")
|
|
97
|
+
assert not rep["ok"] and any("opening" in e for e in rep["errors"])
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def test_summary_reports_coverage(tmp_path):
|
|
101
|
+
led = make(tmp_path)
|
|
102
|
+
led.declare("a", 0.9, "x", "1"); led.declare("b", 0.9, "x", "1"); led.resolve("a", True)
|
|
103
|
+
s = led.summary("x")
|
|
104
|
+
assert s["declared"] == 2 and s["resolved"] == 1 and s["coverage"] == 0.5
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def test_private_dir_not_needed_for_verification(tmp_path):
|
|
108
|
+
led = make(tmp_path)
|
|
109
|
+
led.declare("t1", 0.9, "a", "1"); led.resolve("t1", True)
|
|
110
|
+
import shutil; shutil.rmtree(tmp_path / "priv")
|
|
111
|
+
assert verify_ledger_dir(tmp_path / "pub")["ok"]
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
# --- metrics ---------------------------------------------------------------
|
|
115
|
+
def test_brier_ece_known_values():
|
|
116
|
+
pairs = [(1.0, 1), (0.0, 0), (0.5, 1), (0.5, 0)]
|
|
117
|
+
assert metrics.brier(pairs) == pytest.approx(0.125)
|
|
118
|
+
assert metrics.ece([(0.7, 1)] * 7 + [(0.7, 0)] * 3) == pytest.approx(0.0)
|
|
119
|
+
assert metrics.ece([(0.9, 0)] * 10) == pytest.approx(0.9)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def test_log_loss_clipped():
|
|
123
|
+
assert math.isfinite(metrics.log_loss([(0.0, 1), (1.0, 0)]))
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def test_wilson_matches_reference():
|
|
127
|
+
r = metrics.accuracy_wilson([(0.9, 1)] * 8 + [(0.9, 0)] * 2) # 8/10 correct
|
|
128
|
+
assert r["accuracy"] == 0.8
|
|
129
|
+
assert r["ci95"][0] == pytest.approx(0.4902, abs=1e-3) and r["ci95"][1] == pytest.approx(0.9433, abs=1e-3)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def test_bootstrap_deterministic_and_brackets_mean():
|
|
133
|
+
pairs = [(0.8, 1), (0.6, 0), (0.9, 1), (0.3, 0), (0.7, 1), (0.4, 1)] * 5
|
|
134
|
+
a = metrics.brier_interval(pairs); b = metrics.brier_interval(pairs)
|
|
135
|
+
assert a == b and a["ci95"][0] <= a["brier"] <= a["ci95"][1]
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def test_empty_raises():
|
|
139
|
+
with pytest.raises(ValueError): metrics.brier([])
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
[
|
|
2
|
+
{
|
|
3
|
+
"task_id": "task-1",
|
|
4
|
+
"p": 0.73,
|
|
5
|
+
"agent_id": "agent-a",
|
|
6
|
+
"agent_version": "1.0.0",
|
|
7
|
+
"salt": "00000000000000000000000000000000",
|
|
8
|
+
"p_fmt": "0.730000",
|
|
9
|
+
"commitment": "390c97e32123c75310aa506d533d868d0a94e5063be3bec14e3f89ee75264f1f"
|
|
10
|
+
},
|
|
11
|
+
{
|
|
12
|
+
"task_id": "ünï-任务\"q\"\n",
|
|
13
|
+
"p": 0.0,
|
|
14
|
+
"agent_id": "qants-x",
|
|
15
|
+
"agent_version": "unspecified",
|
|
16
|
+
"salt": "abababababababababababababababab",
|
|
17
|
+
"p_fmt": "0.000000",
|
|
18
|
+
"commitment": "6e86d98656fc0bae6c7cebd69356fdb7a8ba4e13655c400e0c1c1fe3a7d1ed38"
|
|
19
|
+
},
|
|
20
|
+
{
|
|
21
|
+
"task_id": "t",
|
|
22
|
+
"p": 1.0,
|
|
23
|
+
"agent_id": "a",
|
|
24
|
+
"agent_version": "v",
|
|
25
|
+
"salt": "ffffffffffffffffffffffffffffffff",
|
|
26
|
+
"p_fmt": "1.000000",
|
|
27
|
+
"commitment": "6e1661aa29b698fcb5d019c7ef5b604bfe9d45123b4d6e7d3c9de75f7ef7ce6b"
|
|
28
|
+
},
|
|
29
|
+
{
|
|
30
|
+
"task_id": "t",
|
|
31
|
+
"p": 0.123456789,
|
|
32
|
+
"agent_id": "a",
|
|
33
|
+
"agent_version": "v",
|
|
34
|
+
"salt": "0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f",
|
|
35
|
+
"p_fmt": "0.123457",
|
|
36
|
+
"commitment": "9e6efcdbe14745e409e2269921cd6c7f078475650b2e71fe8e4c7873aa4b5d73"
|
|
37
|
+
}
|
|
38
|
+
]
|