chancellor 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- chancellor-1.0.0/LICENSE +20 -0
- chancellor-1.0.0/PKG-INFO +183 -0
- chancellor-1.0.0/README.md +170 -0
- chancellor-1.0.0/pyproject.toml +60 -0
- chancellor-1.0.0/setup.cfg +4 -0
- chancellor-1.0.0/src/chancellor/__init__.py +59 -0
- chancellor-1.0.0/src/chancellor/api_client.py +298 -0
- chancellor-1.0.0/src/chancellor/cli.py +285 -0
- chancellor-1.0.0/src/chancellor/escalation.py +230 -0
- chancellor-1.0.0/src/chancellor/evals/__init__.py +5 -0
- chancellor-1.0.0/src/chancellor/evals/calibration.py +359 -0
- chancellor-1.0.0/src/chancellor/evals/suite_a.py +398 -0
- chancellor-1.0.0/src/chancellor/evals/suite_a_bank.py +235 -0
- chancellor-1.0.0/src/chancellor/evals/suite_a_gold.py +58 -0
- chancellor-1.0.0/src/chancellor/evals/suite_a_key.py +263 -0
- chancellor-1.0.0/src/chancellor/evals/suite_a_probes.py +131 -0
- chancellor-1.0.0/src/chancellor/evals/suite_a_references.py +365 -0
- chancellor-1.0.0/src/chancellor/evals/suite_b.py +650 -0
- chancellor-1.0.0/src/chancellor/evals/suite_b_tasks.py +255 -0
- chancellor-1.0.0/src/chancellor/evals/suite_b_world.py +332 -0
- chancellor-1.0.0/src/chancellor/exhibit.py +559 -0
- chancellor-1.0.0/src/chancellor/gate_report.py +465 -0
- chancellor-1.0.0/src/chancellor/grader.py +168 -0
- chancellor-1.0.0/src/chancellor/instruments.py +206 -0
- chancellor-1.0.0/src/chancellor/judge.py +298 -0
- chancellor-1.0.0/src/chancellor/live_transport.py +233 -0
- chancellor-1.0.0/src/chancellor/orchestrator.py +388 -0
- chancellor-1.0.0/src/chancellor/receipts.py +304 -0
- chancellor-1.0.0/src/chancellor/task_graph.py +397 -0
- chancellor-1.0.0/src/chancellor/verify_log.py +255 -0
- chancellor-1.0.0/src/chancellor.egg-info/PKG-INFO +183 -0
- chancellor-1.0.0/src/chancellor.egg-info/SOURCES.txt +56 -0
- chancellor-1.0.0/src/chancellor.egg-info/dependency_links.txt +1 -0
- chancellor-1.0.0/src/chancellor.egg-info/entry_points.txt +2 -0
- chancellor-1.0.0/src/chancellor.egg-info/requires.txt +3 -0
- chancellor-1.0.0/src/chancellor.egg-info/top_level.txt +1 -0
- chancellor-1.0.0/tests/test_calibration.py +164 -0
- chancellor-1.0.0/tests/test_evals.py +37 -0
- chancellor-1.0.0/tests/test_exhibit.py +45 -0
- chancellor-1.0.0/tests/test_fault_injection.py +236 -0
- chancellor-1.0.0/tests/test_gate_report.py +344 -0
- chancellor-1.0.0/tests/test_governance.py +177 -0
- chancellor-1.0.0/tests/test_grader.py +29 -0
- chancellor-1.0.0/tests/test_grader_properties.py +172 -0
- chancellor-1.0.0/tests/test_instruments_escalation.py +56 -0
- chancellor-1.0.0/tests/test_integration.py +252 -0
- chancellor-1.0.0/tests/test_judge.py +123 -0
- chancellor-1.0.0/tests/test_kill_resume.py +204 -0
- chancellor-1.0.0/tests/test_live_transport.py +213 -0
- chancellor-1.0.0/tests/test_meter.py +118 -0
- chancellor-1.0.0/tests/test_orchestrator.py +88 -0
- chancellor-1.0.0/tests/test_receipts.py +214 -0
- chancellor-1.0.0/tests/test_smoke.py +52 -0
- chancellor-1.0.0/tests/test_suite_a_bank.py +123 -0
- chancellor-1.0.0/tests/test_suite_a_probes.py +96 -0
- chancellor-1.0.0/tests/test_suite_b.py +271 -0
- chancellor-1.0.0/tests/test_task_graph.py +53 -0
- chancellor-1.0.0/tests/test_task_graph_properties.py +254 -0
chancellor-1.0.0/LICENSE
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
PROPRIETARY — Chancellor
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Israel Lee Armstead / E5 Enclave Inc. All rights reserved.
|
|
4
|
+
|
|
5
|
+
This repository is published for the Meta Global AI Developer Hackathon
|
|
6
|
+
(Cerebral Valley) so that the work can be read, run, and verified. Under
|
|
7
|
+
the hackathon's Official Rules (Oct 26), publication grants the review
|
|
8
|
+
rights the competition requires — and nothing further.
|
|
9
|
+
|
|
10
|
+
No license is granted to copy, modify, distribute, or reuse the code,
|
|
11
|
+
documentation, or artwork in this repository, in whole or in part, for
|
|
12
|
+
any other purpose, without prior written permission from the copyright
|
|
13
|
+
holder.
|
|
14
|
+
|
|
15
|
+
The development record in BUILD-LOG.md and the measurements quoted in
|
|
16
|
+
README.md may be cited with attribution.
|
|
17
|
+
|
|
18
|
+
For the avoidance of doubt: nothing in this license limits the rights
|
|
19
|
+
granted to the hackathon's organizers under the event waiver in respect
|
|
20
|
+
of the submitted entry.
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: chancellor
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Chancellor — governance for long-running agents on the Meta Model API. Work runs as a durable task graph that survives crashes; a separate judge model grades every consequential output; and spending, sending, publishing, or using credentials stops at a human gate that leaves a signed receipt.
|
|
5
|
+
Author: Israel Lee Armstead
|
|
6
|
+
License: Proprietary — hackathon build, see Official Rules (Oct 26) for IP terms
|
|
7
|
+
Requires-Python: >=3.11
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Provides-Extra: dev
|
|
11
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
12
|
+
Dynamic: license-file
|
|
13
|
+
|
|
14
|
+
<div align="center">
|
|
15
|
+
|
|
16
|
+
<img src="docs/media/banner.png" alt="Chancellor — the governance-and-eval harness for long-running agents" width="880">
|
|
17
|
+
|
|
18
|
+
[](https://github.com/IAMGODIAM/chancellor/actions/workflows/gates.yml)
|
|
19
|
+
<a href="docs/evidence/run-5/"><img src="docs/media/badge-drift.svg" alt="Suite B: drift 0 / 1,800 — bounds printed"></a>
|
|
20
|
+
<a href="https://github.com/IAMGODIAM/chancellor/actions"><img src="docs/media/badge-tests.svg" alt="Tests: 188 passing — gates green"></a>
|
|
21
|
+
<a href="#results-measured-oct-2931-2026"><img src="docs/media/badge-calibration.svg" alt="Calibration: gate not passed — verdicts labeled"></a>
|
|
22
|
+
<a href="https://github.com/IAMGODIAM/chancellor/releases/tag/v1.0.0"><img src="docs/media/badge-release.svg" alt="Release: v1.0.0"></a>
|
|
23
|
+
|
|
24
|
+
</div>
|
|
25
|
+
|
|
26
|
+
# Chancellor
|
|
27
|
+
|
|
28
|
+
**The governance-and-eval harness for long-running agents.**
|
|
29
|
+
|
|
30
|
+
Built for the Meta Global AI Developer Hackathon (Cerebral Valley), Indie Dev track, on the Meta Model API: `muse-spark-1.3` does the work, `muse-wonder-1.0` grades it — a worker never scores its own output.
|
|
31
|
+
|
|
32
|
+
## For judges — the 60-second path
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
pip install chancellor
|
|
36
|
+
# (until the listing posts, the same package straight from this repo:
|
|
37
|
+
# pip install git+https://github.com/IAMGODIAM/chancellor)
|
|
38
|
+
|
|
39
|
+
chancellor demo # the exhibit, offline — graded work, a halted
|
|
40
|
+
# fault, a human gate, a verified receipt
|
|
41
|
+
python -m chancellor.verify_log .chancellor-exhibit/audit-log.jsonl
|
|
42
|
+
# the demo's own receipt book, verified cold
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
- **The evidence:** [docs/evidence/run-5/](docs/evidence/run-5/) — the RUN-5 record with its bounds printed; a two-file sample you can verify in isolation lives in [docs/evidence/sample/](docs/evidence/sample/).
|
|
46
|
+
- **The film:** the [Demo](#demo) section below — three minutes, a capture of this exact run.
|
|
47
|
+
|
|
48
|
+
One term, defined once: **drift**, here, means a run claiming success for work whose final state failed its check — not model drift. Every drift figure in this README uses that sense.
|
|
49
|
+
|
|
50
|
+
The demo runs offline by design. A `--live` mode exists — the same run against the live Meta models, through the same client — and it refuses to run without credentials rather than falling back silently.
|
|
51
|
+
|
|
52
|
+
## The pitch
|
|
53
|
+
|
|
54
|
+
Long-running agents fail quietly: they drift, mis-grade their own work, trust broken tools, and sail past decisions that were never theirs to make. Frameworks optimize for capability. Chancellor is the other half — **governance**:
|
|
55
|
+
|
|
56
|
+
- **Work is a graph, not a vibe.** Every run is an explicit, durable task graph that persists to disk and resumes after a crash.
|
|
57
|
+
- **Output is graded, not trusted.** Every consequential output is scored against a weighted rubric before acceptance.
|
|
58
|
+
- **Instruments fail closed.** If Chancellor's own checks error, the run halts. A broken gauge is a stop signal, never a pass — and there is no flag to turn this off.
|
|
59
|
+
- **Authority escalates; capability doesn't.** Spend, external sends, publishing, irreversible acts, and credential use stop at a human gate. Every gate decision — human or policy — leaves an append-only receipt.
|
|
60
|
+
|
|
61
|
+
Two eval suites test the claims. **Suite A** asks whether the judge grades remedies for invented harms the way the approved key grades them — consistently, and for the stated reasons. **Suite B** breaks long runs on purpose — crashes, dropped steps, corrupted values — and counts how often an agent claims success for work that did not happen.
|
|
62
|
+
|
|
63
|
+
Model names in this repo are exact API model IDs only; one naming correction is recorded in [ERRATA.md](ERRATA.md).
|
|
64
|
+
|
|
65
|
+
**Status — Oct 31, 2026:** the offline demo is the guaranteed path and needs no key. The live connection to the Meta Model API was established Oct 30 and the transport is validated — the model list returns and both working models answer. Graded live runs are metered and reported in [BUILD-LOG.md](BUILD-LOG.md).
|
|
66
|
+
|
|
67
|
+
## Results (measured, Oct 29–31, 2026)
|
|
68
|
+
|
|
69
|
+
> **The regeneration rule.** This README is a derivative of the measured record — BUILD-LOG.md + docs/evidence/ — re-rendered at every release; a README that trails the record is a stale claim and is corrected as one.
|
|
70
|
+
|
|
71
|
+
**Suite B reliability frontier, at 10x scale** (RUN-5, Oct 31, on the finalized tree — 5,600 Suite B runs in all): governed drift **0 / 1,800 runs** against ungoverned drift **300 / 1,800** on identical fault manifests. The zero travels with its bounds wherever it is quoted: a 95% upper bound of 0.166% at the cell level (n = 1,800), and the weaker manifest-level bound of 25.9% (0 of 10 manifests). A value-cell extension added governed drift **0 / 2,000**. Drift is a bound with its denominator, never a bare zero.
|
|
72
|
+
|
|
73
|
+
The original frontier (Oct 29) had the same shape at one-tenth the scale. Each arm ran the same 10 tasks under the same 180 fault manifests. A run *completes* when it finishes; it *succeeds* when its final state passes the task's check. Drift is the gap that matters — claiming success when the check failed:
|
|
74
|
+
|
|
75
|
+
| (180 runs per arm) | Governed | Ungoverned |
|
|
76
|
+
|---|---:|---:|
|
|
77
|
+
| Drift (claimed success, final state failed) | **0** | **30** |
|
|
78
|
+
| Completion | 60 of 180 (0.333) | 120 of 180 (0.667) |
|
|
79
|
+
| Cost per success | $0.0021 | $0.0018 |
|
|
80
|
+
|
|
81
|
+
Governance is not free, and the price is printed, not hidden: the governed arm completes fewer tasks at a higher cost per success — and never once claims a success that did not happen.
|
|
82
|
+
|
|
83
|
+
**Endurance:** four parallel 8-hour soaks (RUN-5, Oct 31) each completed 480 / 480 segments and 5,760 steps per soak — 48 injected faults per soak, each met by a fail-closed halt rather than a corrupted claim — with drift 0, zero corrupted state files, and every final hash chain verifying. An independent red team then re-derived all four chains from the raw files: 1,846 entries, 0 breaks.
|
|
84
|
+
|
|
85
|
+
**Kill-resume gate (T3):** the fixed harness passes **300 / 300** gates (6,000 trials in RUN-5's S2 stage, on the Lambda box). The original harness's 290 / 300 stands in [BUILD-LOG.md](BUILD-LOG.md) as the measurement of the harness as first written — the defect was in the test harness, not the product, and the diagnosis is recorded there in full.
|
|
86
|
+
|
|
87
|
+
**Tests:** 188 passing, 0 failing on the frozen tree — which also passes its own presentation gates: Ruff format and lint at zero findings, mypy clean, identical commands in pre-commit and CI (see the badge above).
|
|
88
|
+
|
|
89
|
+
**Suite A:** the calibration gate has been run twice under pre-registration and has not passed — and no Suite A score is claimed anywhere in this submission. The calibration of record measured band κ 0.667 and Spearman ρ 0.881, with precedent-consistency (κ 0.294) and reversibility (κ 0.575) below the 0.6 criterion floor. The pre-registered remedy run then failed its own completeness gate — 35 of 36 references scored, one lost to two transport failures — and an incomplete run earns no gate statistics; its diagnostics (not gate statistics) put every criterion above threshold, and the judge's blind self-agreement, measured for the first time, satisfied its gate on all four criteria. What ships is the labeled fallback: live judge verdicts carry the label *uncalibrated* until a pre-registered run passes in full. The gate catching our own judge — twice — is the method working as designed.
|
|
90
|
+
|
|
91
|
+
Don't take the tables' word for any of this — the audit log of a governed run verifies offline, with no credentials and no network (see [Verify the audit log](#verify-the-audit-log)).
|
|
92
|
+
|
|
93
|
+
The full record — measurements, honest negative results, and corrections filed as new entries rather than silent edits — is the development log: [BUILD-LOG.md](BUILD-LOG.md) and [ERRATA.md](ERRATA.md).
|
|
94
|
+
|
|
95
|
+
## Quickstart
|
|
96
|
+
|
|
97
|
+
```bash
|
|
98
|
+
python3 -m venv .venv && source .venv/bin/activate
|
|
99
|
+
pip install -e ".[dev]"
|
|
100
|
+
|
|
101
|
+
pytest # smoke + unit tests (all offline)
|
|
102
|
+
chancellor demo # the exhibit — the hearing, live in the terminal (offline)
|
|
103
|
+
chancellor eval-a # Suite A with stub agents
|
|
104
|
+
chancellor eval-b # Suite B reliability metrics
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
Zero runtime dependencies — stdlib only, Python ≥ 3.11. Allow about two minutes for the demo at stage pace (the gate is meant to be waited at); `--fast` compresses the same run for a quick pass. Judging from a distance: `pip install git+https://github.com/IAMGODIAM/chancellor` installs the package straight from this repo, and `chancellor demo` runs offline.
|
|
108
|
+
|
|
109
|
+
The exhibit writes its state to `./.chancellor-exhibit` in the current directory (task-graph state, the audit log, and its checkpoints). Its verdict block is byte-identical across clean runs. `chancellor demo --classic` runs the original scripted governed run, which keeps its state in `./.chancellor`.
|
|
110
|
+
|
|
111
|
+
## Demo
|
|
112
|
+
|
|
113
|
+
<img src="docs/media/exhibit.gif" alt="The exhibit, animated: the four beats of a governed run — graded work, a halted fault, a human gate, and a receipt chain verified cold" width="880">
|
|
114
|
+
|
|
115
|
+
*The exhibit in half a minute — real output of `chancellor demo`, typeset from the verified capture. The full hearing, narrated, is the film:*
|
|
116
|
+
|
|
117
|
+
https://github.com/user-attachments/assets/8a0843e8-a226-4218-bcd0-ef50a4110e45
|
|
118
|
+
|
|
119
|
+
The demo is the exhibit — `chancellor demo` (Quickstart above) runs the hearing live in the terminal: graded work with a revision, an injected fault halted fail-closed, a human gate the run waits at, and a receipt chain verified cold (with `--tamper`, a quietly altered copy is rejected on screen, the entry named). Its verdict block is byte-identical across clean runs, so the video above is a capture of a fixed target, not a lucky take.
|
|
120
|
+
|
|
121
|
+
## Verify the audit log
|
|
122
|
+
|
|
123
|
+
Every governed run writes a hash-chained audit log with signed checkpoints. Verify one offline — no credentials, no network:
|
|
124
|
+
|
|
125
|
+
```bash
|
|
126
|
+
python -m chancellor.verify_log .chancellor/audit-log.jsonl
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
Exit 0 means the chain is intact and the newest signed checkpoint checks out. Exit 1 names the failure — a mid-log edit, a reordering, a truncated tail, a forged checkpoint. A log with no checkpoint is never reported as verified: tail truncation is undetectable without one, and the verifier says so rather than guessing. The checkpoints are signed with the development signer whose key ships in the source — deliberately public, and labeled in the code where it sits: those signatures prove the chain is internally consistent and unaltered since signing (tamper-evidence), not who signed it (non-repudiation). Separating the signer — a production key held apart from the code — is the named hardening step before any deployment that needs attribution.
|
|
130
|
+
|
|
131
|
+
A ready-made pair to try it on — one exhibit run's log and its signed checkpoint, with the exact commands — lives in [docs/evidence/sample/](docs/evidence/sample/).
|
|
132
|
+
|
|
133
|
+
## Going live
|
|
134
|
+
|
|
135
|
+
The live client talks to the Meta Model API through one transport function; the offline demo uses the same client with a stub transport, which is why the demo's behavior does not change when the live path is used.
|
|
136
|
+
|
|
137
|
+
```bash
|
|
138
|
+
cp .env.example .env # set META_MODEL_API_BASE_URL and META_MODEL_API_KEY
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
With no key configured, `MetaModelClient.chat()` raises `NotConfiguredError` — use `chancellor.api_client.offline_client()` for keyless development. The connection was established and validated on Oct 30, 2026; see [BUILD-LOG.md](BUILD-LOG.md).
|
|
142
|
+
|
|
143
|
+
## Layout
|
|
144
|
+
|
|
145
|
+
```
|
|
146
|
+
src/chancellor/
|
|
147
|
+
task_graph.py explicit DAG, validation, persistence, resume
|
|
148
|
+
orchestrator.py CrewRunner: governed execution of the graph
|
|
149
|
+
grader.py weighted rubrics, verdict bands, fail-closed grading
|
|
150
|
+
instruments.py pre-acceptance checks; errors == failures
|
|
151
|
+
escalation.py authority gates + append-only audit receipts
|
|
152
|
+
api_client.py Meta Model API client interface (env-configured)
|
|
153
|
+
judge.py the Judge: a separate caller that produces the scores
|
|
154
|
+
receipts.py hash-chained receipts + signed Merkle checkpoints
|
|
155
|
+
verify_log.py offline audit-log verifier (python -m chancellor.verify_log)
|
|
156
|
+
cli.py chancellor demo | eval-a | eval-b
|
|
157
|
+
evals/
|
|
158
|
+
suite_a.py normative-judgment eval (remedy design)
|
|
159
|
+
suite_a_bank.py the 12 scenarios, 3 tiers, one-fact-flip twins
|
|
160
|
+
suite_a_gold.py sealed gold stakeholder lists (judge-side only)
|
|
161
|
+
suite_a_probes.py scorer probes: style twins, padding, order, hedging
|
|
162
|
+
suite_a_references.py the 36 hand-authored reference proposals
|
|
163
|
+
suite_a_key.py the calibration key of record (approved)
|
|
164
|
+
calibration.py agreement statistics (κ, ρ, MAE) + the gate
|
|
165
|
+
suite_b.py long-horizon reliability eval (failure injection)
|
|
166
|
+
suite_b_tasks.py the 10 world-state tasks Suite B executes
|
|
167
|
+
suite_b_world.py the assertable world state the tasks run over
|
|
168
|
+
tests/ smoke + unit tests, all offline
|
|
169
|
+
docs/ GLOSSARY.md (the amendment key) + media/ (banner, social preview) + evidence/ (the RUN-5 record)
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
The code cites its governing amendments inline (AM-2, EA-5, …) — the key is [docs/GLOSSARY.md](docs/GLOSSARY.md).
|
|
173
|
+
|
|
174
|
+
## How it maps to the judging criteria
|
|
175
|
+
|
|
176
|
+
- **Impact (30%)** — for anyone who runs long-running agents with real authority: every consequential act is gated before it happens, graded after it lands, and receipted either way — so an operator can hand an agent real work (spend, sends, publishing, irreversible acts) and prove afterward exactly what it did, what it cost, and who approved it.
|
|
177
|
+
- **Tooling — creative use of Meta's tools (25%)** — the Meta Model API supplies both halves of the separation of powers: `muse-spark-1.3` does the work and `muse-wonder-1.0` grades it, so no run is scored by the model that produced it. Exact, versioned IDs only — see [ERRATA.md](ERRATA.md) E-1.
|
|
178
|
+
- **Demo (25%)** — `chancellor demo` runs end-to-end offline today; the identical run executes on the live Meta models through the same client (connection validated Oct 30), so the demo cannot drift from the product. (Video: see the Demo section above.)
|
|
179
|
+
- **Technical Depth (20%)** — durable, crash-atomic task graphs proven under kill-resume gates (300 / 300), rubric-graded outputs from an independent judge, failure-injected reliability measurement with the cost printed beside the frontier, and a hash-chained audit log verifiable offline.
|
|
180
|
+
|
|
181
|
+
## License
|
|
182
|
+
|
|
183
|
+
Proprietary — see [LICENSE](LICENSE). This repository is public so the work can be read and verified; publication is not a license to reuse it.
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
<div align="center">
|
|
2
|
+
|
|
3
|
+
<img src="docs/media/banner.png" alt="Chancellor — the governance-and-eval harness for long-running agents" width="880">
|
|
4
|
+
|
|
5
|
+
[](https://github.com/IAMGODIAM/chancellor/actions/workflows/gates.yml)
|
|
6
|
+
<a href="docs/evidence/run-5/"><img src="docs/media/badge-drift.svg" alt="Suite B: drift 0 / 1,800 — bounds printed"></a>
|
|
7
|
+
<a href="https://github.com/IAMGODIAM/chancellor/actions"><img src="docs/media/badge-tests.svg" alt="Tests: 188 passing — gates green"></a>
|
|
8
|
+
<a href="#results-measured-oct-2931-2026"><img src="docs/media/badge-calibration.svg" alt="Calibration: gate not passed — verdicts labeled"></a>
|
|
9
|
+
<a href="https://github.com/IAMGODIAM/chancellor/releases/tag/v1.0.0"><img src="docs/media/badge-release.svg" alt="Release: v1.0.0"></a>
|
|
10
|
+
|
|
11
|
+
</div>
|
|
12
|
+
|
|
13
|
+
# Chancellor
|
|
14
|
+
|
|
15
|
+
**The governance-and-eval harness for long-running agents.**
|
|
16
|
+
|
|
17
|
+
Built for the Meta Global AI Developer Hackathon (Cerebral Valley), Indie Dev track, on the Meta Model API: `muse-spark-1.3` does the work, `muse-wonder-1.0` grades it — a worker never scores its own output.
|
|
18
|
+
|
|
19
|
+
## For judges — the 60-second path
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
pip install chancellor
|
|
23
|
+
# (until the listing posts, the same package straight from this repo:
|
|
24
|
+
# pip install git+https://github.com/IAMGODIAM/chancellor)
|
|
25
|
+
|
|
26
|
+
chancellor demo # the exhibit, offline — graded work, a halted
|
|
27
|
+
# fault, a human gate, a verified receipt
|
|
28
|
+
python -m chancellor.verify_log .chancellor-exhibit/audit-log.jsonl
|
|
29
|
+
# the demo's own receipt book, verified cold
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
- **The evidence:** [docs/evidence/run-5/](docs/evidence/run-5/) — the RUN-5 record with its bounds printed; a two-file sample you can verify in isolation lives in [docs/evidence/sample/](docs/evidence/sample/).
|
|
33
|
+
- **The film:** the [Demo](#demo) section below — three minutes, a capture of this exact run.
|
|
34
|
+
|
|
35
|
+
One term, defined once: **drift**, here, means a run claiming success for work whose final state failed its check — not model drift. Every drift figure in this README uses that sense.
|
|
36
|
+
|
|
37
|
+
The demo runs offline by design. A `--live` mode exists — the same run against the live Meta models, through the same client — and it refuses to run without credentials rather than falling back silently.
|
|
38
|
+
|
|
39
|
+
## The pitch
|
|
40
|
+
|
|
41
|
+
Long-running agents fail quietly: they drift, mis-grade their own work, trust broken tools, and sail past decisions that were never theirs to make. Frameworks optimize for capability. Chancellor is the other half — **governance**:
|
|
42
|
+
|
|
43
|
+
- **Work is a graph, not a vibe.** Every run is an explicit, durable task graph that persists to disk and resumes after a crash.
|
|
44
|
+
- **Output is graded, not trusted.** Every consequential output is scored against a weighted rubric before acceptance.
|
|
45
|
+
- **Instruments fail closed.** If Chancellor's own checks error, the run halts. A broken gauge is a stop signal, never a pass — and there is no flag to turn this off.
|
|
46
|
+
- **Authority escalates; capability doesn't.** Spend, external sends, publishing, irreversible acts, and credential use stop at a human gate. Every gate decision — human or policy — leaves an append-only receipt.
|
|
47
|
+
|
|
48
|
+
Two eval suites test the claims. **Suite A** asks whether the judge grades remedies for invented harms the way the approved key grades them — consistently, and for the stated reasons. **Suite B** breaks long runs on purpose — crashes, dropped steps, corrupted values — and counts how often an agent claims success for work that did not happen.
|
|
49
|
+
|
|
50
|
+
Model names in this repo are exact API model IDs only; one naming correction is recorded in [ERRATA.md](ERRATA.md).
|
|
51
|
+
|
|
52
|
+
**Status — Oct 31, 2026:** the offline demo is the guaranteed path and needs no key. The live connection to the Meta Model API was established Oct 30 and the transport is validated — the model list returns and both working models answer. Graded live runs are metered and reported in [BUILD-LOG.md](BUILD-LOG.md).
|
|
53
|
+
|
|
54
|
+
## Results (measured, Oct 29–31, 2026)
|
|
55
|
+
|
|
56
|
+
> **The regeneration rule.** This README is a derivative of the measured record — BUILD-LOG.md + docs/evidence/ — re-rendered at every release; a README that trails the record is a stale claim and is corrected as one.
|
|
57
|
+
|
|
58
|
+
**Suite B reliability frontier, at 10x scale** (RUN-5, Oct 31, on the finalized tree — 5,600 Suite B runs in all): governed drift **0 / 1,800 runs** against ungoverned drift **300 / 1,800** on identical fault manifests. The zero travels with its bounds wherever it is quoted: a 95% upper bound of 0.166% at the cell level (n = 1,800), and the weaker manifest-level bound of 25.9% (0 of 10 manifests). A value-cell extension added governed drift **0 / 2,000**. Drift is a bound with its denominator, never a bare zero.
|
|
59
|
+
|
|
60
|
+
The original frontier (Oct 29) had the same shape at one-tenth the scale. Each arm ran the same 10 tasks under the same 180 fault manifests. A run *completes* when it finishes; it *succeeds* when its final state passes the task's check. Drift is the gap that matters — claiming success when the check failed:
|
|
61
|
+
|
|
62
|
+
| (180 runs per arm) | Governed | Ungoverned |
|
|
63
|
+
|---|---:|---:|
|
|
64
|
+
| Drift (claimed success, final state failed) | **0** | **30** |
|
|
65
|
+
| Completion | 60 of 180 (0.333) | 120 of 180 (0.667) |
|
|
66
|
+
| Cost per success | $0.0021 | $0.0018 |
|
|
67
|
+
|
|
68
|
+
Governance is not free, and the price is printed, not hidden: the governed arm completes fewer tasks at a higher cost per success — and never once claims a success that did not happen.
|
|
69
|
+
|
|
70
|
+
**Endurance:** four parallel 8-hour soaks (RUN-5, Oct 31) each completed 480 / 480 segments and 5,760 steps per soak — 48 injected faults per soak, each met by a fail-closed halt rather than a corrupted claim — with drift 0, zero corrupted state files, and every final hash chain verifying. An independent red team then re-derived all four chains from the raw files: 1,846 entries, 0 breaks.
|
|
71
|
+
|
|
72
|
+
**Kill-resume gate (T3):** the fixed harness passes **300 / 300** gates (6,000 trials in RUN-5's S2 stage, on the Lambda box). The original harness's 290 / 300 stands in [BUILD-LOG.md](BUILD-LOG.md) as the measurement of the harness as first written — the defect was in the test harness, not the product, and the diagnosis is recorded there in full.
|
|
73
|
+
|
|
74
|
+
**Tests:** 188 passing, 0 failing on the frozen tree — which also passes its own presentation gates: Ruff format and lint at zero findings, mypy clean, identical commands in pre-commit and CI (see the badge above).
|
|
75
|
+
|
|
76
|
+
**Suite A:** the calibration gate has been run twice under pre-registration and has not passed — and no Suite A score is claimed anywhere in this submission. The calibration of record measured band κ 0.667 and Spearman ρ 0.881, with precedent-consistency (κ 0.294) and reversibility (κ 0.575) below the 0.6 criterion floor. The pre-registered remedy run then failed its own completeness gate — 35 of 36 references scored, one lost to two transport failures — and an incomplete run earns no gate statistics; its diagnostics (not gate statistics) put every criterion above threshold, and the judge's blind self-agreement, measured for the first time, satisfied its gate on all four criteria. What ships is the labeled fallback: live judge verdicts carry the label *uncalibrated* until a pre-registered run passes in full. The gate catching our own judge — twice — is the method working as designed.
|
|
77
|
+
|
|
78
|
+
Don't take the tables' word for any of this — the audit log of a governed run verifies offline, with no credentials and no network (see [Verify the audit log](#verify-the-audit-log)).
|
|
79
|
+
|
|
80
|
+
The full record — measurements, honest negative results, and corrections filed as new entries rather than silent edits — is the development log: [BUILD-LOG.md](BUILD-LOG.md) and [ERRATA.md](ERRATA.md).
|
|
81
|
+
|
|
82
|
+
## Quickstart
|
|
83
|
+
|
|
84
|
+
```bash
|
|
85
|
+
python3 -m venv .venv && source .venv/bin/activate
|
|
86
|
+
pip install -e ".[dev]"
|
|
87
|
+
|
|
88
|
+
pytest # smoke + unit tests (all offline)
|
|
89
|
+
chancellor demo # the exhibit — the hearing, live in the terminal (offline)
|
|
90
|
+
chancellor eval-a # Suite A with stub agents
|
|
91
|
+
chancellor eval-b # Suite B reliability metrics
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
Zero runtime dependencies — stdlib only, Python ≥ 3.11. Allow about two minutes for the demo at stage pace (the gate is meant to be waited at); `--fast` compresses the same run for a quick pass. Judging from a distance: `pip install git+https://github.com/IAMGODIAM/chancellor` installs the package straight from this repo, and `chancellor demo` runs offline.
|
|
95
|
+
|
|
96
|
+
The exhibit writes its state to `./.chancellor-exhibit` in the current directory (task-graph state, the audit log, and its checkpoints). Its verdict block is byte-identical across clean runs. `chancellor demo --classic` runs the original scripted governed run, which keeps its state in `./.chancellor`.
|
|
97
|
+
|
|
98
|
+
## Demo
|
|
99
|
+
|
|
100
|
+
<img src="docs/media/exhibit.gif" alt="The exhibit, animated: the four beats of a governed run — graded work, a halted fault, a human gate, and a receipt chain verified cold" width="880">
|
|
101
|
+
|
|
102
|
+
*The exhibit in half a minute — real output of `chancellor demo`, typeset from the verified capture. The full hearing, narrated, is the film:*
|
|
103
|
+
|
|
104
|
+
https://github.com/user-attachments/assets/8a0843e8-a226-4218-bcd0-ef50a4110e45
|
|
105
|
+
|
|
106
|
+
The demo is the exhibit — `chancellor demo` (Quickstart above) runs the hearing live in the terminal: graded work with a revision, an injected fault halted fail-closed, a human gate the run waits at, and a receipt chain verified cold (with `--tamper`, a quietly altered copy is rejected on screen, the entry named). Its verdict block is byte-identical across clean runs, so the video above is a capture of a fixed target, not a lucky take.
|
|
107
|
+
|
|
108
|
+
## Verify the audit log
|
|
109
|
+
|
|
110
|
+
Every governed run writes a hash-chained audit log with signed checkpoints. Verify one offline — no credentials, no network:
|
|
111
|
+
|
|
112
|
+
```bash
|
|
113
|
+
python -m chancellor.verify_log .chancellor/audit-log.jsonl
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
Exit 0 means the chain is intact and the newest signed checkpoint checks out. Exit 1 names the failure — a mid-log edit, a reordering, a truncated tail, a forged checkpoint. A log with no checkpoint is never reported as verified: tail truncation is undetectable without one, and the verifier says so rather than guessing. The checkpoints are signed with the development signer whose key ships in the source — deliberately public, and labeled in the code where it sits: those signatures prove the chain is internally consistent and unaltered since signing (tamper-evidence), not who signed it (non-repudiation). Separating the signer — a production key held apart from the code — is the named hardening step before any deployment that needs attribution.
|
|
117
|
+
|
|
118
|
+
A ready-made pair to try it on — one exhibit run's log and its signed checkpoint, with the exact commands — lives in [docs/evidence/sample/](docs/evidence/sample/).
|
|
119
|
+
|
|
120
|
+
## Going live
|
|
121
|
+
|
|
122
|
+
The live client talks to the Meta Model API through one transport function; the offline demo uses the same client with a stub transport, which is why the demo's behavior does not change when the live path is used.
|
|
123
|
+
|
|
124
|
+
```bash
|
|
125
|
+
cp .env.example .env # set META_MODEL_API_BASE_URL and META_MODEL_API_KEY
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
With no key configured, `MetaModelClient.chat()` raises `NotConfiguredError` — use `chancellor.api_client.offline_client()` for keyless development. The connection was established and validated on Oct 30, 2026; see [BUILD-LOG.md](BUILD-LOG.md).
|
|
129
|
+
|
|
130
|
+
## Layout
|
|
131
|
+
|
|
132
|
+
```
|
|
133
|
+
src/chancellor/
|
|
134
|
+
task_graph.py explicit DAG, validation, persistence, resume
|
|
135
|
+
orchestrator.py CrewRunner: governed execution of the graph
|
|
136
|
+
grader.py weighted rubrics, verdict bands, fail-closed grading
|
|
137
|
+
instruments.py pre-acceptance checks; errors == failures
|
|
138
|
+
escalation.py authority gates + append-only audit receipts
|
|
139
|
+
api_client.py Meta Model API client interface (env-configured)
|
|
140
|
+
judge.py the Judge: a separate caller that produces the scores
|
|
141
|
+
receipts.py hash-chained receipts + signed Merkle checkpoints
|
|
142
|
+
verify_log.py offline audit-log verifier (python -m chancellor.verify_log)
|
|
143
|
+
cli.py chancellor demo | eval-a | eval-b
|
|
144
|
+
evals/
|
|
145
|
+
suite_a.py normative-judgment eval (remedy design)
|
|
146
|
+
suite_a_bank.py the 12 scenarios, 3 tiers, one-fact-flip twins
|
|
147
|
+
suite_a_gold.py sealed gold stakeholder lists (judge-side only)
|
|
148
|
+
suite_a_probes.py scorer probes: style twins, padding, order, hedging
|
|
149
|
+
suite_a_references.py the 36 hand-authored reference proposals
|
|
150
|
+
suite_a_key.py the calibration key of record (approved)
|
|
151
|
+
calibration.py agreement statistics (κ, ρ, MAE) + the gate
|
|
152
|
+
suite_b.py long-horizon reliability eval (failure injection)
|
|
153
|
+
suite_b_tasks.py the 10 world-state tasks Suite B executes
|
|
154
|
+
suite_b_world.py the assertable world state the tasks run over
|
|
155
|
+
tests/ smoke + unit tests, all offline
|
|
156
|
+
docs/ GLOSSARY.md (the amendment key) + media/ (banner, social preview) + evidence/ (the RUN-5 record)
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
The code cites its governing amendments inline (AM-2, EA-5, …) — the key is [docs/GLOSSARY.md](docs/GLOSSARY.md).
|
|
160
|
+
|
|
161
|
+
## How it maps to the judging criteria
|
|
162
|
+
|
|
163
|
+
- **Impact (30%)** — for anyone who runs long-running agents with real authority: every consequential act is gated before it happens, graded after it lands, and receipted either way — so an operator can hand an agent real work (spend, sends, publishing, irreversible acts) and prove afterward exactly what it did, what it cost, and who approved it.
|
|
164
|
+
- **Tooling — creative use of Meta's tools (25%)** — the Meta Model API supplies both halves of the separation of powers: `muse-spark-1.3` does the work and `muse-wonder-1.0` grades it, so no run is scored by the model that produced it. Exact, versioned IDs only — see [ERRATA.md](ERRATA.md) E-1.
|
|
165
|
+
- **Demo (25%)** — `chancellor demo` runs end-to-end offline today; the identical run executes on the live Meta models through the same client (connection validated Oct 30), so the demo cannot drift from the product. (Video: see the Demo section above.)
|
|
166
|
+
- **Technical Depth (20%)** — durable, crash-atomic task graphs proven under kill-resume gates (300 / 300), rubric-graded outputs from an independent judge, failure-injected reliability measurement with the cost printed beside the frontier, and a hash-chained audit log verifiable offline.
|
|
167
|
+
|
|
168
|
+
## License
|
|
169
|
+
|
|
170
|
+
Proprietary — see [LICENSE](LICENSE). This repository is public so the work can be read and verified; publication is not a license to reuse it.
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "chancellor"
|
|
7
|
+
version = "1.0.0"
|
|
8
|
+
description = "Chancellor — governance for long-running agents on the Meta Model API. Work runs as a durable task graph that survives crashes; a separate judge model grades every consequential output; and spending, sending, publishing, or using credentials stops at a human gate that leaves a signed receipt."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
license = { text = "Proprietary — hackathon build, see Official Rules (Oct 26) for IP terms" }
|
|
12
|
+
authors = [{ name = "Israel Lee Armstead" }]
|
|
13
|
+
# Deliberately zero runtime dependencies: stdlib only, so the demo runs anywhere.
|
|
14
|
+
dependencies = []
|
|
15
|
+
|
|
16
|
+
[project.optional-dependencies]
|
|
17
|
+
dev = ["pytest>=8"]
|
|
18
|
+
|
|
19
|
+
[project.scripts]
|
|
20
|
+
chancellor = "chancellor.cli:main"
|
|
21
|
+
|
|
22
|
+
[tool.setuptools.packages.find]
|
|
23
|
+
where = ["src"]
|
|
24
|
+
|
|
25
|
+
[tool.pytest.ini_options]
|
|
26
|
+
testpaths = ["tests"]
|
|
27
|
+
pythonpath = ["src"]
|
|
28
|
+
|
|
29
|
+
# --- Code-beauty canon (Wisdom Council on Code Beauty, 2026-10-31) --------
|
|
30
|
+
# The surface is surrendered to the formatter; the gates below are the law,
|
|
31
|
+
# run identically by pre-commit and CI (.github/workflows/gates.yml).
|
|
32
|
+
# Prose data files (the Suite A key, bank, and references) carry compact
|
|
33
|
+
# prose literals whose shape IS the readability; by stated policy they sit
|
|
34
|
+
# outside the automated gates entirely — their review is the calibration
|
|
35
|
+
# process, not the linter. Every other file answers to the full canon.
|
|
36
|
+
[tool.ruff]
|
|
37
|
+
target-version = "py311"
|
|
38
|
+
line-length = 88
|
|
39
|
+
extend-exclude = [
|
|
40
|
+
"src/chancellor/evals/suite_a_key.py",
|
|
41
|
+
"src/chancellor/evals/suite_a_bank.py",
|
|
42
|
+
"src/chancellor/evals/suite_a_references.py",
|
|
43
|
+
]
|
|
44
|
+
|
|
45
|
+
[tool.ruff.lint]
|
|
46
|
+
select = ["E", "F", "I", "UP", "B", "SIM", "C4", "RET", "ARG", "ERA"]
|
|
47
|
+
|
|
48
|
+
[tool.ruff.lint.per-file-ignores]
|
|
49
|
+
# Test doubles conform to the interfaces they fake — a stub worker takes
|
|
50
|
+
# the node, a fake transport takes the full call, an approver takes the
|
|
51
|
+
# request — whether or not the double uses them. Unused parameters in
|
|
52
|
+
# tests are the protocol's shape, stated here rather than noqa'd 40 times.
|
|
53
|
+
"tests/**" = ["ARG001", "ARG002", "ARG005"]
|
|
54
|
+
|
|
55
|
+
[tool.mypy]
|
|
56
|
+
python_version = "3.11"
|
|
57
|
+
files = "src/chancellor"
|
|
58
|
+
ignore_missing_imports = true
|
|
59
|
+
warn_unused_ignores = true
|
|
60
|
+
warn_redundant_casts = true
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""Chancellor — governance-and-eval harness for long-running agents.
|
|
2
|
+
|
|
3
|
+
Doctrine:
|
|
4
|
+
* Work is a graph, not a vibe.
|
|
5
|
+
* Output is graded, not trusted.
|
|
6
|
+
* Instruments fail closed.
|
|
7
|
+
* Authority escalates; capability doesn't.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
__version__ = "0.1.0"
|
|
11
|
+
|
|
12
|
+
from .api_client import CostMeter, MetaModelClient, NotConfiguredError
|
|
13
|
+
from .escalation import (
|
|
14
|
+
ActionClass,
|
|
15
|
+
AuditLog,
|
|
16
|
+
AuditWriteError,
|
|
17
|
+
EscalationRequest,
|
|
18
|
+
Gatekeeper,
|
|
19
|
+
)
|
|
20
|
+
from .grader import Criterion, GradeResult, Rubric, grade
|
|
21
|
+
from .instruments import CheckResult, InstrumentPanel, default_panel
|
|
22
|
+
from .judge import Judge, JudgeError
|
|
23
|
+
from .orchestrator import CrewRunner, RunReport
|
|
24
|
+
from .task_graph import (
|
|
25
|
+
SCHEMA_VERSION,
|
|
26
|
+
AttemptCapError,
|
|
27
|
+
IllegalTransitionError,
|
|
28
|
+
NodeStatus,
|
|
29
|
+
TaskGraph,
|
|
30
|
+
TaskNode,
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
__all__ = [
|
|
34
|
+
"TaskGraph",
|
|
35
|
+
"TaskNode",
|
|
36
|
+
"NodeStatus",
|
|
37
|
+
"SCHEMA_VERSION",
|
|
38
|
+
"AttemptCapError",
|
|
39
|
+
"IllegalTransitionError",
|
|
40
|
+
"Rubric",
|
|
41
|
+
"Criterion",
|
|
42
|
+
"GradeResult",
|
|
43
|
+
"grade",
|
|
44
|
+
"InstrumentPanel",
|
|
45
|
+
"CheckResult",
|
|
46
|
+
"default_panel",
|
|
47
|
+
"Gatekeeper",
|
|
48
|
+
"EscalationRequest",
|
|
49
|
+
"ActionClass",
|
|
50
|
+
"AuditLog",
|
|
51
|
+
"AuditWriteError",
|
|
52
|
+
"MetaModelClient",
|
|
53
|
+
"NotConfiguredError",
|
|
54
|
+
"CostMeter",
|
|
55
|
+
"CrewRunner",
|
|
56
|
+
"RunReport",
|
|
57
|
+
"Judge",
|
|
58
|
+
"JudgeError",
|
|
59
|
+
]
|