quantum-cortex 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cortex_c2/__init__.py +161 -0
- cortex_c2/consolidation.py +146 -0
- cortex_c2/router_ablation.py +254 -0
- cortex_c2/router_v2.py +109 -0
- cortex_c2b/__init__.py +724 -0
- cortex_c2b/companion_demo.py +203 -0
- cortex_c2b/crypto.py +102 -0
- cortex_c2b/external_arm.py +591 -0
- cortex_c2b/hm_lm.py +983 -0
- cortex_c2b/hm_protocol.py +308 -0
- cortex_c2b/lifecycle.py +454 -0
- cortex_c2b/lm_bridge.py +338 -0
- cortex_c2b/organ_use.py +254 -0
- cortex_c2b/read_path.py +218 -0
- cortex_c2b/reconstruction_probe.py +471 -0
- cortex_c2b/write_path.py +176 -0
- cortex_data/__init__.py +439 -0
- cortex_data/relay.py +174 -0
- cortex_eval/__init__.py +20 -0
- cortex_eval/ablation_mqar.py +187 -0
- cortex_eval/confirmation_run.py +133 -0
- cortex_eval/domain_map.py +127 -0
- cortex_eval/external_baseline.py +148 -0
- cortex_eval/ingest_confirmation.py +194 -0
- cortex_eval/mqar.py +122 -0
- cortex_eval/multiseed.py +148 -0
- cortex_eval/recurrent_base.py +197 -0
- cortex_eval/recurrent_ladder.py +159 -0
- cortex_eval/resumable_ladder.py +231 -0
- cortex_eval/resumable_rb.py +207 -0
- cortex_eval/run_mqar.py +127 -0
- cortex_eval/serial_position.py +116 -0
- quantum_cortex-1.0.0.dist-info/METADATA +209 -0
- quantum_cortex-1.0.0.dist-info/RECORD +41 -0
- quantum_cortex-1.0.0.dist-info/WHEEL +5 -0
- quantum_cortex-1.0.0.dist-info/licenses/AUTHORS.md +30 -0
- quantum_cortex-1.0.0.dist-info/licenses/LICENSE +73 -0
- quantum_cortex-1.0.0.dist-info/licenses/LICENSE-DOCS.md +34 -0
- quantum_cortex-1.0.0.dist-info/licenses/NOTICE +50 -0
- quantum_cortex-1.0.0.dist-info/top_level.txt +5 -0
- train.py +885 -0
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
"""cortex_eval.ingest_confirmation — grave the confirmation result in one command.
|
|
2
|
+
|
|
3
|
+
Reads the REAL `metrics/mqar/LATEST-confirmation.json` (the Kaggle output — the
|
|
4
|
+
record of truth), applies the whitepaper's admission gate to each comparison,
|
|
5
|
+
and updates three documents mechanically, never inventing a number:
|
|
6
|
+
|
|
7
|
+
1. docs/RESULTS.md — rows 10 (delta vs control) and a new row for Hopfield
|
|
8
|
+
vs control get their real numbers and a status from the
|
|
9
|
+
fixed vocabulary: gated (passes) / held (not established)
|
|
10
|
+
2. docs/WHITEPAPER.md — §4.4's table is rewritten from the artefact; the
|
|
11
|
+
prose states which rows pass, in the same register
|
|
12
|
+
3. metrics/mqar/domain-map.* — regenerated (domain_map.py reads the artefact)
|
|
13
|
+
|
|
14
|
+
Gate applied here (rules 1 and 5 of the six; the others are properties of the
|
|
15
|
+
run, already satisfied or stated): a comparison is **gated** iff the paired
|
|
16
|
+
t-test over seeds is significant at 95% AND n_seeds >= 3; otherwise **held**.
|
|
17
|
+
The reserve line is written for every row, passing or not.
|
|
18
|
+
|
|
19
|
+
Usage: python -m cortex_eval.ingest_confirmation # after committing the artefact
|
|
20
|
+
python -m cortex_eval.ingest_confirmation --dry-run # print what would change
|
|
21
|
+
"""
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import argparse
|
|
25
|
+
import json
|
|
26
|
+
import re
|
|
27
|
+
from datetime import datetime, timezone
|
|
28
|
+
from pathlib import Path
|
|
29
|
+
|
|
30
|
+
REPO = Path(__file__).resolve().parents[1]
|
|
31
|
+
ART = REPO / "metrics" / "mqar" / "LATEST-confirmation.json"
|
|
32
|
+
RESULTS = REPO / "docs" / "RESULTS.md"
|
|
33
|
+
WP = REPO / "docs" / "WHITEPAPER.md"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _status(test: dict, n_seeds: int) -> str:
|
|
37
|
+
return "gated" if (test.get("significant_95") and n_seeds >= 3) else "held"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def read_artefact(path: Path = ART) -> dict:
|
|
41
|
+
r = json.loads(path.read_text())
|
|
42
|
+
seeds = r["regime"]["seeds"]; n = len(seeds)
|
|
43
|
+
pt = r["c2_ablation"]["paired_tests"]; st = r["c2_ablation"]["stats"]
|
|
44
|
+
rows = {}
|
|
45
|
+
for key, a, b in (("hopfield_vs_control", "hopfield", "none"),
|
|
46
|
+
("delta_vs_control", "delta", "none"),
|
|
47
|
+
("delta_vs_hopfield", "delta", "hopfield")):
|
|
48
|
+
# the artefact may name the third comparison the other way round
|
|
49
|
+
# (multiseed.py writes "hopfield_vs_delta"); accept both, flip the sign.
|
|
50
|
+
if key in pt:
|
|
51
|
+
t = dict(pt[key])
|
|
52
|
+
else:
|
|
53
|
+
rev = "_vs_".join(reversed(key.split("_vs_")))
|
|
54
|
+
if rev not in pt:
|
|
55
|
+
raise KeyError(f"artefact has neither {key!r} nor {rev!r}")
|
|
56
|
+
t = dict(pt[rev])
|
|
57
|
+
t["mean_diff"] = -t["mean_diff"]
|
|
58
|
+
if t.get("t") is not None:
|
|
59
|
+
t["t"] = -t["t"]
|
|
60
|
+
rows[key] = {
|
|
61
|
+
"a": a, "b": b, "mean_a": st[a]["mean"], "mean_b": st[b]["mean"],
|
|
62
|
+
"diff": t["mean_diff"], "t": t.get("t"), "df": t.get("df"),
|
|
63
|
+
"t_crit": t.get("t_crit_95"), "sig": bool(t.get("significant_95")),
|
|
64
|
+
"status": _status(t, n),
|
|
65
|
+
"pct": (t["mean_diff"] / st[b]["mean"] * 100) if st[b]["mean"] else None,
|
|
66
|
+
}
|
|
67
|
+
hyb = r.get("hybrid_vs_pure", {})
|
|
68
|
+
lad = r.get("recurrent_ladder", {})
|
|
69
|
+
return {
|
|
70
|
+
"seeds": seeds, "n_seeds": n, "steps": r["regime"]["steps"],
|
|
71
|
+
"tiers": r["regime"]["tiers"], "rows": rows,
|
|
72
|
+
"per_seed": r["c2_ablation"]["per_seed_auc"],
|
|
73
|
+
"hybrid_vs_pure": hyb, "ladder_L4_vs_L3": lad.get("L4_vs_L3"),
|
|
74
|
+
"gate_readout": r.get("gate_readout", {}),
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
# ---------------------------------------------------------------------------- #
|
|
79
|
+
# RESULTS.md #
|
|
80
|
+
# ---------------------------------------------------------------------------- #
|
|
81
|
+
def _results_row(n: int, stmt: str, status: str, number: str, artefact: str, test: str, reserve: str) -> str:
|
|
82
|
+
return f"| {n} | {stmt} | **{status}** | {number} | `{artefact}` | `{test}` | {reserve} |"
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def update_results(a: dict, dry: bool = False) -> str:
|
|
86
|
+
t = RESULTS.read_text(encoding="utf-8")
|
|
87
|
+
n, steps, tiers = a["n_seeds"], a["steps"], a["tiers"]
|
|
88
|
+
tiers_s = ", ".join(f"kv{kv}/seq{sq}" for kv, sq in tiers)
|
|
89
|
+
art = "metrics/mqar/LATEST-confirmation.json"
|
|
90
|
+
d = a["rows"]["delta_vs_control"]; h = a["rows"]["hopfield_vs_control"]
|
|
91
|
+
def num(x):
|
|
92
|
+
return (f"{x['diff']:+.4f} ({x['pct']:+.0f}%), {n}/{n} seeds' mean {x['mean_a']:.4f} vs {x['mean_b']:.4f}, "
|
|
93
|
+
f"t = {x['t']} vs t_crit {x['t_crit']} (df {x['df']})")
|
|
94
|
+
def reserve(x):
|
|
95
|
+
return (f"{n} seeds, {steps} steps, tiers {tiers_s}; " +
|
|
96
|
+
("passes rules 1–6 at this regime" if x["status"] == "gated" else
|
|
97
|
+
"visible signal, not established at this seed count"))
|
|
98
|
+
new_delta = _results_row(10, f"Delta > control on the discriminating tiers, {n} seeds", d["status"], num(d),
|
|
99
|
+
art, "tests/test_multiseed.py", reserve(d))
|
|
100
|
+
new_hop = _results_row("10b", f"Hopfield > control on the discriminating tiers, {n} seeds", h["status"], num(h),
|
|
101
|
+
art, "tests/test_multiseed.py", reserve(h))
|
|
102
|
+
# replace row 10 (whatever its current text) and insert 10b after it
|
|
103
|
+
lines = t.splitlines()
|
|
104
|
+
out = []; done = False
|
|
105
|
+
for l in lines:
|
|
106
|
+
if l.startswith("| 10 |") and not done:
|
|
107
|
+
out.append(new_delta); out.append(new_hop); done = True
|
|
108
|
+
elif l.startswith("| 10b |"):
|
|
109
|
+
continue
|
|
110
|
+
else:
|
|
111
|
+
out.append(l)
|
|
112
|
+
t2 = "\n".join(out) + "\n"
|
|
113
|
+
# gate status line
|
|
114
|
+
gated = [k for k, v in a["rows"].items() if v["status"] == "gated" and k != "delta_vs_hopfield"]
|
|
115
|
+
stamp = datetime.now(timezone.utc).date().isoformat()
|
|
116
|
+
line = (f"**Gate status (whitepaper §2.6):** as of {stamp}, **{len(gated)}** comparison(s) have passed the six-rule gate"
|
|
117
|
+
+ (f" — {', '.join(gated)}" if gated else "") +
|
|
118
|
+
f" (confirmation run: {n} seeds, {steps} steps).")
|
|
119
|
+
t2 = re.sub(r"\*\*Gate status \(whitepaper §2\.6\):\*\*[^\n]*", line, t2, count=1)
|
|
120
|
+
if not dry:
|
|
121
|
+
RESULTS.write_text(t2, encoding="utf-8")
|
|
122
|
+
return t2
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
# ---------------------------------------------------------------------------- #
|
|
126
|
+
# WHITEPAPER §4.4 #
|
|
127
|
+
# ---------------------------------------------------------------------------- #
|
|
128
|
+
def update_whitepaper(a: dict, dry: bool = False) -> str:
|
|
129
|
+
t = WP.read_text(encoding="utf-8")
|
|
130
|
+
start = t.index("**4.4 What passes the gate")
|
|
131
|
+
end = t.index("**4.5 ")
|
|
132
|
+
n, steps = a["n_seeds"], a["steps"]
|
|
133
|
+
tiers_s = " and ".join(f"kv {kv} at seq {sq}" for kv, sq in a["tiers"])
|
|
134
|
+
R = a["rows"]
|
|
135
|
+
def row(label, x):
|
|
136
|
+
verdict = "**passes**" if x["status"] == "gated" else "does not pass"
|
|
137
|
+
return f"| {label} | **{x['diff']:+.4f}** ({x['pct']:+.0f}%) | {x['t']} | {x['t_crit']} | {verdict} |"
|
|
138
|
+
table = "\n".join([
|
|
139
|
+
"| comparison | mean gap | paired t (df = %d) | t_crit 95%% | verdict |" % (n - 1),
|
|
140
|
+
"|:--|--:|--:|--:|:--|",
|
|
141
|
+
row("delta vs control", R["delta_vs_control"]),
|
|
142
|
+
row("Hopfield vs control", R["hopfield_vs_control"]),
|
|
143
|
+
row("delta vs Hopfield", R["delta_vs_hopfield"]),
|
|
144
|
+
])
|
|
145
|
+
gated = [k for k, v in R.items() if v["status"] == "gated" and k != "delta_vs_hopfield"]
|
|
146
|
+
per = a["per_seed"]
|
|
147
|
+
ps = "; ".join(f"{k}: " + " / ".join(f"{v:.3f}" for v in vals) for k, vals in per.items())
|
|
148
|
+
if gated:
|
|
149
|
+
names = " and ".join(k.replace("_vs_control", "") for k in gated)
|
|
150
|
+
prose = (f"At {n} seeds the critical value falls to {R['delta_vs_control']['t_crit']}, and **{names} now pass{'es' if len(gated)==1 else ''} the gate**: "
|
|
151
|
+
f"the comparison{'s' if len(gated)>1 else ''} enter{'s' if len(gated)==1 else ''} §4.3 as a claim on the discriminating regime. "
|
|
152
|
+
f"Per-seed AUC ({ps}). The remaining rows are held as visible signal.")
|
|
153
|
+
else:
|
|
154
|
+
prose = (f"No comparison passes at {n} seeds (per-seed AUC: {ps}). The signal is held, not claimed; "
|
|
155
|
+
f"rule 6 asks which context factor the inter-seed variance carries before a larger run is scheduled.")
|
|
156
|
+
new = (f"**4.4 What passes the gate — the confirmation run ({n} seeds, {steps} steps, tiers {tiers_s}).**\n\n"
|
|
157
|
+
f"{table}\n\n{prose}\n\n")
|
|
158
|
+
t2 = t[:start] + new + t[end:]
|
|
159
|
+
if not dry:
|
|
160
|
+
WP.write_text(t2, encoding="utf-8")
|
|
161
|
+
return new
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def main():
|
|
165
|
+
ap = argparse.ArgumentParser()
|
|
166
|
+
ap.add_argument("--dry-run", action="store_true")
|
|
167
|
+
ap.add_argument("--artefact", default=str(ART))
|
|
168
|
+
ap.add_argument("--expect-seeds", type=int, default=None,
|
|
169
|
+
help="refuse to write unless the artefact has exactly this many seeds "
|
|
170
|
+
"(guards against ingesting the wrong run — lesson of 2026-09-09)")
|
|
171
|
+
args = ap.parse_args()
|
|
172
|
+
a = read_artefact(Path(args.artefact))
|
|
173
|
+
if args.expect_seeds is not None and a["n_seeds"] != args.expect_seeds and not args.dry_run:
|
|
174
|
+
raise SystemExit(f"refusing to write: artefact has {a['n_seeds']} seeds, expected {args.expect_seeds} "
|
|
175
|
+
f"(is this the right run? see metrics/mqar/PROVENANCE-confirmation.json)")
|
|
176
|
+
print(f"=== confirmation artefact: {a['n_seeds']} seeds, {a['steps']} steps, tiers {a['tiers']} ===")
|
|
177
|
+
for k, x in a["rows"].items():
|
|
178
|
+
print(f" {k:22s} diff {x['diff']:+.4f} t={x['t']} crit={x['t_crit']} -> {x['status'].upper()}")
|
|
179
|
+
if a["ladder_L4_vs_L3"]:
|
|
180
|
+
print(f" ladder L4 vs L3: {a['ladder_L4_vs_L3']}")
|
|
181
|
+
print()
|
|
182
|
+
r = update_results(a, dry=args.dry_run)
|
|
183
|
+
w = update_whitepaper(a, dry=args.dry_run)
|
|
184
|
+
if args.dry_run:
|
|
185
|
+
print("--- would write to WHITEPAPER §4.4 ---\n" + w)
|
|
186
|
+
print("--- RESULTS.md gate line ---")
|
|
187
|
+
print([l for l in r.splitlines() if l.startswith("**Gate status")][0])
|
|
188
|
+
else:
|
|
189
|
+
print("RESULTS.md and WHITEPAPER.md §4.4 updated. Now regenerate the domain map:")
|
|
190
|
+
print(" python -m cortex_eval.domain_map")
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
if __name__ == "__main__":
|
|
194
|
+
main()
|
cortex_eval/mqar.py
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
"""cortex_eval.mqar — Multi-Query Associative Recall, by difficulty tiers.
|
|
2
|
+
|
|
3
|
+
The canonical probe for associative memory (Arora et al. 2023, the Zoology
|
|
4
|
+
lineage; validated as the C2 judge by Kimi Linear's KDA experiments, which
|
|
5
|
+
sweep sequence length 256→2048 and report the *curve*, not a point).
|
|
6
|
+
Re-implemented from scratch (the paper's format is simple); zero dependency.
|
|
7
|
+
|
|
8
|
+
Format (Arora): a sequence [k1,v1, ..., kn,vn, SEP, q1, ..., qm] where each
|
|
9
|
+
query qj repeats an earlier key kj, and the model must emit the associated vj.
|
|
10
|
+
|
|
11
|
+
The founder's principle (2026-09-06): never measure at a single point that may
|
|
12
|
+
be flat — sweep a difficulty scale and read the CURVE. Difficulty rises on two
|
|
13
|
+
axes Kimi documents: number of kv pairs and sequence length. An architecture is
|
|
14
|
+
characterised by *where its curve drops off*; the gap between two architectures'
|
|
15
|
+
drop-off points is the clean signal (never flat, because we always read the
|
|
16
|
+
regime where they separate). This also calibrates the (a)->(b) bridge: the tiers
|
|
17
|
+
tell us at which difficulty to expect signal before probing a language model.
|
|
18
|
+
"""
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
from dataclasses import dataclass, field
|
|
22
|
+
|
|
23
|
+
import numpy as np
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
# Reserved control tokens live at the top of the byte vocab (0..255); MQAR keys
|
|
27
|
+
# and values are drawn from a symbol range that leaves a SEP slot free.
|
|
28
|
+
SEP = 255 # separator symbol (distinct from keys/values below)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass
|
|
32
|
+
class MQARTier:
|
|
33
|
+
"""One difficulty tier."""
|
|
34
|
+
kv_pairs: int
|
|
35
|
+
seq_len: int
|
|
36
|
+
n_symbols: int = 128 # key/value alphabet size (< SEP)
|
|
37
|
+
n_queries: int | None = None # default = kv_pairs (query every key once)
|
|
38
|
+
|
|
39
|
+
def queries(self) -> int:
|
|
40
|
+
return self.n_queries if self.n_queries is not None else self.kv_pairs
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def make_batch(tier: MQARTier, batch: int, seed: int) -> tuple[np.ndarray, np.ndarray]:
|
|
44
|
+
"""Generate an MQAR batch → (inputs, targets), both int64 (b, T).
|
|
45
|
+
|
|
46
|
+
targets are -100 (ignore) everywhere except at query positions, where they
|
|
47
|
+
hold the value associated with that query's key. This matches a standard
|
|
48
|
+
cross-entropy-with-ignore training/eval setup.
|
|
49
|
+
"""
|
|
50
|
+
rng = np.random.default_rng(seed)
|
|
51
|
+
k = tier.kv_pairs
|
|
52
|
+
q = tier.queries()
|
|
53
|
+
# layout length: 2*k (pairs) + 1 (SEP) + q (queries). Pad/truncate to seq_len.
|
|
54
|
+
core = 2 * k + 1 + q
|
|
55
|
+
T = max(tier.seq_len, core)
|
|
56
|
+
X = np.zeros((batch, T), dtype=np.int64)
|
|
57
|
+
Y = np.full((batch, T), -100, dtype=np.int64)
|
|
58
|
+
|
|
59
|
+
for b in range(batch):
|
|
60
|
+
# distinct keys, random values (values may repeat across keys)
|
|
61
|
+
keys = rng.choice(tier.n_symbols, size=k, replace=False)
|
|
62
|
+
vals = rng.integers(0, tier.n_symbols, size=k)
|
|
63
|
+
kv = {int(keys[i]): int(vals[i]) for i in range(k)}
|
|
64
|
+
|
|
65
|
+
pos = 0
|
|
66
|
+
for i in range(k):
|
|
67
|
+
X[b, pos] = keys[i]; pos += 1
|
|
68
|
+
X[b, pos] = vals[i]; pos += 1
|
|
69
|
+
X[b, pos] = SEP; pos += 1
|
|
70
|
+
# choose which keys to query (with repetition allowed → "multi-query")
|
|
71
|
+
qkeys = rng.choice(keys, size=q, replace=True)
|
|
72
|
+
for j in range(q):
|
|
73
|
+
X[b, pos] = int(qkeys[j])
|
|
74
|
+
Y[b, pos] = kv[int(qkeys[j])] # supervise the answer at the query slot
|
|
75
|
+
pos += 1
|
|
76
|
+
# remaining positions (if T > core) stay zero, targets stay ignore
|
|
77
|
+
return X, Y
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def standard_curriculum() -> list[MQARTier]:
|
|
81
|
+
"""The tier ladder (Kimi/Zoology regime): kv ∈ {4,8,16,32}, seq ∈ {128,256,512}.
|
|
82
|
+
Easy tiers give non-zero signal even for the vanilla control (avoids the
|
|
83
|
+
flat-zero); hard tiers separate the architectures.
|
|
84
|
+
"""
|
|
85
|
+
tiers = []
|
|
86
|
+
for seq in (128, 256, 512):
|
|
87
|
+
for kv in (4, 8, 16, 32):
|
|
88
|
+
tiers.append(MQARTier(kv_pairs=kv, seq_len=seq))
|
|
89
|
+
return tiers
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def accuracy(logits_at_queries: np.ndarray, targets_at_queries: np.ndarray) -> float:
|
|
93
|
+
"""Exact-match accuracy over query positions. Inputs are already gathered
|
|
94
|
+
at query slots: preds (N,) argmax and targets (N,)."""
|
|
95
|
+
if len(targets_at_queries) == 0:
|
|
96
|
+
return float("nan")
|
|
97
|
+
return float((logits_at_queries == targets_at_queries).mean())
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@dataclass
|
|
101
|
+
class MQARResult:
|
|
102
|
+
"""The deliverable is the curve: accuracy per tier."""
|
|
103
|
+
per_tier: list[dict] = field(default_factory=list)
|
|
104
|
+
|
|
105
|
+
def add(self, tier: MQARTier, acc: float):
|
|
106
|
+
self.per_tier.append({"kv_pairs": tier.kv_pairs, "seq_len": tier.seq_len, "accuracy": acc})
|
|
107
|
+
|
|
108
|
+
def dropoff_kv(self, threshold: float = 0.5) -> int | None:
|
|
109
|
+
"""The kv level (at the smallest seq_len) where accuracy first drops
|
|
110
|
+
below threshold — a scalar summary of *where the curve breaks*."""
|
|
111
|
+
by_seq = min((t["seq_len"] for t in self.per_tier), default=None)
|
|
112
|
+
if by_seq is None:
|
|
113
|
+
return None
|
|
114
|
+
row = sorted((t for t in self.per_tier if t["seq_len"] == by_seq),
|
|
115
|
+
key=lambda t: t["kv_pairs"])
|
|
116
|
+
for t in row:
|
|
117
|
+
if t["accuracy"] < threshold:
|
|
118
|
+
return t["kv_pairs"]
|
|
119
|
+
return None # never dropped below threshold in this range
|
|
120
|
+
|
|
121
|
+
def as_dict(self) -> dict:
|
|
122
|
+
return {"per_tier": self.per_tier, "dropoff_kv_@0.5": self.dropoff_kv()}
|
cortex_eval/multiseed.py
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
"""cortex_eval.multiseed — make N2 unattackable (D19 Level 1a).
|
|
2
|
+
|
|
3
|
+
A single-seed result is an anecdote. This module runs the C2 ablation across N
|
|
4
|
+
seeds and reports, per variant: mean AUC, standard deviation, and a 95%
|
|
5
|
+
confidence interval — then tests whether the gaps (hopfield vs control, delta vs
|
|
6
|
+
control) are STATISTICALLY SIGNIFICANT. That is the difference between
|
|
7
|
+
"Hopfield +14%" and "Hopfield +14% ± 3%, p < 0.05".
|
|
8
|
+
|
|
9
|
+
Statistics kept simple and honest (from scratch, no scipy dependency):
|
|
10
|
+
- CI 95% via the t-distribution for small N (t critical values tabulated).
|
|
11
|
+
- Significance via a paired comparison across seeds (same seed → same data
|
|
12
|
+
order for all variants, so pairing is legitimate): a paired t-test on the
|
|
13
|
+
per-seed AUC differences. We report the t statistic and whether |t| exceeds
|
|
14
|
+
the two-sided 95% critical value — a claim only when it does.
|
|
15
|
+
Honest-data rule: every seed's curve is stored; nothing is dropped.
|
|
16
|
+
"""
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import json
|
|
20
|
+
import math
|
|
21
|
+
from datetime import datetime, timezone
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
|
|
24
|
+
import numpy as np
|
|
25
|
+
|
|
26
|
+
from cortex_eval.mqar import MQARTier, standard_curriculum
|
|
27
|
+
from cortex_eval.run_mqar import run_curriculum
|
|
28
|
+
|
|
29
|
+
VARIANTS = ("none", "hopfield", "delta")
|
|
30
|
+
|
|
31
|
+
# two-sided 95% t critical values by degrees of freedom (df = n-1)
|
|
32
|
+
_T95 = {1: 12.706, 2: 4.303, 3: 3.182, 4: 2.776, 5: 2.571, 6: 2.447,
|
|
33
|
+
7: 2.365, 8: 2.306, 9: 2.262, 10: 2.228, 15: 2.131, 20: 2.086, 30: 2.042}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _t95(df: int) -> float:
|
|
37
|
+
if df in _T95:
|
|
38
|
+
return _T95[df]
|
|
39
|
+
keys = sorted(_T95)
|
|
40
|
+
for k in keys:
|
|
41
|
+
if k >= df:
|
|
42
|
+
return _T95[k]
|
|
43
|
+
return 1.96
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _auc(curve: list[dict]) -> float:
|
|
47
|
+
accs = [t["accuracy"] for t in curve if t["accuracy"] == t["accuracy"]]
|
|
48
|
+
return float(np.mean(accs)) if accs else float("nan")
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _mean_ci(values: list[float]) -> dict:
|
|
52
|
+
v = np.asarray(values, dtype=float)
|
|
53
|
+
n = len(v)
|
|
54
|
+
mean = float(v.mean())
|
|
55
|
+
sd = float(v.std(ddof=1)) if n > 1 else 0.0
|
|
56
|
+
half = _t95(n - 1) * sd / math.sqrt(n) if n > 1 else float("nan")
|
|
57
|
+
return {"mean": round(mean, 4), "sd": round(sd, 4),
|
|
58
|
+
"ci95": [round(mean - half, 4), round(mean + half, 4)] if n > 1 else None,
|
|
59
|
+
"n": n}
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _paired_test(a: list[float], b: list[float]) -> dict:
|
|
63
|
+
"""Paired t-test on per-seed differences a - b. Returns t, df, significant@95%."""
|
|
64
|
+
d = np.asarray(a, dtype=float) - np.asarray(b, dtype=float)
|
|
65
|
+
n = len(d)
|
|
66
|
+
if n < 2:
|
|
67
|
+
return {"mean_diff": round(float(d.mean()), 4), "t": None, "df": n - 1,
|
|
68
|
+
"significant_95": None, "note": "need ≥2 seeds"}
|
|
69
|
+
md = float(d.mean()); sd = float(d.std(ddof=1))
|
|
70
|
+
t = md / (sd / math.sqrt(n)) if sd > 1e-12 else float("inf")
|
|
71
|
+
crit = _t95(n - 1)
|
|
72
|
+
return {"mean_diff": round(md, 4), "t": round(t, 3), "df": n - 1,
|
|
73
|
+
"t_crit_95": crit, "significant_95": bool(abs(t) > crit)}
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def run_multiseed(seeds: list[int], steps: int = 1500,
|
|
77
|
+
tiers: list[MQARTier] | None = None, **kw) -> dict:
|
|
78
|
+
tiers = tiers or standard_curriculum()
|
|
79
|
+
per_seed: dict[str, list[float]] = {v: [] for v in VARIANTS}
|
|
80
|
+
curves: dict[str, list[list[dict]]] = {v: [] for v in VARIANTS}
|
|
81
|
+
for s in seeds:
|
|
82
|
+
for v in VARIANTS:
|
|
83
|
+
res = run_curriculum(c2_variant=v, steps=steps, tiers=tiers, seed=s, **kw)
|
|
84
|
+
per_seed[v].append(_auc(res.per_tier))
|
|
85
|
+
curves[v].append(res.per_tier) # every curve stored, nothing dropped
|
|
86
|
+
print(f"[multiseed] seed={s} " + " ".join(f"{v}={per_seed[v][-1]:.4f}" for v in VARIANTS))
|
|
87
|
+
|
|
88
|
+
stats = {v: _mean_ci(per_seed[v]) for v in VARIANTS}
|
|
89
|
+
tests = {
|
|
90
|
+
"hopfield_vs_control": _paired_test(per_seed["hopfield"], per_seed["none"]),
|
|
91
|
+
"delta_vs_control": _paired_test(per_seed["delta"], per_seed["none"]),
|
|
92
|
+
"hopfield_vs_delta": _paired_test(per_seed["hopfield"], per_seed["delta"]),
|
|
93
|
+
}
|
|
94
|
+
# honest verdict: a claim only where the paired test is significant
|
|
95
|
+
claims = []
|
|
96
|
+
for name, tst in tests.items():
|
|
97
|
+
if tst.get("significant_95"):
|
|
98
|
+
sign = "+" if tst["mean_diff"] > 0 else ""
|
|
99
|
+
claims.append(f"{name}: {sign}{tst['mean_diff']} AUC (t={tst['t']}, df={tst['df']}) — significant at 95%")
|
|
100
|
+
else:
|
|
101
|
+
claims.append(f"{name}: {tst['mean_diff']} AUC — NOT significant at 95% (t={tst.get('t')})")
|
|
102
|
+
return {
|
|
103
|
+
"benchmark": "mqar_multiseed",
|
|
104
|
+
"timestamp_utc": datetime.now(timezone.utc).isoformat(),
|
|
105
|
+
"regime": {"seeds": seeds, "n_seeds": len(seeds), "steps": steps, "tiers": len(tiers)},
|
|
106
|
+
"per_seed_auc": per_seed,
|
|
107
|
+
"stats": stats,
|
|
108
|
+
"paired_tests": tests,
|
|
109
|
+
"claims": claims,
|
|
110
|
+
"curves_per_seed": curves,
|
|
111
|
+
"note": "D19 Level 1a — the single-seed N2 result made statistical. Paired tests are legitimate "
|
|
112
|
+
"because the same seed fixes the data order across variants. A gap is claimed only when "
|
|
113
|
+
"significant at 95%; otherwise reported as not significant (honest data).",
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def write_artefact(result: dict, out_dir: str | Path) -> Path:
|
|
118
|
+
out = Path(out_dir); out.mkdir(parents=True, exist_ok=True)
|
|
119
|
+
stamp = result["timestamp_utc"][:19].replace(":", "").replace("-", "")
|
|
120
|
+
path = out / f"mqar-multiseed-{stamp}.json"
|
|
121
|
+
path.write_text(json.dumps(result, indent=2) + "\n")
|
|
122
|
+
(out / "LATEST-multiseed.json").write_text(json.dumps(result, indent=2) + "\n")
|
|
123
|
+
return path
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _cli():
|
|
127
|
+
import argparse
|
|
128
|
+
ap = argparse.ArgumentParser(description="Multi-seed C2 ablation with CIs (D19 Level 1a)")
|
|
129
|
+
ap.add_argument("--seeds", type=int, nargs="+", default=[1337, 2024, 7, 42, 99])
|
|
130
|
+
ap.add_argument("--steps", type=int, default=1500)
|
|
131
|
+
ap.add_argument("--out", default="metrics/mqar")
|
|
132
|
+
ap.add_argument("--quick", action="store_true")
|
|
133
|
+
args = ap.parse_args()
|
|
134
|
+
tiers = [MQARTier(kv_pairs=4, seq_len=64), MQARTier(kv_pairs=16, seq_len=64)] if args.quick else None
|
|
135
|
+
steps = 300 if args.quick else args.steps
|
|
136
|
+
r = run_multiseed(args.seeds, steps=steps, tiers=tiers)
|
|
137
|
+
p = write_artefact(r, args.out)
|
|
138
|
+
print("\n=== multi-seed stats ===")
|
|
139
|
+
for v, st in r["stats"].items():
|
|
140
|
+
print(f" {v:9s}: mean {st['mean']} sd {st['sd']} CI95 {st['ci95']} (n={st['n']})")
|
|
141
|
+
print("=== claims ===")
|
|
142
|
+
for c in r["claims"]:
|
|
143
|
+
print(" ", c)
|
|
144
|
+
print(f"artefact: {p}")
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
if __name__ == "__main__":
|
|
148
|
+
_cli()
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
"""cortex_eval.recurrent_base -- the recurrent base hybrid family (register Rev38; hypotheses, no result yet).
|
|
2
|
+
|
|
3
|
+
The 8 seed ladder (2026-09-12) showed two regimes: the pure recurrent rung with a
|
|
4
|
+
local convolution (L3) takes off on 5 seeds out of 8 at 8 key value pairs
|
|
5
|
+
(0.47 to 0.67) and stays at the floor at 16; the transformer control and the
|
|
6
|
+
transformer based hybrids are stable around 0.09 and win at 16 pairs. The
|
|
7
|
+
founder's frame inverts the hybridisation: L3 as the BASE, the attention as the
|
|
8
|
+
ADDED path, zero initialised, taken only where it beats the recurrent. The risk
|
|
9
|
+
named before any code: the attention learns the easy solution first and the
|
|
10
|
+
recurrent never takes off (pre emption), as in the transformer based hybrids.
|
|
11
|
+
|
|
12
|
+
Four arms, declared before the run, one variable at a time:
|
|
13
|
+
|
|
14
|
+
bare the added attention open from step zero, no protection -- the base
|
|
15
|
+
line, and the direct measure of pre emption.
|
|
16
|
+
critical the critical period (founder): the attention's gate stays closed
|
|
17
|
+
until the trunk's training loss plateaus (a measured criterion, not a
|
|
18
|
+
step count), then opens. Until it opens the arm IS L3, to the update.
|
|
19
|
+
cls complementary learning systems (the assistant's design): the
|
|
20
|
+
attention learns immediately (the fast system); a consolidation
|
|
21
|
+
term trains the trunk only prediction to reproduce the full model's
|
|
22
|
+
prediction (the replay teaches the slow system); where the trunk
|
|
23
|
+
alone reaches the full model, the gate retracts geometrically (the
|
|
24
|
+
fast trace fades). Ratios: retraction 1/2 per phase, attention width
|
|
25
|
+
1/2 of the model.
|
|
26
|
+
cls-phi the same with phi where a ratio is needed: retraction 1/phi,
|
|
27
|
+
attention width 1/phi^2 (0.382) of the model. A declared arm against
|
|
28
|
+
the 1/2 control -- nowhere else does phi enter.
|
|
29
|
+
|
|
30
|
+
Every unit records the full model's accuracy (the paired comparisons), the trunk
|
|
31
|
+
only accuracy (did the base take off?), the consolidation gap, the gate at the
|
|
32
|
+
end, and when the gate opened or retracted. Written from scratch (our filon).
|
|
33
|
+
"""
|
|
34
|
+
from __future__ import annotations
|
|
35
|
+
|
|
36
|
+
import math
|
|
37
|
+
from dataclasses import dataclass, asdict
|
|
38
|
+
|
|
39
|
+
import numpy as np
|
|
40
|
+
|
|
41
|
+
from cortex_data import _hash_obj
|
|
42
|
+
from cortex_eval.mqar import MQARTier, make_batch
|
|
43
|
+
from cortex_eval.recurrent_ladder import LADDER, Rung, VOCAB, _build
|
|
44
|
+
|
|
45
|
+
PHI = (1.0 + 5 ** 0.5) / 2.0
|
|
46
|
+
L3 = next(r for r in LADDER if r.name == "L3-+local-conv")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@dataclass(frozen=True)
|
|
50
|
+
class RBArm:
|
|
51
|
+
name: str
|
|
52
|
+
mode: str # bare | critical | cls
|
|
53
|
+
retraction: float = 0.5 # gate multiplier per retraction (cls)
|
|
54
|
+
attn_frac: float = 0.5 # attention inner width as a fraction of d_model
|
|
55
|
+
plateau_window: int = 100 # critical: loss window (steps)
|
|
56
|
+
plateau_tol: float = 0.01 # critical: relative improvement under which the gate opens
|
|
57
|
+
plateau_min_steps: int = 200 # critical: never before
|
|
58
|
+
consolidation_weight: float = 1.0 # cls: weight of the replay term
|
|
59
|
+
retract_every: int = 100 # cls: phase length (steps)
|
|
60
|
+
takeoff_threshold: float = 0.3 # trunk only accuracy that counts as a take off
|
|
61
|
+
gap_epsilon: float = 0.02 # cls: retract when full - trunk only < epsilon (and the trunk took off)
|
|
62
|
+
|
|
63
|
+
def config_hash(self) -> str:
|
|
64
|
+
return _hash_obj(asdict(self))
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
ARMS = [
|
|
68
|
+
RBArm("RB-bare", "bare"),
|
|
69
|
+
RBArm("RB-critical", "critical"),
|
|
70
|
+
RBArm("RB-cls", "cls", retraction=0.5, attn_frac=0.5),
|
|
71
|
+
RBArm("RB-cls-phi", "cls", retraction=1.0 / PHI, attn_frac=1.0 / PHI ** 2),
|
|
72
|
+
]
|
|
73
|
+
ARM_BY_NAME = {a.name: a for a in ARMS}
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _build_rb(arm: RBArm, d_model: int, n_head: int, seed: int):
|
|
77
|
+
"""The trunk is built FIRST, under the seed, so its parameters are exactly a
|
|
78
|
+
standalone L3's (the no op law is testable to the bit); the attention path
|
|
79
|
+
draws its parameters after."""
|
|
80
|
+
import torch, torch.nn as nn
|
|
81
|
+
torch.manual_seed(seed)
|
|
82
|
+
trunk = _build(L3, d_model, n_head)
|
|
83
|
+
attn_dim = max(n_head, int(round(arm.attn_frac * d_model)) // n_head * n_head)
|
|
84
|
+
|
|
85
|
+
class RecurrentBaseHybrid(nn.Module):
|
|
86
|
+
def __init__(self):
|
|
87
|
+
super().__init__()
|
|
88
|
+
self.trunk = trunk
|
|
89
|
+
self.ln_a = nn.LayerNorm(d_model)
|
|
90
|
+
self.qkv = nn.Linear(d_model, 3 * attn_dim, bias=False)
|
|
91
|
+
self.proj = nn.Linear(attn_dim, d_model, bias=False)
|
|
92
|
+
nn.init.zeros_(self.proj.weight) # the no op law: at init the model is L3
|
|
93
|
+
self.register_buffer("gate", torch.tensor(1.0 if arm.mode != "critical" else 0.0))
|
|
94
|
+
self.n_head, self.dk = n_head, attn_dim // n_head
|
|
95
|
+
|
|
96
|
+
def trunk_hidden(self, idx):
|
|
97
|
+
"""The recurrent trunk up to its residual (the expensive scan), shared by
|
|
98
|
+
the full and the trunk only paths."""
|
|
99
|
+
t = self.trunk
|
|
100
|
+
x = t.emb(idx)
|
|
101
|
+
b, T, c = x.shape
|
|
102
|
+
z = t.ln(x)
|
|
103
|
+
z = t.conv(z.transpose(1, 2))[:, :, :T].transpose(1, 2)
|
|
104
|
+
q, k, v = t.qkv(z).split(c, dim=2)
|
|
105
|
+
nh, dk = n_head, c // n_head
|
|
106
|
+
q = torch.nn.functional.normalize(q.view(b, T, nh, dk), dim=-1)
|
|
107
|
+
k = torch.nn.functional.normalize(k.view(b, T, nh, dk), dim=-1)
|
|
108
|
+
v = v.view(b, T, nh, dk)
|
|
109
|
+
g = torch.sigmoid(t.gate(z)).view(b, T, nh, dk)
|
|
110
|
+
S = torch.zeros(b, nh, dk, dk, device=x.device)
|
|
111
|
+
outs = []
|
|
112
|
+
for i in range(T):
|
|
113
|
+
S = S * g[:, i].unsqueeze(2)
|
|
114
|
+
S = S + torch.einsum('bhi,bhj->bhij', k[:, i], v[:, i])
|
|
115
|
+
outs.append(torch.einsum('bhi,bhij->bhj', q[:, i], S).reshape(b, c))
|
|
116
|
+
return x + t.out(torch.stack(outs, 1))
|
|
117
|
+
|
|
118
|
+
def attention(self, h):
|
|
119
|
+
b, T, _ = h.shape
|
|
120
|
+
q, k, v = self.qkv(self.ln_a(h)).split(self.n_head * self.dk, dim=2)
|
|
121
|
+
q = q.view(b, T, self.n_head, self.dk).transpose(1, 2)
|
|
122
|
+
k = k.view(b, T, self.n_head, self.dk).transpose(1, 2)
|
|
123
|
+
v = v.view(b, T, self.n_head, self.dk).transpose(1, 2)
|
|
124
|
+
att = (q @ k.transpose(-2, -1)) * self.dk ** -0.5
|
|
125
|
+
mask = torch.tril(torch.ones(T, T, dtype=torch.bool, device=h.device))
|
|
126
|
+
att = att.masked_fill(~mask, float("-inf")).softmax(-1)
|
|
127
|
+
return self.proj((att @ v).transpose(1, 2).reshape(b, T, self.n_head * self.dk))
|
|
128
|
+
|
|
129
|
+
def forward(self, idx, gate=None):
|
|
130
|
+
"""Returns (full logits, trunk only logits). The trunk is computed once;
|
|
131
|
+
the two paths differ by the gated attention term only."""
|
|
132
|
+
g = self.gate if gate is None else torch.as_tensor(float(gate))
|
|
133
|
+
t = self.trunk
|
|
134
|
+
h = self.trunk_hidden(idx)
|
|
135
|
+
trunk_only = t.head(h + t.mlp(t.ln2(h)))
|
|
136
|
+
if float(g) == 0.0:
|
|
137
|
+
return trunk_only, trunk_only
|
|
138
|
+
hf = h + g * self.attention(h)
|
|
139
|
+
full = t.head(hf + t.mlp(t.ln2(hf)))
|
|
140
|
+
return full, trunk_only
|
|
141
|
+
|
|
142
|
+
return RecurrentBaseHybrid()
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _acc(model, tier: MQARTier, seed: int, batch: int, which: str) -> float:
|
|
146
|
+
import torch
|
|
147
|
+
Xe, Ye = make_batch(tier, batch, seed=seed)
|
|
148
|
+
with torch.no_grad():
|
|
149
|
+
full, trunk = model(torch.from_numpy(Xe))
|
|
150
|
+
preds = (full if which == "full" else trunk).argmax(-1).numpy()
|
|
151
|
+
mask = Ye != -100
|
|
152
|
+
return float((preds[mask] == Ye[mask]).mean())
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def run_rb_unit(arm: RBArm, tier: MQARTier, steps: int, seed: int, d_model: int = 64, n_head: int = 2,
|
|
156
|
+
batch: int = 32, lr: float = 3e-3, eval_batch: int = 64) -> dict:
|
|
157
|
+
"""One unit: the arm on one tier, one seed. Same data, same optimiser, same
|
|
158
|
+
steps as the pure ladder, so the paired comparison against L3 is exact."""
|
|
159
|
+
import torch, torch.nn as nn
|
|
160
|
+
model = _build_rb(arm, d_model, n_head, seed)
|
|
161
|
+
opt = torch.optim.AdamW(model.parameters(), lr=lr)
|
|
162
|
+
losses, opened_at, retractions = [], None, []
|
|
163
|
+
for step in range(steps):
|
|
164
|
+
X, Y = make_batch(tier, batch, seed=seed + step)
|
|
165
|
+
idx, tgt = torch.from_numpy(X), torch.from_numpy(Y)
|
|
166
|
+
full, trunk = model(idx)
|
|
167
|
+
loss = nn.functional.cross_entropy(full.view(-1, VOCAB), tgt.view(-1), ignore_index=-100)
|
|
168
|
+
if arm.mode == "cls" and float(model.gate) > 0.0:
|
|
169
|
+
# the replay: the trunk alone learns the full model's prediction at the queries
|
|
170
|
+
m = (tgt != -100)
|
|
171
|
+
teacher = full.detach()[m].softmax(-1)
|
|
172
|
+
student = trunk[m].log_softmax(-1)
|
|
173
|
+
loss = loss + arm.consolidation_weight * nn.functional.kl_div(student, teacher, reduction="batchmean")
|
|
174
|
+
if not torch.isfinite(loss):
|
|
175
|
+
return {"accuracy": float("nan"), "status": "diverged"}
|
|
176
|
+
opt.zero_grad(); loss.backward()
|
|
177
|
+
nn.utils.clip_grad_norm_(model.parameters(), 1.0); opt.step()
|
|
178
|
+
losses.append(float(loss.detach()))
|
|
179
|
+
# the critical period: open the gate when the trunk's loss has plateaued
|
|
180
|
+
if arm.mode == "critical" and opened_at is None and step + 1 >= max(arm.plateau_min_steps, 2 * arm.plateau_window):
|
|
181
|
+
w = arm.plateau_window
|
|
182
|
+
recent, before = np.mean(losses[-w:]), np.mean(losses[-2 * w:-w])
|
|
183
|
+
if before - recent < arm.plateau_tol * before:
|
|
184
|
+
model.gate.fill_(1.0); opened_at = step + 1
|
|
185
|
+
# the retraction: where the trunk alone has caught up, the fast trace fades
|
|
186
|
+
if arm.mode == "cls" and (step + 1) % arm.retract_every == 0 and float(model.gate) > 1e-3:
|
|
187
|
+
af = _acc(model, tier, seed + 20_000, eval_batch, "full")
|
|
188
|
+
at = _acc(model, tier, seed + 20_000, eval_batch, "trunk")
|
|
189
|
+
if at >= arm.takeoff_threshold and af - at < arm.gap_epsilon:
|
|
190
|
+
model.gate.mul_(arm.retraction); retractions.append(step + 1)
|
|
191
|
+
acc_full = _acc(model, tier, seed + 10_000, eval_batch, "full")
|
|
192
|
+
acc_trunk = _acc(model, tier, seed + 10_000, eval_batch, "trunk")
|
|
193
|
+
return {"accuracy": acc_full, "accuracy_trunk_only": acc_trunk,
|
|
194
|
+
"consolidation_gap": round(acc_full - acc_trunk, 4),
|
|
195
|
+
"took_off": bool(acc_trunk >= arm.takeoff_threshold),
|
|
196
|
+
"gate_final": round(float(model.gate), 4), "opened_at": opened_at, "retractions": retractions,
|
|
197
|
+
"arm_hash": arm.config_hash(), "status": "done"}
|