quantum-cortex 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cortex_c2/__init__.py +161 -0
- cortex_c2/consolidation.py +146 -0
- cortex_c2/router_ablation.py +254 -0
- cortex_c2/router_v2.py +109 -0
- cortex_c2b/__init__.py +724 -0
- cortex_c2b/companion_demo.py +203 -0
- cortex_c2b/crypto.py +102 -0
- cortex_c2b/external_arm.py +591 -0
- cortex_c2b/hm_lm.py +983 -0
- cortex_c2b/hm_protocol.py +308 -0
- cortex_c2b/lifecycle.py +454 -0
- cortex_c2b/lm_bridge.py +338 -0
- cortex_c2b/organ_use.py +254 -0
- cortex_c2b/read_path.py +218 -0
- cortex_c2b/reconstruction_probe.py +471 -0
- cortex_c2b/write_path.py +176 -0
- cortex_data/__init__.py +439 -0
- cortex_data/relay.py +174 -0
- cortex_eval/__init__.py +20 -0
- cortex_eval/ablation_mqar.py +187 -0
- cortex_eval/confirmation_run.py +133 -0
- cortex_eval/domain_map.py +127 -0
- cortex_eval/external_baseline.py +148 -0
- cortex_eval/ingest_confirmation.py +194 -0
- cortex_eval/mqar.py +122 -0
- cortex_eval/multiseed.py +148 -0
- cortex_eval/recurrent_base.py +197 -0
- cortex_eval/recurrent_ladder.py +159 -0
- cortex_eval/resumable_ladder.py +231 -0
- cortex_eval/resumable_rb.py +207 -0
- cortex_eval/run_mqar.py +127 -0
- cortex_eval/serial_position.py +116 -0
- quantum_cortex-1.0.0.dist-info/METADATA +209 -0
- quantum_cortex-1.0.0.dist-info/RECORD +41 -0
- quantum_cortex-1.0.0.dist-info/WHEEL +5 -0
- quantum_cortex-1.0.0.dist-info/licenses/AUTHORS.md +30 -0
- quantum_cortex-1.0.0.dist-info/licenses/LICENSE +73 -0
- quantum_cortex-1.0.0.dist-info/licenses/LICENSE-DOCS.md +34 -0
- quantum_cortex-1.0.0.dist-info/licenses/NOTICE +50 -0
- quantum_cortex-1.0.0.dist-info/top_level.txt +5 -0
- train.py +885 -0
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
"""cortex_eval.ablation_mqar — Slice C: the C2 ablation on MQAR.
|
|
2
|
+
|
|
3
|
+
Runs the MQAR difficulty curriculum on the three variants (control / hopfield /
|
|
4
|
+
delta) at the *same* regime, produces the three accuracy curves, compares them,
|
|
5
|
+
and emits the verdict — the second point on the Pareto frontier, the first that
|
|
6
|
+
*decides* between architectures.
|
|
7
|
+
|
|
8
|
+
Per the founder (2026-09-06): full curriculum first for a solid base, then a
|
|
9
|
+
fast pass driven by insights. Results are a SEPARATE artefact (metrics/mqar/),
|
|
10
|
+
not a run-v1 ledger record — MQAR is an architecture bench, distinct from the
|
|
11
|
+
language-training runs whose `comparison` slot fills at the real FineWeb-Edu
|
|
12
|
+
ablations (W1 proper). NOW-6: benchmarks as open, versioned deliverables.
|
|
13
|
+
|
|
14
|
+
The reading rule (D16 + the founder's curve principle): an architecture is
|
|
15
|
+
characterised by *where its curve drops off*; the gap between drop-off points is
|
|
16
|
+
the clean signal. The circuit breaker (ADR-006 D7) applies per tier per variant.
|
|
17
|
+
|
|
18
|
+
Usage:
|
|
19
|
+
python -m cortex_eval.ablation_mqar --steps 1500 --out metrics/mqar
|
|
20
|
+
python -m cortex_eval.ablation_mqar --quick # tiny, for a smoke check
|
|
21
|
+
"""
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import argparse
|
|
25
|
+
import json
|
|
26
|
+
import time
|
|
27
|
+
from datetime import datetime, timezone
|
|
28
|
+
from pathlib import Path
|
|
29
|
+
|
|
30
|
+
from cortex_eval.mqar import MQARTier, standard_curriculum
|
|
31
|
+
from cortex_eval.run_mqar import run_curriculum
|
|
32
|
+
|
|
33
|
+
VARIANTS = ("none", "hopfield", "delta")
|
|
34
|
+
REPO = Path(__file__).resolve().parents[1]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _dropoff(per_tier: list[dict], threshold: float, axis: str = "kv_pairs") -> dict:
|
|
38
|
+
"""Where each curve first drops below threshold, per the smallest other-axis."""
|
|
39
|
+
other = "seq_len" if axis == "kv_pairs" else "kv_pairs"
|
|
40
|
+
base = min((t[other] for t in per_tier), default=None)
|
|
41
|
+
if base is None:
|
|
42
|
+
return {}
|
|
43
|
+
row = sorted((t for t in per_tier if t[other] == base), key=lambda t: t[axis])
|
|
44
|
+
for t in row:
|
|
45
|
+
if t["accuracy"] < threshold:
|
|
46
|
+
return {"axis": axis, other: base, "dropoff": t[axis], "below": threshold}
|
|
47
|
+
return {"axis": axis, other: base, "dropoff": None, "below": threshold} # never dropped
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _auc(per_tier: list[dict]) -> float:
|
|
51
|
+
"""Mean accuracy across all tiers — area under the curve. A threshold-free
|
|
52
|
+
reading: even when no curve crosses a fixed threshold (the v1 "held" artefact),
|
|
53
|
+
AUC separates architectures by their whole-curve strength (the founder's
|
|
54
|
+
"read the curve, not a point" principle applied to the verdict itself)."""
|
|
55
|
+
accs = [t["accuracy"] for t in per_tier if t["accuracy"] == t["accuracy"]] # drop NaN
|
|
56
|
+
return float(sum(accs) / len(accs)) if accs else float("nan")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def run_ablation(steps: int = 1500, tiers: list[MQARTier] | None = None,
|
|
60
|
+
threshold: float = 0.5, seed: int = 1337, **kw) -> dict:
|
|
61
|
+
tiers = tiers or standard_curriculum()
|
|
62
|
+
curves: dict[str, list[dict]] = {}
|
|
63
|
+
t0 = time.time()
|
|
64
|
+
for v in VARIANTS:
|
|
65
|
+
res = run_curriculum(c2_variant=v, steps=steps, tiers=tiers, seed=seed, **kw)
|
|
66
|
+
curves[v] = res.per_tier
|
|
67
|
+
|
|
68
|
+
# comparison: drop-off per variant, and the gap vs control
|
|
69
|
+
dropoffs = {v: _dropoff(curves[v], threshold) for v in VARIANTS}
|
|
70
|
+
ctrl_do = dropoffs["none"].get("dropoff")
|
|
71
|
+
deltas = {}
|
|
72
|
+
for v in ("hopfield", "delta"):
|
|
73
|
+
vdo = dropoffs[v].get("dropoff")
|
|
74
|
+
if ctrl_do is None and vdo is None:
|
|
75
|
+
deltas[v] = "both never dropped (raise difficulty)"
|
|
76
|
+
elif vdo is None:
|
|
77
|
+
deltas[v] = "later than control (never dropped)"
|
|
78
|
+
elif ctrl_do is None:
|
|
79
|
+
deltas[v] = "earlier than control (control never dropped)"
|
|
80
|
+
else:
|
|
81
|
+
g = vdo - ctrl_do
|
|
82
|
+
deltas[v] = f"{'+' if g >= 0 else ''}{g} kv vs control ({vdo} vs {ctrl_do})"
|
|
83
|
+
|
|
84
|
+
# AUC per variant — the threshold-free reading (fixes the v1 "held" artefact,
|
|
85
|
+
# where Hopfield beat the control on the whole curve but not at the 0.5 cutoff)
|
|
86
|
+
auc = {v: _auc(curves[v]) for v in VARIANTS}
|
|
87
|
+
ctrl_auc = auc["none"]
|
|
88
|
+
auc_delta = {v: (auc[v] - ctrl_auc if auc[v] == auc[v] and ctrl_auc == ctrl_auc else None)
|
|
89
|
+
for v in ("hopfield", "delta")}
|
|
90
|
+
auc_winners = [v for v in ("hopfield", "delta")
|
|
91
|
+
if auc[v] == auc[v] and ctrl_auc == ctrl_auc and auc[v] > ctrl_auc]
|
|
92
|
+
|
|
93
|
+
# honest verdict: which variant extends the frontier on this axis
|
|
94
|
+
winners = [v for v in ("hopfield", "delta")
|
|
95
|
+
if isinstance(dropoffs[v].get("dropoff"), int)
|
|
96
|
+
and (ctrl_do is None or dropoffs[v]["dropoff"] > ctrl_do)]
|
|
97
|
+
if auc_winners:
|
|
98
|
+
deltas_str = ", ".join(f"{v} (+{auc[v]-ctrl_auc:.3f} AUC)" for v in auc_winners)
|
|
99
|
+
verdict = f"advances (AUC): {deltas_str} beat the control's whole-curve MQAR strength"
|
|
100
|
+
elif winners:
|
|
101
|
+
verdict = f"advances (drop-off): {', '.join(winners)} recall further than control"
|
|
102
|
+
else:
|
|
103
|
+
verdict = "held — no associative variant beat the control on AUC or drop-off"
|
|
104
|
+
|
|
105
|
+
return {
|
|
106
|
+
"benchmark": "mqar",
|
|
107
|
+
"timestamp_utc": datetime.now(timezone.utc).isoformat(),
|
|
108
|
+
"regime": {"steps": steps, "tiers": len(tiers), "threshold": threshold, "seed": seed},
|
|
109
|
+
"curves": curves,
|
|
110
|
+
"dropoffs": dropoffs,
|
|
111
|
+
"auc": auc,
|
|
112
|
+
"auc_vs_control": auc_delta,
|
|
113
|
+
"control_delta": deltas,
|
|
114
|
+
"verdict": verdict,
|
|
115
|
+
"wall_s": round(time.time() - t0, 1),
|
|
116
|
+
"note": "architecture bench (separate artefact, not a run-v1 record); "
|
|
117
|
+
"circuit breaker per tier; the gap between drop-offs is the signal.",
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def write_artefact(result: dict, out_dir: str | Path) -> Path:
|
|
122
|
+
out = Path(out_dir)
|
|
123
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
124
|
+
stamp = result["timestamp_utc"][:19].replace(":", "").replace("-", "")
|
|
125
|
+
path = out / f"mqar-ablation-{stamp}.json"
|
|
126
|
+
path.write_text(json.dumps(result, indent=2) + "\n")
|
|
127
|
+
# also refresh a LATEST pointer
|
|
128
|
+
(out / "LATEST.json").write_text(json.dumps(result, indent=2) + "\n")
|
|
129
|
+
return path
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _cli():
|
|
133
|
+
ap = argparse.ArgumentParser(description="MQAR C2 ablation (Slice C, ADR-006)")
|
|
134
|
+
ap.add_argument("--steps", type=int, default=1500)
|
|
135
|
+
ap.add_argument("--threshold", type=float, default=0.5)
|
|
136
|
+
ap.add_argument("--seed", type=int, default=1337)
|
|
137
|
+
ap.add_argument("--out", default="metrics/mqar")
|
|
138
|
+
ap.add_argument("--quick", action="store_true", help="tiny curriculum + few steps")
|
|
139
|
+
args = ap.parse_args()
|
|
140
|
+
tiers = None
|
|
141
|
+
steps = args.steps
|
|
142
|
+
if args.quick:
|
|
143
|
+
tiers = [MQARTier(kv_pairs=4, seq_len=64), MQARTier(kv_pairs=16, seq_len=64)]
|
|
144
|
+
steps = 300
|
|
145
|
+
result = run_ablation(steps=steps, tiers=tiers, threshold=args.threshold, seed=args.seed)
|
|
146
|
+
path = write_artefact(result, args.out)
|
|
147
|
+
print("\n=== MQAR ablation verdict ===")
|
|
148
|
+
print(result["verdict"])
|
|
149
|
+
print(f"drop-offs: {json.dumps(result['dropoffs'])}")
|
|
150
|
+
print(f"vs control: {json.dumps(result['control_delta'])}")
|
|
151
|
+
print(f"artefact: {path}")
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
if __name__ == "__main__":
|
|
155
|
+
_cli()
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def run_hopfield_capacity_sweep(steps: int = 1500, seed: int = 1337,
|
|
159
|
+
slots: tuple[int, ...] = (32, 64, 128, 256),
|
|
160
|
+
tier: MQARTier | None = None, **kw) -> dict:
|
|
161
|
+
"""The founder's question: is Hopfield limited by its memory SIZE or by its
|
|
162
|
+
structure? Sweep c2_mem_slots at a fixed hard tier. If accuracy RISES with
|
|
163
|
+
slots → the limit is dimensioning (fixable). If it PLATEAUS → the structure
|
|
164
|
+
is the ceiling (delta's expressivity may be the better bet). Reads the curve.
|
|
165
|
+
"""
|
|
166
|
+
from datetime import datetime, timezone
|
|
167
|
+
tier = tier or MQARTier(kv_pairs=16, seq_len=256) # a hard-enough tier to separate
|
|
168
|
+
points = []
|
|
169
|
+
for s in slots:
|
|
170
|
+
res = run_curriculum(c2_variant="hopfield", steps=steps, tiers=[tier],
|
|
171
|
+
seed=seed, c2_mem_slots=s, **kw)
|
|
172
|
+
acc = res.per_tier[0]["accuracy"]
|
|
173
|
+
points.append({"mem_slots": s, "accuracy": acc})
|
|
174
|
+
print(f"[hopfield-sweep] slots={s} kv={tier.kv_pairs} seq={tier.seq_len} -> acc={acc:.3f}")
|
|
175
|
+
# verdict: monotone increase → dimensioning-limited; flat → structure-limited
|
|
176
|
+
accs = [p["accuracy"] for p in points]
|
|
177
|
+
rising = accs[-1] > accs[0] + 0.02
|
|
178
|
+
reading = ("dimensioning-limited (accuracy rises with capacity — add slots)"
|
|
179
|
+
if rising else
|
|
180
|
+
"structure-limited (accuracy plateaus — capacity is not the ceiling)")
|
|
181
|
+
return {
|
|
182
|
+
"benchmark": "hopfield_capacity_sweep",
|
|
183
|
+
"timestamp_utc": datetime.now(timezone.utc).isoformat(),
|
|
184
|
+
"tier": {"kv_pairs": tier.kv_pairs, "seq_len": tier.seq_len},
|
|
185
|
+
"points": points,
|
|
186
|
+
"reading": reading,
|
|
187
|
+
}
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
"""cortex_eval.confirmation_run — the run that unlocks whitepaper v0.
|
|
2
|
+
|
|
3
|
+
One GPU job that turns today's single-seed signals into gate-passing claims
|
|
4
|
+
(WHITEPAPER.md admission gate: ≥3 seeds, significant, anchored, real, reproducible).
|
|
5
|
+
|
|
6
|
+
Three parts, all at the same regime (steps, seeds), on the DISCRIMINATING tiers —
|
|
7
|
+
those where the variants actually separated in the 12-tier run (kv 8/16 at
|
|
8
|
+
seq 128/256) — so the job fits one Kaggle session:
|
|
9
|
+
1. C2 ablation × 5 seeds → paired t-tests, CIs (hopfield/delta vs control)
|
|
10
|
+
2. Recurrent ladder × 5 seeds → is "delta-rule pays only when hybridised" real?
|
|
11
|
+
3. External anchor × 5 seeds → the three families, statistically
|
|
12
|
+
|
|
13
|
+
Everything is written to metrics/mqar/confirmation-*.json. Nothing is claimed
|
|
14
|
+
here; the artefact feeds the whitepaper's §4 only where the gate passes.
|
|
15
|
+
"""
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import argparse
|
|
19
|
+
import json
|
|
20
|
+
from datetime import datetime, timezone
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
|
|
23
|
+
import numpy as np
|
|
24
|
+
|
|
25
|
+
from cortex_eval.mqar import MQARTier
|
|
26
|
+
from cortex_eval.multiseed import run_multiseed, _mean_ci, _paired_test
|
|
27
|
+
from cortex_eval.recurrent_ladder import run_ladder, LADDER
|
|
28
|
+
from cortex_eval.external_baseline import _run_mlp_baseline
|
|
29
|
+
|
|
30
|
+
# the discriminating tiers (from the 12-tier ablation: where the paths separated)
|
|
31
|
+
DISCRIMINATING = [MQARTier(kv_pairs=8, seq_len=128), MQARTier(kv_pairs=16, seq_len=128),
|
|
32
|
+
MQARTier(kv_pairs=8, seq_len=256), MQARTier(kv_pairs=16, seq_len=256)]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _ladder_multiseed(seeds, tiers, steps, **kw) -> dict:
|
|
36
|
+
per_rung: dict[str, list[float]] = {r.name: [] for r in LADDER}
|
|
37
|
+
for s in seeds:
|
|
38
|
+
res = run_ladder(tiers=tiers, steps=steps, seed=s, **kw)
|
|
39
|
+
for pt in res["curve"]:
|
|
40
|
+
per_rung[pt["rung"]].append(pt["auc"])
|
|
41
|
+
stats = {r: _mean_ci([v for v in vals if v == v]) for r, vals in per_rung.items()}
|
|
42
|
+
# the candidate finding: L4 (pure delta) vs L3 (best pure) — does delta regress when pure?
|
|
43
|
+
l3, l4 = per_rung["L3-+local-conv"], per_rung["L4-+delta-rule"]
|
|
44
|
+
return {"per_seed": per_rung, "stats": stats,
|
|
45
|
+
"L4_vs_L3": _paired_test(l4, l3)}
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _floor_multiseed(seeds, tiers, steps, **kw) -> dict:
|
|
49
|
+
vals = []
|
|
50
|
+
for s in seeds:
|
|
51
|
+
accs = [_run_mlp_baseline(t, steps, s, d_model=kw.get("d_model", 64)) for t in tiers]
|
|
52
|
+
vals.append(float(np.mean(accs)))
|
|
53
|
+
return {"per_seed": vals, "stats": _mean_ci(vals)}
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def run_confirmation(seeds=(1337, 2024, 7, 42, 99), steps: int = 1500,
|
|
57
|
+
tiers=None, **kw) -> dict:
|
|
58
|
+
tiers = tiers or DISCRIMINATING
|
|
59
|
+
print("=== 1/3 C2 ablation × seeds ===")
|
|
60
|
+
abl = run_multiseed(list(seeds), steps=steps, tiers=tiers, **kw)
|
|
61
|
+
print("=== 2/3 recurrent ladder × seeds ===")
|
|
62
|
+
lad = _ladder_multiseed(seeds, tiers, steps, **kw)
|
|
63
|
+
print("=== 3/3 no-attention floor × seeds ===")
|
|
64
|
+
flo = _floor_multiseed(seeds, tiers, steps, **kw)
|
|
65
|
+
|
|
66
|
+
# hybrid vs best pure recurrent (the hybrid thesis, statistically)
|
|
67
|
+
best_pure = max(lad["per_seed"], key=lambda r: np.nanmean(lad["per_seed"][r]))
|
|
68
|
+
hyb_vs_pure = {
|
|
69
|
+
"best_pure_rung": best_pure,
|
|
70
|
+
"hopfield_vs_best_pure": _paired_test(abl["per_seed_auc"]["hopfield"], lad["per_seed"][best_pure]),
|
|
71
|
+
"delta_hybrid_vs_best_pure": _paired_test(abl["per_seed_auc"]["delta"], lad["per_seed"][best_pure]),
|
|
72
|
+
}
|
|
73
|
+
return {
|
|
74
|
+
"benchmark": "confirmation_run",
|
|
75
|
+
"timestamp_utc": datetime.now(timezone.utc).isoformat(),
|
|
76
|
+
"regime": {"seeds": list(seeds), "steps": steps,
|
|
77
|
+
"tiers": [(t.kv_pairs, t.seq_len) for t in tiers], "note": "discriminating tiers"},
|
|
78
|
+
"c2_ablation": {"stats": abl["stats"], "paired_tests": abl["paired_tests"], "claims": abl["claims"],
|
|
79
|
+
"per_seed_auc": abl["per_seed_auc"]},
|
|
80
|
+
"recurrent_ladder": lad,
|
|
81
|
+
"floor_mlp": flo,
|
|
82
|
+
"hybrid_vs_pure": hyb_vs_pure,
|
|
83
|
+
"gate_readout": {
|
|
84
|
+
"hopfield_beats_control_significant": abl["paired_tests"]["hopfield_vs_control"].get("significant_95"),
|
|
85
|
+
"delta_beats_control_significant": abl["paired_tests"]["delta_vs_control"].get("significant_95"),
|
|
86
|
+
"delta_regresses_when_pure_significant": lad["L4_vs_L3"].get("significant_95") and lad["L4_vs_L3"]["mean_diff"] < 0,
|
|
87
|
+
"hybrid_beats_best_pure_significant": hyb_vs_pure["hopfield_vs_best_pure"].get("significant_95"),
|
|
88
|
+
},
|
|
89
|
+
"note": "Feeds WHITEPAPER.md §4 only where the gate passes; otherwise §5 open questions.",
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def write_artefact(result: dict, out_dir="metrics/mqar") -> Path:
|
|
94
|
+
out = Path(out_dir); out.mkdir(parents=True, exist_ok=True)
|
|
95
|
+
stamp = result["timestamp_utc"][:19].replace(":", "").replace("-", "")
|
|
96
|
+
p = out / f"confirmation-{stamp}.json"
|
|
97
|
+
p.write_text(json.dumps(result, indent=2) + "\n")
|
|
98
|
+
(out / "LATEST-confirmation.json").write_text(json.dumps(result, indent=2) + "\n")
|
|
99
|
+
return p
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _cli():
|
|
103
|
+
ap = argparse.ArgumentParser()
|
|
104
|
+
ap.add_argument("--steps", type=int, default=1500)
|
|
105
|
+
ap.add_argument("--seeds", type=int, nargs="+", default=[1337, 2024, 7, 42, 99])
|
|
106
|
+
ap.add_argument("--tiers", type=int, default=4,
|
|
107
|
+
help="how many discriminating tiers to use (1-4, sharpest first) — "
|
|
108
|
+
"use 2 to fit a ~3h GPU budget with 3 seeds at 1000 steps")
|
|
109
|
+
ap.add_argument("--quick", action="store_true")
|
|
110
|
+
args = ap.parse_args()
|
|
111
|
+
if args.quick:
|
|
112
|
+
r = run_confirmation(seeds=(1, 2, 3), steps=150,
|
|
113
|
+
tiers=[MQARTier(kv_pairs=4, seq_len=48)], d_model=48)
|
|
114
|
+
else:
|
|
115
|
+
# sharpest-first ordering: the tiers where paths separated most in the 12-tier run
|
|
116
|
+
sharpest = [MQARTier(kv_pairs=16, seq_len=128), MQARTier(kv_pairs=8, seq_len=128),
|
|
117
|
+
MQARTier(kv_pairs=16, seq_len=256), MQARTier(kv_pairs=8, seq_len=256)]
|
|
118
|
+
tiers = sharpest[:max(1, min(4, args.tiers))]
|
|
119
|
+
print(f"[confirmation] seeds={args.seeds} steps={args.steps} tiers={[(t.kv_pairs,t.seq_len) for t in tiers]}")
|
|
120
|
+
r = run_confirmation(seeds=tuple(args.seeds), steps=args.steps, tiers=tiers)
|
|
121
|
+
p = write_artefact(r)
|
|
122
|
+
print("\n=== GATE READOUT ===")
|
|
123
|
+
print(json.dumps(r["gate_readout"], indent=2))
|
|
124
|
+
print("=== C2 claims ===")
|
|
125
|
+
for c in r["c2_ablation"]["claims"]:
|
|
126
|
+
print(" ", c)
|
|
127
|
+
print("=== hybrid vs best pure recurrent ===")
|
|
128
|
+
print(json.dumps(r["hybrid_vs_pure"], indent=2))
|
|
129
|
+
print("artefact:", p)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
if __name__ == "__main__":
|
|
133
|
+
_cli()
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"""cortex_eval.domain_map — the per-domain reading of the C2 ablation.
|
|
2
|
+
|
|
3
|
+
The confirmation run showed that the "winner" flips between regimes: on the
|
|
4
|
+
12-tier average Hopfield leads; on the hard/short discriminating tiers delta
|
|
5
|
+
leads (and Hopfield sinks to the control). That is not a contradiction — it is
|
|
6
|
+
RES-20 in action: the ranking depends on Σ = which tiers are held. The honest
|
|
7
|
+
object is therefore not a ranking but a DOMAIN MAP: for each tier, which path
|
|
8
|
+
wins and by what margin, with the evidence basis (single-seed / multi-seed).
|
|
9
|
+
|
|
10
|
+
This map is (a) what §4 of the whitepaper displays, and (b) exactly what the
|
|
11
|
+
RES-18 router must learn — the map is the router's training target (ADR-007
|
|
12
|
+
Slice C). "No single memory dominates" is the result; the router is the answer.
|
|
13
|
+
|
|
14
|
+
Inputs: the 12-tier ablation artefact (single seed) and, when present, the
|
|
15
|
+
confirmation artefact (multi-seed on the discriminating tiers). Output: a JSON
|
|
16
|
+
map + a markdown table for the paper.
|
|
17
|
+
"""
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import json
|
|
21
|
+
from datetime import datetime, timezone
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
|
|
24
|
+
import numpy as np
|
|
25
|
+
|
|
26
|
+
PATHS = ("none", "hopfield", "delta")
|
|
27
|
+
LABEL = {"none": "control", "hopfield": "hopfield", "delta": "delta"}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _load(path):
|
|
31
|
+
p = Path(path)
|
|
32
|
+
return json.loads(p.read_text()) if p.exists() else None
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def build_domain_map(ablation12_path="metrics/mqar/mqar-ablation-final-2026-09-07.json",
|
|
36
|
+
confirmation_path="metrics/mqar/LATEST-confirmation.json") -> dict:
|
|
37
|
+
a12 = _load(ablation12_path)
|
|
38
|
+
conf = _load(confirmation_path)
|
|
39
|
+
rows = []
|
|
40
|
+
|
|
41
|
+
# --- basis 1: the 12-tier single-seed curves ---
|
|
42
|
+
if a12:
|
|
43
|
+
curves = a12["curves"]
|
|
44
|
+
n = len(curves["none"])
|
|
45
|
+
for i in range(n):
|
|
46
|
+
kv, sq = curves["none"][i]["kv_pairs"], curves["none"][i]["seq_len"]
|
|
47
|
+
sc = {p: curves[p][i]["accuracy"] for p in PATHS}
|
|
48
|
+
best = max(sc, key=sc.get)
|
|
49
|
+
ranked = sorted(sc.values(), reverse=True)
|
|
50
|
+
margin = ranked[0] - ranked[1]
|
|
51
|
+
rows.append({"kv_pairs": kv, "seq_len": sq, "scores": sc, "winner": LABEL[best],
|
|
52
|
+
"margin": round(margin, 4), "basis": "12-tier, 1 seed",
|
|
53
|
+
"separable": margin >= 0.01})
|
|
54
|
+
|
|
55
|
+
# --- basis 2: confirmation (multi-seed) on discriminating tiers — overrides where present ---
|
|
56
|
+
domain_multiseed = {}
|
|
57
|
+
if conf and "c2_ablation" in conf:
|
|
58
|
+
st = conf["c2_ablation"]["stats"] # per-variant mean over the held tiers
|
|
59
|
+
tests = conf["c2_ablation"]["paired_tests"]
|
|
60
|
+
tiers = conf["regime"]["tiers"]
|
|
61
|
+
means = {p: st[p]["mean"] for p in PATHS}
|
|
62
|
+
best = max(means, key=means.get)
|
|
63
|
+
sig = {
|
|
64
|
+
"hopfield_vs_control": tests["hopfield_vs_control"].get("significant_95"),
|
|
65
|
+
"delta_vs_control": tests["delta_vs_control"].get("significant_95"),
|
|
66
|
+
"hopfield_vs_delta": tests["hopfield_vs_delta"].get("significant_95"),
|
|
67
|
+
}
|
|
68
|
+
domain_multiseed = {
|
|
69
|
+
"tiers": tiers, "means": means, "winner": LABEL[best],
|
|
70
|
+
"n_seeds": conf["regime"]["n_seeds"] if "n_seeds" in conf["regime"] else len(conf["regime"].get("seeds", [])),
|
|
71
|
+
"significant": sig,
|
|
72
|
+
"basis": f"confirmation, {len(conf['regime'].get('seeds', []))} seeds, {conf['regime'].get('steps')} steps",
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
# --- the reading ---
|
|
76
|
+
winners = [r["winner"] for r in rows if r["separable"]]
|
|
77
|
+
counts = {LABEL[p]: winners.count(LABEL[p]) for p in PATHS}
|
|
78
|
+
dominated = max(counts, key=counts.get) if winners else None
|
|
79
|
+
no_single = len([c for c in counts.values() if c > 0]) >= 2
|
|
80
|
+
reading = ("no single memory dominates — each path has a domain; the per-tier map is the result "
|
|
81
|
+
"and the multi-path router (RES-18) is the answer" if no_single else
|
|
82
|
+
f"{dominated} dominates the separable tiers")
|
|
83
|
+
|
|
84
|
+
return {
|
|
85
|
+
"benchmark": "domain_map",
|
|
86
|
+
"timestamp_utc": datetime.now(timezone.utc).isoformat(),
|
|
87
|
+
"law": "RES-20: a ranking that flips between regimes is not a contradiction — it is Σ (which tiers are held). Read the map, not a ranking.",
|
|
88
|
+
"per_tier": rows,
|
|
89
|
+
"separable_winner_counts": counts,
|
|
90
|
+
"discriminating_multiseed": domain_multiseed,
|
|
91
|
+
"reading": reading,
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def to_markdown(dm: dict) -> str:
|
|
96
|
+
lines = ["| kv | seq | control | hopfield | delta | winner | margin | basis |",
|
|
97
|
+
"|---:|---:|---:|---:|---:|:---|---:|:---|"]
|
|
98
|
+
for r in dm["per_tier"]:
|
|
99
|
+
s = r["scores"]
|
|
100
|
+
mark = "" if r["separable"] else " (noise-level)"
|
|
101
|
+
lines.append(f"| {r['kv_pairs']} | {r['seq_len']} | {s['none']:.3f} | {s['hopfield']:.3f} | {s['delta']:.3f} | "
|
|
102
|
+
f"**{r['winner']}**{mark} | {r['margin']:.3f} | {r['basis']} |")
|
|
103
|
+
ms = dm.get("discriminating_multiseed")
|
|
104
|
+
if ms:
|
|
105
|
+
m = ms["means"]
|
|
106
|
+
lines.append("")
|
|
107
|
+
lines.append(f"**Discriminating tiers {ms['tiers']} — {ms['basis']}:** control {m['none']:.4f} · "
|
|
108
|
+
f"hopfield {m['hopfield']:.4f} · delta {m['delta']:.4f} → winner **{ms['winner']}**; "
|
|
109
|
+
f"significance@95%: {ms['significant']}")
|
|
110
|
+
lines.append("")
|
|
111
|
+
lines.append(f"**Reading:** {dm['reading']}")
|
|
112
|
+
return "\n".join(lines)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def write_artefact(dm: dict, out_dir="metrics/mqar") -> Path:
|
|
116
|
+
out = Path(out_dir); out.mkdir(parents=True, exist_ok=True)
|
|
117
|
+
p = out / "domain-map.json"
|
|
118
|
+
p.write_text(json.dumps(dm, indent=2) + "\n")
|
|
119
|
+
(out / "domain-map.md").write_text(to_markdown(dm) + "\n")
|
|
120
|
+
return p
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
if __name__ == "__main__":
|
|
124
|
+
dm = build_domain_map()
|
|
125
|
+
p = write_artefact(dm)
|
|
126
|
+
print(to_markdown(dm))
|
|
127
|
+
print(f"\nartefact: {p}")
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
"""cortex_eval.external_baseline — D19 Level 1c: an external anchor.
|
|
2
|
+
|
|
3
|
+
A number nobody can situate is not a result. This module runs, on OUR MQAR
|
|
4
|
+
protocol at EQUAL size, two reference points anyone can reproduce:
|
|
5
|
+
- a standard GPT-2-style transformer (the nanoGPT / VanillaGPT lineage —
|
|
6
|
+
which is exactly our control, so this makes the "control = a known public
|
|
7
|
+
architecture" statement explicit and checkable);
|
|
8
|
+
- an absolute floor: a position-wise MLP with NO attention (cannot do
|
|
9
|
+
associative recall by construction) — the chance-level anchor.
|
|
10
|
+
So our variants sit between a public reference (the transformer) and a hard
|
|
11
|
+
floor (no-attention MLP). Same tiers, same steps, same seeds — equal at
|
|
12
|
+
everything (hub-019). Nothing borrowed: the MLP baseline is written here.
|
|
13
|
+
"""
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from datetime import datetime, timezone
|
|
17
|
+
|
|
18
|
+
import numpy as np
|
|
19
|
+
|
|
20
|
+
from cortex_eval.mqar import MQARTier, make_batch
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _run_mlp_baseline(tier: MQARTier, steps: int, seed: int, d_model: int = 64,
|
|
24
|
+
batch: int = 32, lr: float = 3e-3, eval_batch: int = 64) -> float:
|
|
25
|
+
"""A position-wise MLP over token embeddings — no attention, no recurrence.
|
|
26
|
+
It sees each position independently, so it CANNOT look back at the key/value
|
|
27
|
+
pairs: an honest floor for associative recall."""
|
|
28
|
+
import torch, torch.nn as nn
|
|
29
|
+
torch.manual_seed(seed)
|
|
30
|
+
vocab = 256 + 8
|
|
31
|
+
emb = nn.Embedding(vocab, d_model)
|
|
32
|
+
mlp = nn.Sequential(nn.Linear(d_model, 4 * d_model), nn.GELU(), nn.Linear(4 * d_model, vocab))
|
|
33
|
+
params = list(emb.parameters()) + list(mlp.parameters())
|
|
34
|
+
opt = torch.optim.AdamW(params, lr=lr)
|
|
35
|
+
for step in range(steps):
|
|
36
|
+
X, Y = make_batch(tier, batch, seed=seed + step)
|
|
37
|
+
logits = mlp(emb(torch.from_numpy(X)))
|
|
38
|
+
loss = nn.functional.cross_entropy(logits.view(-1, vocab),
|
|
39
|
+
torch.from_numpy(Y).view(-1), ignore_index=-100)
|
|
40
|
+
opt.zero_grad(); loss.backward(); opt.step()
|
|
41
|
+
Xe, Ye = make_batch(tier, eval_batch, seed=seed + 10_000)
|
|
42
|
+
with torch.no_grad():
|
|
43
|
+
preds = mlp(emb(torch.from_numpy(Xe))).argmax(-1).numpy()
|
|
44
|
+
mask = Ye != -100
|
|
45
|
+
return float((preds[mask] == Ye[mask]).mean())
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _run_gla_baseline(tier: MQARTier, steps: int, seed: int, d_model: int = 64,
|
|
49
|
+
n_head: int = 2, batch: int = 32, lr: float = 3e-3,
|
|
50
|
+
eval_batch: int = 64) -> float:
|
|
51
|
+
"""A PURE gated linear-attention recurrent model (the Mamba/RWKV/GLA lineage),
|
|
52
|
+
written from scratch in its canonical minimal form:
|
|
53
|
+
S_t = g_t ⊙ S_{t-1} + k_t v_tᵀ (gated fixed-size state)
|
|
54
|
+
o_t = q_t S_t (linear read)
|
|
55
|
+
No softmax attention anywhere. This is the *incumbent* of the fixed-state
|
|
56
|
+
memory family our C2 layers belong to, and the reference the MQAR literature
|
|
57
|
+
(Zoology, Kimi Linear) expects. Equal size to the control."""
|
|
58
|
+
import torch, torch.nn as nn
|
|
59
|
+
torch.manual_seed(seed)
|
|
60
|
+
vocab = 256 + 8
|
|
61
|
+
dk = d_model // n_head
|
|
62
|
+
|
|
63
|
+
class GLA(nn.Module):
|
|
64
|
+
def __init__(self):
|
|
65
|
+
super().__init__()
|
|
66
|
+
self.emb = nn.Embedding(vocab, d_model)
|
|
67
|
+
self.ln = nn.LayerNorm(d_model)
|
|
68
|
+
self.qkv = nn.Linear(d_model, 3 * d_model, bias=False)
|
|
69
|
+
self.gate = nn.Linear(d_model, n_head * dk, bias=True)
|
|
70
|
+
self.out = nn.Linear(d_model, d_model, bias=False)
|
|
71
|
+
self.ln2 = nn.LayerNorm(d_model)
|
|
72
|
+
self.mlp = nn.Sequential(nn.Linear(d_model, 4 * d_model), nn.GELU(),
|
|
73
|
+
nn.Linear(4 * d_model, d_model))
|
|
74
|
+
self.head = nn.Linear(d_model, vocab, bias=False)
|
|
75
|
+
|
|
76
|
+
def forward(self, idx):
|
|
77
|
+
x = self.emb(idx)
|
|
78
|
+
b, T, c = x.shape
|
|
79
|
+
z = self.ln(x)
|
|
80
|
+
q, k, v = self.qkv(z).split(c, dim=2)
|
|
81
|
+
q = torch.nn.functional.normalize(q.view(b, T, n_head, dk), dim=-1)
|
|
82
|
+
k = torch.nn.functional.normalize(k.view(b, T, n_head, dk), dim=-1)
|
|
83
|
+
v = v.view(b, T, n_head, dk)
|
|
84
|
+
g = torch.sigmoid(self.gate(z)).view(b, T, n_head, dk)
|
|
85
|
+
S = torch.zeros(b, n_head, dk, dk, device=x.device)
|
|
86
|
+
outs = []
|
|
87
|
+
for i in range(T):
|
|
88
|
+
S = S * g[:, i].unsqueeze(2) + torch.einsum('bhi,bhj->bhij', k[:, i], v[:, i])
|
|
89
|
+
outs.append(torch.einsum('bhi,bhij->bhj', q[:, i], S).reshape(b, c))
|
|
90
|
+
o = torch.stack(outs, 1)
|
|
91
|
+
x = x + self.out(o)
|
|
92
|
+
x = x + self.mlp(self.ln2(x))
|
|
93
|
+
return self.head(x)
|
|
94
|
+
|
|
95
|
+
model = GLA()
|
|
96
|
+
opt = torch.optim.AdamW(model.parameters(), lr=lr)
|
|
97
|
+
for step in range(steps):
|
|
98
|
+
X, Y = make_batch(tier, batch, seed=seed + step)
|
|
99
|
+
logits = model(torch.from_numpy(X))
|
|
100
|
+
loss = nn.functional.cross_entropy(logits.view(-1, vocab),
|
|
101
|
+
torch.from_numpy(Y).view(-1), ignore_index=-100)
|
|
102
|
+
if not torch.isfinite(loss):
|
|
103
|
+
return float("nan") # circuit-breaker spirit: report, do not hide
|
|
104
|
+
opt.zero_grad(); loss.backward()
|
|
105
|
+
nn.utils.clip_grad_norm_(model.parameters(), 1.0); opt.step()
|
|
106
|
+
Xe, Ye = make_batch(tier, eval_batch, seed=seed + 10_000)
|
|
107
|
+
with torch.no_grad():
|
|
108
|
+
preds = model(torch.from_numpy(Xe)).argmax(-1).numpy()
|
|
109
|
+
mask = Ye != -100
|
|
110
|
+
return float((preds[mask] == Ye[mask]).mean())
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def run_external_anchor(tiers: list[MQARTier] | None = None, steps: int = 1500,
|
|
114
|
+
seed: int = 1337, **kw) -> dict:
|
|
115
|
+
from cortex_eval.run_mqar import run_curriculum
|
|
116
|
+
tiers = tiers or [MQARTier(kv_pairs=4, seq_len=128), MQARTier(kv_pairs=8, seq_len=128),
|
|
117
|
+
MQARTier(kv_pairs=16, seq_len=128)]
|
|
118
|
+
out = {"mlp_no_attention": [], "gla_pure_recurrent": [], "transformer_control": [],
|
|
119
|
+
"hopfield": [], "delta": []}
|
|
120
|
+
for tier in tiers:
|
|
121
|
+
out["mlp_no_attention"].append(
|
|
122
|
+
{"kv_pairs": tier.kv_pairs, "seq_len": tier.seq_len,
|
|
123
|
+
"accuracy": _run_mlp_baseline(tier, steps, seed)})
|
|
124
|
+
out["gla_pure_recurrent"].append(
|
|
125
|
+
{"kv_pairs": tier.kv_pairs, "seq_len": tier.seq_len,
|
|
126
|
+
"accuracy": _run_gla_baseline(tier, steps, seed)})
|
|
127
|
+
for name, v in (("transformer_control", "none"), ("hopfield", "hopfield"), ("delta", "delta")):
|
|
128
|
+
res = run_curriculum(c2_variant=v, steps=steps, tiers=tiers, seed=seed, **kw)
|
|
129
|
+
out[name] = res.per_tier
|
|
130
|
+
auc = {k: round(float(np.mean([t["accuracy"] for t in v])), 4) for k, v in out.items()}
|
|
131
|
+
return {
|
|
132
|
+
"benchmark": "mqar_external_anchor",
|
|
133
|
+
"timestamp_utc": datetime.now(timezone.utc).isoformat(),
|
|
134
|
+
"regime": {"tiers": [(t.kv_pairs, t.seq_len) for t in tiers], "steps": steps, "seed": seed},
|
|
135
|
+
"curves": out,
|
|
136
|
+
"auc": auc,
|
|
137
|
+
"reading": {
|
|
138
|
+
"floor": "mlp_no_attention — cannot recall by construction (chance-level anchor)",
|
|
139
|
+
"recurrent_reference": "gla_pure_recurrent — a PURE gated linear-attention model (Mamba/RWKV/GLA "
|
|
140
|
+
"lineage, from scratch): the incumbent of the fixed-state memory family our C2 "
|
|
141
|
+
"layers belong to, and the reference the MQAR literature expects",
|
|
142
|
+
"attention_reference": "transformer_control — a standard GPT-2-style model (nanoGPT lineage), "
|
|
143
|
+
"identical to our control: our 'control' IS a known public architecture",
|
|
144
|
+
"our_variants": "hopfield / delta — the C2 HYBRIDS (attention + memory), measured against all three "
|
|
145
|
+
"families: no-memory floor, pure-recurrent, pure-attention",
|
|
146
|
+
},
|
|
147
|
+
"note": "D19 Level 1c — equal size, tiers, steps, seed. The MLP floor is written from scratch here.",
|
|
148
|
+
}
|