anima-python 0.13.1__py3-none-any.whl

anima_py/cli/serialize.py

@@ -0,0 +1,174 @@
+#!/usr/bin/env python3
+# serialize.py — anima STANDALONE re-serialize entry (.pt → .clm v0.3 + held-out gate).
+#
+# WHY THIS FILE (recovery/re-export · a_clm_gen_pipeline): `anima train` ALREADY
+# auto-serializes its trained weights to .clm v0.3 and runs the held-out mirror-DESCENT
+# gate at the end of a run (cli/train.py / cli/train.hexa post-serialize block). This
+# standalone command exists for the SEPARATE case: re-exporting an ALREADY-TRAINED torch
+# .pt checkpoint (recovery, re-export, re-verification) without re-training.
+#
+# REFERENCE-MATCH (NOT a reimplementation): the byte layout comes from the GROUND-TRUTH
+# unified serializer core/serialize.py::serialize_v3 (the SAME byte grammar core/decode.hexa
+# parses and the golden reexport_d768_v2_fast.clm uses), and the held-out gate is the KEPT
+# torch-interop sibling verify_clm_v2.py descent (math.log mirror, dt_ln-immune, under the
+# CLM model dir). This file only loads the .pt, derives L/E from the state-dict keys, and
+# calls those two backends — it invents no new layout and no new gate.
+#
+# TORCH IS ALLOWED HERE (training family): unlike cli/evaluate.py (torch-free
+# measurement) this is the LEARNING-side re-export and must read a torch .pt. The gate
+# enforcer's torch grep targets the measurement surfaces (evaluate / anima dispatch), not
+# the trainer/serializer family (a_engine_native_learning: train/ may use torch).
+#
+# USAGE (installed `anima` PATH command after `hx install anima`)
+#   anima serialize <ckpt.pt> <out.clm> [--heldout <path>] [--train <path>] [--nwin N]
+#     <ckpt.pt>    torch state_dict (or {"state_dict": ...}/{"model": ...}) checkpoint
+#     <out.clm>    output .clm v0.3 path (engine-loadable, CLM magic)
+#     --heldout    held-out byte corpus for the post-serialize mirror-DESCENT gate
+#                  (a_clm_gen_pipeline: structure round-trip is NOT enough)
+#     --train      optional train corpus (gap-vs-heldout overfit advisory)
+#     --nwin N     descent window length (default 64)
+
+import os
+import sys
+
+_HERE = os.path.dirname(os.path.abspath(__file__))
+_REPO = os.path.dirname(_HERE)
+# UNIFIED serializer = core/serialize.py (CLM serialize_v3 + ByteGPT .pt→.bin bridge),
+# parallel to core/decode.py — add core/ so `import serialize` resolves (same resolution
+# cli/evaluate.py uses for `import decode`). The held-out DESCENT verifier verify_clm_v2.py
+# and the .pt→.clm bridge serialize_standalone.py are CORE-owned siblings in core/ too
+# (owner directive: core-related lives in core/; no archive/train import — a_no_archive_import).
+sys.path.insert(0, os.path.join(_REPO, "core"))
+
+
+def serialize_usage():
+    """Print the canonical usage banner (installed `anima serialize` command form)."""
+    print("anima serialize — re-export a torch .pt to .clm v0.3 + held-out DESCENT gate.")
+    print("")
+    print("usage:")
+    print("  anima serialize <ckpt.pt> <out.clm> [--heldout <path>] [--train <path>] [--nwin N]")
+    print("")
+    print("  re-serialize an ALREADY-TRAINED torch checkpoint to an engine-loadable .clm")
+    print("  (ground-truth bridge serialize_v3) and verify it models held-out text")
+    print("  (verify_clm_v2 mirror-DESCENT gate). `anima train` does this automatically;")
+    print("  this command is for recovery / re-export of an existing .pt.")
+
+
+def _strval(argv, flag, dflt):
+    i = 0
+    while i < len(argv):
+        if argv[i] == flag and i + 1 < len(argv):
+            return argv[i + 1]
+        i += 1
+    return dflt
+
+
+def _intval(argv, flag, dflt):
+    v = _strval(argv, flag, None)
+    return int(v) if v is not None else dflt
+
+
+def _derive_LE(sd):
+    """Derive (L trunk layers, E experts) from the state-dict keys (reference-match
+    of serialize_v2._general_block_keymap naming: trunk.{i}.* and moe.experts.{j}.*)."""
+    L = 0
+    E = 0
+    for k in sd.keys():
+        if k.startswith("trunk.") and k.endswith(".conv.conv.weight"):
+            try:
+                L = max(L, int(k.split(".")[1]) + 1)
+            except (ValueError, IndexError):
+                pass
+        if k.startswith("moe.experts.") and k.endswith(".conv.conv.weight"):
+            try:
+                E = max(E, int(k.split(".")[2]) + 1)
+            except (ValueError, IndexError):
+                pass
+    return L, E
+
+
+def serialize_run(argv):
+    if len(argv) < 2:
+        serialize_usage()
+        return 2
+
+    pt = argv[0]
+    out = argv[1]
+    heldout = _strval(argv[2:], "--heldout", None)
+    train_corpus = _strval(argv[2:], "--train", None)
+    nwin = _intval(argv[2:], "--nwin", 64)
+
+    if not os.path.exists(pt):
+        print("ERROR: torch checkpoint not found: " + pt)
+        return 2
+
+    import torch                                          # training family — torch OK here
+    import serialize as S                                 # core/serialize.py — serialize_v3 = bridge SSOT
+    import verify_clm_v2 as VC                            # clm_decodable / descent (train/clm/model/)
+
+    print("=== anima serialize — .pt → .clm v0.3 (ground-truth bridge) ===")
+    print("pt:      " + pt)
+    print("out:     " + out)
+
+    raw = torch.load(pt, map_location="cpu")
+    # normalize to a flat state_dict (mirror serialize_v2._resolve_state_dict inputs).
+    if isinstance(raw, dict) and "state_dict" in raw and isinstance(raw["state_dict"], dict):
+        sd = raw["state_dict"]
+    elif isinstance(raw, dict) and "model" in raw and isinstance(raw["model"], dict):
+        sd = raw["model"]
+    else:
+        sd = raw
+    sd = {k: (v.detach().cpu() if hasattr(v, "detach") else v) for k, v in sd.items()}
+
+    L, E = _derive_LE(sd)
+    if L < 1 or E < 1:
+        print("ERROR: could not derive L/E from state-dict keys "
+              "(expected trunk.{i}.conv.conv.weight / moe.experts.{j}.conv.conv.weight). "
+              "Is this an anima CLMConvMoE checkpoint?")
+        return 2
+    print("derived: L(trunk)=" + str(L) + "  E(experts)=" + str(E))
+
+    S.serialize_v3(sd, n_trunk_layers=L, n_experts=E, out_path=out)
+    nbytes = os.path.getsize(out)
+    print("WRITTEN " + str(nbytes) + " bytes -> " + out)
+
+    # ── CORE-loadable self-check (mirror of core/decode.hexa CONV mouth) ─────
+    rb = open(out, "rb").read()
+    decodable = VC.clm_decodable(rb)
+    print("")
+    print("=== CORE-loadable self-check (mirror of core/decode.hexa CONV mouth) ===")
+    print("  clm_decodable = " + str(decodable))
+    if not decodable:
+        print("FAIL — serialized .clm is NOT CORE-loadable (see above).")
+        return 1
+
+    # ── held-out mirror-DESCENT gate (a_clm_gen_pipeline) ────────────────────
+    if heldout:
+        if not os.path.exists(heldout):
+            print("ERROR: --heldout corpus not found: " + heldout)
+            return 2
+        print("")
+        print("post-serialize held-out DESCENT gate (verify_clm_v2 mirror, dt_ln-immune):")
+        rc = VC.run_descent_gate_cli(out, heldout, train_corpus, nwin)
+        if rc != 0:
+            print("  ⛔ DESCENT GATE FAIL — serialized .clm broken/overfit; do NOT mark "
+                  "done / HF-upload (a_clm_gen_pipeline).")
+            return 1
+        print("  ✅ DESCENT GATE PASS — serialized .clm models held-out text.")
+    else:
+        print("")
+        print("NOTE: no --heldout corpus given — structure round-trip only. The .clm is "
+              "CORE-loadable but its held-out predictive descent is UNVERIFIED "
+              "(a_clm_gen_pipeline: decodable != generalizing). Pass --heldout to gate it.")
+    return 0
+
+
+def main(argv):
+    if len(argv) >= 1 and argv[0] in ("-h", "--help"):
+        serialize_usage()
+        return 0
+    return serialize_run(argv)
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]) or 0)

anima_py/cli/serialize_bind.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+# serialize_bind.py — anima STANDALONE injected-bind re-serialize entry
+# (base ByteGPT .bin + injected torch .pt → engine .bin with a "BGB\x01" trailer).
+#
+# WHY THIS FILE (H_9027 · a_clm_gen_pipeline sibling for the BYTE mouth): a trained
+# BindAttn model = the FROZEN base ByteGPT + N appended GATED transformer blocks
+# (applied after the L base blocks, before ln_f: x = x + gate*(block(x)-x)). This
+# command bridges the injected torch .pt back onto the base .bin so the combined
+# model is engine-loadable and scoreable via `anima-python evaluate <bin>` with ZERO
+# changes to cli/evaluate.py (bg_is_bytegpt stays true; bg_load reads the trailer).
+#
+# REFERENCE-MATCH (NOT a reimplementation): the byte layout is core/serialize.py::
+# serialize_bind (mirrors `serialize`'s per-layer write + the CLM "CLMB" bind-trailer
+# precedent), and it is the SAME grammar core/decode.py bg_load parses. This file only
+# reads argv, then calls that one backend — it invents no new layout.
+#
+# TORCH IS ALLOWED HERE (training family): reading the injected .pt (unpickle) is the
+# only torch touch. This is the LEARNING-side re-export, NOT the torch-free measurement
+# surface (cli/evaluate.py) the gate enforcer targets.
+#
+# USAGE (installed `anima` PATH command after `hx install anima`)
+#   anima serialize-bind <base.bin> <injected.pt> <out.bin>
+#     <base.bin>      plain engine ByteGPT .bin (5xu32 header, NO trailer)
+#     <injected.pt>   torch {"bind": <Block state_dict | list | dict>,
+#                            "gate": <scalar | list | dict>, "config": {...}}
+#     <out.bin>       output engine .bin = base bytes verbatim + BGB trailer
+
+import os
+import sys
+
+_HERE = os.path.dirname(os.path.abspath(__file__))
+_REPO = os.path.dirname(_HERE)
+# UNIFIED serializer = core/serialize.py (same resolution cli/serialize.py uses).
+sys.path.insert(0, os.path.join(_REPO, "core"))
+
+
+def serialize_bind_usage():
+    print("anima serialize-bind — base ByteGPT .bin + injected torch .pt → .bin + BGB trailer.")
+    print("")
+    print("usage:")
+    print("  anima serialize-bind <base.bin> <injected.pt> <out.bin>")
+    print("")
+    print("  splice N appended gated transformer blocks (BindAttn .pt) onto a base ByteGPT")
+    print("  .bin so the combined model is engine-loadable + scoreable via")
+    print("  `anima-python evaluate <out.bin>`. gate=0 => byte-identical to the base .bin.")
+
+
+def serialize_bind_run(argv):
+    if len(argv) < 3:
+        serialize_bind_usage()
+        return 2
+    base_bin, injected_pt, out_bin = argv[0], argv[1], argv[2]
+    if not os.path.exists(base_bin):
+        print("ERROR: base .bin not found: " + base_bin)
+        return 2
+    if not os.path.exists(injected_pt):
+        print("ERROR: injected .pt not found: " + injected_pt)
+        return 2
+
+    import serialize as S       # core/serialize.py — serialize_bind = bind-trailer SSOT
+    import decode as D          # core/decode.py — CORE-loadable self-check (bg_load)
+
+    print("=== anima serialize-bind — base .bin + injected .pt → .bin + BGB ===")
+    print("base:     " + base_bin)
+    print("injected: " + injected_pt)
+    print("out:      " + out_bin)
+
+    S.serialize_bind(base_bin, injected_pt, out_bin)
+    nbytes = os.path.getsize(out_bin)
+    print("WRITTEN " + str(nbytes) + " bytes -> " + out_bin)
+
+    # ── CORE-loadable self-check (mirror of core/decode.py BYTE mouth) ─────────
+    print("")
+    print("=== CORE-loadable self-check (core/decode.py bg_load BYTE mouth) ===")
+    is_bg = D.bg_is_bytegpt(out_bin)
+    print("  bg_is_bytegpt = " + str(is_bg))
+    if not is_bg:
+        print("FAIL — output .bin is NOT recognized as a ByteGPT mouth.")
+        return 1
+    W = D.bg_load(out_bin)
+    nb = len(W.get("bind", []))
+    print("  bg_load ok    = " + str(W.get("ok")) + "   n_bind blocks = " + str(nb))
+    if not W.get("ok") or nb < 1:
+        print("FAIL — output .bin did not round-trip a BGB trailer.")
+        return 1
+    print("  ✅ CORE-loadable with " + str(nb) + " injected-bind block(s).")
+    print("  NOTE: this is a STRUCTURE round-trip only — the engine-native ρ-AXON reach")
+    print("        verdict (former G0-G6) is `anima-python evaluate <out.bin>` (a_engine_native_learning).")
+    return 0
+
+
+def main(argv):
+    if len(argv) >= 1 and argv[0] in ("-h", "--help"):
+        serialize_bind_usage()
+        return 0
+    return serialize_bind_run(argv)
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]) or 0)