model-d 0.4.0__py3-none-any.whl

ocm/build.py

@@ -0,0 +1,77 @@
+"""Build a workspace over one or more repositories (and their docs).
+
+    python -m ocm build <folder> [<folder> ...] --out <workspace folder>
+
+Creates <workspace>/memory (the fingerprints: 1,024 bits per unit, from a small embedding model that runs on this
+machine), <workspace>/keyword.db (a keyword index for the router's keyword lane), <workspace>/spans.json (the first and
+last line of every unit, so a taught question's vector can be put on the unit holding a given line) and
+<workspace>/workspace.json (which folders, how many units, the router's rare-word threshold). Rebuild after large code changes; teaching later only adds."""
+import argparse, json, os, shutil, sqlite3, time
+
+from .cmlm_vector import Memory
+from .units import files, units
+
+THETA_DEFAULT = 9.349                     # the router law's 44% rule on code questions (measured on transformers)
+
+
+def build(roots, out, model=None, device=None, log=print, carry=None, prune_after=None, cache=None):
+    if os.path.exists(os.path.join(out, "memory", "manifest.json")):
+        raise SystemExit(f"{out} already holds a workspace; remove it or choose another --out")
+    os.makedirs(out, exist_ok=True)
+    roots = {os.path.basename(os.path.abspath(r).rstrip("/")) or "root": os.path.abspath(r) for r in roots}
+    m = Memory.create(os.path.join(out, "memory"), bits=1024, seed=0, model=model, device=device)
+    cache = cache or os.environ.get("MODELD_CACHE")
+    if cache:                                  # rebuilds embed only the units whose text changed
+        from .embed_cache import CachedEmbedder
+        m._embedder = CachedEmbedder(m.embedder, os.path.expanduser(cache), m.man.get("model", ""))
+    F = files(roots); U = []
+    for name, rel, path in F:
+        try:
+            text = open(path, encoding="utf-8").read()
+        except (UnicodeDecodeError, OSError):
+            continue
+        us = units(rel, text)
+        m.sources.append({"id": len(m.sources), "path": rel, "sha256": "", "units": len(us)})
+        U.extend((t, len(m.sources) - 1, a, b) for t, a, b in us)
+    if not U:
+        raise SystemExit("no code or documentation found in those folders")
+    step = max(1, len(U) // 20000)
+    m.freeze_mean([u[0] for u in U[::step][:20000]])
+    log(f"{len(m.sources):,} files, {len(U):,} units; fingerprinting (first run downloads the embedding model)")
+    t0, seen, spans = time.time(), set(), []
+    for s in range(0, len(U), 2048):
+        chunk = [u for u in U[s:s + 2048] if u[0] not in seen]; seen.update(u[0] for u in chunk)
+        for (t, sid, a, last), b in zip(chunk, m.measure([u[0] for u in chunk])):
+            m._add_fact(t, sid, a, 0, True, b); spans.append([a, last])
+        if len(m.pending_f) >= 200_000:
+            m.save()
+        log(f"  {min(s + 2048, len(U)):,} / {len(U):,} units, {time.time() - t0:.0f}s")
+    m.save()
+    db = os.path.join(out, "keyword.db")
+    con = sqlite3.connect(db); con.execute("create virtual table s using fts5(t)")
+    con.executemany("insert into s(rowid, t) values (?, ?)", ((j + 1, t) for j, t in enumerate(m.texts))); con.commit(); con.close()
+    json.dump(spans, open(os.path.join(out, "spans.json"), "w"))
+    json.dump({"roots": roots, "files": len(m.sources), "units": len(m.texts), "theta": THETA_DEFAULT,
+               "built": time.strftime("%Y-%m-%dT%H:%M:%S")}, open(os.path.join(out, "workspace.json"), "w"), indent=1)
+    shutil.copytree(os.path.join(out, "memory"), os.path.join(out, "memory_at_build"))
+    reuse = f"; {m._embedder.embedded:,} newly embedded, the rest from the cache" if cache else ""
+    log(f"workspace ready: {out} ({len(m.texts):,} units from {len(m.sources):,} files{reuse})")
+    if carry:
+        from .server import Workspace
+        r = Workspace(out).carry(carry, prune_after)
+        log(f"carried from {carry}: {r['carried']} of {r['entries']} taught entries ({r['moved']} moved, "
+            f"{r['reformatted']} found after reformatting), {r['abstained']} abstained (their line is gone), "
+            f"{r['pruned']} pruned; Life sha {r['life sha']}")
+
+
+def main(argv=None):
+    ap = argparse.ArgumentParser(prog="ocm build"); ap.add_argument("folders", nargs="+"); ap.add_argument("--out", required=True)
+    ap.add_argument("--model"); ap.add_argument("--device")
+    ap.add_argument("--cache", help="an embedding cache file ($MODELD_CACHE): a rebuild embeds only changed code")
+    ap.add_argument("--carry", help="an earlier workspace whose taught knowledge (its Life) is carried into this one")
+    ap.add_argument("--prune-after", type=int, metavar="N",
+                    help="with --carry: remove a taught entry whose line has not been found in N carries in a row")
+    a = ap.parse_args(argv)
+    if a.prune_after is not None and (a.prune_after < 1 or not a.carry):
+        ap.error("--prune-after needs --carry and a whole number of at least 1")
+    build(a.folders, a.out, a.model, a.device, carry=a.carry, prune_after=a.prune_after, cache=a.cache)

ocm/calc.py

@@ -0,0 +1,51 @@
+"""The savings calculator: expected model requests with and without the workspace, from per-question costs measured in
+live tests (ocm/data/pools.json), with a 95% bootstrap interval.
+
+    python -m ocm calc --repeat 0.5 --deep 0.3 [--questions-per-month 16800]
+
+  --repeat   share of questions that repeat something a teammate already asked (in any words)
+  --deep     share of questions that are multi-file deep dives (the rest are lookups and short scans)
+Requests per question = search-and-read calls + a fixed session overhead per question (both arms) + batched teaching
+(one call per session of about six questions, workspace arm). This arithmetic predicted a held-out live run before it
+ran: 83 requests predicted, 84 measured; 1.66x predicted, 1.66x measured (results/RESULTS.md)."""
+import argparse, json, os, random
+
+POOLS = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data", "pools.json")      # shipped inside the package
+
+
+def expect(P, p, d, over, teach, rng=None):
+    pick = (lambda xs: [rng.choice(xs) for _ in xs]) if rng else (lambda xs: xs)
+    m = {k: sum(x) / len(x) for k, x in ((k, pick(v)) for k, v in P.items())}
+    def arm(a):
+        look = (1 - p) * m[f"lookup first {a}"] + p * m[f"lookup repeat {a}"]
+        deep = (1 - p) * m[f"deep first {a}"] + p * m[f"deep repeat {a}"]
+        return (1 - d) * look + d * deep + over + (teach if a == "memory" else 0)
+    return arm("tools"), arm("memory")
+
+
+def main(argv=None):
+    ap = argparse.ArgumentParser(prog="ocm calc")
+    ap.add_argument("--repeat", type=float); ap.add_argument("--deep", type=float, default=0.0)
+    ap.add_argument("--questions-per-month", type=int, default=0); ap.add_argument("--seed", type=int, default=20261010)
+    ap.add_argument("--table", action="store_true", help="the published table: repeats 0 to 90%%, with and without 30%% deep dives")
+    a = ap.parse_args(argv)
+    D = json.load(open(POOLS)); P = D["pools"]; over, teach = D["overhead per question"], D["teach per question, batched"]
+    if a.table:                                     # one generator over every row, in this order, as the published table
+        rng = random.Random(a.seed); q = a.questions_per_month or 16800
+        print(f"{'deep dives':>10} {'repeats':>8} {'without':>8} {'with':>6} {'saving':>7}  95% interval   fewer per month at {q:,} questions")
+        for d in (0.0, 0.3):
+            for p in (0.0, 0.25, 0.5, 0.75, 0.9):
+                t, m = expect(P, p, d, over, teach)
+                bs = sorted(x / y for x, y in (expect(P, p, d, over, teach, rng) for _ in range(5000)))
+                print(f"{d:>10.0%} {p:>8.0%} {t:>8.2f} {m:>6.2f} {t / m:>6.2f}x  [{bs[125]:.2f}, {bs[4875]:.2f}]   {q * (t - m):>9,.0f}")
+        return
+    if a.repeat is None:
+        ap.error("--repeat is required (or use --table)")
+    t, m = expect(P, a.repeat, a.deep, over, teach)
+    rng = random.Random(a.seed); boots = sorted(x / y for x, y in (expect(P, a.repeat, a.deep, over, teach, rng) for _ in range(5000)))
+    print(f"requests per question: without the workspace {t:.2f}, with it {m:.2f}")
+    print(f"saving: {t / m:.2f}x fewer requests (95% interval {boots[125]:.2f} to {boots[4875]:.2f})")
+    if a.questions_per_month:
+        q = a.questions_per_month
+        print(f"per month at {q:,} questions: {q * t:,.0f} requests without, {q * m:,.0f} with, {q * (t - m):,.0f} fewer")
+    print("billed input falls less than requests (measured: 1.40x when requests fell 1.66x), because readouts add reading.")

ocm/cli.py

@@ -0,0 +1,182 @@
+"""modeld: map a repository once, ask it where the answer is, teach it what was found.
+
+  modeld build [path] [--out DIR]       map the repository into <path>/modeld-out (alias: modeld .)
+  modeld ask "question" [--scope code]  the ranked places to look; --scope code leaves documentation out
+  modeld next "question"                the next readout for the same question
+  modeld teach "question" path:line     teach the line the answer is on
+  modeld stats                          units, files, questions taught
+  modeld install [--team URL]           register Model D with the coding assistants found on this machine
+  modeld init <claude|kiro|codex|gemini|all> [repo] [--url URL]
+                                        wire one repository's agent to a team server
+  modeld serve [workspace] [--port 7700] [--host 127.0.0.1] [--build REPO ...]
+                                        the team's shared workspace server (--build maps the repos first if empty)
+  modeld mcp                            the MCP server over stdio (a local workspace, or $OCM_URL)
+  modeld fetch-model [--to DIR]         download the embedding model once, for offline and air-gapped machines
+  modeld hybrid | measure | calc | hook ...
+                                        as in `python -m ocm` (see README)
+
+ask, next, teach and stats use the nearest modeld-out folder (this folder or a parent), $MODELD_WORKSPACE, or
+--workspace; with --url (or $OCM_URL) they ask a team server instead."""
+import argparse, json, os, sys, urllib.request
+
+OUT = "modeld-out"
+
+
+def find_workspace(start=None):
+    """--workspace, else $MODELD_WORKSPACE, else the nearest modeld-out above the current folder, else None."""
+    env = os.environ.get("MODELD_WORKSPACE")
+    if env:
+        return env
+    d = os.path.abspath(start or os.getcwd())
+    while True:
+        cand = os.path.join(d, OUT)
+        if os.path.exists(os.path.join(cand, "workspace.json")):
+            return cand
+        parent = os.path.dirname(d)
+        if parent == d:
+            return None
+        d = parent
+
+
+def _post(url, path, body):
+    req = urllib.request.Request(f"{url.rstrip('/')}/{path}", data=json.dumps(body).encode(),
+                                 headers={"Content-Type": "application/json"})
+    return urllib.request.urlopen(req, timeout=300).read().decode()
+
+
+def _call(args, path, body, then=None):
+    """Run one workspace call (or two, for `next`) locally or on the team server; print the result."""
+    url = args.url or os.environ.get("OCM_URL")
+    if url:
+        print(_post(url, then or path, body)); return
+    ws = args.workspace or find_workspace()
+    if not ws:
+        sys.exit(f"no {OUT} folder here or above: run `modeld build .` first, or pass --workspace or --url")
+    os.environ.setdefault("OCM_OFFLINE", "1")
+    from .server import Workspace
+    w = Workspace(ws)
+    out = w.handle(path, body)
+    if then:                                   # a local process has no earlier readout to advance from: ask, then next
+        out = w.handle(then, body)
+    print(out)
+
+
+def _common(ap):
+    ap.add_argument("--workspace"); ap.add_argument("--url")
+
+
+def build_cmd(argv):
+    ap = argparse.ArgumentParser(prog="modeld build")
+    ap.add_argument("path", nargs="*", default=["."]); ap.add_argument("--out")
+    ap.add_argument("--model"); ap.add_argument("--device"); ap.add_argument("--carry")
+    ap.add_argument("--cache", help="an embedding cache file ($MODELD_CACHE): a rebuild embeds only changed code")
+    a = ap.parse_args(argv)
+    roots = [os.path.abspath(p) for p in a.path]
+    out = os.path.abspath(a.out or os.path.join(roots[0], OUT))
+    carry = a.carry or (out if os.path.exists(os.path.join(out, "workspace.json")) else None)
+    from .build import build
+    if carry and os.path.abspath(carry) == out:                # a rebuild in place carries what was taught
+        import shutil, tempfile
+        tmp = tempfile.mkdtemp(prefix="modeld-out.new-")
+        build(roots, tmp, a.model, a.device, carry=out, cache=a.cache)
+        for name in os.listdir(out):                           # replace the contents, not the folder: it may be a mount
+            p = os.path.join(out, name)
+            shutil.rmtree(p) if os.path.isdir(p) and not os.path.islink(p) else os.remove(p)
+        for name in os.listdir(tmp):
+            shutil.move(os.path.join(tmp, name), os.path.join(out, name))
+        os.rmdir(tmp)
+    else:
+        build(roots, out, a.model, a.device, carry=carry, cache=a.cache)
+    open(os.path.join(out, ".gitignore"), "w").write("*\n")    # the workspace never lands in git by accident
+    print(f"\nmapped into {out}\nnext: modeld ask \"your question\" --scope code")
+
+
+def ask_cmd(argv, path="ask"):
+    ap = argparse.ArgumentParser(prog=f"modeld {path}"); ap.add_argument("question")
+    ap.add_argument("--scope", choices=["all", "code"]); _common(ap)
+    a = ap.parse_args(argv)
+    body = {"question": a.question, **({"scope": a.scope} if a.scope else {})}
+    if path == "next":
+        _call(a, "ask", body, then="next")
+    else:
+        _call(a, "ask", body)
+
+
+def teach_cmd(argv):
+    ap = argparse.ArgumentParser(prog="modeld teach"); ap.add_argument("question"); ap.add_argument("at")
+    _common(ap)
+    a = ap.parse_args(argv)
+    _call(a, "teach", {"pairs": [{"question": a.question, "at": a.at}]})
+
+
+def stats_cmd(argv):
+    ap = argparse.ArgumentParser(prog="modeld stats"); _common(ap)
+    _call(ap.parse_args(argv), "stats", {})
+
+
+def serve_cmd(argv):
+    ap = argparse.ArgumentParser(prog="modeld serve"); ap.add_argument("workspace", nargs="?")
+    ap.add_argument("--port", type=int, default=7700); ap.add_argument("--host", default="127.0.0.1")
+    ap.add_argument("--build", nargs="+", metavar="REPO", help="map these repositories into the workspace first, if it is empty")
+    a = ap.parse_args(argv)
+    ws = a.workspace or find_workspace()
+    if a.build and ws and not os.path.exists(os.path.join(ws, "workspace.json")):
+        build_cmd([*a.build, "--out", ws])
+    if not ws or not os.path.exists(os.path.join(ws, "workspace.json")):
+        sys.exit(f"no workspace at {ws or 'a ' + OUT + ' folder here or above'}: run `modeld build <repo>` first, or pass --build")
+    os.environ.setdefault("OCM_OFFLINE", "1")
+    from .server import serve
+    print(f"serving {ws} at http://{a.host}:{a.port}"); sys.stdout.flush()
+    serve(ws, a.port, a.host)
+
+
+def fetch_model_cmd(argv):
+    ap = argparse.ArgumentParser(prog="modeld fetch-model")
+    ap.add_argument("--to", help="a folder to hold the model (used as HF_HOME); default: the normal Hugging Face cache")
+    a = ap.parse_args(argv)
+    if a.to:
+        os.environ["HF_HOME"] = os.path.abspath(a.to)
+    os.environ.pop("HF_HUB_OFFLINE", None); os.environ.pop("TRANSFORMERS_OFFLINE", None)
+    from .cmlm_vector.embed import DEFAULT_MODEL
+    from huggingface_hub import snapshot_download
+    path = snapshot_download(DEFAULT_MODEL, allow_patterns=["*.json", "*.txt", "model.safetensors"])   # weights + tokenizer only
+    print(f"{DEFAULT_MODEL} is at {path}")
+    if a.to:
+        print(f"on the offline machine: export HF_HOME={os.path.abspath(a.to)} OCM_OFFLINE=1")
+
+
+def main(argv=None):
+    argv = list(sys.argv[1:] if argv is None else argv)
+    if not argv or argv[0] in ("-h", "--help", "help"):
+        print(__doc__); return
+    cmd, rest = argv[0], argv[1:]
+    if cmd == "--version":
+        from . import __version__
+        print(f"modeld {__version__}"); return
+    if cmd not in COMMANDS and os.path.isdir(cmd):            # modeld .  ==  modeld build .
+        cmd, rest = "build", argv
+    if cmd not in COMMANDS:
+        print(__doc__); sys.exit(2)
+    COMMANDS[cmd](rest)
+
+
+def _module(name):
+    def run(rest):
+        import importlib
+        importlib.import_module(f"ocm.{name}").main(rest)
+    return run
+
+
+def install_cmd(argv):
+    from .install import main as install_main
+    install_main(argv)
+
+
+COMMANDS = {"build": build_cmd, "ask": ask_cmd, "next": lambda r: ask_cmd(r, "next"), "teach": teach_cmd,
+            "stats": stats_cmd, "serve": serve_cmd, "fetch-model": fetch_model_cmd, "install": install_cmd,
+            "init": _module("init"), "mcp": _module("mcp"), "hybrid": _module("hybrid"), "measure": _module("measure"),
+            "calc": _module("calc"), "hook": _module("loop")}
+
+
+if __name__ == "__main__":
+    main()

ocm/cmlm_vector/__init__.py

@@ -0,0 +1,6 @@
+"""cmlm-vector: the CMLM memory with vector generalization and a learned law, as an installable package."""
+__version__ = "0.2.0"
+
+from .memory import Memory  # noqa: E402
+
+__all__ = ["Memory", "__version__"]

ocm/cmlm_vector/chunk.py

@@ -0,0 +1,38 @@
+"""Text into units: the sentences a memory stores and returns.
+
+Paragraphs are split on blank lines, then into sentences; a line break inside a paragraph is kept as a boundary
+only when the next line starts a new thought (a capital, a digit or a bullet), so wrapped prose is rejoined. Units
+under `min_words` words are dropped; a unit over `max_chars` is cut at word boundaries. Deterministic: the same
+text gives the same units in the same order."""
+import re
+
+_SENT = re.compile(r"(?<=[.!?])\s+(?=[A-Z0-9\"'(\[])")
+# mail and news metadata: near-identical lines in every message, which crowd a readout and rarely answer anything
+HEADER = re.compile(r"^(From|Subject|Organi[sz]ation|Lines|Reply-To|NNTP-Posting-Host|Distribution|Keywords|Summary|"
+                    r"Article-I\.D\.|Expires|X-[\w-]+|Sender|Path|Newsgroups|Message-ID|References|Date|To|Cc):|^In article <|"
+                    r"(writes|wrote):$", re.I)
+
+
+def units(text, min_words=4, max_chars=400, drop_headers=False):
+    if drop_headers:
+        return [u for u in units(text, min_words, max_chars) if not HEADER.search(u)]
+    out = []
+    for para in re.split(r"\n\s*\n", text.replace("\r", "")):
+        lines = [l.strip() for l in para.split("\n") if l.strip()]
+        if not lines:
+            continue
+        joined = lines[0]
+        for l in lines[1:]:
+            joined += ("\n" if re.match(r"[A-Z0-9\-*>•]", l) and re.search(r"[.!?:]$", joined) else " ") + l
+        for piece in joined.split("\n"):
+            for s in _SENT.split(piece):
+                s = " ".join(s.split())
+                while len(s) > max_chars:
+                    cut = s.rfind(" ", 0, max_chars)
+                    cut = cut if cut > max_chars // 2 else max_chars
+                    head, s = s[:cut], s[cut:].strip()
+                    if len(head.split()) >= min_words:
+                        out.append(head)
+                if len(s.split()) >= min_words:
+                    out.append(s)
+    return out

ocm/cmlm_vector/cli.py

@@ -0,0 +1,75 @@
+"""cmlm-vector: a memory a frontier model asks and teaches, instead of a transcript it rereads.
+
+    cmlm-vector init MEM [--bits 1024] [--seed 0] [--model BAAI/bge-base-en-v1.5] [--device cpu|mps|cuda]
+    cmlm-vector ingest MEM PATH... [--glob '*.txt'] [--drop-headers]    files or folders; already-held files are skipped
+    cmlm-vector ask MEM "question" [-k 5] [--context 1] [--json]        single shot: the top k facts with their sources
+    cmlm-vector teach MEM --cue "how a person might ask" --fact "the one sentence that answers it"
+    cmlm-vector info MEM
+    cmlm-vector verify MEM                                              every file against its sha256, planes against seed
+
+A memory is a folder; copy it and it answers the same way on another machine (use the same device for bit-exact
+replication). Nothing is sent anywhere: embeddings are computed locally."""
+import argparse, glob as globmod, json, os, sys
+
+from .memory import Memory
+
+
+def _files(paths, pattern):
+    out = []
+    for p in paths:
+        if os.path.isdir(p):
+            out += [f for f in globmod.glob(os.path.join(p, "**", pattern), recursive=True) if os.path.isfile(f)]
+        elif os.path.isfile(p):
+            out.append(p)
+    return sorted(set(out))
+
+
+def main(argv=None):
+    p = argparse.ArgumentParser(prog="cmlm-vector", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    sub = p.add_subparsers(dest="cmd", required=True)
+    a = sub.add_parser("init"); a.add_argument("mem"); a.add_argument("--bits", type=int, default=1024)
+    a.add_argument("--seed", type=int, default=0); a.add_argument("--model"); a.add_argument("--device")
+    a = sub.add_parser("ingest"); a.add_argument("mem"); a.add_argument("paths", nargs="+")
+    a.add_argument("--glob", default="*.txt"); a.add_argument("--keep-float", action="store_true")
+    a.add_argument("--drop-headers", action="store_true", help="skip mail/news metadata lines (From:, Subject:, ...)")
+    a.add_argument("--no-center", action="store_true", help="do not centre the bits on the corpus mean")
+    a = sub.add_parser("ask"); a.add_argument("mem"); a.add_argument("question"); a.add_argument("-k", type=int, default=5)
+    a.add_argument("--context", type=int, default=1); a.add_argument("--json", action="store_true")
+    a = sub.add_parser("teach"); a.add_argument("mem"); a.add_argument("--cue", required=True); a.add_argument("--fact", required=True)
+    a = sub.add_parser("info"); a.add_argument("mem")
+    a = sub.add_parser("verify"); a.add_argument("mem")
+    x = p.parse_args(argv)
+
+    if x.cmd == "init":
+        m = Memory.create(x.mem, bits=x.bits, seed=x.seed, model=x.model, device=x.device)
+        print(json.dumps(m.info(), indent=1)); return 0
+    if x.cmd == "verify":                              # before opening: a damaged memory may not open at all
+        bad = Memory.verify_path(x.mem)
+        print("verified: every file matches its sha256 and the planes match the seed" if not bad else f"MISMATCH: {bad}")
+        return 0 if not bad else 1
+    m = Memory(x.mem)
+    if x.cmd == "ingest":
+        fs = _files(x.paths, x.glob)
+        print(f"ingesting {len(fs):,} files into {x.mem}", flush=True)
+        r = m.ingest(fs, keep_float=x.keep_float, log=lambda s: print(s, flush=True), drop_headers=x.drop_headers, center=not x.no_center)
+        print(json.dumps({**r, **m.info()}, indent=1)); return 0
+    if x.cmd == "ask":
+        r = m.recall(x.question, k=x.k, context=x.context)
+        if x.json:
+            print(json.dumps(r, indent=1)); return 0
+        if r["status"] == "abstain":
+            print("NOTHING CLOSE: no stored fact stands out for this question; these are the nearest. Ask again in other words if none answers it.")
+        for i, row in enumerate(r["rows"], 1):
+            flag = " [ambiguous: check against the question]" if row["flag"] else ""
+            print(f"{i}. {row['fact']}{flag}\n   source: {row['source']} (unit {row['unit']})")
+            if row["context"]:
+                print("   context: " + " | ".join(row["context"]))
+        return 0
+    if x.cmd == "teach":
+        r = m.teach([(x.cue, x.fact)]); m.save(); print(json.dumps(r)); return 0
+    if x.cmd == "info":
+        print(json.dumps(m.info(), indent=1)); return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

ocm/cmlm_vector/embed.py

@@ -0,0 +1,66 @@
+"""Sentence embeddings from a small local model, at no token cost: nothing leaves the machine.
+
+BAAI/bge-base-en-v1.5 by default (768 dimensions, CLS pooled, normalized), read from the local Hugging Face cache.
+Batched in order of length so padding stays small, on Apple's GPU or CUDA when present. The device is recorded in
+the memory's manifest: CPU and GPU arithmetic differ in the last bits, so an exact replication uses the same device
+(or --device cpu everywhere)."""
+import os
+
+DEFAULT_MODEL = os.environ.get("CMLM_EMBED_MODEL", "BAAI/bge-base-en-v1.5")
+# The instruction a model's card says to put before a search query matched against passages (not before passages,
+# and not before a query matched against other questions).
+QUERY_PREFIX = {"BAAI/bge-small-en-v1.5": "Represent this sentence for searching relevant passages: ",
+                "BAAI/bge-base-en-v1.5": "Represent this sentence for searching relevant passages: ",
+                "BAAI/bge-large-en-v1.5": "Represent this sentence for searching relevant passages: "}
+
+
+def pick_device(device=None):
+    import torch
+    if device:
+        return device
+    if torch.cuda.is_available():
+        return "cuda"
+    if getattr(torch.backends, "mps", None) and torch.backends.mps.is_available():
+        return "mps"
+    return "cpu"
+
+
+class Embedder:
+    def __init__(self, name=DEFAULT_MODEL, device=None, offline=None, max_length=128):
+        # The first run downloads the embedding model (about 440 MB) from Hugging Face; set OCM_OFFLINE=1 afterwards to
+        # guarantee nothing is fetched. Your code and questions are never sent anywhere: embedding runs on this machine.
+        offline = os.environ.get("OCM_OFFLINE") == "1" if offline is None else offline
+        if offline:
+            os.environ.setdefault("HF_HUB_OFFLINE", "1"); os.environ.setdefault("TRANSFORMERS_OFFLINE", "1")
+        import torch
+        from transformers import AutoModel, AutoTokenizer
+        self.torch, self.name, self.max_length = torch, name, max_length
+        self.device = pick_device(device)
+        self.tok = AutoTokenizer.from_pretrained(name)
+        self.model = AutoModel.from_pretrained(name).train(False).to(self.device)   # inference only: nothing learns
+        self.dim = self.model.config.hidden_size
+
+    def __call__(self, texts, batch=256):
+        import numpy as np
+        texts = list(texts)
+        out = np.zeros((len(texts), self.dim), dtype=np.float32)
+        order = sorted(range(len(texts)), key=lambda i: len(texts[i]))
+        with self.torch.no_grad():
+            for s in range(0, len(order), batch):
+                idx = order[s:s + batch]
+                b = self.tok([texts[i] for i in idx], padding=True, truncation=True, max_length=self.max_length, return_tensors="pt")
+                b = {k: v.to(self.device) for k, v in b.items()}
+                v = self.model(**b).last_hidden_state[:, 0]
+                v = self.torch.nn.functional.normalize(v.float(), dim=-1).cpu().numpy()
+                out[idx] = v
+        return out
+
+
+_SHARED = {}
+
+
+def shared(name=DEFAULT_MODEL, device=None):
+    key = (name, device)
+    if key not in _SHARED:
+        _SHARED[key] = Embedder(name, device)
+    return _SHARED[key]

ocm/cmlm_vector/lawsearch.py

@@ -0,0 +1,146 @@
+"""The learned law: fluidnet's searched integer law over the vector law's top candidates, learned from Claude's teaching.
+
+The vector law ranks by closeness alone. The learned law decides which of its top N candidates comes first, from an
+8-bit word measured per candidate, and it is SEARCHED, not trained: each lane is STRONG, WEAK or OFF (3^8 = 6,561
+laws), the rank is the lexicographic (strong count, weak count, the vector law's order), one strong beats any number
+of weak because the rank says so, and every law is monotone (a lane turning on never lowers a candidate), which
+`monotone` checks on all 256 words as fluidnet checks its laws.
+
+    b0 CUE1   best of all facts by cue bits          b1 FACT1  best by fact-text bits
+    b2 CUE~   within 1% of the bits of the best cue  b3 FACT~  the same by fact text
+    b4 CUE3   top 3 by cue                           b5 FACT3  top 3 by fact text
+    b6 TYPE   the question asks for a date / amount / count / person / place / percent and the fact holds one;
+              silent when it asks for none (fluidnet's revision 2: unmeasured is silent, not weak)
+    b7 BOTH3  top 3 by cue and by fact text
+
+The examples are the pairs Claude teaches for a fact the memory already holds: a new wording of a known fact, which
+is what learning from a miss produces. Each is measured before its cue is stored, so it cannot match itself. The
+gate is prequential and honest: every example is first scored by the law in use and by the plain vector order, and
+the law takes over only when it has put the right fact first more often, on examples it had not yet learned from,
+over the last 50 examples (and it is searched on those 50), so a law that stops winning loses its place.
+sandbox/general_check/law_search.py is the same law judged on held-out facts and topics."""
+import itertools
+import re
+
+import numpy as np
+
+LANES = ("CUE1", "FACT1", "CUE~", "FACT~", "CUE3", "FACT3", "TYPE", "BOTH3")
+NEAR = 0.01
+
+ASK = {
+    "date": r"\bwhen\b|\bdates?\b|\bdays?\b|\bdeadline\b|\bwhat time\b|\bmoment\b|\bwhich year\b|\bwhat year\b",
+    "money": r"\bhow much\b|\bmoney\b|\bprice\b|\bcosts?\b|\bfees?\b|\bcharges?\b|\bspend|\bbudget|\bbill\b|\bpay\b|\bpaid\b|\bowe\b"
+             r"|\bsalary\b|\bcompensation\b|\bdollar|\bfunding\b|\ballowance\b|\breimburse|\bbonus\b|\bstipend\b|\bearn\b|\bpayout\b",
+    "count": r"\bhow many\b|\bnumber of\b|\bcapacity\b|\bheadcount\b|\bhold\b|\bfit\b|\bcount\b|\bsample size\b|\battendance\b",
+    "person": r"\bwho\b|\bwhom\b|\bwhose\b|\bwhich (colleague|individual|person|people|employee|staff|hire|specialist)\b",
+    "place": r"\bwhere\b|\baddress\b|\blocation\b|\bvenue\b|\bwhich city\b|\bregion\b|\bzone\b",
+    "percent": r"\bpercent|\bproportion\b|\bshare of\b|\bfraction\b|\bportion\b",
+}
+MONTHS = "January|February|March|April|May|June|July|August|September|October|November|December"
+HOLDS = {
+    "date": rf"\b({MONTHS}|Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday)\b|\b(19|20)\d\d\b|\bQ[1-4]\b",
+    "money": r"\$\s?\d",
+    "count": r"(?<![$\d.,])\b\d[\d,]*\b(?!\s*(percent|%))",
+    "person": r"\b[A-Z][a-z]+ [A-Z][a-z]+\b",
+    "place": r"\b(Street|Avenue|Road|Boulevard|floor|Room|Suite|region)\b",
+    "percent": r"\bpercent\b|%",
+}
+
+
+def asks(q):
+    return {t for t, rx in ASK.items() if re.search(rx, q.lower())}
+
+
+def holds(f):
+    return {t for t, rx in HOLDS.items() if re.search(rx, f)}
+
+
+LAWS = np.array(list(itertools.product((0, 1, 2), repeat=8)), dtype=np.int16)          # 0 off, 1 weak, 2 strong
+BASE = int(np.where(np.all(LAWS == 0, axis=1))[0][0])                                 # every lane off: the vector order
+_BITS = ((np.arange(256)[:, None] >> np.arange(8)[None, :]) & 1).astype(np.int16)
+VALUE = ((_BITS[None] * (LAWS[:, None, :] == 2)).sum(-1) * 16
+         + (_BITS[None] * (LAWS[:, None, :] == 1)).sum(-1)).astype(np.int16)           # [law, word]
+
+
+def words(order, strong, weak, bits, qtypes, ftypes, n=8):
+    """The 8-bit word of each of the top-n candidates in `order` (the vector law's order)."""
+    sr = np.empty(len(strong), dtype=np.int64); sr[np.argsort(-strong, kind="stable")] = np.arange(len(strong))
+    wr = np.empty(len(weak), dtype=np.int64); wr[np.argsort(-weak, kind="stable")] = np.arange(len(weak))
+    bs, bw = int(strong.max()), int(weak.max())
+    out = []
+    for j in order[:n]:
+        b = (sr[j] == 0, wr[j] == 0, strong[j] >= bs - NEAR * bits, weak[j] >= bw - NEAR * bits,
+             sr[j] < 3, wr[j] < 3, bool(qtypes & ftypes[j]), sr[j] < 3 and wr[j] < 3)
+        out.append(sum(int(v) << i for i, v in enumerate(b)))
+    return out
+
+
+def rank_of(law, ws, right):
+    """Where `law` puts the candidate at position `right` (0 = first); ties keep the vector law's order."""
+    if right < 0:
+        return 99
+    v = VALUE[law][ws].astype(np.int32) * 64 - np.arange(len(ws))
+    return int((v > v[right]).sum())
+
+
+def apply(law, ws):
+    """The permutation of the candidates that `law` ranks."""
+    v = VALUE[law][ws].astype(np.int32) * 64 - np.arange(len(ws))
+    return list(np.argsort(-v, kind="stable"))
+
+
+def search(examples, n=8):
+    """fluidnet: the law that puts the right fact first most often; ties -> better top-3, then fewer lanes on."""
+    ex = [(w + [0] * (n - len(w)), r) for w, r in examples if r >= 0]
+    if not ex:
+        return BASE
+    Wd = np.array([w for w, _ in ex]); R = np.array([r for _, r in ex])
+    v = VALUE[:, Wd].astype(np.int32) * 64 - np.arange(n, dtype=np.int32)[None, None, :]
+    vr = np.take_along_axis(v, np.broadcast_to(R[None, :, None], v.shape[:2] + (1,)), 2)
+    got = (v > vr).sum(-1)
+    key = (got == 0).mean(1) * 1e6 + (got < 3).mean(1) * 1e3 - (LAWS > 0).sum(1)
+    return int(np.argmax(key))
+
+
+def monotone(law):
+    return all(VALUE[law][w | (1 << b)] >= VALUE[law][w] for w in range(256) for b in range(8))
+
+
+def describe(law):
+    s = [x for x, t in zip(LANES, LAWS[law]) if t == 2]; w = [x for x, t in zip(LANES, LAWS[law]) if t == 1]
+    return f"strong {{{', '.join(s)}}} weak {{{', '.join(w)}}}" if (s or w) else "the vector order"
+
+
+class LearnedLaw:
+    def __init__(self, n=8, min_examples=12, window=50):
+        """window: the law is searched on, and must be winning over, the last `window` examples (None: all of
+        them). The memory changes as Claude teaches (cues pile up and the vector order gets better), so a law
+        that won early must keep winning to keep its place."""
+        self.n, self.min_examples, self.window = n, min_examples, window
+        self.examples = []                         # (words, position of the right fact among the candidates, or -1)
+        self.scored = []                           # prequential, per example: (law in use first, vector order first)
+        self.law = BASE
+
+    @property
+    def hits(self):
+        recent = self.scored[-self.window:] if self.window else self.scored
+        return {"law": sum(a for a, _ in recent), "vector": sum(b for _, b in recent)}
+
+    @property
+    def active(self):
+        h = self.hits
+        return len(self.examples) >= self.min_examples and self.law != BASE and h["law"] > h["vector"]
+
+    def observe(self, ws, right):
+        self.scored.append((int(rank_of(self.law, ws, right) == 0), int(right == 0)))
+        self.examples.append((list(ws), int(right)))
+        self.law = search(self.examples[-self.window:] if self.window else self.examples, self.n)
+        assert monotone(self.law)
+
+    def order(self, cand, ws):
+        """The candidates reordered by the law when it has earned it, else as they came."""
+        return [cand[i] for i in apply(self.law, ws)] if self.active else list(cand)
+
+    def state(self):
+        return {"law": describe(self.law), "active": self.active, "examples": len(self.examples),
+                "prequential first-right": dict(self.hits)}