ms-moe-maker 0.4.0__py3-none-any.whl

ms_moe_maker/__init__.py

@@ -0,0 +1,28 @@
+"""Ms.MoE - Multi-Specified Mixture of Experts.
+
+Five deliberate experts instead of a hundred lottery tickets. The design thesis
+is the INVERSE of a frontier MoE: hand-assign the domains so every expert has a
+guaranteed constituency, which eliminates dead and collapsed experts by
+construction rather than fighting them with a load-balancing auxiliary loss.
+Because each expert does exactly one thing, you can retrain ONE and re-splice
+without touching the others - which is what makes it maintainable by one
+person.
+
+The real product is the factory, not the model. Swap the expert list and
+someone else gets their own Ms.MoE, shaped like THEIR stack.
+
+This package deliberately depends on nothing of Seren's. seren-theatre can
+watch a run, and seren-theatre[stagehand] can start one, but the arrow only
+points that way - and even then the two never speak, they share a directory.
+Opt in, never opt out.
+"""
+from __future__ import annotations
+
+try:
+    from ._version import version as __version__
+except Exception:  # noqa: BLE001 - source checkout without a build
+    __version__ = "0.0.0+unknown"
+
+from ._describe import DESCRIBE, NAME  # noqa: F401  (stdlib-only, safe here)
+
+__all__ = ["DESCRIBE", "NAME", "__version__"]

ms_moe_maker/__main__.py

@@ -0,0 +1,253 @@
+"""The ms-moe-maker CLI - the command stagehand forks and a person types.
+
+Three verbs:
+
+    ms-moe-maker describe                     one line of JSON, exit 0, no side effects
+    ms-moe-maker validate recipe.yaml         parse + check, touching nothing
+    ms-moe-maker build recipe.yaml            translate, fork the pipeline, report
+
+`ms-moe-maker build recipe.yaml` is deliberately the literal string in the README and
+the literal string seren-theatre[stagehand] forks. Not a Python API call, not
+an internal entry point with different defaults - the same command. If the two
+ever diverge, the hand-run path is the one that rots, because it is the one
+with no automated users; making them identical removes the possibility.
+
+--describe is scanned before argparse for the same reason every Seren installer
+does it: it has to answer on a broken install, so nothing may run first.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+
+from ._describe import DESCRIBE
+from .events import Events
+
+
+def _force_utf8_stdio() -> None:
+    """UTF-8 regardless of console codepage. Windows defaults to legacy, and
+    the pipeline prints emoji milestones - a UnicodeEncodeError mid-build would
+    kill a run over a decorative character."""
+    for stream in (sys.stdout, sys.stderr):
+        try:
+            stream.reconfigure(encoding="utf-8", errors="replace")  # type: ignore[union-attr]
+        except (AttributeError, ValueError):
+            pass
+
+
+def _find_pipeline(explicit: str | None, recipe_path: Path,
+                   required: bool) -> Path | None:
+    """Locate fraunkenstein_universal.py. None if absent and not required.
+
+    Looked up rather than assumed, and reported when missing, because "wrapped
+    a script that isn't there" should be one clear error and not a traceback
+    from subprocess. Order: --pipeline, beside the recipe, cwd, then upward.
+
+    `required` is the difference between the two verbs, and it is not a
+    convenience:
+
+      build    - REQUIRED. There is nothing to fork without it.
+      validate - OPTIONAL. The README promises `ms-moe-maker validate` runs on a
+                 laptop with no GPU so you can check a recipe BEFORE going
+                 near a machine that can run it. Demanding the pipeline made
+                 that promise false: a stranger with a recipe and no checkout
+                 got "could not find fraunkenstein_universal.py" and no
+                 validation at all. Recipe SHAPE is checkable on its own; only
+                 the refusal analysis needs a pipeline to compare against.
+
+    An explicit --pipeline that does not exist is always an error, for either
+    verb. Being told where it is and being wrong is different from not saying.
+    """
+    from .levers import DEFAULT_PIPELINE
+
+    if explicit:
+        p = Path(explicit)
+        if not p.is_file():
+            raise SystemExit(f"--pipeline {p} does not exist")
+        return p.resolve()
+
+    for candidate in (recipe_path.parent / DEFAULT_PIPELINE,
+                      Path.cwd() / DEFAULT_PIPELINE):
+        if candidate.is_file():
+            return candidate.resolve()
+    for parent in [Path.cwd(), *Path.cwd().parents]:
+        candidate = parent / DEFAULT_PIPELINE
+        if candidate.is_file():
+            return candidate.resolve()
+    if required:
+        raise SystemExit(
+            f"could not find {DEFAULT_PIPELINE}. Pass --pipeline PATH, or run "
+            f"from the directory that holds it.")
+    return None
+
+
+def main(argv: list[str] | None = None) -> int:
+    _force_utf8_stdio()
+    argv = list(sys.argv[1:] if argv is None else argv)
+
+    # Before argparse: zero side effects, works half-installed.
+    if "--describe" in argv or (argv and argv[0] == "describe"):
+        print(json.dumps(DESCRIBE))
+        return 0
+
+    ap = argparse.ArgumentParser(
+        prog="ms-moe-maker",
+        description="Build a mixture of experts from a recipe.")
+    ap.add_argument("command", choices=["build", "validate", "describe"])
+    ap.add_argument("recipe", nargs="?", help="path to the recipe .yaml")
+    ap.add_argument("--pipeline", default=None,
+                    help="path to fraunkenstein_universal.py (default: found "
+                         "beside the recipe, then upward from cwd)")
+    ap.add_argument("--json", action="store_true",
+                    help="JSON Lines events on stdout, prose on stderr")
+    ap.add_argument("--python", default=None,
+                    help="interpreter to run the pipeline with (default: the "
+                         "one running ms-moe-maker). Use this when the trainer lives "
+                         "in a different venv - which is the normal case, "
+                         "since ms-moe-maker is deliberately small and torch is not.")
+    ap.add_argument("--dryrun", action="store_true",
+                    help="FRAUNK_DRYRUN=1 - the whole pipeline, small")
+    ap.add_argument("--force", action="store_true",
+                    help="FRAUNK_FORCE=1 - redo stages whose artifacts exist")
+    ap.add_argument("--allow-refusals", action="store_true",
+                    help="run even though some recipe fields cannot be "
+                         "honoured. They are recorded in the manifest either "
+                         "way; this only removes the stop.")
+    a = ap.parse_args(argv)
+
+    ev = Events(enabled=a.json)
+
+    if a.command == "describe":
+        print(json.dumps(DESCRIBE))
+        return 0
+
+    if not a.recipe:
+        ap.error("a recipe path is required")
+
+    from .recipe import load, resolve, validate
+
+    recipe_path = Path(a.recipe).resolve()
+    try:
+        rec, parse_warns = load(str(recipe_path))
+    except Exception as exc:  # noqa: BLE001 - the message IS the product
+        ev.error("parse", str(exc))
+        ev.say(f"FAILED to parse {recipe_path}: {exc}")
+        return 2
+
+    errs, warns = validate(rec)
+    warns = parse_warns + warns
+    for w in warns:
+        ev.warning(w)
+        ev.say(f"   WARN  {w}")
+    for e in errs:
+        ev.error("validate", e)
+        ev.say(f"   ERROR {e}")
+    if errs:
+        ev.done(ok=False, stage="validate")
+        return 1
+
+    # RESOLVED BEFORE THE PIPELINE LOOKUP, on purpose. Both are explicit
+    # arguments, and an explicit argument that is wrong should say so no matter
+    # what else is also missing - same rule as --pipeline. Reporting "could not
+    # find fraunkenstein_universal.py" to someone who mistyped --python sends
+    # them to fix the wrong thing.
+    #
+    # MSMOE_PYTHON as well as --python: the interpreter is a property of the
+    # BOX, not of the run, so it belongs somewhere you set once. Same shape as
+    # the family's SEREN_<X>_* levers - a flag for the one-off, an env var for
+    # the machine.
+    interpreter = a.python or os.environ.get("MSMOE_PYTHON") or None
+    if interpreter:
+        ipath = Path(interpreter)
+        if not ipath.is_file():
+            raise SystemExit(f"--python {ipath} does not exist")
+        # abspath, NEVER resolve(). A venv's bin/python is a SYMLINK to the
+        # base interpreter, and resolving it throws the venv away: you asked
+        # for /lab/bin/python and got /usr/bin/python3.12, whose sys.prefix is
+        # /usr and whose site-packages has none of your training deps. The
+        # failure then reads as "No module named 'torch'" from an interpreter
+        # you never named, which is about as misleading as it gets.
+        #
+        # Measured: running the symlink gives sys.prefix=/tmp/venvtest;
+        # running its target gives sys.prefix=/usr. Same file, different venv.
+        # abspath normalises the path without following the link.
+        interpreter = os.path.abspath(str(ipath))
+
+    pipeline = _find_pipeline(a.pipeline, recipe_path,
+                              required=(a.command == "build"))
+
+    from .levers import Translation, translate
+
+    if pipeline is None:
+        # Recipe-only validation. Say so LOUDLY rather than reporting a clean
+        # bill of health: "valid" and "valid, and nothing checked whether the
+        # pipeline can honour it" are different answers, and quietly giving
+        # the first when you mean the second is how a document that lies gets
+        # blessed on its way out the door.
+        tr = Translation()
+        no_pipeline_note = (
+            "no pipeline found, so ONLY the recipe's own shape was checked. "
+            "Refusals could not be computed - run this again beside "
+            "fraunkenstein_universal.py, or pass --pipeline PATH, to find out "
+            "whether a build would actually honour these fields.")
+        ev.warning(no_pipeline_note)
+    else:
+        no_pipeline_note = ""
+        tr = translate(rec, pipeline, force=a.force)
+
+    if tr.refusals:
+        ev.refused(tr.refusals)
+        ev.say("")
+        ev.say(f"   {len(tr.refusals)} recipe field(s) cannot be honoured by "
+               f"{pipeline.name}:")
+        for r in tr.refusals:
+            ev.say(f"     · {r}")
+        ev.say("")
+        ev.say("   These are not warnings. A recipe is a document you hand to "
+               "someone so they get YOUR run, and a field that is silently "
+               "ignored makes it a document that lies. Fix the recipe, carve "
+               "the stage out, or pass --allow-refusals to proceed knowing "
+               "the build will not match the file.")
+
+    if a.command == "validate":
+        eff = resolve(rec)
+        ev.emit("resolved", **eff)
+        ev.say("")
+        ev.say(f"Ms.MoE recipe  {rec.name}  [{eff['recipe_id']}]")
+        ev.say(f"   pipeline {pipeline if pipeline else '(none found)'}")
+        ev.say(f"   honoured {len(tr.agreed)} field(s), "
+               f"{len(tr.env)} env lever(s) set")
+        ev.say(f"   refused  {len(tr.refusals)} field(s)")
+        if no_pipeline_note:
+            ev.say("")
+            ev.say(f"   NOTE  {no_pipeline_note}")
+        ok = not tr.refusals
+        ev.done(ok=ok, refusals=len(tr.refusals), agreed=len(tr.agreed),
+                env=tr.env, pipeline=str(pipeline) if pipeline else None,
+                # The consumer needs to be able to tell "no refusals" from
+                # "refusals were never computed". Same key set either way, one
+                # honest flag - the alternative is a caller inferring depth of
+                # analysis from an empty list, which it cannot do.
+                refusals_checked=pipeline is not None)
+        return 0 if ok else 1
+
+    # build
+    if tr.refusals and not a.allow_refusals:
+        ev.done(ok=False, stage="translate", refusals=len(tr.refusals))
+        ev.say("   REFUSED - nothing was run.")
+        return 3
+
+    from .runner import Runner
+
+    if interpreter:
+        ev.say(f"   pipeline interpreter: {interpreter}")
+    runner = Runner(rec, pipeline, tr, ev, cwd=pipeline.parent,
+                    dryrun=a.dryrun, python=interpreter)
+    return runner.run()
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

ms_moe_maker/_describe.py

@@ -0,0 +1,47 @@
+"""ms-moe-maker's identity card. STDLIB ONLY - nothing imported, nothing read.
+
+Same contract and the same reason as every Seren service's `_describe`, even
+though ms-moe-maker is NOT a Seren package and never imports one: `--describe` has to
+answer on a half-installed tool, so it cannot need torch, pydantic, or yaml.
+The moment you most want something to be able to say its own name is when its
+install is broken.
+
+ms-moe-maker is deliberately Seren-agnostic. No seren-* dependency, no assumption
+that Lodestar exists, no Seren in the name. It is a pipeline for building a
+specified mixture of experts, usable by someone who has never heard of any of
+this. seren-theatre[stagehand] depends on ms-moe-maker; ms-moe-maker depends on nothing of
+Chad's. Mandate is not ethos - the connection is opt-in from the Seren side,
+and the run DIRECTORY is the only thing the two ever share.
+"""
+from __future__ import annotations
+
+NAME = "ms-moe-maker"
+DESCRIPTION = ("Build a mixture of experts from deliberately chosen "
+               "specialists. Not a coding model - a coding model shaped like "
+               "your stack.")
+
+# The three verbs. Named here so a front-end can offer them without parsing
+# --help, and so `stagehand` can check the tool it forked speaks the version of
+# the contract it expects.
+COMMANDS = ("build", "validate", "describe")
+
+# The event vocabulary emitted under --json. A consumer that does not know an
+# event kind must ignore it, so adding one is not a breaking change; removing
+# or renaming one is.
+EVENTS = ("started", "stage", "progress", "refused", "warning", "error", "done")
+
+DESCRIBE = {
+    "name": NAME,
+    "kind": "pipeline",
+    "description": DESCRIPTION,
+    "commands": list(COMMANDS),
+    "events": list(EVENTS),
+    # The manifest schema this build writes into a run directory. A reader
+    # (seren-theatre) can check compatibility before it trusts a file.
+    "manifest_schema_version": 1,
+    "recipe_schema_version": 1,
+    # Requires nothing, of anyone. The heavy deps (torch, transformers,
+    # datasets) belong to the PIPELINE it forks, not to this CLI - which is
+    # what lets `ms-moe-maker validate` run on a laptop with no GPU and no CUDA.
+    "requires": [],
+}

ms_moe_maker/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.4.0'
+__version_tuple__ = version_tuple = (0, 4, 0)
+
+__commit_id__ = commit_id = 'g9c59d38f4'

ms_moe_maker/events.py

@@ -0,0 +1,70 @@
+"""JSON Lines events - the machine-readable twin of the human output.
+
+The Starwright contract, reused: human prose on stderr, one JSON object per
+line on stdout when --json is on. Two channels, never interleaved, so a
+consumer can parse stdout without a heuristic for "is this line prose".
+
+WHY STDERR FOR THE PROSE. The other direction - prose on stdout, events on a
+side channel - means anything piping the tool has to know about the side
+channel. Putting the machine stream on stdout makes `ms-moe-maker build r.yaml --json
+| jq` work with no ceremony, which is the whole point of having it.
+
+ONE RULE, and it is the one that gets broken: flush every line. A consumer
+following a build is reading a pipe, and Python block-buffers a pipe by
+default. Without the flush, a run that takes six hours emits its first event
+when the buffer fills or the process exits, and a dashboard watching it shows
+nothing at all for hours while everything is completely fine. That failure
+looks exactly like a hang.
+"""
+from __future__ import annotations
+
+import json
+import sys
+from typing import Any, TextIO
+
+
+class Events:
+    """Emitter. Inert unless --json was passed, so call sites need no branch."""
+
+    def __init__(self, enabled: bool = False, stream: TextIO | None = None,
+                 prose: TextIO | None = None) -> None:
+        self.enabled = enabled
+        self._out = stream or sys.stdout
+        self._prose = prose or sys.stderr
+
+    def emit(self, kind: str, **fields: Any) -> None:
+        if not self.enabled:
+            return
+        # default=str so a Path or a dataclass never turns a progress report
+        # into a crash. An event stream that can kill the build it is
+        # describing is worse than no event stream.
+        self._out.write(json.dumps({"event": kind, **fields}, default=str) + "\n")
+        self._out.flush()
+
+    def say(self, message: str) -> None:
+        """Human line. Always stderr, never the event stream."""
+        self._prose.write(message + "\n")
+        self._prose.flush()
+
+    # -- the vocabulary, as methods so typos are import errors not silence ---
+
+    def started(self, **kw: Any) -> None:
+        self.emit("started", **kw)
+
+    def stage(self, id: str, status: str, **kw: Any) -> None:
+        self.emit("stage", id=id, status=status, **kw)
+
+    def progress(self, id: str, **kw: Any) -> None:
+        self.emit("progress", id=id, **kw)
+
+    def refused(self, reasons: list[str]) -> None:
+        self.emit("refused", reasons=reasons, count=len(reasons))
+
+    def warning(self, message: str) -> None:
+        self.emit("warning", message=message)
+
+    def error(self, stage: str, message: str) -> None:
+        self.emit("error", stage=stage, message=message)
+
+    def done(self, ok: bool, **kw: Any) -> None:
+        self.emit("done", ok=ok, **kw)