model-gear 0.5.0__py3-none-any.whl

model_gear/__init__.py

@@ -0,0 +1,11 @@
+"""model-gear — run, assess, and switch the local vLLM model."""
+
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version as _v
+
+try:
+    __version__ = _v("model-gear")
+except PackageNotFoundError:  # editable install without metadata
+    __version__ = "0.0.0+local"
+
+__all__ = ["__version__"]

model_gear/__main__.py

@@ -0,0 +1,8 @@
+"""Allow running model-gear as ``python -m model_gear``."""
+
+import sys
+
+from model_gear.cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

model_gear/assess.py

@@ -0,0 +1,268 @@
+"""API-side assessment and benchmark of a vLLM-served model (stdlib only).
+
+Talks only to the OpenAI-compatible endpoint (``urllib``, no third-party deps).
+Ported from the original ``_assess.py`` and split into two concerns:
+
+* :func:`run_correctness` — fixed correctness probes + reasoning-trace detection
+  (drives ``model assess``);
+* :func:`run_benchmark` — decode throughput + prefill latency (drives
+  ``model benchmark``).
+
+Host-side facts (image tag, GPU memory) are gathered by the command handlers via
+:mod:`model_gear.runtime._compose` and printed alongside this output.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import time
+import urllib.request
+
+from model_gear.cli._errors import EXIT_ENV_ERROR, ModelGearError
+
+# urllib.error.URLError is a subclass of OSError, so `except OSError` covers
+# connection failures, timeouts, and HTTPError without listing it redundantly.
+
+
+@contextlib.contextmanager
+def _api_errors(what: str):
+    """Turn raw HTTP / JSON / response-shape failures into a structured error.
+
+    Without this, an ``HTTPError``/``URLError`` or an unexpected payload
+    (``KeyError``/``JSONDecodeError``) bubbles to the dispatcher's catch-all and
+    appears as ``unexpected: ...`` with no remediation.
+    """
+    try:
+        yield
+    except ModelGearError:
+        raise
+    except OSError as exc:
+        raise ModelGearError(
+            code=EXIT_ENV_ERROR,
+            message=f"{what} failed: {exc}",
+            remediation="check 'model status' / 'docker logs model-gear-vllm'",
+        ) from exc
+    except (json.JSONDecodeError, KeyError, IndexError, TypeError) as exc:
+        raise ModelGearError(
+            code=EXIT_ENV_ERROR,
+            message=f"{what}: unexpected response shape ({exc.__class__.__name__}: {exc})",
+            remediation="the served model returned an unexpected payload; check the vLLM logs",
+        ) from exc
+
+
+# (prompt, expected-substring, table-label) — the two fixed correctness probes.
+_PROBES = [
+    ("What is 17 * 23?", "391", "`17 * 23 = 391`"),
+    (
+        "If a train leaves at 14:45 and arrives at 17:10, how long is the journey in minutes?",
+        "145",
+        "train 14:45→17:10 = 145 min",
+    ),
+]
+
+
+def _post(url: str, payload: dict, timeout: int = 300) -> dict:
+    data = json.dumps(payload).encode()
+    req = urllib.request.Request(
+        url + "/v1/chat/completions",
+        data=data,
+        headers={"Content-Type": "application/json"},
+    )
+    with urllib.request.urlopen(req, timeout=timeout) as r:  # local endpoint only
+        return json.load(r)
+
+
+def _get(url: str, path: str, timeout: int = 10):
+    with urllib.request.urlopen(url + path, timeout=timeout) as r:  # local endpoint only
+        if r.headers.get("content-type", "").startswith("application/json"):
+            return r.status, json.load(r)
+        return r.status, r.read().decode()
+
+
+def _trace_field(msg: dict) -> tuple[str | None, int]:
+    """Return ``(field_name, length)`` of the reasoning trace, whichever key holds it.
+
+    vLLM builds vary: the ``<think>`` trace lands in ``reasoning`` on the nv26.04
+    image, ``reasoning_content`` on older builds.
+    """
+    for key in ("reasoning", "reasoning_content"):
+        val = msg.get(key)
+        if isinstance(val, str) and val:
+            return key, len(val)
+    return None, 0
+
+
+def health_status(url: str) -> int:
+    """Return the ``/health`` status code, or raise if the endpoint is unreachable."""
+    try:
+        status, _ = _get(url, "/health")
+    except OSError as exc:
+        raise ModelGearError(
+            code=EXIT_ENV_ERROR,
+            message=f"/health unreachable at {url} ({exc})",
+            remediation="start the server with 'model serve --apply'",
+        ) from exc
+    return status
+
+
+def served_model(url: str, override: str | None = None) -> tuple[str, object]:
+    """Return ``(model_id, max_model_len)`` from ``/v1/models``. Raises if none served."""
+    with _api_errors("/v1/models"):
+        _, models = _get(url, "/v1/models")
+        data = models.get("data") if isinstance(models, dict) else None
+        if not data:
+            raise ModelGearError(
+                code=EXIT_ENV_ERROR,
+                message=f"/v1/models returned no models at {url}",
+                remediation="check 'model status' / 'docker logs model-gear-vllm'",
+            )
+        first = data[0]
+        return (override or first["id"]), first.get("max_model_len")
+
+
+def _probe(url: str, model: str, prompt: str, expect: str) -> dict:
+    d = _post(
+        url,
+        {
+            "model": model,
+            "messages": [{"role": "user", "content": prompt}],
+            "max_tokens": 2048,
+            "temperature": 0.3,
+        },
+    )
+    msg = d["choices"][0]["message"]
+    content = msg.get("content") or ""
+    field, tlen = _trace_field(msg)
+    return {
+        "ok": expect in content,
+        "expect": expect,
+        "trace_field": field,
+        "trace_len": tlen,
+        "finish": d["choices"][0].get("finish_reason"),
+        "completion_tokens": d.get("usage", {}).get("completion_tokens"),
+    }
+
+
+def _decode_throughput(url: str, model: str, n_tokens: int, runs: int = 2) -> list[float]:
+    rates = []
+    for _ in range(runs):
+        t0 = time.monotonic()
+        d = _post(
+            url,
+            {
+                "model": model,
+                "messages": [
+                    {"role": "user", "content": "Write a detailed essay about distributed systems."}
+                ],
+                "max_tokens": n_tokens,
+                "temperature": 0,
+                "ignore_eos": True,
+            },
+        )
+        dt = time.monotonic() - t0
+        ct = d["usage"]["completion_tokens"]
+        rates.append(round(ct / dt, 1))
+    return rates
+
+
+def _prefill(url: str, model: str) -> dict:
+    prompt = "Summarize this. " + "The system processes events. " * 400
+    t0 = time.monotonic()
+    d = _post(
+        url,
+        {
+            "model": model,
+            "messages": [{"role": "user", "content": prompt}],
+            "max_tokens": 16,
+            "temperature": 0,
+        },
+    )
+    dt = time.monotonic() - t0
+    return {"prompt_tokens": d["usage"]["prompt_tokens"], "seconds": round(dt, 2)}
+
+
+def run_correctness(url: str, model: str | None = None) -> dict:
+    """Run the fixed correctness probes; return a structured result."""
+    url = url.rstrip("/")
+    hstatus = health_status(url)
+    model, max_len = served_model(url, model)
+    probes = []
+    with _api_errors("correctness probe"):
+        for prompt, expect, label in _PROBES:
+            result = _probe(url, model, prompt, expect)
+            result["label"] = label
+            probes.append(result)
+    trace_field = next((p["trace_field"] for p in probes if p["trace_field"]), None)
+    trace_len = max((p["trace_len"] for p in probes), default=0)
+    return {
+        "model": model,
+        "endpoint": url,
+        "health": hstatus,
+        "max_model_len": max_len,
+        "probes": probes,
+        "trace_field": trace_field or "(none)",
+        "trace_len": trace_len,
+        "passed": all(p["ok"] for p in probes),
+    }
+
+
+def run_benchmark(
+    url: str, model: str | None = None, decode_tokens: int = 512, runs: int = 2
+) -> dict:
+    """Measure decode throughput + prefill latency; return a structured result."""
+    url = url.rstrip("/")
+    health_status(url)
+    model, max_len = served_model(url, model)
+    with _api_errors("benchmark"):
+        rates = _decode_throughput(url, model, decode_tokens, runs)
+        pf = _prefill(url, model)
+    return {
+        "model": model,
+        "endpoint": url,
+        "max_model_len": max_len,
+        "decode_tokens": decode_tokens,
+        "decode_rates": rates,
+        "prefill": pf,
+    }
+
+
+def render_correctness(result: dict) -> str:
+    """Render :func:`run_correctness` output as a markdown block for a per-model doc."""
+    lines = [
+        f"## Assessment — `{result['model']}`",
+        "",
+        f"- Endpoint: `{result['endpoint']}` · `/health` {result['health']} · "
+        f"`max_model_len` {result['max_model_len']}",
+        "",
+        "| Check | Result |",
+        "|---|---|",
+    ]
+    for p in result["probes"]:
+        mark = "PASS" if p["ok"] else "FAIL"
+        lines.append(
+            f"| {p['label']} | {mark} (finish={p['finish']}, {p['completion_tokens']} tok) |"
+        )
+    lines.append(
+        f"| reasoning trace field | `{result['trace_field']}` (len {result['trace_len']}) |"
+    )
+    return "\n".join(lines)
+
+
+def render_benchmark(result: dict) -> str:
+    """Render :func:`run_benchmark` output as a markdown block for a per-model doc."""
+    rates = "/".join(str(r) for r in result["decode_rates"])
+    pf = result["prefill"]
+    return "\n".join(
+        [
+            f"## Benchmark — `{result['model']}`",
+            "",
+            f"- Endpoint: `{result['endpoint']}` · `max_model_len` {result['max_model_len']}",
+            "",
+            "| Metric | Result |",
+            "|---|---|",
+            f"| **decode throughput** | **{rates} tok/s** (batch=1, greedy, "
+            f"{result['decode_tokens']} tok forced) |",
+            f"| prefill | {pf['prompt_tokens']} prompt tokens + 16 gen in {pf['seconds']} s |",
+        ]
+    )

model_gear/cli/__init__.py

@@ -0,0 +1,149 @@
+"""Unified CLI entry point for model-gear (binary: ``model``).
+
+The model-ops verbs (``switch``, ``serve``/``stop``, ``status``, ``assess``,
+``benchmark``, ``init``) are the heart of the tool; the agent-first verbs
+(``whoami``, ``learn``, ``explain``, ``overview``, ``doctor``, ``cli``) keep the
+sibling rubric satisfied. Each verb module exposes ``register(sub)`` following
+the same pattern.
+
+Error propagation contract
+--------------------------
+Every handler raises :class:`model_gear.cli._errors.ModelGearError` on failure;
+``main()`` catches it via :func:`_dispatch` and routes through
+:mod:`model_gear.cli._output`. Unknown exceptions are wrapped into a
+``ModelGearError`` so no Python traceback leaks to stderr.
+
+Argparse errors (unknown verb, missing arg) also route through the structured
+format — ``_ModelGearArgumentParser`` overrides ``.error()`` and the subparsers
+are built with ``parser_class=_ModelGearArgumentParser``. Whether errors render
+as text or JSON depends on whether ``--json`` appears in the raw argv
+(:func:`main` sets ``_json_hint`` before ``parse_args``).
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+
+from model_gear import __version__
+from model_gear.cli._errors import EXIT_USER_ERROR, ModelGearError
+from model_gear.cli._output import emit_error
+
+
+class _ModelGearArgumentParser(argparse.ArgumentParser):
+    """ArgumentParser that routes errors through :func:`emit_error`.
+
+    Argparse's default error handler writes ``prog: error: <msg>`` to stderr
+    and exits 2, skipping the ModelGearError plumbing (and the ``hint:`` line
+    agents look for). This subclass emits the structured format and exits with
+    :attr:`EXIT_USER_ERROR`.
+
+    JSON mode: parse-time errors happen before ``args.json`` exists, so we rely
+    on a class-level ``_json_hint`` that :func:`main` pre-populates by scanning
+    raw argv for ``--json``. Shared across all subparser instances.
+    """
+
+    _json_hint: bool = False
+
+    def error(self, message: str) -> None:  # type: ignore[override]
+        err = ModelGearError(
+            code=EXIT_USER_ERROR,
+            message=message,
+            remediation=f"run '{self.prog} --help' to see valid arguments",
+        )
+        emit_error(err, json_mode=type(self)._json_hint)
+        raise SystemExit(err.code)
+
+
+def _argv_has_json(argv: list[str] | None) -> bool:
+    tokens = argv if argv is not None else sys.argv[1:]
+    return any(t == "--json" or t.startswith("--json=") for t in tokens)
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    from model_gear.cli._commands import assess as _assess_cmd
+    from model_gear.cli._commands import benchmark as _benchmark_cmd
+    from model_gear.cli._commands import cli as _cli_group
+    from model_gear.cli._commands import doctor as _doctor_cmd
+    from model_gear.cli._commands import explain as _explain_cmd
+    from model_gear.cli._commands import init as _init_cmd
+    from model_gear.cli._commands import learn as _learn_cmd
+    from model_gear.cli._commands import overview as _overview_cmd
+    from model_gear.cli._commands import serve as _serve_cmd
+    from model_gear.cli._commands import status as _status_cmd
+    from model_gear.cli._commands import stop as _stop_cmd
+    from model_gear.cli._commands import switch as _switch_cmd
+    from model_gear.cli._commands import whoami as _whoami_cmd
+
+    parser = _ModelGearArgumentParser(
+        prog="model",
+        description="model-gear — run, assess, and switch the local vLLM model",
+    )
+    parser.add_argument(
+        "--version",
+        action="version",
+        version=f"%(prog)s {__version__}",
+    )
+    # parser_class propagates to every subparser so their .error() routes
+    # through _ModelGearArgumentParser too.
+    sub = parser.add_subparsers(dest="command", parser_class=_ModelGearArgumentParser)
+
+    # Model-ops verbs (the heart of the tool).
+    _switch_cmd.register(sub)
+    _serve_cmd.register(sub)
+    _stop_cmd.register(sub)
+    _status_cmd.register(sub)
+    _assess_cmd.register(sub)
+    _benchmark_cmd.register(sub)
+    _init_cmd.register(sub)
+
+    # Agent-first / introspection verbs (sibling rubric).
+    _whoami_cmd.register(sub)
+    _learn_cmd.register(sub)
+    _explain_cmd.register(sub)
+    _overview_cmd.register(sub)
+    _doctor_cmd.register(sub)
+    _cli_group.register(sub)
+
+    return parser
+
+
+def _dispatch(args: argparse.Namespace) -> int:
+    """Invoke the registered handler and translate exceptions to exit codes.
+
+    A handler may return ``None`` (success, exit 0) or an ``int`` exit code.
+    Failures MUST raise :class:`ModelGearError`; any other exception is wrapped
+    into one so no Python traceback leaks.
+    """
+    json_mode = bool(getattr(args, "json", False))
+    try:
+        rc = args.func(args)
+    except ModelGearError as err:
+        emit_error(err, json_mode=json_mode)
+        return err.code
+    except Exception as err:  # noqa: BLE001 - last-resort; wrap and route cleanly
+        wrapped = ModelGearError(
+            code=EXIT_USER_ERROR,
+            message=f"unexpected: {err.__class__.__name__}: {err}",
+            remediation="file a bug at https://github.com/agentculture/model-gear/issues",
+        )
+        emit_error(wrapped, json_mode=json_mode)
+        return wrapped.code
+    return rc if rc is not None else 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    # Pre-parse peek so argparse-level errors honour --json.
+    _ModelGearArgumentParser._json_hint = _argv_has_json(argv)
+    parser = _build_parser()
+    args = parser.parse_args(argv)
+
+    if args.command is None:
+        parser.print_help()
+        return 0
+
+    return _dispatch(args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

model_gear/cli/_commands/assess.py

@@ -0,0 +1,53 @@
+"""``model assess`` — correctness probes against the served model.
+
+Read-only. Runs the two fixed correctness probes and detects the reasoning-trace
+field, then emits a markdown block (plus host-side facts) ready to paste into a
+per-model doc under ``docs/``. Throughput lives in ``model benchmark``.
+"""
+
+from __future__ import annotations
+
+import argparse
+
+from model_gear import assess as _assess
+from model_gear.cli import _runtime_ops
+from model_gear.cli._output import emit_result
+from model_gear.runtime import _compose, _env
+
+
+def cmd_assess(args: argparse.Namespace) -> int:
+    json_mode = bool(getattr(args, "json", False))
+    port, deploy_dir = _runtime_ops.resolve_port_soft(args)
+    model = args.model
+    if model is None and deploy_dir is not None:
+        model = _env.read_env(deploy_dir / _compose.ENV_FILE, "VLLM_SERVED_NAME")
+
+    url = f"http://localhost:{port}"
+    result = _assess.run_correctness(url, model)
+    host = {"image": _compose.container_image(), "gpu_memory": _compose.gpu_engine_mem()}
+
+    if json_mode:
+        emit_result({**result, "host": host}, json_mode=True)
+    else:
+        header = (
+            "### Host-side\n"
+            f"- Image: `{host['image']}`  ·  GPU memory (EngineCore): {host['gpu_memory']}\n"
+        )
+        emit_result(header + "\n" + _assess.render_correctness(result), json_mode=False)
+    return 0
+
+
+def register(sub: argparse._SubParsersAction) -> None:
+    p = sub.add_parser(
+        "assess",
+        help="Correctness probes against the served model (markdown for a per-model doc).",
+    )
+    p.add_argument("--port", type=int, help="Host port (default: VLLM_PORT in .env, else 8000).")
+    p.add_argument(
+        "--model", help="Served model name (default: VLLM_SERVED_NAME, else first /v1/models)."
+    )
+    p.add_argument(
+        "--compose-dir", help="Deployment dir (default: $MODEL_GEAR_DIR or ~/.model-gear)."
+    )
+    p.add_argument("--json", action="store_true", help="Emit structured JSON.")
+    p.set_defaults(func=cmd_assess)

model_gear/cli/_commands/benchmark.py

@@ -0,0 +1,57 @@
+"""``model benchmark`` — decode throughput + prefill latency for the served model.
+
+Read-only. Forces a fixed decode length over a couple of runs and measures a
+large-prompt prefill, then emits a markdown block (plus host-side facts) for a
+per-model doc under ``docs/``. Correctness lives in ``model assess``.
+"""
+
+from __future__ import annotations
+
+import argparse
+
+from model_gear import assess as _assess
+from model_gear.cli import _runtime_ops
+from model_gear.cli._output import emit_result
+from model_gear.runtime import _compose, _env
+
+
+def cmd_benchmark(args: argparse.Namespace) -> int:
+    json_mode = bool(getattr(args, "json", False))
+    port, deploy_dir = _runtime_ops.resolve_port_soft(args)
+    model = args.model
+    if model is None and deploy_dir is not None:
+        model = _env.read_env(deploy_dir / _compose.ENV_FILE, "VLLM_SERVED_NAME")
+
+    url = f"http://localhost:{port}"
+    result = _assess.run_benchmark(url, model, decode_tokens=args.decode_tokens, runs=args.runs)
+    host = {"image": _compose.container_image(), "gpu_memory": _compose.gpu_engine_mem()}
+
+    if json_mode:
+        emit_result({**result, "host": host}, json_mode=True)
+    else:
+        header = (
+            "### Host-side\n"
+            f"- Image: `{host['image']}`  ·  GPU memory (EngineCore): {host['gpu_memory']}\n"
+        )
+        emit_result(header + "\n" + _assess.render_benchmark(result), json_mode=False)
+    return 0
+
+
+def register(sub: argparse._SubParsersAction) -> None:
+    p = sub.add_parser(
+        "benchmark",
+        help="Decode throughput + prefill latency for the served model (markdown for a doc).",
+    )
+    p.add_argument("--port", type=int, help="Host port (default: VLLM_PORT in .env, else 8000).")
+    p.add_argument(
+        "--model", help="Served model name (default: VLLM_SERVED_NAME, else first /v1/models)."
+    )
+    p.add_argument(
+        "--decode-tokens", type=int, default=512, help="Forced decode length (default 512)."
+    )
+    p.add_argument("--runs", type=int, default=2, help="Decode-throughput repetitions (default 2).")
+    p.add_argument(
+        "--compose-dir", help="Deployment dir (default: $MODEL_GEAR_DIR or ~/.model-gear)."
+    )
+    p.add_argument("--json", action="store_true", help="Emit structured JSON.")
+    p.set_defaults(func=cmd_benchmark)

model_gear/cli/_commands/cli.py

@@ -0,0 +1,38 @@
+"""``model cli`` — noun grouping CLI-surface introspection.
+
+Exists to satisfy the agent-first rubric's ``overview_cli_noun_exists`` check.
+``model cli overview`` describes the CLI surface itself (distinct from the global
+``overview``, which describes the tool and the served model).
+"""
+
+from __future__ import annotations
+
+import argparse
+
+from model_gear.cli._commands.overview import cli_sections, emit_overview
+
+
+def cmd_cli_overview(args: argparse.Namespace) -> int:
+    emit_overview("model cli", cli_sections(), json_mode=bool(getattr(args, "json", False)))
+    return 0
+
+
+def _no_verb(args: argparse.Namespace) -> int:
+    # `model cli` with no sub-verb prints the noun's overview.
+    return cmd_cli_overview(args)
+
+
+def register(sub: argparse._SubParsersAction) -> None:
+    p = sub.add_parser(
+        "cli",
+        help="CLI-surface introspection (see 'model cli overview').",
+    )
+    p.add_argument("--json", action="store_true", help="Emit structured JSON.")
+    p.set_defaults(func=_no_verb, json=False)
+    # `p` is a _ModelGearArgumentParser (the top-level subparsers were built with
+    # that parser_class); propagate it so `cli overview` parse errors route through
+    # the structured error contract instead of argparse's default stderr/exit 2.
+    noun_sub = p.add_subparsers(dest="cli_command", parser_class=type(p))
+    ov = noun_sub.add_parser("overview", help="Describe the model-gear CLI surface.")
+    ov.add_argument("--json", action="store_true", help="Emit structured JSON.")
+    ov.set_defaults(func=cmd_cli_overview)