benchmax 0.1.2.dev35__tar.gz → 0.1.2.dev36__tar.gz

{benchmax-0.1.2.dev35/src/benchmax.egg-info → benchmax-0.1.2.dev36}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchmax
-Version: 0.1.2.dev35
+Version: 0.1.2.dev36
 Summary: Framework-Agnostic RL Environments for LLM Fine-Tuning
 Author: castie@castform.com
 Classifier: Programming Language :: Python :: 3

{benchmax-0.1.2.dev35 → benchmax-0.1.2.dev36}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchmax"
-version = "0.1.2.dev35"
+version = "0.1.2.dev36"
 description = "Framework-Agnostic RL Environments for LLM Fine-Tuning"
 readme = "README.md"
 authors = [{ name = "castie@castform.com" }]
@@ -32,9 +32,11 @@ where = ["src"]
 
 [tool.setuptools.package-data]
 # Ship the `castform setup` scaffold templates in the wheel (read at runtime via
-# importlib.resources): docs (*.md), the starter run.py + datasets (*.py/*.jsonl),
+# importlib.resources): docs (*.md), the starter main.py + seed datasets (*.py/*.jsonl),
 # and the per-skill SKILL.md. Skill dirs are hyphenated (data, not Python packages).
 "benchmax.cli.scaffold" = ["*.md", "*.py", "*.jsonl", "skills/*/SKILL.md"]
+# Ship the `castform data view` HTML viewer (read at runtime via importlib.resources).
+"benchmax.cli" = ["templates/*.html"]
 
 [project.optional-dependencies]
 mcp = [

{benchmax-0.1.2.dev35 → benchmax-0.1.2.dev36}/src/benchmax/cli/__init__.py

@@ -18,6 +18,7 @@ from benchmax.cli import (
     control,
     corpus,
     data,
+    doctor,
     help,
     launch,
     runs,
@@ -43,6 +44,7 @@ def build_parser() -> argparse.ArgumentParser:
     data.register(sub)
     corpus.register(sub)
     setup.register(sub)
+    doctor.register(sub)
 
     # `guide` renders the getting-started walkthrough (the renderer lives in
     # ``benchmax.cli.help``). Named `guide`, not `quickstart`, because `setup`

{benchmax-0.1.2.dev35 → benchmax-0.1.2.dev36}/src/benchmax/cli/_output.py

@@ -132,3 +132,20 @@ def fmt_value(value: Any) -> str:
     if isinstance(value, float):
         return f"{value:.4g}"
     return str(value)
+
+
+def truncate(text: object, n: int) -> str:
+    """Collapse whitespace and clip to ``n`` chars (…-suffixed) for one-line display.
+
+    ``None`` → ``""``, but a falsy-yet-real value is preserved (a gold answer of
+    ``0`` must render as ``0``, not blank)."""
+    s = " ".join(("" if text is None else str(text)).split())
+    return s if len(s) <= n else s[: n - 1] + "…"
+
+
+def final_answer(messages: list | None) -> str | None:
+    """The last assistant-turn content in a transcript — the model's committed answer."""
+    for m in reversed(messages or []):
+        if isinstance(m, dict) and m.get("role") == "assistant" and m.get("content"):
+            return m["content"]
+    return None

benchmax-0.1.2.dev36/src/benchmax/cli/_preflight.py

@@ -0,0 +1,104 @@
+"""Preflight: turn a missing-env-dependency ImportError into a copy-paste fix.
+
+``castform validate``/``launch`` import the project's ``main.py`` in-process (see
+:func:`benchmax.cli._project.load_project`). A ``main.py`` that uses a corpus-provider
+backend (turbopuffer / pinecone / chroma) or the data-generation helpers imports a
+package that lives behind a ``castform[...]`` extra, not in base castform — so a base
+install fails with ``ModuleNotFoundError`` mid-import. Rather than surface a raw
+traceback, map the missing top-level module back to the extra that ships it and print
+the exact ``uv pip install`` line.
+
+The default *postgres* rag env imports on base castform (openai + httpx); this is for
+the provider backends and the ``castform data qa-gen`` / chunking path, which pull the
+heavier deps.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import sys
+
+# Import-name → the ``castform[<extra>]`` extra that provides it. Import names differ
+# from the pip/extra names (scikit-learn→``sklearn``; chromadb ships under the
+# ``[chroma]`` extra), and ``ModuleNotFoundError.name`` reports the *import* name — so
+# this is keyed on that. Provider rows mirror ``_providers.PROVIDER_PIP``'s keys; the
+# rest are the ``[rag]`` extra's data-generation + chunking deps.
+_MODULE_EXTRA: dict[str, str] = {
+    # provider search backends — benchmax.rag.corpus.{turbopuffer,pinecone,chroma}
+    "turbopuffer": "turbopuffer",
+    "pinecone": "pinecone",
+    "chromadb": "chroma",
+    "snowballstemmer": "chroma",
+    # data generation (qa-gen) + chunking / ingest — the [rag] extra
+    "sentence_transformers": "rag",
+    "sklearn": "rag",
+    "keybert": "rag",
+    "nest_asyncio": "rag",
+    "langchain_text_splitters": "rag",
+    "ragas": "rag",
+    "ruamel": "rag",
+    "tqdm": "rag",
+}
+
+# One sentinel import name per extra — used by ``castform doctor`` to report whether an
+# extra is installed without importing the (heavy) package itself.
+_EXTRA_SENTINEL: dict[str, str] = {
+    "rag": "keybert",
+    "turbopuffer": "turbopuffer",
+    "pinecone": "pinecone",
+    "chroma": "chromadb",
+}
+
+
+def _missing_module(exc: BaseException) -> str | None:
+    """Top-level name of the first ``ModuleNotFoundError`` in the exception chain."""
+    seen: set[int] = set()
+    cur: BaseException | None = exc
+    while cur is not None and id(cur) not in seen:
+        seen.add(id(cur))
+        if isinstance(cur, ModuleNotFoundError) and cur.name:
+            return cur.name.split(".")[0]
+        cur = cur.__cause__ or cur.__context__
+    return None
+
+
+def install_hint_for_import_error(exc: BaseException) -> str | None:
+    """A copy-paste install line if ``exc`` (or its cause chain) is a missing import.
+
+    Returns ``None`` when the error isn't a ``ModuleNotFoundError`` — callers then
+    just print the base error and leave it at that.
+    """
+    module = _missing_module(exc)
+    if module is None:
+        return None
+    extra = _MODULE_EXTRA.get(module)
+    if extra:
+        return f"→ install the env deps: uv pip install 'castform[{extra}]'"
+    # A third-party module the env imports that isn't one of our extras — still nudge
+    # toward installing it rather than leaving a bare "no module named …".
+    return (
+        f"→ missing dependency {module!r}: install it into this environment "
+        f"(e.g. uv pip install {module})"
+    )
+
+
+def print_project_error(exc: Exception) -> None:
+    """Print a project-load error to stderr, appending an install hint for a
+    missing env dependency (so a base-install ImportError is a one-line fix, not a
+    traceback). Shared by ``validate`` and ``launch``."""
+    print(f"Error: {exc}", file=sys.stderr)
+    hint = install_hint_for_import_error(exc)
+    if hint:
+        print(hint, file=sys.stderr)
+
+
+def extra_is_installed(extra: str) -> bool:
+    """True if ``extra``'s sentinel module resolves (no import executed)."""
+    sentinel = _EXTRA_SENTINEL.get(extra)
+    if sentinel is None:
+        return False
+    try:
+        return importlib.util.find_spec(sentinel) is not None
+    except (ImportError, ValueError):
+        # A parent package missing (or a namespace edge case) => treat as absent.
+        return False

{benchmax-0.1.2.dev35 → benchmax-0.1.2.dev36}/src/benchmax/cli/_project.py

@@ -1,10 +1,10 @@
 """Load a benchmax project (env class + datasets) from a directory.
 
-Convention mirrors the web-app scaffold (``buildAgentContextBody``): ``run.py``
+Convention mirrors the web-app scaffold (``buildAgentContextBody``): ``main.py``
 defines a single :class:`BaseEnv` subclass; ``train_dataset.jsonl`` /
 ``eval_dataset.jsonl`` hold one JSON object per line. ``validate`` and ``launch``
 share this loader. An importable module path (``--module``) is an alternative to
-``run.py`` for shipped envs / fixtures.
+``main.py`` for shipped envs / fixtures.
 """
 
 from __future__ import annotations
@@ -21,7 +21,7 @@ from typing import Any
 
 
 class ProjectError(Exception):
-    """A project couldn't be loaded (missing run.py/dataset, or no/ambiguous env)."""
+    """A project couldn't be loaded (missing main.py/dataset, or no/ambiguous env)."""
 
 
 @dataclass
@@ -31,8 +31,56 @@ class LoadedProject:
     eval_dataset: list[dict[str, Any]]
     module: ModuleType
     from_file: (
-        bool  # loaded from a run.py path (pickle env by value) vs an importable module
+        bool  # loaded from a main.py path (pickle env by value) vs an importable module
     )
+    # Optional module-level config dicts baked into main.py so a run is
+    # reproducible from the file (validate/launch read these; CLI flags override).
+    launch_config: dict[str, Any]
+    validate_config: dict[str, Any]
+
+
+def row_question_and_gold(row: Any) -> tuple[object, object]:
+    """``(question, gold)`` from a dataset row, across the on-disk shapes.
+
+    Question: ``prompt`` (a chat-list prompt → its last user turn) else ``question``.
+    Gold: ``ground_truth`` else ``answer``. One definition so ``runs rollout`` and
+    ``validate --reward-audit`` can't drift when a dataset field is renamed (see the
+    ``castform-dataset-ondisk-shapes`` note)."""
+    if not isinstance(row, dict):
+        return None, None
+    q = row.get("prompt")
+    if not q:
+        q = row.get("question")
+    if isinstance(q, list):  # chat-style prompt → last user turn
+        q = next(
+            (
+                m.get("content")
+                for m in reversed(q)
+                if isinstance(m, dict) and m.get("role") == "user" and m.get("content")
+            ),
+            None,
+        )
+    gold = row.get("ground_truth")
+    if gold is None:
+        gold = row.get("answer")
+    return q, gold
+
+
+def _read_config(module: ModuleType, name: str) -> dict[str, Any]:
+    """A module-level config dict (``LAUNCH_CONFIG`` / ``VALIDATE_CONFIG``) from
+    main.py — the knobs the file bakes in so the run reproduces without remembering
+    CLI flags. Absent → ``{}`` (the block is optional); present-but-not-a-dict is a
+    user error we fail loudly on rather than silently drop (a dropped budget wastes
+    GPU with no explanation)."""
+    value = getattr(module, name, None)
+    if value is None:
+        return {}
+    if not isinstance(value, dict):
+        raise ProjectError(
+            f"{name} must be a dict (got {type(value).__name__}); it bakes the "
+            "validate/launch knobs into main.py. Fix or remove it."
+        )
+    return dict(value)
 
 
 def _load_module_from_file(path: Path) -> ModuleType:
@@ -97,7 +145,7 @@ def _load_jsonl(path: Path) -> list[dict[str, Any]]:
 def load_project(
     *,
     directory: str = ".",
-    run_file: str = "run.py",
+    run_file: str = "main.py",
     module_path: str | None = None,
     env_class_name: str | None = None,
     train_file: str = "train_dataset.jsonl",
@@ -135,4 +183,6 @@ def load_project(
         eval_dataset=eval_dataset,
         module=module,
         from_file=from_file,
+        launch_config=_read_config(module, "LAUNCH_CONFIG"),
+        validate_config=_read_config(module, "VALIDATE_CONFIG"),
     )

{benchmax-0.1.2.dev35 → benchmax-0.1.2.dev36}/src/benchmax/cli/corpus.py

@@ -2,7 +2,7 @@
 
 ``corpus ingest <folder>`` chunks a local document folder and uploads it to the
 deployed Corpora backend (BM25/lexical search), yielding a named corpus that
-``data qa-gen`` and the rag ``run.py`` template resolve by name. ``corpus list`` /
+``data qa-gen`` and the rag ``main.py`` template resolve by name. ``corpus list`` /
 ``corpus delete`` manage them against the 5-corpus-per-user cap (the cap is why
 ``ingest`` can refuse — delete one to free a slot). Thin wrappers over
 ``benchmax.rag.corpus.postgres`` — the rag bits import lazily so users without the
@@ -18,7 +18,9 @@ under cap, or raises ``CorpusLimitError`` — surfaced here as a clean message.
 from __future__ import annotations
 
 import argparse
+import os
 import sys
+import tempfile
 from pathlib import Path
 
 from benchmax import config
@@ -186,6 +188,74 @@ def _cmd_corpus_search(args: argparse.Namespace) -> int:
     return 0
 
 
+@handle_errors
+def _cmd_corpus_browse(args: argparse.Namespace) -> int:
+    # Recover the wizard's "preview the corpus" step: pull the stored chunks and
+    # render them in the local HTML viewer (same one `data view` uses).
+    from benchmax.cli.dataview import build_view_model, write_html
+    from benchmax.platform.browser import maybe_open_browser
+
+    try:
+        client = _corpus_client()
+    except ImportError as exc:
+        print(f"Error: {exc}. {_RAG_INSTALL_HINT}", file=sys.stderr)
+        return 1
+
+    corpora = client.list_corpora()
+    target = next(
+        (c for c in corpora if c.id == args.corpus or c.name == args.corpus), None
+    )
+    if target is None:
+        have = ", ".join(c.name for c in corpora) or "(none)"
+        print(f"Error: no corpus matching {args.corpus!r}. Have: {have}", file=sys.stderr)
+        return 1
+
+    # Page through chunks up to --limit (0 = all). `more` tracks whether a
+    # next-page cursor remained when we stopped, so we can say so honestly.
+    want = args.limit
+    chunks: list = []
+    cursor: str | None = None
+    while True:
+        if want and want > 0 and len(chunks) >= want:
+            break
+        page_size = 500 if not (want and want > 0) else min(500, want - len(chunks))
+        page, cursor = client.list_corpus_chunks(target.id, limit=page_size, cursor=cursor)
+        chunks.extend(page)
+        if not cursor or not page:
+            break
+    more = cursor is not None
+
+    records = [
+        {"id": c.id, "content": c.content, "metadata": c.metadata, "score": c.score}
+        for c in chunks
+    ]
+    if args.json:
+        from benchmax.cli._output import print_json
+
+        print_json(records)
+        return 0
+
+    model = build_view_model(
+        records, source=f"{target.name} (chunks)", type_override="chunks", limit=0
+    )
+    if args.out:
+        out = Path(args.out)
+    else:
+        fd, tmp = tempfile.mkstemp(prefix="castform-chunks-", suffix=".html")
+        os.close(fd)
+        out = Path(tmp)
+    write_html(model, out, view=args.view)
+
+    n = len(records)
+    print(f"✓ corpus '{target.name}' — {n} chunk{'' if n == 1 else 's'}")
+    print(f"  {out}")
+    if more:
+        print(f"  (showing first {n}; corpus has more — re-run with --limit 0 for all)")
+    if not args.no_open:
+        maybe_open_browser(out.resolve().as_uri())
+    return 0
+
+
 def register(sub: argparse._SubParsersAction) -> None:
     """Attach the `corpus` group to the top-level subparsers."""
     corpus = sub.add_parser("corpus", help="Corpus utilities for RAG envs")
@@ -228,3 +298,33 @@ def register(sub: argparse._SubParsersAction) -> None:
     )
     p_se.add_argument("--json", action="store_true", help="Emit raw JSON")
     p_se.set_defaults(func=_cmd_corpus_search)
+
+    p_br = corpus_sub.add_parser(
+        "browse",
+        help="Browse a corpus's chunks in an HTML viewer (opens in your browser)",
+        description="Page a corpus's stored chunks into the local HTML viewer — recovering "
+        "the wizard's 'preview corpus' step. Resolves the corpus by name or id.",
+        epilog="Examples:\n"
+        "  castform corpus browse my-corpus\n"
+        "  castform corpus browse my-corpus --limit 0          # all chunks\n"
+        "  castform corpus browse my-corpus --json > chunks.jsonl",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    p_br.add_argument("corpus", help="Corpus name or id")
+    p_br.add_argument(
+        "--limit", type=int, default=200, help="Max chunks to fetch (default: 200; 0 = all)"
+    )
+    p_br.add_argument(
+        "--view",
+        choices=["auto", "table", "messages", "raw"],
+        default="auto",
+        help="Initial display mode (default: auto)",
+    )
+    p_br.add_argument("--out", help="Write the HTML here (default: a temp file)")
+    p_br.add_argument(
+        "--no-open", action="store_true", help="Write the file but don't open a browser"
+    )
+    p_br.add_argument(
+        "--json", action="store_true", help="Emit raw chunk JSON instead of the viewer"
+    )
+    p_br.set_defaults(func=_cmd_corpus_browse)

{benchmax-0.1.2.dev35 → benchmax-0.1.2.dev36}/src/benchmax/cli/data.py

@@ -15,6 +15,7 @@ import argparse
 import json
 import os
 import sys
+import tempfile
 from pathlib import Path
 from typing import Any
 
@@ -36,7 +37,7 @@ def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
 def _provider_qa_source_factory(provider: str) -> tuple[Any, str]:
     """Build a ``(source_factory, corpus_label)`` for ``qa-gen --provider``.
 
-    Reads the same ``DATA_*`` env vars the wizard / run.py use (``DATA_api_key`` plus
+    Reads the same ``DATA_*`` env vars the wizard / main.py use (``DATA_api_key`` plus
     per-provider resource fields). qa-gen reads stored chunks directly, so no ``embed_fn``
     is wired here. Raises ``ValueError`` with a clean message on a missing env var or a
     missing provider SDK (the caller turns that into an ``Error:`` line).
@@ -280,6 +281,32 @@ def _cmd_data_traces(args: argparse.Namespace) -> int:
         print(f"Error: no traces fetched from project {project_id}.", file=sys.stderr)
         return 1
 
+    if args.preview:
+        # Recover the wizard's "preview imported traces" step: show the NORMALIZED
+        # traces before any filtering, and stop (don't build the train/eval set).
+        from benchmax.cli.dataview import build_view_model, write_html
+        from benchmax.platform.browser import maybe_open_browser
+
+        out_dir = Path(args.out)
+        out_dir.mkdir(parents=True, exist_ok=True)
+        records = [t.to_dict() for t in traces]
+        snapshot = out_dir / "traces_preview.jsonl"
+        _write_jsonl(snapshot, records)
+        html = out_dir / "traces_preview.html"
+        model = build_view_model(records, source=f"{project_id} (traces preview)", limit=0)
+        write_html(model, html)
+        n = len(records)
+        if args.json:
+            print_json(
+                {"preview": True, "count": n, "snapshot": str(snapshot), "html": str(html)}
+            )
+        else:
+            print(f"✓ preview: {n} normalized trace{'' if n == 1 else 's'} (not built into a dataset)")
+            print(f"  snapshot: {snapshot}")
+            print(f"  viewer:   {html}")
+        maybe_open_browser(html.resolve().as_uri())
+        return 0
+
     pipeline = TracesPipeline(
         traces=traces,
         system_prompt=args.system_prompt,
@@ -346,15 +373,61 @@ def _cmd_data_traces(args: argparse.Namespace) -> int:
             print(f"  detected system prompt ({len(system_prompt)} chars): {preview}")
         if tools:
             print(f"  detected tools: {', '.join(t.get('name', '?') for t in tools)}")
-        # The agent authors run.py's dataset_preprocess to match the pulled rows.
+        # The agent authors main.py's dataset_preprocess to match the pulled rows.
         print(
             "  Rows are {prompt_messages, ground_truth, init_rollout_args} — author "
-            "run.py's dataset_preprocess to match."
+            "main.py's dataset_preprocess to match."
         )
         print("  Next: castform setup   (then castform validate)")
     return 0
 
 
+@handle_errors
+def _cmd_data_view(args: argparse.Namespace) -> int:
+    # Lazy import — `dataview` only pulls the traces models for trace files, and
+    # then only on demand; RAG/generic viewing stays free of the [traces] extra.
+    from benchmax.cli.dataview import load_view_model, write_html
+    from benchmax.platform.browser import maybe_open_browser
+
+    src = Path(args.file)
+    if not src.exists():
+        print(f"Error: file not found: {src}", file=sys.stderr)
+        return 1
+
+    try:
+        model = load_view_model(src, type_override=args.type, limit=args.limit)
+    except (OSError, UnicodeDecodeError) as exc:  # binary / unreadable file
+        print(f"Error: could not read {src}: {exc}", file=sys.stderr)
+        return 1
+
+    if args.out:
+        out = Path(args.out)
+    else:
+        # A persistent temp file (not auto-deleted) so the browser can open it;
+        # printed below so headless users can grab the path.
+        fd, tmp = tempfile.mkstemp(prefix="castform-view-", suffix=".html")
+        os.close(fd)
+        out = Path(tmp)
+    try:
+        write_html(model, out, view=args.view)
+    except OSError as exc:
+        print(f"Error: could not write {out}: {exc}", file=sys.stderr)
+        return 1
+
+    noun = "row" if model["total"] == 1 else "rows"
+    print(
+        f"✓ {model['kind']}/{model['variant']} — "
+        f"{model['shown']} of {model['total']} {noun}"
+    )
+    print(f"  {out}")
+    if model["capped"]:
+        print(f"  (capped at {args.limit}; re-run with --limit 0 to embed all)")
+
+    if not args.no_open:
+        maybe_open_browser(out.resolve().as_uri())
+    return 0
+
+
 def register(sub: argparse._SubParsersAction) -> None:
     """Attach the `data` group to the top-level subparsers."""
     data = sub.add_parser("data", help="Dataset utilities")
@@ -437,5 +510,53 @@ def register(sub: argparse._SubParsersAction) -> None:
         help="Detect-only: print the full detected system prompt + tools and write "
         "nothing — confirm them before generating the dataset",
     )
+    p_tr.add_argument(
+        "--preview",
+        action="store_true",
+        help="Open the imported (normalized) traces in the browser and exit, "
+        "without building the train/eval dataset",
+    )
     p_tr.add_argument("--json", action="store_true", help="Emit raw JSON")
     p_tr.set_defaults(func=_cmd_data_traces)
+
+    p_view = data_sub.add_parser(
+        "view",
+        help="Render a local dataset/traces JSONL as an HTML viewer (opens in your browser)",
+        description="Render a dataset, traces, corpus-chunk, or Claude session-transcript "
+        "JSONL as a self-contained HTML page and open it in the browser. The shape is "
+        "auto-detected; anything unrecognized falls back to a generic record table (never "
+        "errors). The path is always printed, so it works headless (open it yourself).",
+        epilog="Examples:\n"
+        "  castform data view dataset.jsonl\n"
+        "  castform data view traces.jsonl --view raw\n"
+        "  castform data view ~/.claude/projects/<proj>/<session-id>.jsonl   # a Claude session\n"
+        "  castform data view data.jsonl --type chunks --out chunks.html --no-open",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    p_view.add_argument("file", help="Local .jsonl dataset or traces file to view")
+    p_view.add_argument(
+        "--type",
+        choices=["rag", "traces", "chunks", "claude", "generic"],
+        default=None,
+        help="Force the classification (default: auto-detect from the first row; "
+        "`claude` = a Claude Code session transcript)",
+    )
+    p_view.add_argument(
+        "--view",
+        choices=["auto", "table", "messages", "raw"],
+        default="auto",
+        help="Initial display mode (default: auto — the detected kind's view)",
+    )
+    p_view.add_argument("--out", help="Write the HTML here (default: a temp file)")
+    p_view.add_argument(
+        "--limit",
+        type=int,
+        default=1000,
+        help="Max rows to embed (default: 1000; 0 = all)",
+    )
+    p_view.add_argument(
+        "--no-open",
+        action="store_true",
+        help="Write the file but don't open a browser",
+    )
+    p_view.set_defaults(func=_cmd_data_view)