corethink 1.0.0__py3-none-any.whl

corethink/commands/capture.py

@@ -0,0 +1,564 @@
+"""`corethink capture`: an environment made from live traffic (companion/harnessCapture.ts).
+
+Start a capture, point your app at `<program>@capture` (a strong model answers meanwhile), label what it caught (good, fix,
+skip; a judge can suggest), then build the dataset: the labelled conversations become the environment's tasks.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any
+
+import typer
+from rich.text import Text
+
+from corethink import output as ui
+from corethink.client import ApiError
+from corethink.commands.env import (
+    BASE,
+    _check_verdict,
+    _clip,
+    _plural,
+    _q,
+    find,
+    list_environments,
+    print_check,
+)
+from corethink.context import client
+
+app = typer.Typer(
+    help="Make an environment from live traffic: capture, label, build its tasks.",
+    no_args_is_help=True,
+)
+
+ENV_ARGUMENT = typer.Argument(..., help="The capture's environment: its ID, the start of it, or its name.")
+CONVERSATION_ID = re.compile(r"^[0-9a-f]{24}$")
+FILTERS = ("all", "unlabelled", "good", "fix", "skip", "suggested")
+DEFAULTS = {"days": 2, "budget": 25.0, "teacher": "anthropic/claude-sonnet-5.5"}
+MAX_BUDGET_USD = 100
+PAGE = 50
+JUDGE_AT_ONCE = 25
+
+STATE = {
+    "running": ("Capturing", "cyan"),
+    "stopped": ("Stopped", "dim"),
+    "ended": ("Ended", "dim"),
+    "limit": ("At its spending limit", "yellow"),
+    "none": ("Not started", "dim"),
+}
+
+
+def _env_id(given: str) -> str:
+    return str(find(list_environments(), given)["id"])
+
+
+def _status(env_id: str) -> dict[str, Any]:
+    return client().get(f"{BASE}/usecases/{_q(env_id)}/capture") or {}
+
+
+def _gateway(env_id: str) -> str | None:
+    try:
+        return (client().get(f"{BASE}/usecases/{_q(env_id)}/endpoint") or {}).get("gateway")
+    except ApiError:
+        return None
+
+
+def _snippet(program: str, gateway: str | None) -> str:
+    return f"""import os
+from openai import OpenAI
+
+client = OpenAI(base_url="{gateway or "<gateway>"}/v1", api_key=os.environ["CORETHINK_API_KEY"])
+client.chat.completions.create(
+    model="{program}@capture",
+    messages=messages,   # with your system prompt, as your app sends it today
+    tools=tools,         # your tools; your code runs the calls, as with any model
+    user=conversation_id,
+)"""
+
+
+@app.command("teachers")
+def teachers() -> None:
+    """The models that can answer your traffic while it is captured."""
+    rows = (client().get(f"{BASE}/usecases:capture") or {}).get("teachers", [])
+
+    def human() -> None:
+        ui.table(rows, [("Model", "id"), ("Name", "label"), ("Note", "note")], empty="No models offered.")
+
+    ui.emit(rows, human)
+
+
+@app.command("start")
+def start(
+    name: str = typer.Option(..., "--name", help="The environment's name."),
+    description: str = typer.Option("", "--description", help="What it is for."),
+    days: int = typer.Option(DEFAULTS["days"], "--days", min=1, max=7, help="How long to capture (1 to 7 days)."),
+    budget: float = typer.Option(
+        DEFAULTS["budget"],
+        "--budget",
+        min=1,
+        max=MAX_BUDGET_USD,
+        help="Spending limit in USD (held from your credit).",
+    ),
+    teacher: str = typer.Option(
+        DEFAULTS["teacher"],
+        "--teacher",
+        help="The model that answers meanwhile (`capture teachers`).",
+    ),
+) -> None:
+    """Start capturing live traffic into a new environment.
+
+    Example: corethink capture start --name "Support chat" --days 3 --budget 40
+    """
+    started = client().post(
+        f"{BASE}/usecases:capture",
+        {
+            "name": name,
+            "description": description,
+            "days": days,
+            "budgetUsd": budget,
+            "teacher": teacher,
+        },
+    )
+    meta = started.get("meta") or {}
+    program = started.get("program") or ""
+    gateway = _gateway(str(meta.get("id"))) if meta.get("id") else None
+
+    def human() -> None:
+        plan = started.get("plan") or {}
+        ui.success(
+            f"Capturing into {meta.get('name')} ({meta.get('id')}) until {ui.when(plan.get('until'))}, up to {ui.usd(plan.get('budget_usd'))}."
+        )
+        ui.out.print(f"Point your app at model [bold]{program}@capture[/bold]:")
+        ui.out.print(_snippet(program, gateway), markup=False)
+        ui.info(
+            "It needs an API key of this environment's endpoint (see `corethink endpoint --help`, or its page). "
+            f"Then: corethink capture status {meta.get('id')}"
+        )
+
+    ui.emit({**started, "gateway": gateway, "model": f"{program}@capture"}, human)
+
+
+@app.command("status")
+def status(env: str = ENV_ARGUMENT) -> None:
+    """Is a capture running, what it caught and spent, and how to point an app at it."""
+    env_id = _env_id(env)
+    found = _status(env_id)
+
+    def human() -> None:
+        plan = found.get("plan") or {}
+        label, style = STATE.get(str(found.get("state")), (str(found.get("state")), ""))
+        labelled = found.get("labelled") or {}
+        usable = int(labelled.get("good") or 0) + int(labelled.get("fix") or 0)
+        ui.fields(
+            [
+                ("State", Text(label, style=style)),
+                ("Answered by", plan.get("teacher_model")),
+                ("Until", ui.when(plan.get("until")) if plan.get("until") else None),
+                (
+                    "Conversations",
+                    f"{found.get('conversations', 0)} ({found.get('requests', 0)} requests)",
+                ),
+                (
+                    "Spent",
+                    f"{ui.usd(found.get('spentUsd'))} of {ui.usd(plan.get('budget_usd'))}"
+                    if plan
+                    else ui.usd(found.get("spentUsd")),
+                ),
+                ("Last request", ui.ago(found.get("lastAt")) if found.get("lastAt") else None),
+                (
+                    "Labelled",
+                    f"{usable} usable ({labelled.get('good', 0)} right, {labelled.get('fix', 0)} fixed), "
+                    f"{labelled.get('skip', 0)} skipped, {labelled.get('suggested', 0)} reviewed by the judge",
+                ),
+                ("Model", f"{found.get('program')}@capture" if found.get("program") else None),
+            ]
+        )
+        ui.info("A dataset needs at least 20 conversations for the eval set and 10 to train on.")
+
+    ui.emit(found, human)
+
+
+@app.command("change")
+def change(
+    env: str = ENV_ARGUMENT,
+    stop: bool = typer.Option(False, "--stop", help="Stop capturing now."),
+    extend_days: int = typer.Option(
+        None,
+        "--extend-days",
+        min=1,
+        max=7,
+        help="Capture this many more days (resumes a stopped one).",
+    ),
+    budget: float = typer.Option(None, "--budget", min=1, max=MAX_BUDGET_USD, help="A new spending limit in USD."),
+) -> None:
+    """Stop a capture, extend (or resume) it, or change its spending limit.
+
+    Example: corethink capture change support-chat --extend-days 1 --budget 50
+    """
+    body: dict[str, Any] = {}
+    if stop:
+        body["stop"] = True
+    if extend_days:
+        body["extendDays"] = extend_days
+    if budget is not None:
+        body["budgetUsd"] = budget
+    if not body:
+        ui.error("Give --stop, --extend-days or --budget.")
+        raise typer.Exit(2)
+    if stop and extend_days:
+        ui.error("Give --stop or --extend-days, not both.")
+        raise typer.Exit(2)
+    env_id = _env_id(env)
+    _apply(env_id, body)
+
+
+def _apply(env_id: str, body: dict[str, Any]) -> None:
+    plan = client().post(f"{BASE}/usecases/{_q(env_id)}/capture", body)
+
+    def human() -> None:
+        if plan.get("stopped"):
+            ui.success("Capture stopped.")
+        else:
+            ui.success(f"Capturing until {ui.when(plan.get('until'))}, up to {ui.usd(plan.get('budget_usd'))}.")
+
+    ui.emit(plan, human)
+
+
+@app.command("stop")
+def stop_capture(env: str = ENV_ARGUMENT) -> None:
+    """Stop capturing (`capture resume` starts it again)."""
+    _apply(_env_id(env), {"stop": True})
+
+
+@app.command("resume")
+def resume(
+    env: str = ENV_ARGUMENT,
+    days: int = typer.Option(1, "--days", min=1, max=7, help="Capture this many more days."),
+) -> None:
+    """Capture again, for one more day or --days more."""
+    _apply(_env_id(env), {"extendDays": days})
+
+
+# ---- conversations ---------------------------------------------------------------------------------------------------------
+
+
+def _page(env_id: str, offset: int, limit: int, filter_: str) -> dict[str, Any]:
+    return (
+        client().get(
+            f"{BASE}/usecases/{_q(env_id)}/conversations",
+            offset=offset,
+            limit=limit,
+            filter=filter_,
+        )
+        or {}
+    )
+
+
+def _conversation_id(env_id: str, given: str) -> str:
+    """A conversation's ID from all of it or its start (searched page by page)."""
+    given = given.strip().lower()
+    if CONVERSATION_ID.match(given):
+        return given
+    matches: list[str] = []
+    offset = 0
+    while True:
+        page = _page(env_id, offset, PAGE, "all")
+        items = page.get("items") or []
+        matches += [str(item["id"]) for item in items if str(item.get("id", "")).startswith(given)]
+        offset += len(items)
+        if not items or offset >= int(page.get("total") or 0) or len(matches) > 1:
+            break
+    if len(matches) == 1:
+        return matches[0]
+    ui.error(
+        f"No conversation starts with {given!r}."
+        if not matches
+        else f"More than one conversation starts with {given!r}."
+    )
+    raise typer.Exit(5 if not matches else 2)
+
+
+def _verdict(label: dict[str, Any] | None) -> Text | str:
+    if not label:
+        return "—"
+    verdict = label.get("verdict")
+    if verdict:
+        return Text(verdict, style={"good": "green", "fix": "yellow", "skip": "dim"}.get(verdict, ""))
+    suggestion = label.get("suggestion") or {}
+    if suggestion:
+        return Text(f"judge: {suggestion.get('verdict')}", style="cyan")
+    return "—"
+
+
+@app.command("conversations")
+def conversations(
+    env: str = ENV_ARGUMENT,
+    filter_: str = typer.Option("all", "--filter", help="all, unlabelled, good, fix, skip or suggested."),
+    limit: int = typer.Option(20, "--limit", min=1, max=PAGE, help="How many (at most 50)."),
+    offset: int = typer.Option(0, "--offset", min=0, help="Skip this many."),
+) -> None:
+    """The conversations a capture caught, newest first, with their labels.
+
+    Example: corethink capture conversations support-chat --filter unlabelled
+    """
+    if filter_ not in FILTERS:
+        ui.error(f"--filter is one of {', '.join(FILTERS)}.")
+        raise typer.Exit(2)
+    env_id = _env_id(env)
+    page = _page(env_id, offset, limit, filter_)
+    items = page.get("items") or []
+
+    def human() -> None:
+        ui.table(
+            items,
+            [
+                ("ID", lambda c: ui.short_id(c.get("id"), 10)),
+                ("When", lambda c: ui.when(c.get("at"))),
+                ("User", lambda c: _clip(c.get("user"), 50)),
+                ("Answer", lambda c: _clip(c.get("answer"), 40)),
+                ("Calls", lambda c: len(c.get("calls") or [])),
+                ("Label", lambda c: _verdict(c.get("label"))),
+            ],
+            empty="No conversations here yet." if filter_ == "all" else f"No {filter_} conversations.",
+        )
+        if items:
+            ui.info(f"{offset + 1}–{offset + len(items)} of {page.get('total', len(items))}.")
+
+    ui.emit(page, human)
+
+
+def _content(value: Any) -> str:
+    if value is None:
+        return ""
+    if isinstance(value, str):
+        return value
+    if isinstance(value, list):
+        return "\n".join(part.get("text", "") if isinstance(part, dict) else str(part) for part in value)
+    return json.dumps(value, ensure_ascii=False)
+
+
+@app.command("show")
+def show(
+    env: str = ENV_ARGUMENT,
+    conversation: str = typer.Argument(..., help="The conversation's ID (or its start)."),
+) -> None:
+    """One captured conversation: every message and tool call, its label and the judge's suggestion."""
+    env_id = _env_id(env)
+    conv = _conversation_id(env_id, conversation)
+    detail = client().get(f"{BASE}/usecases/{_q(env_id)}/conversations/{_q(conv)}")
+
+    def human() -> None:
+        ui.fields(
+            [
+                ("Conversation", detail.get("id")),
+                ("When", ui.when(detail.get("at"))),
+                ("Answered by", detail.get("teacher")),
+                ("Requests", detail.get("requests")),
+                ("Cost", ui.usd(detail.get("costUsd"), 4)),
+            ]
+        )
+        if detail.get("system"):
+            ui.out.print(Text("system", style="bold dim"))
+            ui.out.print(_clip(detail["system"], 400), markup=False)
+        for message in detail.get("messages") or []:
+            role = str(message.get("role"))
+            ui.out.print(
+                Text(
+                    role,
+                    style="bold " + {"user": "cyan", "assistant": "green", "tool": "magenta"}.get(role, "dim"),
+                )
+            )
+            text = _content(message.get("content"))
+            if text:
+                ui.out.print(text, markup=False)
+            for call in message.get("tool_calls") or []:
+                function = call.get("function") or {}
+                arguments = function.get("arguments")
+                ui.out.print(
+                    f"→ {function.get('name')}({arguments if isinstance(arguments, str) else json.dumps(arguments, ensure_ascii=False)})",
+                    markup=False,
+                )
+        label = detail.get("label") or {}
+        if label.get("verdict"):
+            ui.fields(
+                [
+                    ("Label", label.get("verdict")),
+                    (
+                        "Expected",
+                        json.dumps(label["expected"], ensure_ascii=False) if "expected" in label else None,
+                    ),
+                    ("Goals", "; ".join(label.get("goals") or []) or None),
+                    ("Split", label.get("split")),
+                    ("Note", label.get("note")),
+                    ("By", label.get("by")),
+                ],
+                title="Label",
+            )
+        suggestion = label.get("suggestion") or {}
+        if suggestion:
+            ui.fields(
+                [
+                    ("Verdict", suggestion.get("verdict")),
+                    ("Score", ui.score(suggestion.get("score"))),
+                    ("Why", suggestion.get("reason")),
+                    (
+                        "Expected",
+                        json.dumps(suggestion.get("expected"), ensure_ascii=False)
+                        if suggestion.get("expected") is not None
+                        else None,
+                    ),
+                    ("Goals", "; ".join(suggestion.get("goals") or []) or None),
+                ],
+                title="The judge suggests",
+            )
+
+    ui.emit(detail, human)
+
+
+def _parse_expected(text: str) -> Any:
+    """A JSON value when it reads as one, else the text itself."""
+    try:
+        return json.loads(text)
+    except ValueError:
+        return text
+
+
+@app.command("label")
+def label(
+    env: str = ENV_ARGUMENT,
+    conversation: str = typer.Argument(..., help="The conversation's ID (or its start)."),
+    verdict: str = typer.Option(
+        None, "--verdict", help="good (the answer is right), fix (give the right one) or skip."
+    ),
+    expected: str = typer.Option(None, "--expected", help="The right final answer (JSON or text), for fix."),
+    goal: list[str] = typer.Option(None, "--goal", help="What a right answer does (repeatable, at most 10)."),
+    split: str = typer.Option("auto", "--split", help="train, test or auto."),
+    note: str = typer.Option(None, "--note", help="A note kept with the label."),
+    accept: bool = typer.Option(False, "--accept", help="Take the judge's suggestion (its verdict, answer and goals)."),
+) -> None:
+    """Label a captured conversation: good, fix (with the right answer or goals) or skip.
+
+    Example: corethink capture label support-chat 6f1c --verdict fix --expected '{"decision": "deny"}'
+    """
+    if split not in ("train", "test", "auto"):
+        ui.error("--split is train, test or auto.")
+        raise typer.Exit(2)
+    env_id = _env_id(env)
+    conv = _conversation_id(env_id, conversation)
+    body: dict[str, Any] = {"split": split}
+    if accept:
+        detail = client().get(f"{BASE}/usecases/{_q(env_id)}/conversations/{_q(conv)}")
+        suggestion = (detail.get("label") or {}).get("suggestion")
+        if not suggestion:
+            ui.error(
+                "The judge has not reviewed this conversation: `corethink capture judge` first, or give --verdict."
+            )
+            raise typer.Exit(2)
+        body["verdict"] = suggestion.get("verdict")
+        if suggestion.get("verdict") == "fix":
+            if suggestion.get("expected") is not None:
+                body["expected"] = suggestion["expected"]
+            if suggestion.get("goals"):
+                body["goals"] = suggestion["goals"]
+    if verdict is not None:
+        if verdict not in ("good", "fix", "skip"):
+            ui.error("--verdict is good, fix or skip.")
+            raise typer.Exit(2)
+        body["verdict"] = verdict
+    if "verdict" not in body:
+        ui.error("Give --verdict (good, fix or skip), or --accept the judge's suggestion.")
+        raise typer.Exit(2)
+    if expected is not None:
+        body["expected"] = _parse_expected(expected)
+    if goal:
+        body["goals"] = list(goal)
+    if note:
+        body["note"] = note
+    if body["verdict"] == "fix" and "expected" not in body and not body.get("goals"):
+        ui.error("A fix needs the right answer (--expected) or at least one --goal.")
+        raise typer.Exit(2)
+    saved = client().put(f"{BASE}/usecases/{_q(env_id)}/conversations/{_q(conv)}/label", body)
+    ui.emit(saved, lambda: ui.success(f"Labelled {conv[:10]} {saved.get('verdict', body['verdict'])}."))
+
+
+@app.command("judge")
+def judge(
+    env: str = ENV_ARGUMENT,
+    conversation: list[str] = typer.Argument(
+        None,
+        help="Conversations to review (IDs); default: the next unlabelled ones the judge has not seen.",
+    ),
+    count: int = typer.Option(
+        JUDGE_AT_ONCE, "--count", min=1, max=JUDGE_AT_ONCE, help="How many by default (at most 25)."
+    ),
+) -> None:
+    """Ask the judge to review conversations and suggest labels (about $0.10 each, from your credit).
+
+    Example: corethink capture judge support-chat
+    """
+    env_id = _env_id(env)
+    if conversation:
+        ids = [_conversation_id(env_id, given) for given in conversation][:JUDGE_AT_ONCE]
+    else:
+        ids = []
+        offset = 0
+        while len(ids) < count:
+            page = _page(env_id, offset, PAGE, "unlabelled")
+            items = page.get("items") or []
+            # only what nobody has decided and the judge has not seen: reviewing twice costs twice
+            ids += [item["id"] for item in items if not (item.get("label") or {}).get("suggestion")]
+            offset += len(items)
+            if not items or offset >= int(page.get("total") or 0):
+                break
+        ids = ids[:count]
+        if not ids:
+            ui.info("Nothing to review: every conversation is labelled or reviewed.")
+            ui.emit({"judged": 0, "failed": 0, "costUsd": 0}, lambda: None)
+            return
+    ui.info(f"Reviewing {_plural(len(ids), 'conversation')}…")
+    result = client().post(f"{BASE}/usecases/{_q(env_id)}/conversations:judge", {"conversations": ids})
+
+    def human() -> None:
+        failed = f", {result.get('failed')} failed" if result.get("failed") else ""
+        ui.success(f"Reviewed {result.get('judged', 0)}{failed} for {ui.usd(result.get('costUsd'))}.")
+        ui.info(f"See the suggestions: corethink capture conversations {env_id} --filter suggested")
+
+    ui.emit(result, human)
+
+
+@app.command("dataset")
+def dataset(
+    env: str = ENV_ARGUMENT,
+    rubric: str = typer.Option(None, "--rubric", help="How answers are judged (for goals)."),
+) -> None:
+    """Build the environment's tasks from the labelled conversations; checked like an upload.
+
+    Example: corethink capture dataset support-chat
+    """
+    env_id = _env_id(env)
+    built = client().post(f"{BASE}/usecases/{_q(env_id)}/dataset", {"rubric": rubric} if rubric else {})
+    check = built.get("check") or {}
+    counts = built.get("counts") or {}
+
+    def human() -> None:
+        ui.fields(
+            [
+                ("Tasks", counts.get("scenarios")),
+                ("Train / test", f"{counts.get('train', 0)} / {counts.get('test', 0)}"),
+                ("Traces", counts.get("traces")),
+                ("Duplicates left out", counts.get("duplicates") or None),
+            ]
+        )
+        print_check(check)
+        _check_verdict(check, "The dataset")
+        if check.get("ok") and built.get("meta"):
+            ui.info(f"Next: corethink env show {env_id}")
+
+    ui.emit(built, human)
+    if not check.get("ok"):
+        raise typer.Exit(1)
+
+
+def register(root: typer.Typer) -> None:
+    root.add_typer(app, name="capture", rich_help_panel="Build")