corethink 1.0.0__py3-none-any.whl

corethink/cli.py

@@ -0,0 +1,1058 @@
+"""Thin CoreThink command-line front door."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from collections.abc import Sequence
+from dataclasses import replace
+from pathlib import Path
+
+import httpx
+from corethink_evals import (
+    DEFAULT_SYSTEM_INSTRUCTIONS,
+    ArtifactReader,
+    EvaluationRunner,
+    NetworkMode,
+    OpenAICompatibleModelClient,
+    ProgramSource,
+    ProgramSpec,
+    PullPolicy,
+    RunConfig,
+    RuntimeResources,
+    RuntimeSelection,
+    TaskStatus,
+    initialize_program_definition,
+    load_program_definition,
+    load_taskset,
+)
+from corethink_evals.audit import RequestCorrelation, build_audit_bundle, write_audit_bundle
+from corethink_evals.program_budget import DIAL_FIELDS, ReasoningEffort, resolve_effort
+from corethink_experiments.errors import ExperimentError
+
+from corethink import experiment_cli, jobs_cli
+from corethink.account import AccountError
+from corethink.config import ConfigError
+from corethink.jobs_cli import JobsError
+
+DEFAULT_BASE_URL = "https://nre-kernel-xfsavzevuq-uc.a.run.app/v1"
+DEFAULT_MODEL = "corethink/nre-kernel"
+DEFAULT_PROGRAM_ID = "corethink/local-eval"
+DEFAULT_PROGRAM_VERSION = "0.1.0"
+DEFAULT_MODEL_API_KEY_ENV = "CORETHINK_MODEL_API_KEY"
+
+
+#: Named endpoints, so the common case is a word rather than three flags.
+#: Deliberately small and explicit: a preset that guesses a base URL from a model
+#: name would send a customer's prompts somewhere they did not choose.
+AGENT_PRESETS: dict[str, dict[str, str]] = {
+    "corethink": {
+        "base_url": DEFAULT_BASE_URL,
+        "model": DEFAULT_MODEL,
+        "description": "the public CoreThink Reasoner; needs no key",
+    },
+    "openrouter": {
+        "base_url": "https://openrouter.ai/api/v1",
+        # Match Workbench's hosted evaluation default. Explicit --model wins;
+        # the paid preset never silently falls back to a different/free model.
+        "model": "deepseek/deepseek-v4.1-flash",
+        "api_key_env": "OPENROUTER_API_KEY",
+        "description": "paid DeepSeek V4.1 Flash via OpenRouter; requires API key and credits",
+    },
+    "fake": {
+        "adapter": "fake",
+        "description": "the taskset's own build_fake_client; no key, no network",
+    },
+}
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="corethink",
+        description="Evaluate versioned agent programs with replayable stateful traces.",
+    )
+    commands = parser.add_subparsers(dest="command", required=True)
+
+    evaluate = commands.add_parser("eval", help="run a local taskset evaluation")
+    evaluate.add_argument("taskset", help="taskset.py file or directory containing it")
+    evaluate.add_argument("--adapter", choices=("live", "fake"), default="live")
+    # Off by default. Requesting logprobs from an endpoint that does not support
+    # them can be rejected outright, so an evaluation that never wanted them must
+    # not start failing. Deliberately not a program-config key: config feeds the
+    # program hash, so capture there would make the same program with capture on a
+    # different program, and its RL data incomparable with its own baseline.
+    evaluate.add_argument(
+        "--temperature",
+        type=float,
+        default=0.0,
+        help="sampling temperature. 0 is deterministic, which makes every sample of "
+        "a prompt identical — and a GRPO group of identical answers has no gradient "
+        "signal, so collecting RL data needs this above 0.",
+    )
+    evaluate.add_argument("--top-p", type=float, default=None)
+    evaluate.add_argument(
+        "--seed",
+        type=int,
+        default=None,
+        help="reproduce one rollout exactly. Not for building groups: one seed makes "
+        "k samples identical, which is the thing a group must not be.",
+    )
+    evaluate.add_argument(
+        "--capture-logprobs",
+        action="store_true",
+        help="record per-token log-probabilities, which a policy gradient needs "
+        "and supervised fine-tuning does not",
+    )
+    # No parser default: with one, an explicit flag and the fallback are
+    # indistinguishable at resolution time, and a stored base_url could never
+    # take effect. The default is applied last, in `_resolved`.
+    evaluate.add_argument("--base-url")
+    evaluate.add_argument("--model")
+    evaluate.add_argument(
+        "--model-api-key-env",
+        default=DEFAULT_MODEL_API_KEY_ENV,
+        help="environment variable holding the model API key for --adapter live; "
+        "never passed on argv. Unset means the endpoint is called unauthenticated, "
+        "which is correct for the public CoreThink Reasoner and wrong for anything else.",
+    )
+    evaluate.add_argument(
+        "agent",
+        nargs="?",
+        help=(
+            "a named endpoint instead of --adapter/--base-url/--model: "
+            + ", ".join(sorted(AGENT_PRESETS))
+            + ". Explicit flags still win over the preset."
+        ),
+    )
+    evaluate.add_argument("--out", default=".corethink", help="artifact root directory")
+    evaluate.add_argument(
+        "--group",
+        type=_positive_int,
+        default=1,
+        metavar="K",
+        help=(
+            "sample the whole taskset K times instead of once. A group-relative "
+            "baseline compares a completion against its own prompt's other "
+            "attempts, so K>1 is what produces an advantage at all — one attempt "
+            "per prompt has nothing to be relative to. Needs --temperature above "
+            "0, and is refused without it."
+        ),
+    )
+    evaluate.add_argument("--limit", type=_positive_int)
+    evaluate.add_argument("--program", help="program.json file or directory containing it")
+    evaluate.add_argument("--program-id")
+    evaluate.add_argument("--program-version")
+    evaluate.add_argument("--system-instructions")
+    evaluate.add_argument("--max-model-calls", type=_positive_int)
+    evaluate.add_argument("--max-tool-calls", type=_positive_int)
+    evaluate.add_argument("--task-timeout", type=_positive_float)
+    evaluate.add_argument("--model-timeout", type=_positive_float)
+    evaluate.add_argument("--tool-timeout", type=_positive_float)
+    evaluate.add_argument("--retries", type=_nonnegative_int)
+    evaluate.add_argument("--repair-calls", type=_nonnegative_int)
+    evaluate.add_argument("--token-budget", type=_positive_int)
+    evaluate.add_argument("--cost-budget", type=_positive_float)
+    evaluate.add_argument("--fail-fast", action="store_true")
+    evaluate.add_argument("--runtime", choices=("auto", "subprocess", "docker"), default="auto")
+    evaluate.add_argument("--runtime-image")
+    evaluate.add_argument("--runtime-network", choices=("public", "none"))
+    evaluate.add_argument("--runtime-cpu", type=_positive_float)
+    evaluate.add_argument("--runtime-memory-mb", type=_positive_int)
+    evaluate.add_argument("--runtime-pids", type=_positive_int)
+    evaluate.add_argument(
+        "--runtime-pull", choices=("missing", "always", "never"), default="missing"
+    )
+    evaluate.add_argument("--json", action="store_true", help="print the result as JSON")
+    evaluate.add_argument(
+        "--effort",
+        choices=[str(item) for item in ReasoningEffort],
+        default=None,
+        help="program-time budget dial. Sets model/tool call, wall-time, and token "
+        "allowances together. Any budget you pass explicitly is preserved. Effort "
+        "permits more reasoning; it does not promise a better score.",
+    )
+    evaluate.set_defaults(handler=_run_eval)
+
+    login = commands.add_parser(
+        "login",
+        help="sign in to your CoreThink account through the browser",
+        description="Authorize this CLI installation against your CoreThink account. "
+        "With no options it opens the browser you already sign in with; the CLI "
+        "never sees your Google credential. The options below configure endpoints "
+        "and a model provider's key instead, which is a separate thing and stays "
+        "separate.",
+    )
+    login.add_argument(
+        "--provider-key",
+        action="store_true",
+        help="prompt for a MODEL PROVIDER's key (OpenRouter and the like) and store "
+        "it for direct provider evaluation. Not a CoreThink credential and never "
+        "sent to CoreThink.",
+    )
+    login.add_argument(
+        "--api-key",
+        help=argparse.SUPPRESS,  # the old, ambiguous spelling of --provider-key
+    )
+    login.add_argument("--api-url", help="the CoreThink API to sign in to")
+    login.add_argument(
+        "--no-browser",
+        action="store_true",
+        help="print the URL instead of opening it, for a machine with no browser",
+    )
+    login.add_argument("--base-url", help="model endpoint, used when --base-url is not passed")
+    login.add_argument("--model", help="default model")
+    login.add_argument("--registry-url", help="hosted program registry")
+    login.add_argument("--hub-url", help="environments hub")
+    login.add_argument(
+        "--hub-token",
+        action="store_true",
+        help="securely prompt for a temporary environment access token from Workbench",
+    )
+    login.set_defaults(handler=_run_login)
+
+    logout = commands.add_parser("logout", help="remove the stored credential")
+    logout.add_argument(
+        "--all",
+        action="store_true",
+        help="remove every stored setting, not only the credential",
+    )
+    logout.set_defaults(handler=_run_logout)
+
+    config = commands.add_parser("config", help="show or change stored settings")
+    config_commands = config.add_subparsers(dest="config_command", required=True)
+    config_list = config_commands.add_parser("list", help="every setting and where it came from")
+    config_list.add_argument("--json", action="store_true")
+    config_list.set_defaults(handler=_run_config_list)
+    config_get = config_commands.add_parser("get", help="one setting's value in force")
+    config_get.add_argument("key")
+    config_get.set_defaults(handler=_run_config_get)
+    config_set = config_commands.add_parser("set", help="store one setting")
+    config_set.add_argument("key")
+    config_set.add_argument("value")
+    config_set.set_defaults(handler=_run_config_set)
+
+    models = commands.add_parser("models", help="the endpoints this CLI can reach")
+    models_commands = models.add_subparsers(dest="models_command", required=True)
+    models_list = models_commands.add_parser("list", help="named endpoints and their readiness")
+    models_list.add_argument("--json", action="store_true")
+    models_list.set_defaults(handler=_run_models_list)
+
+    program = commands.add_parser("program", help="create portable reasoning programs")
+    program_commands = program.add_subparsers(dest="program_command", required=True)
+    initialize = program_commands.add_parser("init", help="create an editable program.json")
+    initialize.add_argument("path", help="target JSON file or directory")
+    initialize.add_argument("--program-id")
+    initialize.add_argument("--program-version", default=DEFAULT_PROGRAM_VERSION)
+    initialize.set_defaults(handler=_run_program_init)
+
+    from corethink import account
+
+    account.register(commands)
+
+    experiment_cli.register(commands, program_commands)
+    jobs_cli.register(commands)
+    from corethink import rl_cli
+
+    rl_cli.register(commands)
+
+    trace = commands.add_parser("trace", help="inspect evaluation artifacts")
+    trace_commands = trace.add_subparsers(dest="trace_command", required=True)
+    show = trace_commands.add_parser("show", help="show one completed run")
+    show.add_argument("run_id")
+    show.add_argument("--out", default=".corethink", help="artifact root directory")
+    show.add_argument("--json", action="store_true", help="include all events as JSON")
+    show.set_defaults(handler=_show_trace)
+
+    export = trace_commands.add_parser("export", help="export a redacted audit bundle for one run")
+    export.add_argument("run_id")
+    export.add_argument("--out", default=".", help="artifact root containing runs/")
+    export.add_argument("--to", default=None, help="write the bundle to this path")
+    export.add_argument("--request-id", default=None, help="production request this run answered")
+    export.add_argument("--received-at", default=None, help="when that request arrived")
+    export.add_argument("--subject-reference", default=None, help="opaque customer reference")
+    export.add_argument("--channel", default=None, help="how the request arrived")
+    export.add_argument(
+        "--all-events",
+        action="store_true",
+        help="include operational events outside the audit set",
+    )
+    export.add_argument("--json", action="store_true")
+    export.set_defaults(handler=_export_audit)
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    try:
+        return int(args.handler(args))
+    except (
+        ImportError,
+        OSError,
+        ValueError,
+        AccountError,
+        ConfigError,
+        ExperimentError,
+        JobsError,
+    ) as error:
+        print(f"corethink: configuration error: {error}", file=sys.stderr)
+        return 2
+    except Exception as error:
+        print(f"corethink: evaluation infrastructure failed: {error}", file=sys.stderr)
+        return 1
+
+
+def _live_model_client(args: argparse.Namespace) -> OpenAICompatibleModelClient:
+    """The `--adapter live` transport, authenticated only when a key is configured.
+
+    The key comes from an environment variable, never an argument, so it never
+    lands in a shell history or process listing. Absent means unauthenticated —
+    correct for the public CoreThink Reasoner, and the reason this must never
+    default to requiring a key: that would break the one endpoint that needs none.
+    """
+
+    # The named environment variable first, because a caller who passed
+    # --model-api-key-env is naming where the key lives for this run. Then the
+    # stored credential, which is what `corethink login` exists to provide.
+    api_key = os.environ.get(args.model_api_key_env, "").strip() or (_provider_key() or "")
+    http_client = httpx.Client(headers={"Authorization": f"Bearer {api_key}"}) if api_key else None
+    return OpenAICompatibleModelClient(
+        _resolved(args, "base_url", DEFAULT_BASE_URL),
+        client=http_client,
+        capture_logprobs=getattr(args, "capture_logprobs", False),
+    )
+
+
+def _provider_key() -> str | None:
+    """The model provider's key, with a broken config treated as unset.
+
+    Never the CoreThink account credential: that lives in its own file, has its
+    own prefix, and is refused by `corethink.account` if it ever turns up here.
+    """
+
+    from corethink.config import ConfigError, provider_key
+
+    try:
+        return provider_key()
+    except ConfigError:
+        return None
+
+
+def _stored(key: str) -> str | None:
+    """One stored setting, with a missing or unreadable config treated as unset.
+
+    A broken config file must not stop an evaluation that was fully specified on
+    the command line; it is a convenience layer, and a convenience that can
+    prevent work is not one.
+    """
+
+    from corethink.config import ConfigError, resolve
+
+    try:
+        return resolve(key)
+    except ConfigError:
+        return None
+
+
+def _resolved(args: argparse.Namespace, key: str, fallback: str | None = None) -> str | None:
+    """Flag, then environment variable, then stored config, then the built-in."""
+
+    return getattr(args, key, None) or _stored(key) or fallback
+
+
+def _sampling_config(args: argparse.Namespace) -> dict:
+    """Sampling parameters, as part of the program's identity.
+
+    Only what was asked for is included, so an unset option leaves both the request
+    payload and the program hash exactly as earlier clients produced them.
+    """
+
+    config: dict = {"temperature": getattr(args, "temperature", 0) or 0}
+    top_p = getattr(args, "top_p", None)
+    if top_p is not None:
+        config["top_p"] = top_p
+    seed = getattr(args, "seed", None)
+    if seed is not None:
+        # A seed makes stochastic sampling reproducible, which sounds strictly good
+        # and is not: k samples of one prompt under one seed are k identical
+        # answers. Useful for reproducing a single rollout, useless for building a
+        # group, so it is opt-in and never defaulted.
+        config["seed"] = seed
+    return config
+
+
+def _run_eval(args: argparse.Namespace) -> int:
+    if args.program and any(
+        value is not None
+        for value in (args.program_id, args.program_version, args.system_instructions)
+    ):
+        raise ValueError(
+            "--program cannot be combined with --program-id, --program-version, "
+            "or --system-instructions"
+        )
+    _apply_agent_preset(args)
+    group_size = getattr(args, "group", 1) or 1
+    if group_size > 1:
+        # The *effective* temperature, not the flag. `--program` supplies its own
+        # config, and a program pinned at temperature 0 made `--temperature 1.0`
+        # a no-op: the guard passed on the flag, every sample of a prompt was
+        # identical, and 70 tasks came back perfectly flat — which reads as a
+        # taskset with no headroom rather than as sampling that never varied.
+        temperature = _effective_temperature(args)
+        if temperature <= 0:
+            raise ValueError(
+                "--group requires --temperature above 0. At temperature 0 every "
+                "sample of a prompt is the same completion, so a group of K is K "
+                "copies of one answer: identical rewards, zero advantage, nothing "
+                "to learn from. Refused here rather than after K model calls have "
+                "been paid for."
+            )
+        if getattr(args, "seed", None) is not None:
+            raise ValueError(
+                "--group cannot be combined with --seed. One seed makes every "
+                "sample of a prompt identical, which is exactly what a group must "
+                "not be. Seed a single rollout to reproduce it; leave it unset to "
+                "collect a group."
+            )
+    loaded = load_taskset(args.taskset)
+    taskset = loaded.taskset
+    if args.limit is not None:
+        taskset = replace(taskset, tasks=taskset.tasks[: args.limit])
+        if not taskset.tasks:
+            raise ValueError("--limit selected no tasks")
+
+    if args.adapter == "fake":
+        if loaded.fake_client_factory is None:
+            raise ValueError(
+                f"{loaded.source} does not expose build_fake_client() required by --adapter fake"
+            )
+        client = loaded.fake_client_factory()
+        provider = "scripted"
+        model = args.model or "corethink/fake-tool-router"
+    else:
+        client = _live_model_client(args)
+        provider = "openai-compatible"
+        model = _resolved(args, "model", DEFAULT_MODEL)
+
+    try:
+        if args.program:
+            loaded_program = load_program_definition(args.program)
+            program = loaded_program.definition.bind(provider=provider, model=model)
+            program_source = ProgramSource.manifest(loaded_program.definition.content_hash)
+            program = _with_sampling_override(program, args)
+        else:
+            program = ProgramSpec(
+                program_id=args.program_id or DEFAULT_PROGRAM_ID,
+                version=args.program_version or DEFAULT_PROGRAM_VERSION,
+                provider=provider,
+                model=model,
+                system_instructions=args.system_instructions or DEFAULT_SYSTEM_INSTRUCTIONS,
+                # Temperature belongs in config, unlike logprob capture: it changes
+                # what the program *does*, so two temperatures are two programs and
+                # the hash should say so. The default stays 0, because every
+                # evaluation to date was deterministic and silently making them
+                # stochastic would make historical scores irreproducible.
+                #
+                # It has to be reachable, though. At temperature 0 every sample of
+                # a prompt is identical, so a GRPO group of k is k copies of one
+                # answer: flat, zero advantage, no gradient at all. Sampling k
+                # *different* completions is the entire premise of a group-relative
+                # baseline, and until now there was no way to ask for it.
+                config=_sampling_config(args),
+            )
+            program_source = ProgramSource.inline()
+        config = _run_config(args)
+        # Each attempt is a full run with its own artifacts, rather than K
+        # results inside one. The corpus builder already merges run directories
+        # and groups by task id, so this reuses the whole path instead of
+        # inventing a second artifact shape that only grouped runs would use.
+        results = [
+            EvaluationRunner(
+                taskset,
+                program,
+                client,
+                config=config,
+                artifact_root=Path(args.out),
+                program_source=program_source,
+                runtime_selection=_runtime_selection(args),
+            ).evaluate()
+            for _ in range(group_size)
+        ]
+        result = results[-1]
+    finally:
+        close = getattr(client, "close", None)
+        if callable(close):
+            close()
+
+    if args.json:
+        payload = (
+            result.to_dict() if group_size == 1 else {"group": [item.to_dict() for item in results]}
+        )
+        print(json.dumps(payload, indent=2, sort_keys=True))
+    elif group_size > 1:
+        _print_group(results)
+    else:
+        passed = sum(item.status is TaskStatus.PASSED for item in result.task_results)
+        aggregate = "null" if result.primary_score is None else f"{result.primary_score:.3f}"
+        print(
+            f"run {result.run_id}: score={aggregate} "
+            f"passed={passed}/{len(result.task_results)} "
+            f"scored={result.scored_task_count} errors={result.errored_task_count}"
+        )
+        for item in result.task_results:
+            score = "null" if item.score is None else f"{item.score.primary:.3f}"
+            print(
+                f"  {item.task_id}: {item.status.value} "
+                f"score={score} stop={item.stop_reason.value} "
+                f"runtime={item.runtime_spec.provider if item.runtime_spec else 'unknown'}"
+            )
+        print(f"artifacts: {result.artifact_dir}")
+    return (
+        1 if any(item.errored_task_count or item.infrastructure_warnings for item in results) else 0
+    )
+
+
+def _apply_agent_preset(args: argparse.Namespace) -> None:
+    """Expand a named agent into the flags it stands for.
+
+    Applied before anything reads them, and only where the caller left a value
+    unset — naming a preset must never override something typed explicitly, or
+    `corethink eval tasks.py corethink --model other` would quietly run the
+    wrong model.
+    """
+
+    name = getattr(args, "agent", None)
+    if not name:
+        return
+    preset = AGENT_PRESETS.get(name)
+    if preset is None:
+        raise ValueError(
+            f"unknown agent {name!r}; known: {', '.join(sorted(AGENT_PRESETS))}. "
+            f"Pass --base-url and --model to reach anything else."
+        )
+    if preset.get("adapter") and args.adapter == "live":
+        args.adapter = preset["adapter"]
+    for key in ("base_url", "model"):
+        if preset.get(key) and not getattr(args, key, None):
+            setattr(args, key, preset[key])
+    # The key's *location* comes from the preset, never the key itself.
+    if preset.get("api_key_env") and args.model_api_key_env == DEFAULT_MODEL_API_KEY_ENV:
+        args.model_api_key_env = preset["api_key_env"]
+
+
+def _effective_temperature(args: argparse.Namespace) -> float:
+    """The temperature the request will actually carry.
+
+    A flag on the command line, or — when a program file is named and the flag
+    is not — whatever that program pins. Read here rather than at the call site
+    so the group guard and the request cannot disagree about it.
+    """
+
+    flag = getattr(args, "temperature", None)
+    if flag is not None:
+        return float(flag)
+    if getattr(args, "program", None):
+        try:
+            definition = load_program_definition(args.program).definition
+        except (OSError, ValueError):
+            return 0.0
+        return float(definition.config.get("temperature", 0) or 0)
+    return 0.0
+
+
+def _with_sampling_override(program, args: argparse.Namespace):
+    """Apply `--temperature`/`--top-p` over a program file's own config.
+
+    Sampling is part of a program's identity by design — two temperatures are
+    two programs and the hash says so — so this produces a *different* program
+    rather than the named one with a changed knob, and says so on stderr. The
+    alternative was silence, which is what let a group be collected at
+    temperature 0 while the caller had asked for 1.0.
+    """
+
+    overrides = {
+        key: getattr(args, key)
+        for key in ("temperature", "top_p")
+        if getattr(args, key, None) is not None
+    }
+    if not overrides:
+        return program
+    changed = {key: value for key, value in overrides.items() if program.config.get(key) != value}
+    if not changed:
+        return program
+    updated = replace(program, config={**program.config, **overrides})
+    print(
+        f"corethink: {args.program} pins "
+        + ", ".join(f"{key}={program.config.get(key)!r}" for key in changed)
+        + "; overridden to "
+        + ", ".join(f"{key}={value!r}" for key, value in changed.items())
+        + f". Sampling is part of a program's identity, so this is a different "
+        f"program: {updated.content_hash[:12]} rather than "
+        f"{program.content_hash[:12]}.",
+        file=sys.stderr,
+    )
+    return updated
+
+
+def _print_group(results: list) -> None:
+    """What a group is collected for: whether any prompt's attempts disagree.
+
+    A task that *errored* is not a task that scored zero, and the two are easy
+    to confuse because the scorer still returns a `Score` for an errored task —
+    an empty answer matches nothing, so it reads 0.000. Counting those as wrong
+    answers invents disagreement out of a rate limit: eight correct completions
+    and four HTTP 429s read as a spread worth training on, which is exactly the
+    shape of failure this platform exists to refuse.
+
+    Errored attempts are therefore excluded from the numbers and reported on
+    their own, and a group missing attempts is called incomplete rather than
+    usable.
+    """
+
+    from statistics import fmean, pstdev
+
+    per_task: dict[str, list[float]] = {}
+    errored: dict[str, int] = {}
+    for result in results:
+        for item in result.task_results:
+            if getattr(item.status, "value", str(item.status)) == "error":
+                errored[item.task_id] = errored.get(item.task_id, 0) + 1
+                continue
+            if item.score is not None:
+                per_task.setdefault(item.task_id, []).append(item.score.primary)
+
+    attempts = len(results)
+    task_ids = sorted(set(per_task) | set(errored))
+    spread = [task for task, scores in per_task.items() if len(set(scores)) > 1]
+    print(f"group of {attempts} over {len(task_ids)} task(s)")
+    for task in task_ids:
+        scores = per_task.get(task, [])
+        failures = errored.get(task, 0)
+        if not scores:
+            print(f"  {task}: no scored attempt — {failures}/{attempts} errored")
+            continue
+        mean = fmean(scores)
+        deviation = pstdev(scores) if len(scores) > 1 else 0.0
+        marker = "spread" if task in spread else "flat"
+        rendered = " ".join(f"{score:.3f}" for score in scores)
+        suffix = f"  ({failures} errored, excluded)" if failures else ""
+        print(f"  {task}: mean={mean:.3f} sd={deviation:.3f} {marker:<6} [{rendered}]{suffix}")
+
+    total_errors = sum(errored.values())
+    if total_errors:
+        # Before the verdict, because an incomplete group is the thing most
+        # likely to be mistaken for a result.
+        print(
+            f"{total_errors} attempt(s) errored and were excluded — "
+            f"`corethink trace show <run-id>` names the cause"
+        )
+        print(
+            "incomplete: some prompts have fewer attempts than others, so the "
+            "baseline they would be compared against is not the one you asked for"
+        )
+        return
+
+    verdict = (
+        "a group-relative baseline has something to work with"
+        if spread
+        else "nothing here can produce an advantage"
+    )
+    print(f"{len(spread)}/{len(task_ids)} task(s) produced a spread — {verdict}")
+
+
+def _run_models_list(args: argparse.Namespace) -> int:
+    """Which endpoints are reachable, and which are missing a credential.
+
+    Deliberately not a catalogue of models a hosted service offers, because
+    there is no such service: these are the endpoints this CLI knows how to
+    address. Naming them here is the difference between "unknown agent" and
+    knowing what to type.
+
+    There is no `fork` to match HUD's trainable models. Creating one would be a
+    new word for machinery that already exists under a truer name, so the row
+    below points at it instead.
+    """
+
+    rows = []
+    for name in sorted(AGENT_PRESETS):
+        preset = AGENT_PRESETS[name]
+        key_env = preset.get("api_key_env")
+        if preset.get("adapter") == "fake":
+            readiness = "ready (offline)"
+        elif key_env is None:
+            readiness = "ready (no key needed)"
+        elif os.environ.get(key_env, "").strip():
+            readiness = f"ready (${key_env})"
+        elif _provider_key():
+            readiness = "ready (stored credential)"
+        else:
+            readiness = f"needs ${key_env} or `corethink login`"
+        rows.append(
+            {
+                "agent": name,
+                "model": preset.get("model", "—"),
+                "base_url": preset.get("base_url", "—"),
+                "readiness": readiness,
+                "description": preset.get("description", ""),
+            }
+        )
+
+    if args.json:
+        print(json.dumps(rows, indent=2, sort_keys=True))
+        return 0
+
+    print("named endpoints — use as `corethink eval <taskset> <agent>`")
+    for row in rows:
+        print(f"  {row['agent']:<12} {row['model']:<34} {row['readiness']}")
+        print(f"  {'':<12} {row['description']}")
+    stored_model = _stored("model")
+    if stored_model:
+        print(f"  (config: model={stored_model}, base_url={_stored('base_url') or '—'})")
+    print(
+        "  a trained adapter is not created here: `corethink jobs submit --train-on "
+        "corpus.jsonl` produces one, and `jobs adopt` makes it a governed candidate"
+    )
+    print(
+        "  a free-tier model can be withdrawn without notice; if every task scores "
+        "0.000 flat, read the trace — a 404 is recorded as model_attempt_error"
+    )
+    return 0
+
+
+def _run_login(args: argparse.Namespace) -> int:
+    """Sign in to a CoreThink account, or configure the things that are not one.
+
+    One command with two jobs, because `login` is the word people reach for and
+    splitting it would leave the old spelling doing nothing. The rule is stated
+    rather than guessed at: options mean "store this setting", no options means
+    "sign me in". What is gone is the old default — a bare `login` that prompted
+    for a credential without ever saying which service it belonged to.
+    """
+
+    from getpass import getpass
+
+    from corethink.account import AccountError, run_login
+    from corethink.config import CONFIG_KEYS, load_config, masked, save_config
+
+    values = load_config()
+    stored_any = False
+    provider_key = None
+    if args.api_key:
+        # Accepted, never silently: it used to mean the provider key, and a
+        # credential quietly filed under a new meaning is the failure this whole
+        # change exists to prevent.
+        print(
+            "note: --api-key means a model provider's key and is now --provider-key. "
+            "Storing it as provider_api_key; it is never sent to CoreThink.",
+            file=sys.stderr,
+        )
+        provider_key = args.api_key
+    elif args.provider_key:
+        provider_key = getpass(
+            "Model provider API key (input hidden, blank to keep the stored one): "
+        ).strip()
+    if provider_key:
+        values["provider_api_key"] = provider_key
+        stored_any = True
+    if args.hub_token:
+        hub_token = getpass("Temporary environment token (input hidden): ").strip()
+        if not hub_token:
+            raise ValueError("No environment token provided")
+        values["hub_token"] = hub_token
+        stored_any = True
+
+    for key in ("base_url", "model", "registry_url", "hub_url", "api_url"):
+        supplied = getattr(args, key, None)
+        if supplied:
+            values[key] = supplied
+            stored_any = True
+
+    if not stored_any:
+        try:
+            return run_login(args)
+        except AccountError as error:
+            raise ValueError(str(error)) from None
+
+    written = save_config(values)
+    print(f"stored: {written}")
+    for key in sorted(values):
+        if key in CONFIG_KEYS:
+            print(f"  {key:<17} {masked(key, str(values[key]))}")
+    print("  (file mode 0600; no command prints a credential back in full)")
+    if provider_key:
+        print("  provider keys are used for direct provider evaluation only.")
+        print("  Run `corethink login` with no options to sign in to CoreThink.")
+    return 0
+
+
+def _run_logout(args: argparse.Namespace) -> int:
+    """Sign out of CoreThink and remove the credentials stored on this machine.
+
+    Both halves, every time. Somebody typing `logout` wants this laptop to stop
+    being able to act as them, and leaving either the CoreThink session or the
+    provider key behind would not be that.
+    """
+
+    from corethink.account import AccountError, load_credential
+    from corethink.account import run_logout as account_logout
+    from corethink.config import config_path, load_config, save_config
+
+    if load_credential():
+        try:
+            account_logout(args)
+        except AccountError as error:
+            # The local credential still goes. A CLI that kept a usable token
+            # because it could not reach the network has not logged anybody out.
+            print(f"note: {error}", file=sys.stderr)
+
+    values = load_config()
+    if not values:
+        print(f"nothing stored at {config_path()}")
+        return 0
+    if args.all:
+        remaining: dict = {}
+    else:
+        remaining = {
+            key: value
+            for key, value in values.items()
+            if key not in {"api_key", "provider_api_key", "hub_token"}
+        }
+    save_config(remaining)
+    print(f"removed {'every setting' if args.all else 'stored credentials'} from {config_path()}")
+    return 0
+
+
+def _run_config_list(args: argparse.Namespace) -> int:
+    from corethink.config import CONFIG_KEYS, config_path, describe, masked
+
+    settings = describe()
+    if args.json:
+        print(
+            json.dumps(
+                {
+                    setting.key: {
+                        "value": None
+                        if setting.value is None
+                        else masked(setting.key, setting.value),
+                        "source": setting.source,
+                    }
+                    for setting in settings
+                },
+                indent=2,
+                sort_keys=True,
+            )
+        )
+        return 0
+    print(f"config: {config_path()}")
+    for setting in settings:
+        shown = "—" if setting.value is None else masked(setting.key, setting.value)
+        # The source is the useful column: a stale environment variable beating
+        # the file someone just edited is the confusion this exists to end.
+        print(f"  {setting.key:<13} {shown:<28} {setting.source}")
+    print(f"  ({len(CONFIG_KEYS)} settings; an environment variable always wins over the file)")
+    return 0
+
+
+def _run_config_get(args: argparse.Namespace) -> int:
+    from corethink.config import resolve
+
+    value = resolve(args.key)
+    if value is None:
+        print(f"{args.key} is unset", file=sys.stderr)
+        return 1
+    # Printed in full, unmasked, because this is the command whose entire job is
+    # to hand the value to something else.
+    print(value)
+    return 0
+
+
+def _run_config_set(args: argparse.Namespace) -> int:
+    from corethink.config import CONFIG_KEYS, load_config, masked, save_config
+
+    if args.key not in CONFIG_KEYS:
+        raise ValueError(f"unknown setting {args.key!r}; known: {', '.join(sorted(CONFIG_KEYS))}")
+    values = load_config()
+    values[args.key] = args.value
+    written = save_config(values)
+    print(f"{args.key} = {masked(args.key, args.value)}  ({written})")
+    return 0
+
+
+def _run_program_init(args: argparse.Namespace) -> int:
+    loaded = initialize_program_definition(
+        args.path,
+        program_id=args.program_id,
+        version=args.program_version,
+    )
+    print(f"created program: {loaded.source}")
+    print(f"next: corethink eval <taskset> --program {loaded.source}")
+    return 0
+
+
+def _show_trace(args: argparse.Namespace) -> int:
+    reader = ArtifactReader(args.out, args.run_id)
+    manifest = reader.manifest()
+    result = reader.result()
+    events = reader.events()
+    if args.json:
+        print(
+            json.dumps(
+                {
+                    "manifest": manifest,
+                    "result": result.to_dict(),
+                    "events": [event.to_dict() for event in events],
+                },
+                indent=2,
+                sort_keys=True,
+            )
+        )
+        return 0
+
+    requested = manifest.get("requested_program", {})
+    print(f"run {result.run_id}: taskset={result.taskset_id}@{result.taskset_version}")
+    print(
+        "requested program: "
+        f"{requested.get('program_id')}@{requested.get('version')} "
+        f"model={requested.get('model')} sha256={requested.get('content_hash')}"
+    )
+    print(f"events: {len(events)}  usage: {json.dumps(result.usage.to_dict(), sort_keys=True)}")
+    print("timeline:")
+    for event in events:
+        duration = f" {event.duration_ms:.1f}ms" if event.duration_ms is not None else ""
+        print(f"  [{event.sequence:04d}] {event.task_id} {event.kind}{duration}")
+    for item in result.task_results:
+        identities = [identity.to_dict() for identity in item.observed_identities]
+        print(f"\n{item.task_id}: {item.status.value} / {item.stop_reason.value}")
+        score = item.score.to_dict() if item.score else None
+        print(f"  score: {json.dumps(score, sort_keys=True)}")
+        print(f"  final state: {json.dumps(item.final_state.to_dict(), sort_keys=True)}")
+        print(f"  observed engine: {json.dumps(identities, sort_keys=True)}")
+        requested_runtime = item.runtime_spec.to_dict() if item.runtime_spec else None
+        observed_runtime = item.runtime_info.to_dict() if item.runtime_info else None
+        print(
+            "  runtime: "
+            f"requested={json.dumps(requested_runtime, sort_keys=True)} "
+            f"observed={json.dumps(observed_runtime, sort_keys=True)}"
+        )
+        print(f"  cleanup: ok={item.cleanup_ok}")
+    return 0
+
+
+def _export_audit(args: argparse.Namespace) -> int:
+    reader = ArtifactReader(args.out, args.run_id)
+    correlation = None
+    if args.request_id:
+        correlation = RequestCorrelation(
+            request_id=args.request_id,
+            received_at=args.received_at or _iso_now(),
+            subject_reference=args.subject_reference,
+            channel=args.channel,
+        )
+    bundle = build_audit_bundle(
+        reader,
+        run_id=args.run_id,
+        correlation=correlation,
+        include_all_events=args.all_events,
+    )
+    if args.to:
+        path = write_audit_bundle(bundle, Path(args.to))
+        if args.json:
+            print(json.dumps({"path": str(path), "content_hash": bundle.content_hash}, indent=2))
+        else:
+            print(f"wrote audit bundle: {path}")
+            print(f"digest: {bundle.content_hash}")
+            for note in bundle.omissions:
+                print(f"omission: {note}")
+        return 0
+
+    if args.json:
+        print(json.dumps(bundle.to_dict(), indent=2, sort_keys=True))
+    else:
+        print(f"run:     {bundle.run_id}")
+        print(f"digest:  {bundle.content_hash}")
+        if bundle.correlation:
+            print(f"request: {bundle.correlation.request_id}")
+        print(f"events:  {len(bundle.events)}")
+        for note in bundle.omissions:
+            print(f"omission: {note}")
+    return 0
+
+
+def _iso_now() -> str:
+    import datetime as _dt
+
+    return _dt.datetime.now(tz=_dt.UTC).isoformat().replace("+00:00", "Z")
+
+
+def _run_config(args: argparse.Namespace) -> RunConfig:
+    overrides = {
+        "max_model_calls": args.max_model_calls,
+        "max_tool_calls": args.max_tool_calls,
+        "task_timeout_seconds": args.task_timeout,
+        "model_timeout_seconds": args.model_timeout,
+        "tool_timeout_seconds": args.tool_timeout,
+        "transient_retries": args.retries,
+        "malformed_repair_calls": args.repair_calls,
+        "token_budget": args.token_budget,
+        "cost_budget": args.cost_budget,
+        "fail_fast": args.fail_fast,
+    }
+    explicit = {name: value for name, value in overrides.items() if value is not None}
+    config = RunConfig(**explicit)
+
+    effort = getattr(args, "effort", None)
+    if effort is None:
+        return config
+
+    # Pin only the dial-controlled budgets the caller actually set. Store-true
+    # flags like --fail-fast are always present in `explicit` and are none of the
+    # dial's business, so they must not be offered as pinned fields.
+    pinned = tuple(name for name in explicit if name in DIAL_FIELDS)
+    resolution = resolve_effort(effort, base=config, pinned=pinned)
+    if not getattr(args, "json", False):
+        print(f"effort: {resolution.effort}")
+        if resolution.caller_pinned:
+            print(f"  preserved: {', '.join(resolution.caller_pinned)}")
+        print(f"  {resolution.notice}")
+    return resolution.config
+
+
+def _runtime_selection(args: argparse.Namespace) -> RuntimeSelection:
+    return RuntimeSelection(
+        provider=args.runtime,
+        image=args.runtime_image,
+        network=NetworkMode(args.runtime_network) if args.runtime_network else None,
+        resources=RuntimeResources(
+            cpu=args.runtime_cpu,
+            memory_mb=args.runtime_memory_mb,
+            pids=args.runtime_pids,
+        ),
+        pull_policy=PullPolicy(args.runtime_pull),
+    )
+
+
+def _positive_int(value: str) -> int:
+    parsed = int(value)
+    if parsed <= 0:
+        raise argparse.ArgumentTypeError("must be positive")
+    return parsed
+
+
+def _nonnegative_int(value: str) -> int:
+    parsed = int(value)
+    if parsed < 0:
+        raise argparse.ArgumentTypeError("must be nonnegative")
+    return parsed
+
+
+def _positive_float(value: str) -> float:
+    parsed = float(value)
+    if parsed <= 0:
+        raise argparse.ArgumentTypeError("must be positive")
+    return parsed
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())