evalroute 0.6.0__py3-none-any.whl

evalroute/__init__.py

@@ -0,0 +1,18 @@
+"""evalroute — model routing with a verified-success flywheel.
+
+The library behind the Hermes plugin of the same name: a keyword-first
+classifier that routes a task description to a (model, reasoning effort)
+arm, a ledger that labels routes with pass/fail outcomes, and the Tier-A
+harness that measures arms (cost per verified success) and feeds the
+results back into the route table.
+
+The Hermes plugin is a thin adapter: keppy/hermes-plugin-evalroute.
+"""
+
+from __future__ import annotations
+
+try:
+    from importlib.metadata import version as _version
+    __version__ = _version("evalroute")
+except Exception:  # pragma: no cover - running from a checkout, not installed
+    __version__ = "0.6.0"

evalroute/adjudicate.py

@@ -0,0 +1,109 @@
+"""Gonogo adjudication for evalroute: is a route flip statistically real?
+
+Reuses the merged gonogo plugin's stats (McNemar paired comparison, Wilson
+intervals, INSUFFICIENT-EVIDENCE decisions) so evalroute never treats a
+pilot-scale point-estimate gap as a measurement. Import is lazy and optional:
+without gonogo installed, every stamp degrades to "unverified".
+
+Three stamps:
+  distinguishable(winner, runner, per_task_outcomes) -> str|None
+      McNemar on paired per-task outcomes: "gap: +0.10 [95% CI -0.19,+0.39],
+      p=0.48 - NOT distinguishable at n=10" or "... distinguishable (p=0.03)".
+  observed_verdict(passes, trials) -> str|None
+      gonogo_decide semantics for flywheel observed rows: verdict + what
+      sample size would be needed.
+  route_stamp(lane, winner, runner, ...) -> str|None
+      The provenance line suffix for a measured lane.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Optional
+
+logger = logging.getLogger(__name__)
+
+
+def _gonogo():
+    """Import gonogo lazily; None when not installed (stamps degrade)."""
+    try:
+        import gonogo as mod
+        return mod
+    except Exception as exc:  # pragma: no cover - depends on environment
+        logger.debug("gonogo unavailable: %s", exc)
+        return None
+
+
+def _paired_arm_outcomes(runs: list[dict[str, Any]], arm_a: str, arm_b: str
+                         ) -> Optional[dict[tuple[str, str], tuple[bool, bool]]]:
+    """Per-task paired outcomes for two arms: {(task): (a_passed, b_passed)}.
+
+    Uses graded samples only; a task with k>1 samples is a task-pass iff any
+    sample passed (coverage semantics - same rule the report's cov uses).
+    Tasks not present in BOTH arms are dropped (McNemar needs pairs).
+    """
+    per: dict[str, dict[str, list[bool]]] = {}
+    for r in runs:
+        if "text" not in r or r.get("passed") is None:
+            continue
+        per.setdefault(r["task"], {}).setdefault(r["model"], []).append(bool(r["passed"]))
+    pairs = {t: (any(arms[arm_a]), any(arms[arm_b]))
+             for t, arms in per.items() if arm_a in arms and arm_b in arms}
+    return pairs or None
+
+
+def distinguishable(runs: list[dict[str, Any]], arm_a: str, arm_b: str,
+                    level: float = 0.95) -> Optional[str]:
+    """McNemar verdict on whether arm_a really beats arm_b, or None if
+    gonogo is unavailable or the arms share no task pairs."""
+    g = _gonogo()
+    if g is None:
+        return None
+    pairs = _paired_arm_outcomes(runs, arm_a, arm_b)
+    if pairs is None:
+        return None
+    try:
+        rep_a = {"cases": [{"id": t, "passed": a} for t, (a, _) in pairs.items()]}
+        rep_b = {"cases": [{"id": t, "passed": b} for t, (_, b) in pairs.items()]}
+        cmp = g.compare(rep_a, rep_b, level=level)
+        return str(cmp)  # "B vs A (+x% [lo,hi], p=..) — verdict on N shared cases"
+    except Exception as exc:
+        logger.warning("gonogo compare failed: %s", exc)
+        return None
+
+
+def observed_verdict(passes: int, trials: int, target: float = 0.9) -> Optional[str]:
+    """gonogo decide() semantics for an observed lane's pass rate.
+
+    decide() takes per-trial (confidence, passed) tuples; flywheel outcomes
+    carry no confidence signal, so every trial uses the same value and gonogo
+    itself will note no abstention threshold can be derived. Honest.
+    """
+    g = _gonogo()
+    if g is None or trials <= 0:
+        return None
+    results = [(0.5, bool(i < passes)) for i in range(trials)]
+    try:
+        d = g.decide(results=results, target=target, unit="tasks")
+        verdict = getattr(d, "verdict", None)
+        name = getattr(verdict, "name", str(verdict))
+        reason = getattr(d, "reason", "")
+        return f"gonogo {name}: {reason}" if reason else f"gonogo {name}"
+    except Exception as exc:
+        logger.warning("gonogo decide failed: %s", exc)
+        return None
+
+
+def route_stamp(runs: list[dict[str, Any]], winner: str, runner_up: Optional[str],
+                level: float = 0.95) -> str:
+    """Provenance suffix for a measured lane: was the winner's edge real?
+
+    Never raises; degrades to "unverified" so provenance stays honest about
+    what was actually adjudicated rather than silently omitting the check.
+    """
+    if runner_up is None:
+        return "no runner-up arm to compare"
+    stamp = distinguishable(runs, winner, runner_up, level)
+    if stamp is None:
+        return "gap unverified (gonogo absent or arms share no tasks)"
+    return f"gap vs runner-up: {stamp}"

evalroute/cli.py

@@ -0,0 +1,140 @@
+from __future__ import annotations
+
+import argparse
+
+from . import dataset, dispatch, flywheel
+from .routing import _route_for_args, _tool_result, install_routes
+
+_WORKFLOW_EPILOG = """\
+workflow (route -> arm -> rate, in the session that runs the task):
+  1. /route <task>            classify; prints the card (lane, model, effort)
+  2. /model <model>          set the arm from the card's "run:" line
+                             (/reasoning <effort> too, unless install-routes
+                             already wrote it into agent.reasoning_overrides)
+  3. do the task in that session
+  4. /rate pass|fail [--lane <lane-id>] [--note ...]
+                             label the outcome; --lane files a correction
+                             when the route got the lane wrong
+                             --route-id <id> selects a pending route when overlapping
+same flow from the terminal: hermes evalroute route "<task>" (step 1) and
+hermes evalroute rate pass --note ... (step 4); steps 2-3 are chat commands.
+hermes evalroute dispatch <brief.md> runs steps 1-3 on a subprocess worker and
+prints the rate line for step 4.
+routing data improves only when routes are rated: unrouted tasks cost the
+same as ever, unrated routes teach nothing."""
+
+
+def setup_cli(subparser) -> None:
+    """argparse wiring for `hermes evalroute` (register_cli_command setup_fn)."""
+    subs = subparser.add_subparsers(dest="evalroute_action")
+    route_p = subs.add_parser("route", help="Classify a task and print a route card",
+                              epilog=_WORKFLOW_EPILOG,
+                              formatter_class=argparse.RawDescriptionHelpFormatter)
+    route_p.add_argument("task", nargs="*", help="The task description")
+    route_p.add_argument("--lane", help="Pin a lane id instead of classifying")
+    route_p.add_argument("--replace-route-id", help="Replace a specific pending route (requires --lane)")
+    route_p.add_argument("--json", action="store_true",
+                         help="Print the tool-result JSON envelope instead of the card")
+    rate_p = subs.add_parser("rate", help="Rate the last routed task: pass|fail",
+                             epilog=_WORKFLOW_EPILOG,
+                             formatter_class=argparse.RawDescriptionHelpFormatter)
+    rate_p.add_argument("verdict", nargs="?", choices=["pass", "fail", "skip"],
+                        help="pass | fail | skip")
+    rate_p.add_argument("--lane", help="File a lane correction (the lane it should have been)")
+    rate_p.add_argument("--route-id", help="Select a pending route explicitly (profile-wide ledger)")
+    rate_p.add_argument("--model", help="Confirm the actual arm's model id (diagnostic; with --effort)")
+    rate_p.add_argument("--effort", help="Confirm the actual arm's effort (diagnostic; with --model)")
+    rate_p.add_argument("--note", help="Why — the highest-value part of the label")
+    install_p = subs.add_parser("install-routes", help="Write the route table's effort "
+                                   "column into agent.reasoning_overrides")
+    install_p.add_argument("--dry-run", action="store_true", help="Show the diff, write nothing")
+    status_p = subs.add_parser("sync", help="Pin the published route table "
+                               "(keppy/evalroute-flywheel) under the Hermes home")
+    status_p.add_argument("--revision", help="Pin a specific dataset revision "
+                          "(default: resolve 'main' to its commit sha)")
+    status_p.add_argument("--status", action="store_true",
+                          help="Show which route table is active; no network")
+    status_p.add_argument("--clear", action="store_true",
+                          help="Unpin the dataset table; route on the bundled table")
+    dispatch_p = subs.add_parser("dispatch",
+                                 help="Route a brief, spawn hermes chat on that arm, "
+                                      "print the rate line",
+                                 epilog=_WORKFLOW_EPILOG,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    dispatch_p.add_argument("brief", help="Path to the brief markdown file")
+    dispatch_p.add_argument("--lane", help="Pin a lane id instead of classifying")
+    dispatch_p.add_argument("--in", dest="indir", help="Extra --in dir for the child session")
+    dispatch_p.add_argument("--task", help="Task description (default: the brief's first paragraph)")
+    dispatch_p.add_argument("--out", help="Report path (default: <brief stem>.report.md beside it)")
+    dispatch_p.add_argument("--timeout", type=float, help="Kill the child after SECONDS (exit 124)")
+    dispatch_p.add_argument("--rate-on-exit", choices=["fail"],
+                            help="Auto-rate fail when the child exits non-zero (never auto-passes)")
+    dispatch_p.add_argument("--dry-run", action="store_true",
+                            help="Route and print the argv; spawn nothing")
+    subparser.set_defaults(func=evalroute_cli)
+
+
+def evalroute_cli(args) -> int:
+    """Handler for `hermes evalroute ...` (register_cli_command handler_fn)."""
+    action = getattr(args, "evalroute_action", None)
+    if action == "install-routes":
+        return install_routes(dry_run=bool(getattr(args, "dry_run", False)))
+    if action == "dispatch":
+        return dispatch.run(args)
+    if action == "sync":
+        return dataset.run(args)
+    if action == "rate":
+        parts = [getattr(args, "verdict", None) or ""]
+        if getattr(args, "lane", None):
+            parts.append(f"--lane {args.lane}")
+        if getattr(args, "route_id", None):
+            parts.append(f"--route-id {args.route_id}")
+        if getattr(args, "model", None):
+            parts.append(f"--model {args.model}")
+        if getattr(args, "effort", None):
+            parts.append(f"--effort {args.effort}")
+        if getattr(args, "note", None):
+            parts.append(f"--note {args.note}")
+        print(flywheel.handle_rate(" ".join(parts)))
+        return 0
+    if action == "route":
+        task = " ".join(getattr(args, "task", []) or [])
+        lane = getattr(args, "lane", None)
+        replace_id = getattr(args, "replace_route_id", None)
+        as_json = getattr(args, "json", False)
+        if not task and not lane:
+            print(_WORKFLOW_EPILOG)
+            return 2
+        if replace_id and not lane:
+            print("evalroute: --replace-route-id requires --lane <lane-id>")
+            return 2
+        try:
+            if lane:
+                raw = f"--lane {lane} {f'--replace-route-id {replace_id}' if replace_id else ''} {task}".strip()
+            else:
+                raw = task
+            card, lane_obj, conf, pinned, method, route_id = _route_for_args(raw)
+        except Exception as exc:
+            print(f"evalroute: {exc}")
+            return 1
+        if as_json:
+            print(_tool_result(card, lane_obj, conf, pinned, method=method, route_id=route_id))
+        else:
+            print(card)
+        return 0
+    print(_WORKFLOW_EPILOG)
+    return 2
+
+
+def main() -> int:
+    """Standalone entry point (`evalroute ...`), byte-identical to `hermes evalroute ...`."""
+    import sys
+
+    parser = argparse.ArgumentParser(
+        prog="evalroute",
+        description="Route tasks to the right (model, reasoning effort) arm",
+    )
+    setup_cli(parser)
+    args = parser.parse_args()
+    rc = args.func(args)
+    sys.exit(rc)

evalroute/contract.py

@@ -0,0 +1,32 @@
+"""The plugin-facing contract of the evalroute library.
+
+Everything the Hermes plugin (keppy/hermes-plugin-evalroute) relies on is
+listed here. Those nine names are the public API; everything else in the
+package is private to the library and may change without notice.
+"""
+
+from __future__ import annotations
+
+CONTRACT_VERSION = 1
+
+#: The nine contract names, by module:
+#:   evalroute.routing.set_llm_facade        (called at plugin register time)
+#:   evalroute.routing.evalroute_route       (tool handler for evalroute_route)
+#:   evalroute.routing.handle_route_command  (/route slash command)
+#:   evalroute.cli.setup_cli                 (register_cli_command setup_fn)
+#:   evalroute.cli.evalroute_cli             (register_cli_command handler_fn)
+#:   evalroute.flywheel.handle_rate          (/rate slash command + CLI rate)
+#:   evalroute.flywheel.on_pre_command       (pre_command hook: /model /reasoning)
+#:   evalroute.flywheel.on_post_llm_call     (post_llm_call hook)
+#:   evalroute.schemas.EVALROUTE_ROUTE       (tool schema)
+CONTRACT_NAMES = (
+    "routing.set_llm_facade",
+    "routing.evalroute_route",
+    "routing.handle_route_command",
+    "cli.setup_cli",
+    "cli.evalroute_cli",
+    "flywheel.handle_rate",
+    "flywheel.on_pre_command",
+    "flywheel.on_post_llm_call",
+    "schemas.EVALROUTE_ROUTE",
+)

evalroute/data/facets.yaml

@@ -0,0 +1,56 @@
+# Task facets: orthogonal dimensions a task can occupy simultaneously.
+# Lanes are the ROUTING decision (one arm); facets are the LABEL — what the
+# task actually is along each axis. A task may occupy one facet per axis.
+# Facets are descriptive labels only; the lane classifier chooses the arm.
+version: 1
+
+facets:
+  - id: long-doc
+    axis: input-shape
+    description: input-heavy reading over many pages/files before answering
+    lanes: [long-doc-reading]
+
+  - id: interactive
+    axis: input-shape
+    description: conversational or small-context work; no bulk reading required
+    lanes: []   # default when no long-doc signal; never route ON this facet
+
+  - id: domain-dlml
+    axis: domain
+    description: training/eval harness engineering, RL on reasoning, ML research judgment
+    lanes: [dl-ml-research-engineering]
+
+  - id: domain-alignment
+    axis: domain
+    description: alignment/safety reasoning, paper-claim critique
+    lanes: [alignment-reasoning]
+
+  - id: domain-math
+    axis: domain
+    description: first-principles math, proofs, derivations
+    lanes: [math-first-principles]
+
+  - id: domain-prose
+    axis: domain
+    description: writing quality is the deliverable
+    lanes: [prose]
+
+  - id: domain-research
+    axis: domain
+    description: web/literature research, source gathering
+    lanes: [web-research]
+
+  - id: tier-routine
+    axis: demand-tier
+    description: self-contained, well-specified work; retries are cheap
+    lanes: [routine-coding]
+
+  - id: tier-hard
+    axis: demand-tier
+    description: multi-step, multi-file, or judgment-heavy work where a miss is expensive
+    lanes: [hard-agentic-coding]
+
+  - id: tier-orchestration
+    axis: demand-tier
+    description: the work IS planning and delegation
+    lanes: [orchestration]

evalroute/data/routes.yaml

@@ -0,0 +1,224 @@
+version: 1
+lanes:
+- id: alignment-reasoning
+  label: Alignment reasoning, paper claims
+  model: z-ai/glm-5.3
+  effort: medium
+  provenance: 'measured 10 tasks, cov 1.0, all-in $0.0119/succ, 2026-09-27; v1 runs omit model_id (mapped via models.json); gap vs
+    runner-up: 100.0% vs 100.0% (+0.0% [-33.4%, +33.4%], p=1.000) — not distinguishable
+    on 10 shared cases'
+  keywords:
+  - alignment
+  - reward hacking
+  - goodhart
+  - sycophancy
+  - mesa-optimization
+  - mesaoptimization
+  - outer alignment
+  - inner alignment
+  - interpretability
+  - paper claim
+  - rlhf
+  - rlaif
+  - constitutional
+  match_hint: reasoning about failure modes and paper claims
+  notes: 'Four of five evaluated arms have 30 judged samples each; qwen3.8-max@medium has 30 pending, not graded. No checker audit. The p=1 and zero-width empirical gap describe this ten-task sample, not population equivalence.'
+- id: routine-coding
+  label: Routine coding
+  model: z-ai/glm-5.3-flash
+  effort: medium
+  provenance: 'measured 10 tasks, cov 1.0, all-in $0.0001/succ, 2026-09-27; v1 runs omit model_id (mapped via models.json); gap vs
+    runner-up: 100.0% vs 100.0% (+0.0% [-33.4%, +33.4%], p=1.000) — not distinguishable
+    on 10 shared cases'
+  keywords:
+  - refactor
+  - lint
+  - unit test
+  - unit-test
+  - test
+  - typo
+  - bugfix
+  - bug fix
+  - script
+  - rename
+  - cleanup
+  - format
+  - boilerplate
+  - stub
+  - scaffold
+  - small function
+  - helper
+  match_hint: everyday code changes where a test or compiler checks the result
+  escalation: gpt-6-luna
+  notes: 'Verbose already; tests catch misses cheaply, so retries beat deep thinking. ds-v4.1-flash@medium had pass^k 1.00, but costs more; prefer when determinism matters. p=1 and a zero-width empirical gap at n=10 do not prove population equivalence.'
+- id: dl-ml-research-engineering
+  label: DL / ML research engineering
+  model: z-ai/glm-5.3
+  effort: high
+  provenance: 'measured 10 tasks, cov 1.0, all-in $0.0024/succ, 2026-09-27; v1 runs omit model_id (mapped via models.json); gap vs
+    runner-up: 100.0% vs 100.0% (+0.0% [-33.4%, +33.4%], p=1.000) — not distinguishable
+    on 10 shared cases'
+  keywords:
+  - training run
+  - fine-tune
+  - finetune
+  - finetuning
+  - grpo
+  - trl
+  - lora
+  - sft
+  - rl
+  - reward model
+  - loss curve
+  - gradient
+  - ablation
+  - hyperparameter
+  - hyperparameter
+  - overfitting
+  - checkpoint
+  - dataloader
+  - tokenizer
+  - distillation
+  - colocation
+  - colocate
+  match_hint: training/eval harness engineering
+  escalation: gpt-6-sol
+  notes: 'The ten-task p=1, zero-width empirical gap vs runner-up is not proof of population equivalence; this route is a cost choice on measured coverage.'
+- id: hard-agentic-coding
+  label: Hard agentic coding
+  keywords:
+  - hard agentic
+  - terminal-bench
+  - multi-file
+  - multi file
+  - repo-wide
+  - architecture
+  - migrate
+  - performance
+  - race condition
+  - deadlock
+  - memory leak
+  - integration
+  - e2e
+  - integration test
+  - flaky
+  - build failure
+  - build error
+  match_hint: multi-step repo work where a miss costs many tool-call rounds
+  model: z-ai/glm-5.3
+  effort: max
+  escalation: gpt-6-sol
+  provenance: priors 2026-09-26 (GLM 5.3 42% on TB 4.0 at max, Artificial Analysis)
+  notes: the 42% TB 4.0 figure was at max; escalate to GPT-6 Sol when coverage gaps
+- id: long-doc-reading
+  label: Long-doc reading, lit review
+  keywords:
+  - lit review
+  - literature
+  - paper summary
+  - summarize paper
+  - read paper
+  - long document
+  - long doc
+  - pdf
+  - survey
+  - spec review
+  - rfp
+  - due diligence
+  - technical report
+  match_hint: input-heavy extraction over many pages
+  model: z-ai/glm-5.3-flash
+  effort: medium
+  escalation: null
+  provenance: priors 2026-09-26 (V4.1 Flash AA-LCR 84%, on par with GPT-5.6 Sol)
+  notes: input-heavy; gap to closed ~none
+- id: web-research
+  label: Web research
+  keywords:
+  - web research
+  - browse
+  - search the web
+  - fact-check
+  - factcheck
+  - market research
+  - competitor
+  - news
+  - pricing
+  - sources
+  - citations
+  - open-source intelligence
+  - osint
+  match_hint: search-and-synthesize over live web content
+  model: moonshotai/kimi-k3
+  effort: medium
+  escalation: null
+  provenance: priors 2026-09-26 (K3 BrowseComp 91.2, vendor-reported, at max)
+  notes: input-heavy; max roughly triples output at $10.53/M so medium is the starting
+    arm
+- id: math-first-principles
+  label: Math, first principles
+  keywords:
+  - proof
+  - prove
+  - theorem
+  - derivation
+  - derive
+  - combinatorics
+  - algebraic
+  - calculus
+  - probability bound
+  - asymptotic
+  - complexity bound
+  - lemma
+  - math
+  - inequality
+  match_hint: correctness comes from reasoning, not from tools
+  model: qwen/qwen3.8-max
+  effort: max
+  escalation: gpt-6-sol
+  provenance: priors 2026-09-26 (Qwen3.8-Max MathArena 56% vs GPT-6 Sol 85%, Opus
+    5.5 83%)
+  notes: ~27-30 pts behind closed; skip open for one-shot math except Lean-verified
+    pipelines
+- id: prose
+  label: Prose
+  keywords:
+  - blog post
+  - blog
+  - essay
+  - short story
+  - prose
+  - newsletter
+  - announcement
+  - launch post
+  - copywriting
+  - tagline
+  - bio
+  - readme rewrite
+  match_hint: human is the checker
+  model: moonshotai/kimi-k3
+  effort: medium
+  escalation: claude-opus-5-5
+  provenance: 'priors 2026-09-26 (K3 EQ-Bench creative writing #2 behind Opus 5; judged
+    by a Claude model)'
+  notes: untested prior; blind-test K3 vs Opus 5.5 at low-medium
+- id: orchestration
+  label: Orchestrator / subagents
+  keywords:
+  - orchestrate
+  - orchestration
+  - orchestrator
+  - subagent
+  - multi-agent
+  - multiagent
+  - delegate
+  - fan out
+  - parallel agents
+  - crew
+  - workflow design
+  match_hint: the session plans and delegates, not does
+  model: claude-opus-5-5
+  effort: high
+  escalation: null
+  provenance: priors 2026-09-26 (image 2 only; not in README priors)
+  notes: subagents run at low-medium via delegation.reasoning_effort; set it separately