chipgraph 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- chipgraph/__init__.py +7 -0
- chipgraph/adapters/__init__.py +1 -0
- chipgraph/adapters/format/__init__.py +6 -0
- chipgraph/adapters/format/__main__.py +16 -0
- chipgraph/adapters/format/_yaml_lines.py +54 -0
- chipgraph/adapters/format/chip_yaml.py +323 -0
- chipgraph/adapters/format/qsoc_contract.py +246 -0
- chipgraph/adapters/llm/__init__.py +58 -0
- chipgraph/adapters/llm/_budget.py +173 -0
- chipgraph/adapters/llm/_errors.py +50 -0
- chipgraph/adapters/llm/_nda.py +30 -0
- chipgraph/adapters/llm/_retry.py +85 -0
- chipgraph/adapters/llm/anthropic.py +247 -0
- chipgraph/adapters/llm/decide_backend.py +179 -0
- chipgraph/adapters/llm/fake.py +87 -0
- chipgraph/adapters/llm/guard.py +68 -0
- chipgraph/adapters/parser/__init__.py +22 -0
- chipgraph/adapters/parser/cocotb.py +304 -0
- chipgraph/adapters/parser/generic_regex.py +75 -0
- chipgraph/adapters/parser/verible.py +55 -0
- chipgraph/adapters/parser/verilator.py +46 -0
- chipgraph/adapters/review/__init__.py +7 -0
- chipgraph/adapters/review/file.py +123 -0
- chipgraph/adapters/review/github.py +886 -0
- chipgraph/adapters/runner/__init__.py +7 -0
- chipgraph/adapters/runner/local.py +122 -0
- chipgraph/adapters/runtime/__init__.py +5 -0
- chipgraph/adapters/runtime/claude_code/__init__.py +24 -0
- chipgraph/adapters/runtime/claude_code/agents.py +158 -0
- chipgraph/adapters/runtime/claude_code/decisions.py +280 -0
- chipgraph/adapters/runtime/claude_code/guarded.py +310 -0
- chipgraph/adapters/runtime/claude_code/isolation.py +482 -0
- chipgraph/adapters/runtime/claude_code/loop.py +342 -0
- chipgraph/adapters/runtime/claude_code/previous.py +419 -0
- chipgraph/adapters/runtime/claude_code/review.py +492 -0
- chipgraph/adapters/runtime/claude_code/runtime.py +192 -0
- chipgraph/adapters/runtime/claude_code/service.py +2285 -0
- chipgraph/adapters/tool/__init__.py +7 -0
- chipgraph/adapters/tool/cmd.py +229 -0
- chipgraph/adapters/tool/edalize.py +773 -0
- chipgraph/adapters/tool/filelist.py +259 -0
- chipgraph/adapters/tool/pyslang.py +485 -0
- chipgraph/adapters/vcs/__init__.py +1 -0
- chipgraph/adapters/vcs/git.py +521 -0
- chipgraph/app/__init__.py +10 -0
- chipgraph/app/baseline.py +339 -0
- chipgraph/app/build.py +332 -0
- chipgraph/app/checks.py +202 -0
- chipgraph/app/context.py +191 -0
- chipgraph/app/errors.py +11 -0
- chipgraph/app/executors.py +79 -0
- chipgraph/app/findings.py +203 -0
- chipgraph/app/ingest.py +523 -0
- chipgraph/checks/__init__.py +52 -0
- chipgraph/checks/_cdc_analyze.py +272 -0
- chipgraph/checks/_cdc_elaborate.py +455 -0
- chipgraph/checks/_common.py +268 -0
- chipgraph/checks/_cross.py +96 -0
- chipgraph/checks/_model.py +64 -0
- chipgraph/checks/_naming_pyslang.py +235 -0
- chipgraph/checks/_naming_rules.py +127 -0
- chipgraph/checks/_xref_syntax.py +260 -0
- chipgraph/checks/cdc_struct.py +235 -0
- chipgraph/checks/connect.py +353 -0
- chipgraph/checks/cross_chip.py +454 -0
- chipgraph/checks/duplicate.py +153 -0
- chipgraph/checks/filelist.py +323 -0
- chipgraph/checks/format.py +268 -0
- chipgraph/checks/generated.py +151 -0
- chipgraph/checks/hardcode.py +233 -0
- chipgraph/checks/layout.py +155 -0
- chipgraph/checks/naming.py +311 -0
- chipgraph/checks/plan_check.py +901 -0
- chipgraph/checks/ports_diff.py +203 -0
- chipgraph/checks/spec_schema.py +277 -0
- chipgraph/checks/style.py +910 -0
- chipgraph/checks/tb_static.py +599 -0
- chipgraph/checks/trace.py +224 -0
- chipgraph/checks/vendor_guard.py +369 -0
- chipgraph/checks/wrap_check.py +486 -0
- chipgraph/cli/__init__.py +1688 -0
- chipgraph/cli/next_steps.py +137 -0
- chipgraph/core/__init__.py +1 -0
- chipgraph/core/config/__init__.py +93 -0
- chipgraph/core/config/defaults.py +14 -0
- chipgraph/core/config/errors.py +32 -0
- chipgraph/core/config/loader.py +766 -0
- chipgraph/core/config/models.py +845 -0
- chipgraph/core/config/templates.py +340 -0
- chipgraph/core/contracts/__init__.py +75 -0
- chipgraph/core/contracts/_paths.py +26 -0
- chipgraph/core/contracts/agent.py +49 -0
- chipgraph/core/contracts/artifact.py +68 -0
- chipgraph/core/contracts/check.py +76 -0
- chipgraph/core/contracts/event.py +70 -0
- chipgraph/core/contracts/export.py +128 -0
- chipgraph/core/contracts/finding.py +171 -0
- chipgraph/core/contracts/gate.py +50 -0
- chipgraph/core/contracts/rule.py +177 -0
- chipgraph/core/contracts/types.py +45 -0
- chipgraph/core/engine/__init__.py +70 -0
- chipgraph/core/engine/agent_rule.py +1362 -0
- chipgraph/core/engine/baseline.py +283 -0
- chipgraph/core/engine/decide.py +462 -0
- chipgraph/core/engine/fanout.py +1358 -0
- chipgraph/core/engine/gate.py +170 -0
- chipgraph/core/engine/graph.py +647 -0
- chipgraph/core/engine/records.py +60 -0
- chipgraph/core/engine/rules.py +92 -0
- chipgraph/core/engine/scheduler.py +637 -0
- chipgraph/core/engine/select.py +1239 -0
- chipgraph/core/engine/targets.py +342 -0
- chipgraph/core/model/__init__.py +98 -0
- chipgraph/core/model/entities.py +244 -0
- chipgraph/core/model/export.py +16 -0
- chipgraph/core/model/ingest.py +693 -0
- chipgraph/core/model/json_value.py +4 -0
- chipgraph/core/model/keys.py +95 -0
- chipgraph/core/model/model.py +153 -0
- chipgraph/core/model/provenance.py +27 -0
- chipgraph/core/model/query.py +583 -0
- chipgraph/core/model/registry.py +80 -0
- chipgraph/core/model/relations.py +61 -0
- chipgraph/core/model/store.py +279 -0
- chipgraph/core/plugin_api/__init__.py +81 -0
- chipgraph/core/plugin_api/local.py +167 -0
- chipgraph/core/plugin_api/pack.py +159 -0
- chipgraph/core/plugin_api/protocols.py +177 -0
- chipgraph/core/plugin_api/registry.py +111 -0
- chipgraph/core/plugin_api/types.py +94 -0
- chipgraph/core/runtime/__init__.py +41 -0
- chipgraph/core/runtime/executor.py +79 -0
- chipgraph/core/runtime/queue.py +565 -0
- chipgraph/core/runtime/roles/__init__.py +91 -0
- chipgraph/core/runtime/roles/data/author.md +41 -0
- chipgraph/core/runtime/roles/data/critic.md +56 -0
- chipgraph/core/runtime/roles/data/planner.md +38 -0
- chipgraph/core/runtime/roles/data/researcher.md +37 -0
- chipgraph/core/runtime/roles/data/rtl-author.md +64 -0
- chipgraph/core/runtime/roles/data/tb-author.md +48 -0
- chipgraph/core/runtime/roles/data/triage.md +34 -0
- chipgraph/core/runtime/roles/policy.py +299 -0
- chipgraph/core/runtime/roles/registry.py +52 -0
- chipgraph/core/runtime/roles/review.py +275 -0
- chipgraph/core/runtime/roles/schema.py +82 -0
- chipgraph/core/runtime/roles/skills.py +134 -0
- chipgraph/core/runtime/roles/spec.py +232 -0
- chipgraph/core/state/__init__.py +1 -0
- chipgraph/core/state/artifacts.py +156 -0
- chipgraph/core/state/backend.py +117 -0
- chipgraph/core/state/findings.py +151 -0
- chipgraph/core/state/handoff.py +411 -0
- chipgraph/core/state/idempotency.py +59 -0
- chipgraph/core/state/journal.py +238 -0
- chipgraph/core/state/layout.py +82 -0
- chipgraph/core/state/lock.py +195 -0
- chipgraph/core/state/trace.py +225 -0
- chipgraph/learn/__init__.py +96 -0
- chipgraph/learn/draft.py +187 -0
- chipgraph/learn/infer.py +580 -0
- chipgraph/learn/models.py +145 -0
- chipgraph/learn/try_run.py +228 -0
- chipgraph/mcp/__init__.py +12 -0
- chipgraph/mcp/model_tools.py +167 -0
- chipgraph/mcp/server.py +787 -0
- chipgraph/orgs/qnsc/naming-v1.yml +113 -0
- chipgraph/orgs/qnsc/profile.yml +5 -0
- chipgraph/packs/__init__.py +8 -0
- chipgraph/packs/assist/__init__.py +16 -0
- chipgraph/packs/assist/ask/__init__.py +58 -0
- chipgraph/packs/assist/ask/_db.py +57 -0
- chipgraph/packs/assist/ask/_project.py +230 -0
- chipgraph/packs/assist/ask/answer.py +266 -0
- chipgraph/packs/assist/ask/check.py +182 -0
- chipgraph/packs/assist/ask/contract.py +115 -0
- chipgraph/packs/assist/ask/documents.py +157 -0
- chipgraph/packs/assist/ask/fts.py +80 -0
- chipgraph/packs/assist/ask/retrieve.py +331 -0
- chipgraph/packs/assist/audit/__init__.py +383 -0
- chipgraph/packs/assist/pack.yml +13 -0
- chipgraph/packs/assist/triage/__init__.py +64 -0
- chipgraph/packs/assist/triage/contract.py +94 -0
- chipgraph/packs/assist/triage/parse.py +371 -0
- chipgraph/packs/assist/triage/question.py +384 -0
- chipgraph/packs/assist/triage/report.py +160 -0
- chipgraph/packs/assist/triage/rules.py +447 -0
- chipgraph/packs/assist/triage/run.py +224 -0
- chipgraph/packs/digital_rtl/__init__.py +19 -0
- chipgraph/packs/digital_rtl/layout.py +144 -0
- chipgraph/packs/digital_rtl/pack.yml +25 -0
- chipgraph/packs/digital_rtl/plan/__init__.py +34 -0
- chipgraph/packs/digital_rtl/plan/__main__.py +16 -0
- chipgraph/packs/digital_rtl/plan/context.py +82 -0
- chipgraph/packs/digital_rtl/plan/expand.py +94 -0
- chipgraph/packs/digital_rtl/plan/model.py +150 -0
- chipgraph/packs/digital_rtl/plan/resolver.py +400 -0
- chipgraph/packs/digital_rtl/plan/show.py +176 -0
- chipgraph/packs/digital_rtl/pr/__init__.py +11 -0
- chipgraph/packs/digital_rtl/pr/body.py +165 -0
- chipgraph/packs/digital_rtl/pr/gate.py +110 -0
- chipgraph/packs/digital_rtl/pr/open.py +397 -0
- chipgraph/packs/digital_rtl/rtl/__init__.py +9 -0
- chipgraph/packs/digital_rtl/rtl/context.py +272 -0
- chipgraph/packs/digital_rtl/rules/open_pr.yml +30 -0
- chipgraph/packs/digital_rtl/rules/plan.yml +28 -0
- chipgraph/packs/digital_rtl/rules/plan_expand.yml +29 -0
- chipgraph/packs/digital_rtl/rules/pr_gate.yml +27 -0
- chipgraph/packs/digital_rtl/rules/review.yml +31 -0
- chipgraph/packs/digital_rtl/rules/rtl_module.yml +53 -0
- chipgraph/packs/digital_rtl/skills/lang-sv-rtl.md +58 -0
- chipgraph/packs/digital_rtl/skills/org-naming.md +29 -0
- chipgraph/packs/digital_rtl/skills/plan-modules.md +63 -0
- chipgraph/packs/digital_rtl/skills/review-diff.md +41 -0
- chipgraph/packs/digital_rtl/targets/plan.yml +11 -0
- chipgraph/packs/digital_rtl/targets/rtl.yml +20 -0
- chipgraph/packs/digital_rtl/wrapper/__init__.py +24 -0
- chipgraph/packs/digital_rtl/wrapper/common.py +171 -0
- chipgraph/packs/digital_rtl/wrapper/context.py +222 -0
- chipgraph/packs/digital_rtl/wrapper/emacs.py +626 -0
- chipgraph/packs/digital_rtl/wrapper/expansion.py +304 -0
- chipgraph/packs/digital_rtl/wrapper/lisp.py +504 -0
- chipgraph/packs/digital_rtl/wrapper/resolver.py +60 -0
- chipgraph/packs/digital_rtl/wrapper/rules/wrapper.yml +41 -0
- chipgraph/packs/digital_rtl/wrapper/rules/wrapper_gen.yml +35 -0
- chipgraph/packs/digital_rtl/wrapper/skills/wrapper-auto.md +120 -0
- chipgraph/packs/dv/__init__.py +15 -0
- chipgraph/packs/dv/declaration.py +392 -0
- chipgraph/packs/dv/interface.py +596 -0
- chipgraph/packs/dv/pack.yml +14 -0
- chipgraph/packs/dv/rules/tb_module.yml +42 -0
- chipgraph/packs/dv/skills/dv/cocotb.md +72 -0
- chipgraph/packs/spec_core/__init__.py +9 -0
- chipgraph/packs/spec_core/extract/__init__.py +12 -0
- chipgraph/packs/spec_core/extract/_markdown.py +371 -0
- chipgraph/packs/spec_core/extract/_text.py +116 -0
- chipgraph/packs/spec_core/extract/mas.py +975 -0
- chipgraph/packs/spec_core/gen/__init__.py +11 -0
- chipgraph/packs/spec_core/gen/agents_md/__init__.py +148 -0
- chipgraph/packs/spec_core/gen/agents_md/_context.py +239 -0
- chipgraph/packs/spec_core/gen/agents_md/_markers.py +131 -0
- chipgraph/packs/spec_core/gen/agents_md/agents_md.md.j2 +178 -0
- chipgraph/packs/spec_core/gen/diagram/__init__.py +110 -0
- chipgraph/packs/spec_core/gen/diagram/__main__.py +44 -0
- chipgraph/packs/spec_core/gen/diagram/_diagrams.py +403 -0
- chipgraph/packs/spec_core/gen/diagram/_scene.py +163 -0
- chipgraph/packs/spec_core/pack.yml +6 -0
- chipgraph/py.typed +0 -0
- chipgraph-0.1.0.dist-info/METADATA +53 -0
- chipgraph-0.1.0.dist-info/RECORD +252 -0
- chipgraph-0.1.0.dist-info/WHEEL +4 -0
- chipgraph-0.1.0.dist-info/entry_points.txt +61 -0
- chipgraph-0.1.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,1239 @@
|
|
|
1
|
+
"""Best-of-N: N candidates of one rule inside one budget, chosen by its checks (DESIGN 5.3, F6).
|
|
2
|
+
|
|
3
|
+
```
|
|
4
|
+
BestOfNExecutor.execute_checked(rule, instance)
|
|
5
|
+
N = the rule's opt-in (instance param `best_of`, or `select.rules[rule]`), capped at
|
|
6
|
+
`select.max_n`; no opt-in -> the plain executor, unchanged
|
|
7
|
+
plan_budget: ONE ledger = budget_factor x the rule's budget (tries, tokens, cost);
|
|
8
|
+
each candidate gets a slice (total / N); N drops to what fits (1 -> a plain run)
|
|
9
|
+
run_candidates (fanout.py): N workspaces, seeded identically, same inputs and outputs
|
|
10
|
+
candidate i: the rule, sliced budget, variant hint i, its own seed
|
|
11
|
+
every runtime call reserves one try of the ledger first; none left -> cancelled
|
|
12
|
+
select_best: the rule's checks decide; the Critic only breaks exact ties
|
|
13
|
+
merge only the winner onto the result branch; journal one event per candidate and
|
|
14
|
+
one `select` event
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
The selection key, in order (lower is better; `select_best` is total-ordered):
|
|
18
|
+
|
|
19
|
+
| step | key |
|
|
20
|
+
|----------|---------------------------------------------------------------------------|
|
|
21
|
+
| passed | every check of the rule passed (a failing candidate wins only if none did) |
|
|
22
|
+
| errors | fewer errors: error issues, or 1 for a failing check without one |
|
|
23
|
+
| warnings | fewer `warning` issues over all check results (`info` ignored) |
|
|
24
|
+
| area | smaller area, only when a `MetricProvider` gives one; lacking it sorts after |
|
|
25
|
+
| critic | only among exact ties on the above: approve, fewer blocker, major, minor |
|
|
26
|
+
| lowest_id| still tied (or no Critic, or the ledger cannot pay for it): lowest id |
|
|
27
|
+
|
|
28
|
+
Budget arithmetic: the ledger holds `floor(tries x factor)` tries (at least 1),
|
|
29
|
+
`floor(tokens x factor)` tokens and `max_cost x factor` cost when those are capped; each
|
|
30
|
+
candidate's `AttemptBudget` gets `tries // N` tries, `tokens // N` tokens and
|
|
31
|
+
`cost / N`. A ledger try is one runtime call (an infra retry costs one too); a call
|
|
32
|
+
holds its token cap (lowered to what the ledger has left) while it runs, so the
|
|
33
|
+
candidates cannot overspend tokens together. When a call cannot be paid for, that
|
|
34
|
+
candidate is cancelled there (cooperatively, between calls) and the best candidate that
|
|
35
|
+
finished wins. The Critic costs one try per tied candidate it reviews, plus what it
|
|
36
|
+
reports, from the same ledger. Example, a rule with `tries: 3, tokens: 200000` and a
|
|
37
|
+
`max_cost` of 2 USD: the ledger holds 4 tries, 300000 tokens, 3 USD; with N=2 each
|
|
38
|
+
candidate gets 2 tries, 150000 tokens, 1.5 USD; with N=3, 1 try, 100000 tokens, 1 USD.
|
|
39
|
+
|
|
40
|
+
Candidate outputs never touch the user's working tree: the winner lands on a result
|
|
41
|
+
branch (`chipgraph/<run>/best-of-<instance>`). How that branch reaches the user's tree
|
|
42
|
+
(and the scheduler's output check and records), and the claude-code runtime's own
|
|
43
|
+
wiring (its runtime per workspace, the review rule as the Critic) are M2-11's job.
|
|
44
|
+
|
|
45
|
+
Generic: no project, tool, model or file format is used here; the agent runtime is
|
|
46
|
+
named only to say where its wiring belongs.
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
from __future__ import annotations
|
|
50
|
+
|
|
51
|
+
import contextlib
|
|
52
|
+
import hashlib
|
|
53
|
+
import math
|
|
54
|
+
import re
|
|
55
|
+
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
|
56
|
+
from dataclasses import dataclass, field
|
|
57
|
+
from pathlib import Path
|
|
58
|
+
from typing import Any, Literal, Protocol, runtime_checkable
|
|
59
|
+
|
|
60
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
61
|
+
|
|
62
|
+
from chipgraph.core.config.models import FanoutCfg, SelectCfg
|
|
63
|
+
from chipgraph.core.contracts import AgentResult, Budget, CheckResult, Issue
|
|
64
|
+
from chipgraph.core.contracts.rule import RuleInstance, RuleSpec
|
|
65
|
+
from chipgraph.core.contracts.types import FailureLabel
|
|
66
|
+
from chipgraph.core.engine.fanout import (
|
|
67
|
+
BranchContext,
|
|
68
|
+
BranchOutcome,
|
|
69
|
+
BranchResult,
|
|
70
|
+
FanoutBranch,
|
|
71
|
+
FanoutVcs,
|
|
72
|
+
JoinCheck,
|
|
73
|
+
run_candidates,
|
|
74
|
+
)
|
|
75
|
+
from chipgraph.core.engine.scheduler import (
|
|
76
|
+
CheckingExecutor,
|
|
77
|
+
CheckRunner,
|
|
78
|
+
ExecContext,
|
|
79
|
+
ExecEventType,
|
|
80
|
+
ExecOutcome,
|
|
81
|
+
Executor,
|
|
82
|
+
RewindAware,
|
|
83
|
+
)
|
|
84
|
+
from chipgraph.core.plugin_api.protocols import AgentRuntime
|
|
85
|
+
from chipgraph.core.plugin_api.types import AgentTask
|
|
86
|
+
from chipgraph.core.runtime.roles.review import ReviewReport, Verdict
|
|
87
|
+
from chipgraph.core.state.layout import StateLayout, new_run_id
|
|
88
|
+
|
|
89
|
+
VARIANTS: Mapping[str, str] = {
|
|
90
|
+
"a": "approach a: prefer the simplest structure that meets the spec",
|
|
91
|
+
"b": "approach b: prefer the most explicit structure, every case spelled out",
|
|
92
|
+
"c": "approach c: prefer the fewest registers and the least state that meet the spec",
|
|
93
|
+
}
|
|
94
|
+
"""The default hint per candidate, `AgentTask.context["variant"]`; the key is the id."""
|
|
95
|
+
|
|
96
|
+
BEST_OF_PARAM = "best_of"
|
|
97
|
+
"""The instance parameter a rule instance opts in with (`best_of: <n>`)."""
|
|
98
|
+
|
|
99
|
+
OUTPUTS_CHECK = "best_of.outputs"
|
|
100
|
+
"""The check `BestOfNExecutor` adds for a candidate that left an output missing."""
|
|
101
|
+
|
|
102
|
+
CRITIC = "critic"
|
|
103
|
+
"""The ledger account the Critic's spend is charged to."""
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
# --- the shared ledger ------------------------------------------------------------------
|
|
107
|
+
|
|
108
|
+
LedgerLimit = Literal["tries", "tokens", "cost"]
|
|
109
|
+
"""Which limit of the shared ledger stopped something."""
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
class Spend(BaseModel):
|
|
113
|
+
"""What was spent: tries (runtime calls), tokens and cost (USD)."""
|
|
114
|
+
|
|
115
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
116
|
+
|
|
117
|
+
tries: int = Field(default=0, ge=0, description="Runtime calls (each costs one try).")
|
|
118
|
+
tokens: int = Field(default=0, ge=0, description="Tokens reported.")
|
|
119
|
+
cost: float = Field(default=0.0, ge=0, description="Cost reported, USD.")
|
|
120
|
+
|
|
121
|
+
def plus(
|
|
122
|
+
self, *, tries: int = 0, tokens: int | None = None, cost: float | None = None
|
|
123
|
+
) -> Spend:
|
|
124
|
+
"""This spend with more added (`None`: nothing reported)."""
|
|
125
|
+
return Spend(
|
|
126
|
+
tries=self.tries + tries,
|
|
127
|
+
tokens=self.tokens + (tokens or 0),
|
|
128
|
+
cost=self.cost + (cost or 0.0),
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
class LedgerLimits(BaseModel):
|
|
133
|
+
"""The limits of the one budget all candidates share."""
|
|
134
|
+
|
|
135
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
136
|
+
|
|
137
|
+
tries: int = Field(ge=1, description="Runtime calls over all candidates and the Critic.")
|
|
138
|
+
tokens: int | None = Field(default=None, ge=1, description="Token cap, if capped.")
|
|
139
|
+
cost: float | None = Field(default=None, gt=0, description="Cost cap in USD, if capped.")
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
class LedgerExhausted(BaseException):
|
|
143
|
+
"""Raised by a guarded runtime when the shared ledger cannot pay for another call.
|
|
144
|
+
|
|
145
|
+
A `BaseException`, so a loop that retries every `Exception` as an infra error stops
|
|
146
|
+
instead: the candidate is cancelled cooperatively, at its next call.
|
|
147
|
+
"""
|
|
148
|
+
|
|
149
|
+
def __init__(self, limit: LedgerLimit) -> None:
|
|
150
|
+
super().__init__(f"the shared budget is spent ({limit})")
|
|
151
|
+
self.limit: LedgerLimit = limit
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
class CostLedger:
|
|
155
|
+
"""One budget for N candidates (and the Critic): tries, tokens and cost, per account.
|
|
156
|
+
|
|
157
|
+
A call reserves its try, and holds its token cap, before it starts (`reserve`), so
|
|
158
|
+
tries never exceed the limit and, as long as runtimes keep to the token cap of their
|
|
159
|
+
task, neither do tokens, however the calls interleave. What a call reported is
|
|
160
|
+
charged when it ends (`settle`). Cost cannot be held (a task carries no cost cap): a
|
|
161
|
+
call starts only while cost is under the limit, so it can be passed by the calls
|
|
162
|
+
already running. `limited_by` is the first limit that refused a call or was reached.
|
|
163
|
+
"""
|
|
164
|
+
|
|
165
|
+
def __init__(self, limits: LedgerLimits) -> None:
|
|
166
|
+
self.limits = limits
|
|
167
|
+
self.total = Spend()
|
|
168
|
+
self.accounts: dict[str, Spend] = {}
|
|
169
|
+
self.held_tokens = 0
|
|
170
|
+
self.limited_by: LedgerLimit | None = None
|
|
171
|
+
|
|
172
|
+
@property
|
|
173
|
+
def remaining_tries(self) -> int:
|
|
174
|
+
"""Tries left."""
|
|
175
|
+
return max(self.limits.tries - self.total.tries, 0)
|
|
176
|
+
|
|
177
|
+
@property
|
|
178
|
+
def remaining_tokens(self) -> int | None:
|
|
179
|
+
"""Tokens neither spent nor held by a running call; `None` when not capped."""
|
|
180
|
+
if self.limits.tokens is None:
|
|
181
|
+
return None
|
|
182
|
+
return max(self.limits.tokens - self.total.tokens - self.held_tokens, 0)
|
|
183
|
+
|
|
184
|
+
@property
|
|
185
|
+
def remaining_cost(self) -> float | None:
|
|
186
|
+
"""Cost left, or `None` when cost is not capped."""
|
|
187
|
+
if self.limits.cost is None:
|
|
188
|
+
return None
|
|
189
|
+
return max(self.limits.cost - self.total.cost, 0.0)
|
|
190
|
+
|
|
191
|
+
@property
|
|
192
|
+
def exhausted(self) -> LedgerLimit | None:
|
|
193
|
+
"""The limit that is spent, or `None` while another call may run."""
|
|
194
|
+
if self.total.tries >= self.limits.tries:
|
|
195
|
+
return "tries"
|
|
196
|
+
if self.limits.tokens is not None and self.total.tokens >= self.limits.tokens:
|
|
197
|
+
return "tokens"
|
|
198
|
+
if self.limits.cost is not None and self.total.cost >= self.limits.cost:
|
|
199
|
+
return "cost"
|
|
200
|
+
return None
|
|
201
|
+
|
|
202
|
+
def why_not(self, tries: int = 1, tokens: int = 1) -> LedgerLimit | None:
|
|
203
|
+
"""The limit that keeps `tries` calls (holding `tokens`) from starting, if any."""
|
|
204
|
+
spent = self.exhausted
|
|
205
|
+
if spent is not None:
|
|
206
|
+
return spent
|
|
207
|
+
if self.remaining_tries < tries:
|
|
208
|
+
return "tries"
|
|
209
|
+
left = self.remaining_tokens
|
|
210
|
+
if left is not None and left < max(tokens, 1):
|
|
211
|
+
return "tokens"
|
|
212
|
+
return None
|
|
213
|
+
|
|
214
|
+
def can_afford(self, tries: int = 1, tokens: int = 1) -> bool:
|
|
215
|
+
"""Whether `tries` more calls, holding `tokens`, may start now."""
|
|
216
|
+
return self.why_not(tries, tokens) is None
|
|
217
|
+
|
|
218
|
+
def refuse(self, tries: int = 1, tokens: int = 1) -> LedgerLimit:
|
|
219
|
+
"""Record (once) and return the limit that refuses `tries` calls."""
|
|
220
|
+
limit = self.why_not(tries, tokens) or "tries"
|
|
221
|
+
if self.limited_by is None:
|
|
222
|
+
self.limited_by = limit
|
|
223
|
+
return limit
|
|
224
|
+
|
|
225
|
+
def reserve(self, account: str, tries: int = 1, tokens: int | None = None) -> bool:
|
|
226
|
+
"""Take `tries` tries for `account` and hold `tokens`, before its calls start;
|
|
227
|
+
`False` (nothing taken) when the ledger cannot pay for them."""
|
|
228
|
+
if not self.can_afford(tries, tokens or 1):
|
|
229
|
+
self.refuse(tries, tokens or 1)
|
|
230
|
+
return False
|
|
231
|
+
self._add(account, tries=tries)
|
|
232
|
+
if tokens is not None and self.limits.tokens is not None:
|
|
233
|
+
self.held_tokens += tokens
|
|
234
|
+
return True
|
|
235
|
+
|
|
236
|
+
def settle(
|
|
237
|
+
self,
|
|
238
|
+
account: str,
|
|
239
|
+
*,
|
|
240
|
+
held: int | None = None,
|
|
241
|
+
tokens: int | None = None,
|
|
242
|
+
cost: float | None = None,
|
|
243
|
+
) -> None:
|
|
244
|
+
"""A call of `account` ended: release what it held, charge what it reported."""
|
|
245
|
+
if held is not None and self.limits.tokens is not None:
|
|
246
|
+
self.held_tokens = max(self.held_tokens - held, 0)
|
|
247
|
+
self.charge(account, tokens=tokens, cost=cost)
|
|
248
|
+
|
|
249
|
+
def charge(
|
|
250
|
+
self, account: str, *, tries: int = 0, tokens: int | None = None, cost: float | None = None
|
|
251
|
+
) -> None:
|
|
252
|
+
"""Add what `account` spent (tokens and cost as reported, maybe a try)."""
|
|
253
|
+
self._add(account, tries=tries, tokens=tokens, cost=cost)
|
|
254
|
+
reached = self.exhausted
|
|
255
|
+
if reached is not None and reached != "tries" and self.limited_by is None:
|
|
256
|
+
self.limited_by = reached
|
|
257
|
+
|
|
258
|
+
def spent(self, account: str) -> Spend:
|
|
259
|
+
"""What `account` spent so far."""
|
|
260
|
+
return self.accounts.get(account, Spend())
|
|
261
|
+
|
|
262
|
+
def _add(
|
|
263
|
+
self, account: str, *, tries: int = 0, tokens: int | None = None, cost: float | None = None
|
|
264
|
+
) -> None:
|
|
265
|
+
self.total = self.total.plus(tries=tries, tokens=tokens, cost=cost)
|
|
266
|
+
self.accounts[account] = self.spent(account).plus(tries=tries, tokens=tokens, cost=cost)
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
# --- planning N inside one budget --------------------------------------------------------
|
|
270
|
+
|
|
271
|
+
CapReason = Literal["max_n", "variants", "budget"]
|
|
272
|
+
"""Why fewer candidates run than were asked for."""
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
class BudgetPlan(BaseModel):
|
|
276
|
+
"""How many candidates run, the shared limits, and each candidate's slice."""
|
|
277
|
+
|
|
278
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
279
|
+
|
|
280
|
+
requested: int = Field(ge=1, description="The N the rule asked for.")
|
|
281
|
+
n: int = Field(ge=1, description="The N that runs (1: a plain run).")
|
|
282
|
+
capped_by: CapReason | None = Field(default=None, description="Why n < requested.")
|
|
283
|
+
factor: float = Field(gt=0, description="The shared budget, in single-run budgets.")
|
|
284
|
+
limits: LedgerLimits = Field(description="The shared ledger's limits.")
|
|
285
|
+
slice: Budget = Field(description="Each candidate's budget (tries, tier, tokens).")
|
|
286
|
+
slice_cost: float | None = Field(default=None, description="Each candidate's cost cap.")
|
|
287
|
+
|
|
288
|
+
@property
|
|
289
|
+
def plain(self) -> bool:
|
|
290
|
+
"""True when only one candidate fits: then it is a plain run of the rule."""
|
|
291
|
+
return self.n == 1
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def plan_budget(
|
|
295
|
+
budget: Budget,
|
|
296
|
+
*,
|
|
297
|
+
requested: int,
|
|
298
|
+
cfg: SelectCfg | None = None,
|
|
299
|
+
max_cost: float | None = None,
|
|
300
|
+
variants: int | None = None,
|
|
301
|
+
) -> BudgetPlan:
|
|
302
|
+
"""N, the shared ledger and the slices for a rule with `budget` asking for `requested`.
|
|
303
|
+
|
|
304
|
+
N is capped at `cfg.max_n`, at `variants` (the hints there are), and at what the
|
|
305
|
+
ledger can give at least one try (and one token) each. The candidates start at the
|
|
306
|
+
rule's tier and keep its escalation rule; nothing escalates across candidates.
|
|
307
|
+
"""
|
|
308
|
+
cfg = cfg if cfg is not None else SelectCfg()
|
|
309
|
+
factor = cfg.budget_factor
|
|
310
|
+
asked = max(1, requested)
|
|
311
|
+
n = cfg.effective_n(asked)
|
|
312
|
+
capped: CapReason | None = "max_n" if n < asked else None
|
|
313
|
+
if variants is not None and n > variants:
|
|
314
|
+
n, capped = max(1, variants), "variants"
|
|
315
|
+
tries = max(1, math.floor(budget.tries * factor + 1e-9))
|
|
316
|
+
tokens = max(1, math.floor(budget.tokens * factor + 1e-9)) if budget.tokens else None
|
|
317
|
+
cost = max_cost * factor if max_cost is not None and max_cost > 0 else None
|
|
318
|
+
fits = min(tries, tokens) if tokens is not None else tries
|
|
319
|
+
if n > fits:
|
|
320
|
+
n, capped = fits, "budget"
|
|
321
|
+
slice_budget = Budget(
|
|
322
|
+
tries=max(1, tries // n),
|
|
323
|
+
tier=budget.tier,
|
|
324
|
+
escalate=budget.escalate,
|
|
325
|
+
tokens=max(1, tokens // n) if tokens is not None else None,
|
|
326
|
+
)
|
|
327
|
+
return BudgetPlan(
|
|
328
|
+
requested=asked,
|
|
329
|
+
n=n,
|
|
330
|
+
capped_by=capped,
|
|
331
|
+
factor=factor,
|
|
332
|
+
limits=LedgerLimits(tries=tries, tokens=tokens, cost=cost),
|
|
333
|
+
slice=slice_budget,
|
|
334
|
+
slice_cost=cost / n if cost is not None else None,
|
|
335
|
+
)
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
# --- selection ----------------------------------------------------------------------------
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
class ReviewScore(BaseModel):
|
|
342
|
+
"""The Critic's score of one tied candidate (from its `ReviewReport`), and its spend."""
|
|
343
|
+
|
|
344
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
345
|
+
|
|
346
|
+
verdict: Verdict = Field(description="approve, or changes_requested.")
|
|
347
|
+
blocker: int = Field(default=0, ge=0)
|
|
348
|
+
major: int = Field(default=0, ge=0)
|
|
349
|
+
minor: int = Field(default=0, ge=0)
|
|
350
|
+
tokens: int | None = Field(default=None, ge=0, description="Tokens the review reported.")
|
|
351
|
+
cost: float | None = Field(default=None, ge=0, description="Cost the review reported.")
|
|
352
|
+
|
|
353
|
+
@classmethod
|
|
354
|
+
def from_report(
|
|
355
|
+
cls, report: ReviewReport, *, tokens: int | None = None, cost: float | None = None
|
|
356
|
+
) -> ReviewScore:
|
|
357
|
+
"""The score of a review (nits do not count)."""
|
|
358
|
+
|
|
359
|
+
def count(severity: str) -> int:
|
|
360
|
+
return sum(1 for c in report.comments if c.severity == severity)
|
|
361
|
+
|
|
362
|
+
return cls(
|
|
363
|
+
verdict=report.verdict,
|
|
364
|
+
blocker=count("blocker"),
|
|
365
|
+
major=count("major"),
|
|
366
|
+
minor=count("minor"),
|
|
367
|
+
tokens=tokens,
|
|
368
|
+
cost=cost,
|
|
369
|
+
)
|
|
370
|
+
|
|
371
|
+
def key(self) -> tuple[int, int, int, int]:
|
|
372
|
+
"""Lower is better: approve first, then fewer blocker, major, minor comments."""
|
|
373
|
+
return (0 if self.verdict == "approve" else 1, self.blocker, self.major, self.minor)
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
@dataclass(frozen=True)
|
|
377
|
+
class Candidate:
|
|
378
|
+
"""One finished (or not) candidate as the selection sees it."""
|
|
379
|
+
|
|
380
|
+
id: str
|
|
381
|
+
passed: bool
|
|
382
|
+
"""The candidate's own run succeeded (the agent loop, or the plain executor)."""
|
|
383
|
+
checks: tuple[CheckResult, ...] = ()
|
|
384
|
+
"""The final result of each of the rule's checks on the candidate's outputs."""
|
|
385
|
+
finished: bool = True
|
|
386
|
+
"""False for a candidate that was cancelled, skipped or crashed: never eligible."""
|
|
387
|
+
workspace: Path | None = None
|
|
388
|
+
"""Its workspace, while it exists (for a `MetricProvider` and the Critic)."""
|
|
389
|
+
|
|
390
|
+
@property
|
|
391
|
+
def all_passed(self) -> bool:
|
|
392
|
+
"""Its run succeeded and every check result is ok."""
|
|
393
|
+
return self.passed and all(c.ok for c in self.checks)
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
@runtime_checkable
|
|
397
|
+
class MetricProvider(Protocol):
|
|
398
|
+
"""Measures a candidate's workspace; optional (no provider: the area step is skipped)."""
|
|
399
|
+
|
|
400
|
+
def area(self, workspace: Path) -> float | None:
|
|
401
|
+
"""The candidate's area (lower is better), or `None` when it cannot be had."""
|
|
402
|
+
...
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
Tiebreak = Callable[[tuple[Candidate, ...]], Awaitable[Mapping[str, ReviewScore]]]
|
|
406
|
+
"""The Critic: scores the tied candidates (candidate id -> score). In real use it runs
|
|
407
|
+
the review rule on each tied candidate's workspace; any async callable will do."""
|
|
408
|
+
|
|
409
|
+
SelectStep = Literal["none", "only", "passed", "errors", "warnings", "area", "critic", "lowest_id"]
|
|
410
|
+
"""Which step of the key decided the winner (`none`: no candidate finished)."""
|
|
411
|
+
|
|
412
|
+
_STEPS: tuple[SelectStep, ...] = ("passed", "errors", "warnings", "area")
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
def error_count(checks: Sequence[CheckResult]) -> int:
|
|
416
|
+
"""Error issues over `checks`; a failing check with no error issue counts as one."""
|
|
417
|
+
total = 0
|
|
418
|
+
for result in checks:
|
|
419
|
+
errors = sum(1 for i in result.issues if i.severity == "error")
|
|
420
|
+
total += errors if errors or result.ok else 1
|
|
421
|
+
return total
|
|
422
|
+
|
|
423
|
+
|
|
424
|
+
def warning_count(checks: Sequence[CheckResult]) -> int:
|
|
425
|
+
"""Issues of severity `warning` over `checks` (`info` is ignored)."""
|
|
426
|
+
return sum(1 for r in checks for i in r.issues if i.severity == "warning")
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
class CandidateKey(BaseModel):
|
|
430
|
+
"""A candidate's place in the selection: every component of its key."""
|
|
431
|
+
|
|
432
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
433
|
+
|
|
434
|
+
id: str
|
|
435
|
+
eligible: bool = Field(description="It finished, so it took part.")
|
|
436
|
+
passed: bool = Field(default=False, description="Its run and every check passed.")
|
|
437
|
+
errors: int = Field(default=0, ge=0)
|
|
438
|
+
warnings: int = Field(default=0, ge=0)
|
|
439
|
+
area: float | None = Field(default=None, description="When a MetricProvider gave one.")
|
|
440
|
+
critic: ReviewScore | None = Field(default=None, description="When the Critic scored it.")
|
|
441
|
+
position: int | None = Field(default=None, ge=1, description="1 = the winner.")
|
|
442
|
+
|
|
443
|
+
def key(self, area_used: bool) -> tuple[int, int, int, tuple[int, float]]:
|
|
444
|
+
"""Steps 1-4 as a tuple: lower is better."""
|
|
445
|
+
if not area_used:
|
|
446
|
+
area: tuple[int, float] = (0, 0.0)
|
|
447
|
+
elif self.area is None:
|
|
448
|
+
area = (1, 0.0)
|
|
449
|
+
else:
|
|
450
|
+
area = (0, self.area)
|
|
451
|
+
return (0 if self.passed else 1, self.errors, self.warnings, area)
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
class Selection(BaseModel):
|
|
455
|
+
"""The outcome of `select_best`: the winner, every key, the deciding step, why."""
|
|
456
|
+
|
|
457
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
458
|
+
|
|
459
|
+
schema_version: Literal[1] = Field(default=1, description="Schema version of this model.")
|
|
460
|
+
winner: str | None = Field(description="The chosen candidate; None if none finished.")
|
|
461
|
+
decided_by: SelectStep = Field(description="The step that set the winner apart.")
|
|
462
|
+
none_passed: bool = Field(
|
|
463
|
+
default=False, description="No candidate passed: the winner is only the least bad."
|
|
464
|
+
)
|
|
465
|
+
area_used: bool = Field(default=False, description="Whether the area step applied.")
|
|
466
|
+
critic_used: bool = Field(default=False, description="Whether the Critic was called.")
|
|
467
|
+
critic_note: str = Field(default="", description="Why the Critic was not used, if tied.")
|
|
468
|
+
order: tuple[str, ...] = Field(default=(), description="Eligible candidates, best first.")
|
|
469
|
+
keys: tuple[CandidateKey, ...] = Field(
|
|
470
|
+
default=(), description="Every candidate: the eligible in order, then the rest by id."
|
|
471
|
+
)
|
|
472
|
+
|
|
473
|
+
def key_of(self, candidate_id: str) -> CandidateKey | None:
|
|
474
|
+
"""The key of `candidate_id`."""
|
|
475
|
+
return next((k for k in self.keys if k.id == candidate_id), None)
|
|
476
|
+
|
|
477
|
+
def explain(self) -> str:
|
|
478
|
+
"""The choice and its reasons, one candidate per line."""
|
|
479
|
+
lines: list[str] = []
|
|
480
|
+
if self.winner is None:
|
|
481
|
+
lines.append("no winner: no candidate finished")
|
|
482
|
+
else:
|
|
483
|
+
lines.append(f"winner {self.winner}: {self._reason()}")
|
|
484
|
+
if self.none_passed:
|
|
485
|
+
lines.append("no candidate passed every check: the winner is the least bad one")
|
|
486
|
+
if self.critic_note:
|
|
487
|
+
lines.append(f"critic: {self.critic_note}")
|
|
488
|
+
for k in self.keys:
|
|
489
|
+
if not k.eligible:
|
|
490
|
+
lines.append(f"- {k.id}: did not finish, not eligible")
|
|
491
|
+
continue
|
|
492
|
+
parts = [
|
|
493
|
+
"passed" if k.passed else "failed",
|
|
494
|
+
_plural(k.errors, "error"),
|
|
495
|
+
_plural(k.warnings, "warning"),
|
|
496
|
+
]
|
|
497
|
+
if self.area_used:
|
|
498
|
+
parts.append(f"area {k.area:g}" if k.area is not None else "no area")
|
|
499
|
+
if k.critic is not None:
|
|
500
|
+
c = k.critic
|
|
501
|
+
parts.append(
|
|
502
|
+
f"critic {c.verdict} ({c.blocker} blocker, {c.major} major, {c.minor} minor)"
|
|
503
|
+
)
|
|
504
|
+
lines.append(f"{k.position}. {k.id}: " + ", ".join(parts))
|
|
505
|
+
return "\n".join(lines)
|
|
506
|
+
|
|
507
|
+
def _reason(self) -> str:
|
|
508
|
+
if self.decided_by == "only" or len(self.order) < 2:
|
|
509
|
+
return "the only candidate that finished"
|
|
510
|
+
win, other = self.key_of(self.order[0]), self.key_of(self.order[1])
|
|
511
|
+
assert win is not None and other is not None
|
|
512
|
+
step = self.decided_by
|
|
513
|
+
if step == "passed":
|
|
514
|
+
return f"it passed every check and {other.id} did not"
|
|
515
|
+
if step == "errors":
|
|
516
|
+
return f"fewer errors than {other.id} ({win.errors} < {other.errors})"
|
|
517
|
+
if step == "warnings":
|
|
518
|
+
return f"fewer warnings than {other.id} ({win.warnings} < {other.warnings})"
|
|
519
|
+
if step == "area":
|
|
520
|
+
if other.area is None:
|
|
521
|
+
return f"it has an area and {other.id} has none"
|
|
522
|
+
return f"smaller area than {other.id} ({win.area:g} < {other.area:g})"
|
|
523
|
+
if step == "critic":
|
|
524
|
+
return f"tied with {other.id} on checks, warnings and area; the critic preferred it"
|
|
525
|
+
return f"tied with {other.id} on every step; the lowest id wins"
|
|
526
|
+
|
|
527
|
+
|
|
528
|
+
def _plural(n: int, word: str) -> str:
|
|
529
|
+
return f"{n} {word}{'' if n == 1 else 's'}"
|
|
530
|
+
|
|
531
|
+
|
|
532
|
+
def _area(metrics: MetricProvider | None, candidate: Candidate) -> float | None:
|
|
533
|
+
if metrics is None or candidate.workspace is None:
|
|
534
|
+
return None
|
|
535
|
+
try:
|
|
536
|
+
value = metrics.area(candidate.workspace)
|
|
537
|
+
except Exception: # a metric that cannot be had sorts the candidate after
|
|
538
|
+
return None
|
|
539
|
+
if value is None or not math.isfinite(value):
|
|
540
|
+
return None
|
|
541
|
+
return float(value)
|
|
542
|
+
|
|
543
|
+
|
|
544
|
+
def rank(
|
|
545
|
+
candidates: Sequence[Candidate], metrics: MetricProvider | None = None
|
|
546
|
+
) -> tuple[tuple[CandidateKey, ...], bool]:
|
|
547
|
+
"""Steps 1-4: the eligible candidates' keys, best first (ties by id), and whether
|
|
548
|
+
the area step applied. Pure apart from the metric provider."""
|
|
549
|
+
eligible = sorted((c for c in candidates if c.finished), key=lambda c: c.id)
|
|
550
|
+
keys = [
|
|
551
|
+
CandidateKey(
|
|
552
|
+
id=c.id,
|
|
553
|
+
eligible=True,
|
|
554
|
+
passed=c.all_passed,
|
|
555
|
+
errors=error_count(c.checks),
|
|
556
|
+
warnings=warning_count(c.checks),
|
|
557
|
+
area=_area(metrics, c),
|
|
558
|
+
)
|
|
559
|
+
for c in eligible
|
|
560
|
+
]
|
|
561
|
+
area_used = any(k.area is not None for k in keys)
|
|
562
|
+
keys.sort(key=lambda k: (k.key(area_used), k.id))
|
|
563
|
+
return tuple(keys), area_used
|
|
564
|
+
|
|
565
|
+
|
|
566
|
+
async def select_best(
|
|
567
|
+
candidates: Sequence[Candidate],
|
|
568
|
+
metrics: MetricProvider | None = None,
|
|
569
|
+
tiebreak: Tiebreak | None = None,
|
|
570
|
+
*,
|
|
571
|
+
ledger: CostLedger | None = None,
|
|
572
|
+
) -> Selection:
|
|
573
|
+
"""Choose one candidate: deterministic checks first, the Critic only for exact ties.
|
|
574
|
+
|
|
575
|
+
The key (see the module docstring) orders every finished candidate; the order is
|
|
576
|
+
total (equal keys fall back to the id), so the winner does not depend on the order
|
|
577
|
+
the candidates are given or finished in. `tiebreak` is called only for the
|
|
578
|
+
candidates tied for first place after steps 1-4, so it can never override a
|
|
579
|
+
difference there; its cost is charged to `ledger` (one try per tied candidate, plus
|
|
580
|
+
what it reports), and it is skipped when the ledger cannot pay for it. Still tied,
|
|
581
|
+
no tiebreak, or skipped: the lowest id wins.
|
|
582
|
+
"""
|
|
583
|
+
by_id = {c.id: c for c in candidates}
|
|
584
|
+
if len(by_id) != len(candidates):
|
|
585
|
+
raise ValueError("candidate ids must be unique")
|
|
586
|
+
keys, area_used = rank(candidates, metrics)
|
|
587
|
+
rest = tuple(
|
|
588
|
+
CandidateKey(id=c.id, eligible=False)
|
|
589
|
+
for c in sorted(by_id.values(), key=lambda c: c.id)
|
|
590
|
+
if not c.finished
|
|
591
|
+
)
|
|
592
|
+
if not keys:
|
|
593
|
+
return Selection(winner=None, decided_by="none", keys=rest)
|
|
594
|
+
|
|
595
|
+
top = keys[0].key(area_used)
|
|
596
|
+
tied = [k for k in keys if k.key(area_used) == top]
|
|
597
|
+
ordered = list(keys)
|
|
598
|
+
critic_used = False
|
|
599
|
+
note = ""
|
|
600
|
+
scores: Mapping[str, ReviewScore] = {}
|
|
601
|
+
if len(tied) > 1:
|
|
602
|
+
if tiebreak is None:
|
|
603
|
+
note = "no critic configured: the lowest id wins the tie"
|
|
604
|
+
elif ledger is not None and not ledger.can_afford(len(tied)):
|
|
605
|
+
note = (
|
|
606
|
+
f"the shared budget cannot pay for the critic ({len(tied)} tries needed, "
|
|
607
|
+
f"{ledger.remaining_tries} left, the {ledger.refuse(len(tied))} limit): "
|
|
608
|
+
"the lowest id wins the tie"
|
|
609
|
+
)
|
|
610
|
+
else:
|
|
611
|
+
critic_used = True
|
|
612
|
+
if ledger is not None:
|
|
613
|
+
ledger.reserve(CRITIC, len(tied))
|
|
614
|
+
try:
|
|
615
|
+
scores = dict(await tiebreak(tuple(by_id[k.id] for k in tied)))
|
|
616
|
+
except Exception as exc:
|
|
617
|
+
note = f"the critic failed ({type(exc).__name__}: {exc}): the lowest id wins"
|
|
618
|
+
scores = {}
|
|
619
|
+
if ledger is not None:
|
|
620
|
+
ledger.charge(
|
|
621
|
+
CRITIC,
|
|
622
|
+
tokens=sum(s.tokens or 0 for s in scores.values()) or None,
|
|
623
|
+
cost=sum(s.cost or 0.0 for s in scores.values()) or None,
|
|
624
|
+
)
|
|
625
|
+
|
|
626
|
+
def critic_key(k: CandidateKey) -> tuple[int, tuple[int, int, int, int], str]:
|
|
627
|
+
score = scores.get(k.id)
|
|
628
|
+
return (0, score.key(), k.id) if score is not None else (1, (0, 0, 0, 0), k.id)
|
|
629
|
+
|
|
630
|
+
tied_sorted = sorted(tied, key=critic_key)
|
|
631
|
+
ordered = [*tied_sorted, *keys[len(tied) :]]
|
|
632
|
+
ordered = [
|
|
633
|
+
k.model_copy(update={"position": i + 1, "critic": scores.get(k.id)})
|
|
634
|
+
for i, k in enumerate(ordered)
|
|
635
|
+
]
|
|
636
|
+
|
|
637
|
+
decided: SelectStep
|
|
638
|
+
if len(ordered) == 1:
|
|
639
|
+
decided = "only"
|
|
640
|
+
else:
|
|
641
|
+
win, other = ordered[0], ordered[1]
|
|
642
|
+
wk, ok = win.key(area_used), other.key(area_used)
|
|
643
|
+
if wk != ok:
|
|
644
|
+
decided = next(_STEPS[i] for i in range(4) if wk[i] != ok[i])
|
|
645
|
+
elif win.critic is not None and (
|
|
646
|
+
other.critic is None or win.critic.key() < other.critic.key()
|
|
647
|
+
):
|
|
648
|
+
decided = "critic"
|
|
649
|
+
else:
|
|
650
|
+
decided = "lowest_id"
|
|
651
|
+
return Selection(
|
|
652
|
+
winner=ordered[0].id,
|
|
653
|
+
decided_by=decided,
|
|
654
|
+
none_passed=not ordered[0].passed,
|
|
655
|
+
area_used=area_used,
|
|
656
|
+
critic_used=critic_used,
|
|
657
|
+
critic_note=note,
|
|
658
|
+
order=tuple(k.id for k in ordered),
|
|
659
|
+
keys=(*ordered, *rest),
|
|
660
|
+
)
|
|
661
|
+
|
|
662
|
+
|
|
663
|
+
# --- the executor ------------------------------------------------------------------------
|
|
664
|
+
|
|
665
|
+
|
|
666
|
+
@dataclass(frozen=True)
|
|
667
|
+
class CandidateEnv:
|
|
668
|
+
"""What `inner_factory` gets to build one candidate's executor.
|
|
669
|
+
|
|
670
|
+
Build it to work in `workspace` (its store, runtime and checks rooted there), with
|
|
671
|
+
`budget`/`max_cost` as its caps, and without remembered stops (no state layout:
|
|
672
|
+
candidates share the instance id). Wrap its runtime with `guard`: every call then
|
|
673
|
+
reserves a try of the shared ledger first (none left: the candidate is cancelled),
|
|
674
|
+
has its token cap lowered to what the ledger has left, carries `variant`, `seed` and
|
|
675
|
+
the candidate id in `AgentTask.context`, and is charged what it reports.
|
|
676
|
+
"""
|
|
677
|
+
|
|
678
|
+
candidate_id: str
|
|
679
|
+
variant: str
|
|
680
|
+
seed: str
|
|
681
|
+
workspace: Path
|
|
682
|
+
budget: Budget
|
|
683
|
+
max_cost: float | None
|
|
684
|
+
guard: Callable[[AgentRuntime], AgentRuntime]
|
|
685
|
+
|
|
686
|
+
|
|
687
|
+
@dataclass(frozen=True)
|
|
688
|
+
class CandidateParts:
|
|
689
|
+
"""One candidate's executor, and the check runner for its workspace."""
|
|
690
|
+
|
|
691
|
+
executor: Executor
|
|
692
|
+
checks: CheckRunner | None = None
|
|
693
|
+
|
|
694
|
+
|
|
695
|
+
InnerFactory = Callable[[CandidateEnv], CandidateParts]
|
|
696
|
+
|
|
697
|
+
|
|
698
|
+
class _GuardedRuntime:
|
|
699
|
+
"""An `AgentRuntime` that pays for every call from the shared ledger first."""
|
|
700
|
+
|
|
701
|
+
def __init__(
|
|
702
|
+
self, inner: AgentRuntime, ledger: CostLedger, account: str, context: Mapping[str, str]
|
|
703
|
+
) -> None:
|
|
704
|
+
self.inner = inner
|
|
705
|
+
self.ledger = ledger
|
|
706
|
+
self.account = account
|
|
707
|
+
self.context = dict(context)
|
|
708
|
+
self.name: str = getattr(inner, "name", "runtime")
|
|
709
|
+
|
|
710
|
+
async def run_task(self, task: AgentTask) -> AgentResult:
|
|
711
|
+
budget = task.budget
|
|
712
|
+
left = self.ledger.remaining_tokens
|
|
713
|
+
hold: int | None = None
|
|
714
|
+
if left is not None:
|
|
715
|
+
hold = min(budget.tokens, left) if budget.tokens is not None else left
|
|
716
|
+
if hold >= 1:
|
|
717
|
+
budget = budget.model_copy(update={"tokens": hold})
|
|
718
|
+
if not self.ledger.reserve(self.account, tokens=hold):
|
|
719
|
+
raise LedgerExhausted(self.ledger.why_not(1, hold or 1) or "tries")
|
|
720
|
+
sent = task.model_copy(
|
|
721
|
+
update={"context": {**task.context, **self.context}, "budget": budget}
|
|
722
|
+
)
|
|
723
|
+
try:
|
|
724
|
+
result = await self.inner.run_task(sent)
|
|
725
|
+
except BaseException:
|
|
726
|
+
self.ledger.settle(self.account, held=hold)
|
|
727
|
+
raise
|
|
728
|
+
self.ledger.settle(self.account, held=hold, tokens=result.tokens, cost=result.cost)
|
|
729
|
+
return result
|
|
730
|
+
|
|
731
|
+
|
|
732
|
+
CandidateStatus = Literal["passed", "failed", "rejected", "cancelled", "skipped", "error"]
|
|
733
|
+
|
|
734
|
+
|
|
735
|
+
class CandidateResult(BaseModel):
|
|
736
|
+
"""One candidate's outcome and spend."""
|
|
737
|
+
|
|
738
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
739
|
+
|
|
740
|
+
id: str
|
|
741
|
+
variant: str
|
|
742
|
+
status: CandidateStatus
|
|
743
|
+
message: str = ""
|
|
744
|
+
errors: int = Field(default=0, ge=0)
|
|
745
|
+
warnings: int = Field(default=0, ge=0)
|
|
746
|
+
checks: tuple[CheckResult, ...] = ()
|
|
747
|
+
files: tuple[str, ...] = Field(default=(), description="Files it changed.")
|
|
748
|
+
spend: Spend = Field(default_factory=Spend)
|
|
749
|
+
agent: AgentResult | None = None
|
|
750
|
+
|
|
751
|
+
|
|
752
|
+
BestOfNStatus = Literal["done", "failed", "plain"]
|
|
753
|
+
|
|
754
|
+
|
|
755
|
+
class BestOfNResult(BaseModel):
|
|
756
|
+
"""The outcome of one best-of-N run (the `best_of` payload of the rule's event)."""
|
|
757
|
+
|
|
758
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
759
|
+
|
|
760
|
+
schema_version: Literal[1] = Field(default=1, description="Schema version of this model.")
|
|
761
|
+
status: BestOfNStatus = Field(description="done; failed; plain (only one fitted).")
|
|
762
|
+
instance: str
|
|
763
|
+
message: str = ""
|
|
764
|
+
plan: BudgetPlan
|
|
765
|
+
winner: str | None = None
|
|
766
|
+
candidates: tuple[CandidateResult, ...] = Field(default=(), description="By id.")
|
|
767
|
+
selection: Selection | None = None
|
|
768
|
+
spend: Spend = Field(default_factory=Spend, description="Total, Critic included.")
|
|
769
|
+
critic_spend: Spend = Field(default_factory=Spend)
|
|
770
|
+
limited_by: LedgerLimit | None = Field(
|
|
771
|
+
default=None, description="The shared limit that stopped a call, if any."
|
|
772
|
+
)
|
|
773
|
+
fanout: str | None = Field(default=None, description="The candidates' fan-out run id.")
|
|
774
|
+
branch: str | None = Field(default=None, description="The result branch, when created.")
|
|
775
|
+
commit: str | None = None
|
|
776
|
+
|
|
777
|
+
|
|
778
|
+
@dataclass
|
|
779
|
+
class _State:
|
|
780
|
+
id: str
|
|
781
|
+
variant: str
|
|
782
|
+
status: CandidateStatus = "skipped"
|
|
783
|
+
message: str = "not started"
|
|
784
|
+
outcome: ExecOutcome | None = None
|
|
785
|
+
checks: dict[str, CheckResult] = field(default_factory=dict)
|
|
786
|
+
guarded: bool = False
|
|
787
|
+
|
|
788
|
+
|
|
789
|
+
_SLUG_RE = re.compile(r"[^a-z0-9._-]+")
|
|
790
|
+
|
|
791
|
+
|
|
792
|
+
def _slug(instance_id: str) -> str:
|
|
793
|
+
slug = re.sub(r"\.{2,}", ".", _SLUG_RE.sub("-", instance_id.lower())).strip("-._")
|
|
794
|
+
return f"best-of-{slug}"[:80].rstrip("-._") if slug else "best-of"
|
|
795
|
+
|
|
796
|
+
|
|
797
|
+
def _synthetic(check_id: str, issues: Sequence[Issue]) -> CheckResult:
|
|
798
|
+
digest = hashlib.sha256("|".join(i.msg for i in issues).encode("utf-8")).hexdigest()
|
|
799
|
+
return CheckResult(
|
|
800
|
+
check_id=check_id,
|
|
801
|
+
status="fail" if issues else "pass",
|
|
802
|
+
issues=tuple(issues),
|
|
803
|
+
duration_s=0.0,
|
|
804
|
+
idempotency_key=f"{check_id}:{digest}",
|
|
805
|
+
)
|
|
806
|
+
|
|
807
|
+
|
|
808
|
+
def _noop_emit(_type: ExecEventType, _payload: dict[str, Any]) -> None:
|
|
809
|
+
return None
|
|
810
|
+
|
|
811
|
+
|
|
812
|
+
class BestOfNExecutor:
|
|
813
|
+
"""An executor that runs an opted-in rule as N candidates and keeps the best.
|
|
814
|
+
|
|
815
|
+
A rule opts in with the instance parameter `best_of: <n>` or `select.rules` in the
|
|
816
|
+
profile (`cfg`); every other rule goes to `default`, unchanged. `inner_factory`
|
|
817
|
+
builds each candidate's executor for its workspace (`CandidateEnv`); `vcs` and
|
|
818
|
+
`root` are the fan-out's. `metrics` adds the area step, `tiebreak` the Critic.
|
|
819
|
+
|
|
820
|
+
It is a `CheckingExecutor`: for candidates it runs the rule's checks itself, in each
|
|
821
|
+
candidate's workspace; for a plain run it lets `default` do it, or runs them through
|
|
822
|
+
the scheduler's context. Wiring this into the claude-code runtime (a runtime per
|
|
823
|
+
workspace, the review rule as `tiebreak`, landing the result branch) is M2-11's job.
|
|
824
|
+
"""
|
|
825
|
+
|
|
826
|
+
def __init__(
|
|
827
|
+
self,
|
|
828
|
+
inner_factory: InnerFactory,
|
|
829
|
+
vcs: FanoutVcs,
|
|
830
|
+
*,
|
|
831
|
+
root: Path,
|
|
832
|
+
default: Executor,
|
|
833
|
+
cfg: SelectCfg | None = None,
|
|
834
|
+
fanout: FanoutCfg | None = None,
|
|
835
|
+
max_cost: float | None = None,
|
|
836
|
+
metrics: MetricProvider | None = None,
|
|
837
|
+
tiebreak: Tiebreak | None = None,
|
|
838
|
+
variants: Mapping[str, str] | None = None,
|
|
839
|
+
join_checks: Sequence[JoinCheck] = (),
|
|
840
|
+
tmp_root: Path | None = None,
|
|
841
|
+
layout: StateLayout | None = None,
|
|
842
|
+
make_branch: bool = True,
|
|
843
|
+
) -> None:
|
|
844
|
+
self.inner_factory = inner_factory
|
|
845
|
+
self.vcs = vcs
|
|
846
|
+
self.root = root
|
|
847
|
+
self.default = default
|
|
848
|
+
self.cfg = cfg if cfg is not None else SelectCfg()
|
|
849
|
+
self.fanout = fanout if fanout is not None else FanoutCfg()
|
|
850
|
+
self.max_cost = max_cost
|
|
851
|
+
self.metrics = metrics
|
|
852
|
+
self.tiebreak = tiebreak
|
|
853
|
+
self.variants = dict(variants if variants is not None else VARIANTS)
|
|
854
|
+
if not self.variants:
|
|
855
|
+
raise ValueError("best-of-N needs at least one variant")
|
|
856
|
+
self.join_checks = tuple(join_checks)
|
|
857
|
+
self.tmp_root = tmp_root
|
|
858
|
+
self.layout = layout
|
|
859
|
+
self.make_branch = make_branch
|
|
860
|
+
self.results: dict[str, BestOfNResult] = {}
|
|
861
|
+
|
|
862
|
+
# --- opting in ------------------------------------------------------------------
|
|
863
|
+
|
|
864
|
+
def requested_n(self, rule: RuleSpec, instance: RuleInstance) -> int | None:
|
|
865
|
+
"""The N `instance` asks for, or `None` when its rule did not opt in.
|
|
866
|
+
|
|
867
|
+
Raises `ValueError` for a `best_of` parameter that is not a positive integer.
|
|
868
|
+
"""
|
|
869
|
+
raw = instance.params.get(BEST_OF_PARAM)
|
|
870
|
+
if raw is not None:
|
|
871
|
+
try:
|
|
872
|
+
n = int(raw)
|
|
873
|
+
except ValueError:
|
|
874
|
+
raise ValueError(f"{BEST_OF_PARAM}={raw!r} is not a positive integer") from None
|
|
875
|
+
if n < 1:
|
|
876
|
+
raise ValueError(f"{BEST_OF_PARAM}={raw!r} is not a positive integer")
|
|
877
|
+
return n
|
|
878
|
+
return self.cfg.rules.get(rule.id)
|
|
879
|
+
|
|
880
|
+
def forget(self, instance_id: str) -> None:
|
|
881
|
+
"""Pass a rewind on to the plain executor."""
|
|
882
|
+
if isinstance(self.default, RewindAware):
|
|
883
|
+
self.default.forget(instance_id)
|
|
884
|
+
|
|
885
|
+
# --- the executor protocols ----------------------------------------------------------
|
|
886
|
+
|
|
887
|
+
async def execute(self, rule: RuleSpec, instance: RuleInstance) -> ExecOutcome:
|
|
888
|
+
"""Outside a scheduler: no journal, no check runner for a plain run."""
|
|
889
|
+
ctx = ExecContext(run_id="", run_check=None, emit=_noop_emit)
|
|
890
|
+
return await self.execute_checked(rule, instance, ctx)
|
|
891
|
+
|
|
892
|
+
async def execute_checked(
|
|
893
|
+
self, rule: RuleSpec, instance: RuleInstance, ctx: ExecContext
|
|
894
|
+
) -> ExecOutcome:
|
|
895
|
+
try:
|
|
896
|
+
requested = self.requested_n(rule, instance)
|
|
897
|
+
except ValueError as exc:
|
|
898
|
+
return ExecOutcome(ok=False, failure_label="planning", message=str(exc))
|
|
899
|
+
if requested is None:
|
|
900
|
+
return await self._plain(rule, instance, ctx)
|
|
901
|
+
plan = plan_budget(
|
|
902
|
+
rule.budget,
|
|
903
|
+
requested=requested,
|
|
904
|
+
cfg=self.cfg,
|
|
905
|
+
max_cost=self.max_cost,
|
|
906
|
+
variants=len(self.variants),
|
|
907
|
+
)
|
|
908
|
+
if plan.plain:
|
|
909
|
+
outcome = await self._plain(rule, instance, ctx)
|
|
910
|
+
why = f" ({plan.capped_by})" if plan.capped_by else ""
|
|
911
|
+
result = BestOfNResult(
|
|
912
|
+
status="plain",
|
|
913
|
+
instance=instance.instance_id,
|
|
914
|
+
message=f"best-of-{plan.requested}: only one candidate fits{why}; a plain run",
|
|
915
|
+
plan=plan,
|
|
916
|
+
)
|
|
917
|
+
self.results[instance.instance_id] = result
|
|
918
|
+
return outcome.model_copy(
|
|
919
|
+
update={"payload": {**outcome.payload, "best_of": result.model_dump(mode="json")}}
|
|
920
|
+
)
|
|
921
|
+
return await _BestOfRun(self, rule, instance, ctx, plan).run()
|
|
922
|
+
|
|
923
|
+
async def _plain(self, rule: RuleSpec, instance: RuleInstance, ctx: ExecContext) -> ExecOutcome:
|
|
924
|
+
if isinstance(self.default, CheckingExecutor):
|
|
925
|
+
return await self.default.execute_checked(rule, instance, ctx)
|
|
926
|
+
outcome = await self.default.execute(rule, instance)
|
|
927
|
+
if outcome.ok and ctx.run_check is not None:
|
|
928
|
+
for check_id in rule.checks:
|
|
929
|
+
checked = await ctx.run_check(check_id)
|
|
930
|
+
if not checked.ok:
|
|
931
|
+
return ExecOutcome(
|
|
932
|
+
ok=False, failure_label="verification", message=f"check {check_id} failed"
|
|
933
|
+
)
|
|
934
|
+
return outcome
|
|
935
|
+
|
|
936
|
+
|
|
937
|
+
class _BestOfRun:
|
|
938
|
+
"""One best-of-N run of one instance."""
|
|
939
|
+
|
|
940
|
+
def __init__(
|
|
941
|
+
self,
|
|
942
|
+
executor: BestOfNExecutor,
|
|
943
|
+
rule: RuleSpec,
|
|
944
|
+
instance: RuleInstance,
|
|
945
|
+
ctx: ExecContext,
|
|
946
|
+
plan: BudgetPlan,
|
|
947
|
+
) -> None:
|
|
948
|
+
self.ex = executor
|
|
949
|
+
self.rule = rule
|
|
950
|
+
self.instance = instance
|
|
951
|
+
self.ctx = ctx
|
|
952
|
+
self.plan = plan
|
|
953
|
+
self.ledger = CostLedger(plan.limits)
|
|
954
|
+
self.run_id = new_run_id()
|
|
955
|
+
ids = list(executor.variants)[: plan.n]
|
|
956
|
+
self.states = {cid: _State(id=cid, variant=executor.variants[cid]) for cid in ids}
|
|
957
|
+
self.writes = tuple(r.path for r in instance.outputs if r.path is not None)
|
|
958
|
+
self.inputs = tuple(dict.fromkeys(r.path for r in instance.inputs if r.path is not None))
|
|
959
|
+
self.selection: Selection | None = None
|
|
960
|
+
self.sliced = rule.model_copy(update={"budget": plan.slice})
|
|
961
|
+
|
|
962
|
+
def seed(self, cid: str) -> str:
|
|
963
|
+
digest = hashlib.sha256(f"{self.instance.instance_id}\0{cid}".encode()).hexdigest()
|
|
964
|
+
return f"{cid}-{digest[:12]}"
|
|
965
|
+
|
|
966
|
+
# --- one candidate -------------------------------------------------------------------
|
|
967
|
+
|
|
968
|
+
def candidate_ctx(self, state: _State, checks: CheckRunner | None) -> ExecContext:
|
|
969
|
+
async def run_check(check_id: str) -> CheckResult:
|
|
970
|
+
assert checks is not None
|
|
971
|
+
checked = await checks.run(check_id, self.instance)
|
|
972
|
+
state.checks[check_id] = checked # the last result of each check counts
|
|
973
|
+
return checked
|
|
974
|
+
|
|
975
|
+
def emit(type_: ExecEventType, payload: dict[str, Any]) -> None:
|
|
976
|
+
if type_ == "agent_turn" and not state.guarded:
|
|
977
|
+
# An executor whose runtime is not guarded is charged after the fact.
|
|
978
|
+
self.ledger.charge(
|
|
979
|
+
state.id, tries=1, tokens=payload.get("tokens"), cost=payload.get("cost")
|
|
980
|
+
)
|
|
981
|
+
self.ctx.emit(type_, {**payload, "candidate": state.id, "best_of": self.run_id})
|
|
982
|
+
|
|
983
|
+
return ExecContext(
|
|
984
|
+
run_id=self.ctx.run_id, run_check=run_check if checks is not None else None, emit=emit
|
|
985
|
+
)
|
|
986
|
+
|
|
987
|
+
def branch_fn(self, cid: str) -> Callable[[BranchContext], Awaitable[BranchOutcome]]:
|
|
988
|
+
state = self.states[cid]
|
|
989
|
+
|
|
990
|
+
async def run(bctx: BranchContext) -> BranchOutcome:
|
|
991
|
+
if not self.ledger.can_afford(1):
|
|
992
|
+
self.ledger.refuse()
|
|
993
|
+
state.status, state.message = "skipped", "not started: the shared budget is spent"
|
|
994
|
+
return BranchOutcome(ok=False, message=state.message)
|
|
995
|
+
context = {"variant": state.variant, "seed": self.seed(cid), "candidate": cid}
|
|
996
|
+
|
|
997
|
+
def guard(runtime: AgentRuntime) -> AgentRuntime:
|
|
998
|
+
state.guarded = True
|
|
999
|
+
return _GuardedRuntime(runtime, self.ledger, cid, context)
|
|
1000
|
+
|
|
1001
|
+
env = CandidateEnv(
|
|
1002
|
+
candidate_id=cid,
|
|
1003
|
+
variant=state.variant,
|
|
1004
|
+
seed=context["seed"],
|
|
1005
|
+
workspace=bctx.path,
|
|
1006
|
+
budget=self.plan.slice,
|
|
1007
|
+
max_cost=self.plan.slice_cost,
|
|
1008
|
+
guard=guard,
|
|
1009
|
+
)
|
|
1010
|
+
state.status, state.message = "error", "did not finish"
|
|
1011
|
+
parts = self.ex.inner_factory(env)
|
|
1012
|
+
cctx = self.candidate_ctx(state, parts.checks)
|
|
1013
|
+
try:
|
|
1014
|
+
outcome = await self._run_inner(parts, cctx)
|
|
1015
|
+
except LedgerExhausted as stop:
|
|
1016
|
+
state.status = "cancelled"
|
|
1017
|
+
state.message = f"cancelled: the shared budget is spent ({stop.limit})"
|
|
1018
|
+
return BranchOutcome(ok=False, message=state.message)
|
|
1019
|
+
state.outcome = outcome
|
|
1020
|
+
missing = [
|
|
1021
|
+
Issue(file=p, rule="output.missing", msg=f"output {p} was not written")
|
|
1022
|
+
for p in self.writes
|
|
1023
|
+
if not (bctx.path / p).exists()
|
|
1024
|
+
]
|
|
1025
|
+
if missing:
|
|
1026
|
+
state.checks[OUTPUTS_CHECK] = _synthetic(OUTPUTS_CHECK, missing)
|
|
1027
|
+
unchecked = [c for c in self.rule.checks if c not in state.checks]
|
|
1028
|
+
ok = outcome.ok and not unchecked and all(c.ok for c in state.checks.values())
|
|
1029
|
+
state.status = "passed" if ok else "failed"
|
|
1030
|
+
state.message = outcome.message or (
|
|
1031
|
+
f"checks not run: {', '.join(unchecked)}" if unchecked else ""
|
|
1032
|
+
)
|
|
1033
|
+
return BranchOutcome(ok=outcome.ok, message=state.message, agent=_agent_of(outcome))
|
|
1034
|
+
|
|
1035
|
+
return run
|
|
1036
|
+
|
|
1037
|
+
async def _run_inner(self, parts: CandidateParts, cctx: ExecContext) -> ExecOutcome:
|
|
1038
|
+
inner = parts.executor
|
|
1039
|
+
if isinstance(inner, CheckingExecutor):
|
|
1040
|
+
return await inner.execute_checked(self.sliced, self.instance, cctx)
|
|
1041
|
+
outcome = await inner.execute(self.sliced, self.instance)
|
|
1042
|
+
if outcome.ok and cctx.run_check is not None:
|
|
1043
|
+
for check_id in self.rule.checks:
|
|
1044
|
+
await cctx.run_check(check_id)
|
|
1045
|
+
return outcome
|
|
1046
|
+
|
|
1047
|
+
# --- choosing ----------------------------------------------------------------------
|
|
1048
|
+
|
|
1049
|
+
async def choose(
|
|
1050
|
+
self, results: tuple[BranchResult, ...], workspaces: Mapping[str, Path]
|
|
1051
|
+
) -> str | None:
|
|
1052
|
+
branches = {r.id: r for r in results}
|
|
1053
|
+
candidates: list[Candidate] = []
|
|
1054
|
+
for cid, state in sorted(self.states.items()):
|
|
1055
|
+
branch = branches.get(cid)
|
|
1056
|
+
if branch is not None and branch.status == "rejected":
|
|
1057
|
+
state.status, state.message = "rejected", branch.message
|
|
1058
|
+
elif branch is not None and branch.status in ("error", "timeout"):
|
|
1059
|
+
state.status, state.message = "error", branch.message
|
|
1060
|
+
finished = state.status in ("passed", "failed", "rejected")
|
|
1061
|
+
candidates.append(
|
|
1062
|
+
Candidate(
|
|
1063
|
+
id=cid,
|
|
1064
|
+
passed=state.status == "passed",
|
|
1065
|
+
checks=tuple(state.checks[k] for k in sorted(state.checks)),
|
|
1066
|
+
finished=finished,
|
|
1067
|
+
workspace=workspaces.get(cid),
|
|
1068
|
+
)
|
|
1069
|
+
)
|
|
1070
|
+
self.selection = await select_best(
|
|
1071
|
+
candidates, self.ex.metrics, self.ex.tiebreak, ledger=self.ledger
|
|
1072
|
+
)
|
|
1073
|
+
winner = self.selection.winner
|
|
1074
|
+
if winner is None or self.selection.none_passed:
|
|
1075
|
+
return None
|
|
1076
|
+
return winner
|
|
1077
|
+
|
|
1078
|
+
# --- the run ----------------------------------------------------------------------
|
|
1079
|
+
|
|
1080
|
+
async def run(self) -> ExecOutcome:
|
|
1081
|
+
branches = [
|
|
1082
|
+
FanoutBranch(id=cid, run=self.branch_fn(cid), write_set=self.writes, inputs=self.inputs)
|
|
1083
|
+
for cid in self.states
|
|
1084
|
+
]
|
|
1085
|
+
fan = await run_candidates(
|
|
1086
|
+
branches,
|
|
1087
|
+
choose=self.choose,
|
|
1088
|
+
root=self.ex.root,
|
|
1089
|
+
vcs=self.ex.vcs,
|
|
1090
|
+
name=_slug(self.instance.instance_id),
|
|
1091
|
+
run_id=self.run_id,
|
|
1092
|
+
join_checks=self.ex.join_checks,
|
|
1093
|
+
max_parallel=self.ex.fanout.effective_max_parallel(),
|
|
1094
|
+
branch_timeout_s=self.ex.fanout.branch_timeout_s,
|
|
1095
|
+
tmp_root=self.ex.tmp_root,
|
|
1096
|
+
layout=self.ex.layout,
|
|
1097
|
+
ctx=self.ctx,
|
|
1098
|
+
make_branch=self.ex.make_branch,
|
|
1099
|
+
)
|
|
1100
|
+
by_branch = {b.id: b for b in fan.branches}
|
|
1101
|
+
candidates: list[CandidateResult] = []
|
|
1102
|
+
for cid, state in sorted(self.states.items()):
|
|
1103
|
+
checks = tuple(state.checks[k] for k in sorted(state.checks))
|
|
1104
|
+
branch = by_branch.get(cid)
|
|
1105
|
+
cand = CandidateResult(
|
|
1106
|
+
id=cid,
|
|
1107
|
+
variant=state.variant,
|
|
1108
|
+
status=state.status,
|
|
1109
|
+
message=state.message,
|
|
1110
|
+
errors=error_count(checks),
|
|
1111
|
+
warnings=warning_count(checks),
|
|
1112
|
+
checks=checks,
|
|
1113
|
+
files=tuple(c.path for c in branch.files) if branch is not None else (),
|
|
1114
|
+
spend=self.ledger.spent(cid),
|
|
1115
|
+
agent=_agent_of(state.outcome) if state.outcome is not None else None,
|
|
1116
|
+
)
|
|
1117
|
+
candidates.append(cand)
|
|
1118
|
+
self.ctx.emit(
|
|
1119
|
+
"agent_turn",
|
|
1120
|
+
{
|
|
1121
|
+
"phase": "candidate",
|
|
1122
|
+
"best_of": self.run_id,
|
|
1123
|
+
"candidate": cid,
|
|
1124
|
+
"variant": state.variant,
|
|
1125
|
+
"status": state.status,
|
|
1126
|
+
"message": state.message,
|
|
1127
|
+
"errors": cand.errors,
|
|
1128
|
+
"warnings": cand.warnings,
|
|
1129
|
+
"failed_checks": [c.check_id for c in checks if not c.ok],
|
|
1130
|
+
"spend": cand.spend.model_dump(mode="json"),
|
|
1131
|
+
},
|
|
1132
|
+
)
|
|
1133
|
+
|
|
1134
|
+
selection = self.selection
|
|
1135
|
+
ok = fan.ok and selection is not None and fan.chosen == selection.winner
|
|
1136
|
+
if ok:
|
|
1137
|
+
assert selection is not None
|
|
1138
|
+
message = (
|
|
1139
|
+
f"best-of-{self.plan.n}: candidate {selection.winner} chosen by "
|
|
1140
|
+
f"{selection.decided_by}; result on branch {fan.branch or fan.commit}"
|
|
1141
|
+
)
|
|
1142
|
+
elif selection is None or selection.winner is None:
|
|
1143
|
+
message = f"best-of-{self.plan.n}: no candidate finished ({fan.message})"
|
|
1144
|
+
elif selection.none_passed:
|
|
1145
|
+
message = (
|
|
1146
|
+
f"best-of-{self.plan.n}: no candidate passed; the least bad is "
|
|
1147
|
+
f"{selection.winner}, nothing merged"
|
|
1148
|
+
)
|
|
1149
|
+
else:
|
|
1150
|
+
message = f"best-of-{self.plan.n}: {fan.message}"
|
|
1151
|
+
result = BestOfNResult(
|
|
1152
|
+
status="done" if ok else "failed",
|
|
1153
|
+
instance=self.instance.instance_id,
|
|
1154
|
+
message=message,
|
|
1155
|
+
plan=self.plan,
|
|
1156
|
+
winner=selection.winner if selection is not None else None,
|
|
1157
|
+
candidates=tuple(candidates),
|
|
1158
|
+
selection=selection,
|
|
1159
|
+
spend=self.ledger.total,
|
|
1160
|
+
critic_spend=self.ledger.spent(CRITIC),
|
|
1161
|
+
limited_by=self.ledger.limited_by,
|
|
1162
|
+
fanout=fan.run_id,
|
|
1163
|
+
branch=fan.branch if ok else None,
|
|
1164
|
+
commit=fan.commit if ok else None,
|
|
1165
|
+
)
|
|
1166
|
+
self.ex.results[self.instance.instance_id] = result
|
|
1167
|
+
self.ctx.emit(
|
|
1168
|
+
"tool_call",
|
|
1169
|
+
{
|
|
1170
|
+
"tool": "select",
|
|
1171
|
+
"event": "select",
|
|
1172
|
+
"best_of": self.run_id,
|
|
1173
|
+
"status": result.status,
|
|
1174
|
+
"winner": result.winner,
|
|
1175
|
+
"decided_by": selection.decided_by if selection is not None else "none",
|
|
1176
|
+
"critic_used": selection.critic_used if selection is not None else False,
|
|
1177
|
+
"spend": result.spend.model_dump(mode="json"),
|
|
1178
|
+
"limited_by": result.limited_by,
|
|
1179
|
+
"branch": result.branch,
|
|
1180
|
+
},
|
|
1181
|
+
)
|
|
1182
|
+
payload = {"best_of": result.model_dump(mode="json")}
|
|
1183
|
+
if ok:
|
|
1184
|
+
return ExecOutcome(ok=True, message=message, payload=payload)
|
|
1185
|
+
return ExecOutcome(
|
|
1186
|
+
ok=False, failure_label=self._failure_label(selection), message=message, payload=payload
|
|
1187
|
+
)
|
|
1188
|
+
|
|
1189
|
+
def _failure_label(self, selection: Selection | None) -> FailureLabel:
|
|
1190
|
+
if selection is not None and selection.winner is not None:
|
|
1191
|
+
outcome = self.states[selection.winner].outcome
|
|
1192
|
+
if outcome is not None and outcome.failure_label is not None:
|
|
1193
|
+
return outcome.failure_label
|
|
1194
|
+
return "verification"
|
|
1195
|
+
|
|
1196
|
+
|
|
1197
|
+
def _agent_of(outcome: ExecOutcome) -> AgentResult | None:
|
|
1198
|
+
"""The `AgentResult` an agent rule loop put in its outcome's payload, if any."""
|
|
1199
|
+
raw = outcome.payload.get("result")
|
|
1200
|
+
if not isinstance(raw, Mapping):
|
|
1201
|
+
return None
|
|
1202
|
+
with contextlib.suppress(ValueError):
|
|
1203
|
+
return AgentResult.model_validate(dict(raw))
|
|
1204
|
+
return None
|
|
1205
|
+
|
|
1206
|
+
|
|
1207
|
+
__all__ = [
|
|
1208
|
+
"BEST_OF_PARAM",
|
|
1209
|
+
"CRITIC",
|
|
1210
|
+
"OUTPUTS_CHECK",
|
|
1211
|
+
"VARIANTS",
|
|
1212
|
+
"BestOfNExecutor",
|
|
1213
|
+
"BestOfNResult",
|
|
1214
|
+
"BestOfNStatus",
|
|
1215
|
+
"BudgetPlan",
|
|
1216
|
+
"Candidate",
|
|
1217
|
+
"CandidateEnv",
|
|
1218
|
+
"CandidateKey",
|
|
1219
|
+
"CandidateParts",
|
|
1220
|
+
"CandidateResult",
|
|
1221
|
+
"CandidateStatus",
|
|
1222
|
+
"CapReason",
|
|
1223
|
+
"CostLedger",
|
|
1224
|
+
"InnerFactory",
|
|
1225
|
+
"LedgerExhausted",
|
|
1226
|
+
"LedgerLimit",
|
|
1227
|
+
"LedgerLimits",
|
|
1228
|
+
"MetricProvider",
|
|
1229
|
+
"ReviewScore",
|
|
1230
|
+
"SelectStep",
|
|
1231
|
+
"Selection",
|
|
1232
|
+
"Spend",
|
|
1233
|
+
"Tiebreak",
|
|
1234
|
+
"error_count",
|
|
1235
|
+
"plan_budget",
|
|
1236
|
+
"rank",
|
|
1237
|
+
"select_best",
|
|
1238
|
+
"warning_count",
|
|
1239
|
+
]
|