chipgraph 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (252) hide show
  1. chipgraph/__init__.py +7 -0
  2. chipgraph/adapters/__init__.py +1 -0
  3. chipgraph/adapters/format/__init__.py +6 -0
  4. chipgraph/adapters/format/__main__.py +16 -0
  5. chipgraph/adapters/format/_yaml_lines.py +54 -0
  6. chipgraph/adapters/format/chip_yaml.py +323 -0
  7. chipgraph/adapters/format/qsoc_contract.py +246 -0
  8. chipgraph/adapters/llm/__init__.py +58 -0
  9. chipgraph/adapters/llm/_budget.py +173 -0
  10. chipgraph/adapters/llm/_errors.py +50 -0
  11. chipgraph/adapters/llm/_nda.py +30 -0
  12. chipgraph/adapters/llm/_retry.py +85 -0
  13. chipgraph/adapters/llm/anthropic.py +247 -0
  14. chipgraph/adapters/llm/decide_backend.py +179 -0
  15. chipgraph/adapters/llm/fake.py +87 -0
  16. chipgraph/adapters/llm/guard.py +68 -0
  17. chipgraph/adapters/parser/__init__.py +22 -0
  18. chipgraph/adapters/parser/cocotb.py +304 -0
  19. chipgraph/adapters/parser/generic_regex.py +75 -0
  20. chipgraph/adapters/parser/verible.py +55 -0
  21. chipgraph/adapters/parser/verilator.py +46 -0
  22. chipgraph/adapters/review/__init__.py +7 -0
  23. chipgraph/adapters/review/file.py +123 -0
  24. chipgraph/adapters/review/github.py +886 -0
  25. chipgraph/adapters/runner/__init__.py +7 -0
  26. chipgraph/adapters/runner/local.py +122 -0
  27. chipgraph/adapters/runtime/__init__.py +5 -0
  28. chipgraph/adapters/runtime/claude_code/__init__.py +24 -0
  29. chipgraph/adapters/runtime/claude_code/agents.py +158 -0
  30. chipgraph/adapters/runtime/claude_code/decisions.py +280 -0
  31. chipgraph/adapters/runtime/claude_code/guarded.py +310 -0
  32. chipgraph/adapters/runtime/claude_code/isolation.py +482 -0
  33. chipgraph/adapters/runtime/claude_code/loop.py +342 -0
  34. chipgraph/adapters/runtime/claude_code/previous.py +419 -0
  35. chipgraph/adapters/runtime/claude_code/review.py +492 -0
  36. chipgraph/adapters/runtime/claude_code/runtime.py +192 -0
  37. chipgraph/adapters/runtime/claude_code/service.py +2285 -0
  38. chipgraph/adapters/tool/__init__.py +7 -0
  39. chipgraph/adapters/tool/cmd.py +229 -0
  40. chipgraph/adapters/tool/edalize.py +773 -0
  41. chipgraph/adapters/tool/filelist.py +259 -0
  42. chipgraph/adapters/tool/pyslang.py +485 -0
  43. chipgraph/adapters/vcs/__init__.py +1 -0
  44. chipgraph/adapters/vcs/git.py +521 -0
  45. chipgraph/app/__init__.py +10 -0
  46. chipgraph/app/baseline.py +339 -0
  47. chipgraph/app/build.py +332 -0
  48. chipgraph/app/checks.py +202 -0
  49. chipgraph/app/context.py +191 -0
  50. chipgraph/app/errors.py +11 -0
  51. chipgraph/app/executors.py +79 -0
  52. chipgraph/app/findings.py +203 -0
  53. chipgraph/app/ingest.py +523 -0
  54. chipgraph/checks/__init__.py +52 -0
  55. chipgraph/checks/_cdc_analyze.py +272 -0
  56. chipgraph/checks/_cdc_elaborate.py +455 -0
  57. chipgraph/checks/_common.py +268 -0
  58. chipgraph/checks/_cross.py +96 -0
  59. chipgraph/checks/_model.py +64 -0
  60. chipgraph/checks/_naming_pyslang.py +235 -0
  61. chipgraph/checks/_naming_rules.py +127 -0
  62. chipgraph/checks/_xref_syntax.py +260 -0
  63. chipgraph/checks/cdc_struct.py +235 -0
  64. chipgraph/checks/connect.py +353 -0
  65. chipgraph/checks/cross_chip.py +454 -0
  66. chipgraph/checks/duplicate.py +153 -0
  67. chipgraph/checks/filelist.py +323 -0
  68. chipgraph/checks/format.py +268 -0
  69. chipgraph/checks/generated.py +151 -0
  70. chipgraph/checks/hardcode.py +233 -0
  71. chipgraph/checks/layout.py +155 -0
  72. chipgraph/checks/naming.py +311 -0
  73. chipgraph/checks/plan_check.py +901 -0
  74. chipgraph/checks/ports_diff.py +203 -0
  75. chipgraph/checks/spec_schema.py +277 -0
  76. chipgraph/checks/style.py +910 -0
  77. chipgraph/checks/tb_static.py +599 -0
  78. chipgraph/checks/trace.py +224 -0
  79. chipgraph/checks/vendor_guard.py +369 -0
  80. chipgraph/checks/wrap_check.py +486 -0
  81. chipgraph/cli/__init__.py +1688 -0
  82. chipgraph/cli/next_steps.py +137 -0
  83. chipgraph/core/__init__.py +1 -0
  84. chipgraph/core/config/__init__.py +93 -0
  85. chipgraph/core/config/defaults.py +14 -0
  86. chipgraph/core/config/errors.py +32 -0
  87. chipgraph/core/config/loader.py +766 -0
  88. chipgraph/core/config/models.py +845 -0
  89. chipgraph/core/config/templates.py +340 -0
  90. chipgraph/core/contracts/__init__.py +75 -0
  91. chipgraph/core/contracts/_paths.py +26 -0
  92. chipgraph/core/contracts/agent.py +49 -0
  93. chipgraph/core/contracts/artifact.py +68 -0
  94. chipgraph/core/contracts/check.py +76 -0
  95. chipgraph/core/contracts/event.py +70 -0
  96. chipgraph/core/contracts/export.py +128 -0
  97. chipgraph/core/contracts/finding.py +171 -0
  98. chipgraph/core/contracts/gate.py +50 -0
  99. chipgraph/core/contracts/rule.py +177 -0
  100. chipgraph/core/contracts/types.py +45 -0
  101. chipgraph/core/engine/__init__.py +70 -0
  102. chipgraph/core/engine/agent_rule.py +1362 -0
  103. chipgraph/core/engine/baseline.py +283 -0
  104. chipgraph/core/engine/decide.py +462 -0
  105. chipgraph/core/engine/fanout.py +1358 -0
  106. chipgraph/core/engine/gate.py +170 -0
  107. chipgraph/core/engine/graph.py +647 -0
  108. chipgraph/core/engine/records.py +60 -0
  109. chipgraph/core/engine/rules.py +92 -0
  110. chipgraph/core/engine/scheduler.py +637 -0
  111. chipgraph/core/engine/select.py +1239 -0
  112. chipgraph/core/engine/targets.py +342 -0
  113. chipgraph/core/model/__init__.py +98 -0
  114. chipgraph/core/model/entities.py +244 -0
  115. chipgraph/core/model/export.py +16 -0
  116. chipgraph/core/model/ingest.py +693 -0
  117. chipgraph/core/model/json_value.py +4 -0
  118. chipgraph/core/model/keys.py +95 -0
  119. chipgraph/core/model/model.py +153 -0
  120. chipgraph/core/model/provenance.py +27 -0
  121. chipgraph/core/model/query.py +583 -0
  122. chipgraph/core/model/registry.py +80 -0
  123. chipgraph/core/model/relations.py +61 -0
  124. chipgraph/core/model/store.py +279 -0
  125. chipgraph/core/plugin_api/__init__.py +81 -0
  126. chipgraph/core/plugin_api/local.py +167 -0
  127. chipgraph/core/plugin_api/pack.py +159 -0
  128. chipgraph/core/plugin_api/protocols.py +177 -0
  129. chipgraph/core/plugin_api/registry.py +111 -0
  130. chipgraph/core/plugin_api/types.py +94 -0
  131. chipgraph/core/runtime/__init__.py +41 -0
  132. chipgraph/core/runtime/executor.py +79 -0
  133. chipgraph/core/runtime/queue.py +565 -0
  134. chipgraph/core/runtime/roles/__init__.py +91 -0
  135. chipgraph/core/runtime/roles/data/author.md +41 -0
  136. chipgraph/core/runtime/roles/data/critic.md +56 -0
  137. chipgraph/core/runtime/roles/data/planner.md +38 -0
  138. chipgraph/core/runtime/roles/data/researcher.md +37 -0
  139. chipgraph/core/runtime/roles/data/rtl-author.md +64 -0
  140. chipgraph/core/runtime/roles/data/tb-author.md +48 -0
  141. chipgraph/core/runtime/roles/data/triage.md +34 -0
  142. chipgraph/core/runtime/roles/policy.py +299 -0
  143. chipgraph/core/runtime/roles/registry.py +52 -0
  144. chipgraph/core/runtime/roles/review.py +275 -0
  145. chipgraph/core/runtime/roles/schema.py +82 -0
  146. chipgraph/core/runtime/roles/skills.py +134 -0
  147. chipgraph/core/runtime/roles/spec.py +232 -0
  148. chipgraph/core/state/__init__.py +1 -0
  149. chipgraph/core/state/artifacts.py +156 -0
  150. chipgraph/core/state/backend.py +117 -0
  151. chipgraph/core/state/findings.py +151 -0
  152. chipgraph/core/state/handoff.py +411 -0
  153. chipgraph/core/state/idempotency.py +59 -0
  154. chipgraph/core/state/journal.py +238 -0
  155. chipgraph/core/state/layout.py +82 -0
  156. chipgraph/core/state/lock.py +195 -0
  157. chipgraph/core/state/trace.py +225 -0
  158. chipgraph/learn/__init__.py +96 -0
  159. chipgraph/learn/draft.py +187 -0
  160. chipgraph/learn/infer.py +580 -0
  161. chipgraph/learn/models.py +145 -0
  162. chipgraph/learn/try_run.py +228 -0
  163. chipgraph/mcp/__init__.py +12 -0
  164. chipgraph/mcp/model_tools.py +167 -0
  165. chipgraph/mcp/server.py +787 -0
  166. chipgraph/orgs/qnsc/naming-v1.yml +113 -0
  167. chipgraph/orgs/qnsc/profile.yml +5 -0
  168. chipgraph/packs/__init__.py +8 -0
  169. chipgraph/packs/assist/__init__.py +16 -0
  170. chipgraph/packs/assist/ask/__init__.py +58 -0
  171. chipgraph/packs/assist/ask/_db.py +57 -0
  172. chipgraph/packs/assist/ask/_project.py +230 -0
  173. chipgraph/packs/assist/ask/answer.py +266 -0
  174. chipgraph/packs/assist/ask/check.py +182 -0
  175. chipgraph/packs/assist/ask/contract.py +115 -0
  176. chipgraph/packs/assist/ask/documents.py +157 -0
  177. chipgraph/packs/assist/ask/fts.py +80 -0
  178. chipgraph/packs/assist/ask/retrieve.py +331 -0
  179. chipgraph/packs/assist/audit/__init__.py +383 -0
  180. chipgraph/packs/assist/pack.yml +13 -0
  181. chipgraph/packs/assist/triage/__init__.py +64 -0
  182. chipgraph/packs/assist/triage/contract.py +94 -0
  183. chipgraph/packs/assist/triage/parse.py +371 -0
  184. chipgraph/packs/assist/triage/question.py +384 -0
  185. chipgraph/packs/assist/triage/report.py +160 -0
  186. chipgraph/packs/assist/triage/rules.py +447 -0
  187. chipgraph/packs/assist/triage/run.py +224 -0
  188. chipgraph/packs/digital_rtl/__init__.py +19 -0
  189. chipgraph/packs/digital_rtl/layout.py +144 -0
  190. chipgraph/packs/digital_rtl/pack.yml +25 -0
  191. chipgraph/packs/digital_rtl/plan/__init__.py +34 -0
  192. chipgraph/packs/digital_rtl/plan/__main__.py +16 -0
  193. chipgraph/packs/digital_rtl/plan/context.py +82 -0
  194. chipgraph/packs/digital_rtl/plan/expand.py +94 -0
  195. chipgraph/packs/digital_rtl/plan/model.py +150 -0
  196. chipgraph/packs/digital_rtl/plan/resolver.py +400 -0
  197. chipgraph/packs/digital_rtl/plan/show.py +176 -0
  198. chipgraph/packs/digital_rtl/pr/__init__.py +11 -0
  199. chipgraph/packs/digital_rtl/pr/body.py +165 -0
  200. chipgraph/packs/digital_rtl/pr/gate.py +110 -0
  201. chipgraph/packs/digital_rtl/pr/open.py +397 -0
  202. chipgraph/packs/digital_rtl/rtl/__init__.py +9 -0
  203. chipgraph/packs/digital_rtl/rtl/context.py +272 -0
  204. chipgraph/packs/digital_rtl/rules/open_pr.yml +30 -0
  205. chipgraph/packs/digital_rtl/rules/plan.yml +28 -0
  206. chipgraph/packs/digital_rtl/rules/plan_expand.yml +29 -0
  207. chipgraph/packs/digital_rtl/rules/pr_gate.yml +27 -0
  208. chipgraph/packs/digital_rtl/rules/review.yml +31 -0
  209. chipgraph/packs/digital_rtl/rules/rtl_module.yml +53 -0
  210. chipgraph/packs/digital_rtl/skills/lang-sv-rtl.md +58 -0
  211. chipgraph/packs/digital_rtl/skills/org-naming.md +29 -0
  212. chipgraph/packs/digital_rtl/skills/plan-modules.md +63 -0
  213. chipgraph/packs/digital_rtl/skills/review-diff.md +41 -0
  214. chipgraph/packs/digital_rtl/targets/plan.yml +11 -0
  215. chipgraph/packs/digital_rtl/targets/rtl.yml +20 -0
  216. chipgraph/packs/digital_rtl/wrapper/__init__.py +24 -0
  217. chipgraph/packs/digital_rtl/wrapper/common.py +171 -0
  218. chipgraph/packs/digital_rtl/wrapper/context.py +222 -0
  219. chipgraph/packs/digital_rtl/wrapper/emacs.py +626 -0
  220. chipgraph/packs/digital_rtl/wrapper/expansion.py +304 -0
  221. chipgraph/packs/digital_rtl/wrapper/lisp.py +504 -0
  222. chipgraph/packs/digital_rtl/wrapper/resolver.py +60 -0
  223. chipgraph/packs/digital_rtl/wrapper/rules/wrapper.yml +41 -0
  224. chipgraph/packs/digital_rtl/wrapper/rules/wrapper_gen.yml +35 -0
  225. chipgraph/packs/digital_rtl/wrapper/skills/wrapper-auto.md +120 -0
  226. chipgraph/packs/dv/__init__.py +15 -0
  227. chipgraph/packs/dv/declaration.py +392 -0
  228. chipgraph/packs/dv/interface.py +596 -0
  229. chipgraph/packs/dv/pack.yml +14 -0
  230. chipgraph/packs/dv/rules/tb_module.yml +42 -0
  231. chipgraph/packs/dv/skills/dv/cocotb.md +72 -0
  232. chipgraph/packs/spec_core/__init__.py +9 -0
  233. chipgraph/packs/spec_core/extract/__init__.py +12 -0
  234. chipgraph/packs/spec_core/extract/_markdown.py +371 -0
  235. chipgraph/packs/spec_core/extract/_text.py +116 -0
  236. chipgraph/packs/spec_core/extract/mas.py +975 -0
  237. chipgraph/packs/spec_core/gen/__init__.py +11 -0
  238. chipgraph/packs/spec_core/gen/agents_md/__init__.py +148 -0
  239. chipgraph/packs/spec_core/gen/agents_md/_context.py +239 -0
  240. chipgraph/packs/spec_core/gen/agents_md/_markers.py +131 -0
  241. chipgraph/packs/spec_core/gen/agents_md/agents_md.md.j2 +178 -0
  242. chipgraph/packs/spec_core/gen/diagram/__init__.py +110 -0
  243. chipgraph/packs/spec_core/gen/diagram/__main__.py +44 -0
  244. chipgraph/packs/spec_core/gen/diagram/_diagrams.py +403 -0
  245. chipgraph/packs/spec_core/gen/diagram/_scene.py +163 -0
  246. chipgraph/packs/spec_core/pack.yml +6 -0
  247. chipgraph/py.typed +0 -0
  248. chipgraph-0.1.0.dist-info/METADATA +53 -0
  249. chipgraph-0.1.0.dist-info/RECORD +252 -0
  250. chipgraph-0.1.0.dist-info/WHEEL +4 -0
  251. chipgraph-0.1.0.dist-info/entry_points.txt +61 -0
  252. chipgraph-0.1.0.dist-info/licenses/LICENSE +202 -0
@@ -0,0 +1,1239 @@
1
+ """Best-of-N: N candidates of one rule inside one budget, chosen by its checks (DESIGN 5.3, F6).
2
+
3
+ ```
4
+ BestOfNExecutor.execute_checked(rule, instance)
5
+ N = the rule's opt-in (instance param `best_of`, or `select.rules[rule]`), capped at
6
+ `select.max_n`; no opt-in -> the plain executor, unchanged
7
+ plan_budget: ONE ledger = budget_factor x the rule's budget (tries, tokens, cost);
8
+ each candidate gets a slice (total / N); N drops to what fits (1 -> a plain run)
9
+ run_candidates (fanout.py): N workspaces, seeded identically, same inputs and outputs
10
+ candidate i: the rule, sliced budget, variant hint i, its own seed
11
+ every runtime call reserves one try of the ledger first; none left -> cancelled
12
+ select_best: the rule's checks decide; the Critic only breaks exact ties
13
+ merge only the winner onto the result branch; journal one event per candidate and
14
+ one `select` event
15
+ ```
16
+
17
+ The selection key, in order (lower is better; `select_best` is total-ordered):
18
+
19
+ | step | key |
20
+ |----------|---------------------------------------------------------------------------|
21
+ | passed | every check of the rule passed (a failing candidate wins only if none did) |
22
+ | errors | fewer errors: error issues, or 1 for a failing check without one |
23
+ | warnings | fewer `warning` issues over all check results (`info` ignored) |
24
+ | area | smaller area, only when a `MetricProvider` gives one; lacking it sorts after |
25
+ | critic | only among exact ties on the above: approve, fewer blocker, major, minor |
26
+ | lowest_id| still tied (or no Critic, or the ledger cannot pay for it): lowest id |
27
+
28
+ Budget arithmetic: the ledger holds `floor(tries x factor)` tries (at least 1),
29
+ `floor(tokens x factor)` tokens and `max_cost x factor` cost when those are capped; each
30
+ candidate's `AttemptBudget` gets `tries // N` tries, `tokens // N` tokens and
31
+ `cost / N`. A ledger try is one runtime call (an infra retry costs one too); a call
32
+ holds its token cap (lowered to what the ledger has left) while it runs, so the
33
+ candidates cannot overspend tokens together. When a call cannot be paid for, that
34
+ candidate is cancelled there (cooperatively, between calls) and the best candidate that
35
+ finished wins. The Critic costs one try per tied candidate it reviews, plus what it
36
+ reports, from the same ledger. Example, a rule with `tries: 3, tokens: 200000` and a
37
+ `max_cost` of 2 USD: the ledger holds 4 tries, 300000 tokens, 3 USD; with N=2 each
38
+ candidate gets 2 tries, 150000 tokens, 1.5 USD; with N=3, 1 try, 100000 tokens, 1 USD.
39
+
40
+ Candidate outputs never touch the user's working tree: the winner lands on a result
41
+ branch (`chipgraph/<run>/best-of-<instance>`). How that branch reaches the user's tree
42
+ (and the scheduler's output check and records), and the claude-code runtime's own
43
+ wiring (its runtime per workspace, the review rule as the Critic) are M2-11's job.
44
+
45
+ Generic: no project, tool, model or file format is used here; the agent runtime is
46
+ named only to say where its wiring belongs.
47
+ """
48
+
49
+ from __future__ import annotations
50
+
51
+ import contextlib
52
+ import hashlib
53
+ import math
54
+ import re
55
+ from collections.abc import Awaitable, Callable, Mapping, Sequence
56
+ from dataclasses import dataclass, field
57
+ from pathlib import Path
58
+ from typing import Any, Literal, Protocol, runtime_checkable
59
+
60
+ from pydantic import BaseModel, ConfigDict, Field
61
+
62
+ from chipgraph.core.config.models import FanoutCfg, SelectCfg
63
+ from chipgraph.core.contracts import AgentResult, Budget, CheckResult, Issue
64
+ from chipgraph.core.contracts.rule import RuleInstance, RuleSpec
65
+ from chipgraph.core.contracts.types import FailureLabel
66
+ from chipgraph.core.engine.fanout import (
67
+ BranchContext,
68
+ BranchOutcome,
69
+ BranchResult,
70
+ FanoutBranch,
71
+ FanoutVcs,
72
+ JoinCheck,
73
+ run_candidates,
74
+ )
75
+ from chipgraph.core.engine.scheduler import (
76
+ CheckingExecutor,
77
+ CheckRunner,
78
+ ExecContext,
79
+ ExecEventType,
80
+ ExecOutcome,
81
+ Executor,
82
+ RewindAware,
83
+ )
84
+ from chipgraph.core.plugin_api.protocols import AgentRuntime
85
+ from chipgraph.core.plugin_api.types import AgentTask
86
+ from chipgraph.core.runtime.roles.review import ReviewReport, Verdict
87
+ from chipgraph.core.state.layout import StateLayout, new_run_id
88
+
89
+ VARIANTS: Mapping[str, str] = {
90
+ "a": "approach a: prefer the simplest structure that meets the spec",
91
+ "b": "approach b: prefer the most explicit structure, every case spelled out",
92
+ "c": "approach c: prefer the fewest registers and the least state that meet the spec",
93
+ }
94
+ """The default hint per candidate, `AgentTask.context["variant"]`; the key is the id."""
95
+
96
+ BEST_OF_PARAM = "best_of"
97
+ """The instance parameter a rule instance opts in with (`best_of: <n>`)."""
98
+
99
+ OUTPUTS_CHECK = "best_of.outputs"
100
+ """The check `BestOfNExecutor` adds for a candidate that left an output missing."""
101
+
102
+ CRITIC = "critic"
103
+ """The ledger account the Critic's spend is charged to."""
104
+
105
+
106
+ # --- the shared ledger ------------------------------------------------------------------
107
+
108
+ LedgerLimit = Literal["tries", "tokens", "cost"]
109
+ """Which limit of the shared ledger stopped something."""
110
+
111
+
112
+ class Spend(BaseModel):
113
+ """What was spent: tries (runtime calls), tokens and cost (USD)."""
114
+
115
+ model_config = ConfigDict(extra="forbid", frozen=True)
116
+
117
+ tries: int = Field(default=0, ge=0, description="Runtime calls (each costs one try).")
118
+ tokens: int = Field(default=0, ge=0, description="Tokens reported.")
119
+ cost: float = Field(default=0.0, ge=0, description="Cost reported, USD.")
120
+
121
+ def plus(
122
+ self, *, tries: int = 0, tokens: int | None = None, cost: float | None = None
123
+ ) -> Spend:
124
+ """This spend with more added (`None`: nothing reported)."""
125
+ return Spend(
126
+ tries=self.tries + tries,
127
+ tokens=self.tokens + (tokens or 0),
128
+ cost=self.cost + (cost or 0.0),
129
+ )
130
+
131
+
132
+ class LedgerLimits(BaseModel):
133
+ """The limits of the one budget all candidates share."""
134
+
135
+ model_config = ConfigDict(extra="forbid", frozen=True)
136
+
137
+ tries: int = Field(ge=1, description="Runtime calls over all candidates and the Critic.")
138
+ tokens: int | None = Field(default=None, ge=1, description="Token cap, if capped.")
139
+ cost: float | None = Field(default=None, gt=0, description="Cost cap in USD, if capped.")
140
+
141
+
142
+ class LedgerExhausted(BaseException):
143
+ """Raised by a guarded runtime when the shared ledger cannot pay for another call.
144
+
145
+ A `BaseException`, so a loop that retries every `Exception` as an infra error stops
146
+ instead: the candidate is cancelled cooperatively, at its next call.
147
+ """
148
+
149
+ def __init__(self, limit: LedgerLimit) -> None:
150
+ super().__init__(f"the shared budget is spent ({limit})")
151
+ self.limit: LedgerLimit = limit
152
+
153
+
154
+ class CostLedger:
155
+ """One budget for N candidates (and the Critic): tries, tokens and cost, per account.
156
+
157
+ A call reserves its try, and holds its token cap, before it starts (`reserve`), so
158
+ tries never exceed the limit and, as long as runtimes keep to the token cap of their
159
+ task, neither do tokens, however the calls interleave. What a call reported is
160
+ charged when it ends (`settle`). Cost cannot be held (a task carries no cost cap): a
161
+ call starts only while cost is under the limit, so it can be passed by the calls
162
+ already running. `limited_by` is the first limit that refused a call or was reached.
163
+ """
164
+
165
+ def __init__(self, limits: LedgerLimits) -> None:
166
+ self.limits = limits
167
+ self.total = Spend()
168
+ self.accounts: dict[str, Spend] = {}
169
+ self.held_tokens = 0
170
+ self.limited_by: LedgerLimit | None = None
171
+
172
+ @property
173
+ def remaining_tries(self) -> int:
174
+ """Tries left."""
175
+ return max(self.limits.tries - self.total.tries, 0)
176
+
177
+ @property
178
+ def remaining_tokens(self) -> int | None:
179
+ """Tokens neither spent nor held by a running call; `None` when not capped."""
180
+ if self.limits.tokens is None:
181
+ return None
182
+ return max(self.limits.tokens - self.total.tokens - self.held_tokens, 0)
183
+
184
+ @property
185
+ def remaining_cost(self) -> float | None:
186
+ """Cost left, or `None` when cost is not capped."""
187
+ if self.limits.cost is None:
188
+ return None
189
+ return max(self.limits.cost - self.total.cost, 0.0)
190
+
191
+ @property
192
+ def exhausted(self) -> LedgerLimit | None:
193
+ """The limit that is spent, or `None` while another call may run."""
194
+ if self.total.tries >= self.limits.tries:
195
+ return "tries"
196
+ if self.limits.tokens is not None and self.total.tokens >= self.limits.tokens:
197
+ return "tokens"
198
+ if self.limits.cost is not None and self.total.cost >= self.limits.cost:
199
+ return "cost"
200
+ return None
201
+
202
+ def why_not(self, tries: int = 1, tokens: int = 1) -> LedgerLimit | None:
203
+ """The limit that keeps `tries` calls (holding `tokens`) from starting, if any."""
204
+ spent = self.exhausted
205
+ if spent is not None:
206
+ return spent
207
+ if self.remaining_tries < tries:
208
+ return "tries"
209
+ left = self.remaining_tokens
210
+ if left is not None and left < max(tokens, 1):
211
+ return "tokens"
212
+ return None
213
+
214
+ def can_afford(self, tries: int = 1, tokens: int = 1) -> bool:
215
+ """Whether `tries` more calls, holding `tokens`, may start now."""
216
+ return self.why_not(tries, tokens) is None
217
+
218
+ def refuse(self, tries: int = 1, tokens: int = 1) -> LedgerLimit:
219
+ """Record (once) and return the limit that refuses `tries` calls."""
220
+ limit = self.why_not(tries, tokens) or "tries"
221
+ if self.limited_by is None:
222
+ self.limited_by = limit
223
+ return limit
224
+
225
+ def reserve(self, account: str, tries: int = 1, tokens: int | None = None) -> bool:
226
+ """Take `tries` tries for `account` and hold `tokens`, before its calls start;
227
+ `False` (nothing taken) when the ledger cannot pay for them."""
228
+ if not self.can_afford(tries, tokens or 1):
229
+ self.refuse(tries, tokens or 1)
230
+ return False
231
+ self._add(account, tries=tries)
232
+ if tokens is not None and self.limits.tokens is not None:
233
+ self.held_tokens += tokens
234
+ return True
235
+
236
+ def settle(
237
+ self,
238
+ account: str,
239
+ *,
240
+ held: int | None = None,
241
+ tokens: int | None = None,
242
+ cost: float | None = None,
243
+ ) -> None:
244
+ """A call of `account` ended: release what it held, charge what it reported."""
245
+ if held is not None and self.limits.tokens is not None:
246
+ self.held_tokens = max(self.held_tokens - held, 0)
247
+ self.charge(account, tokens=tokens, cost=cost)
248
+
249
+ def charge(
250
+ self, account: str, *, tries: int = 0, tokens: int | None = None, cost: float | None = None
251
+ ) -> None:
252
+ """Add what `account` spent (tokens and cost as reported, maybe a try)."""
253
+ self._add(account, tries=tries, tokens=tokens, cost=cost)
254
+ reached = self.exhausted
255
+ if reached is not None and reached != "tries" and self.limited_by is None:
256
+ self.limited_by = reached
257
+
258
+ def spent(self, account: str) -> Spend:
259
+ """What `account` spent so far."""
260
+ return self.accounts.get(account, Spend())
261
+
262
+ def _add(
263
+ self, account: str, *, tries: int = 0, tokens: int | None = None, cost: float | None = None
264
+ ) -> None:
265
+ self.total = self.total.plus(tries=tries, tokens=tokens, cost=cost)
266
+ self.accounts[account] = self.spent(account).plus(tries=tries, tokens=tokens, cost=cost)
267
+
268
+
269
+ # --- planning N inside one budget --------------------------------------------------------
270
+
271
+ CapReason = Literal["max_n", "variants", "budget"]
272
+ """Why fewer candidates run than were asked for."""
273
+
274
+
275
+ class BudgetPlan(BaseModel):
276
+ """How many candidates run, the shared limits, and each candidate's slice."""
277
+
278
+ model_config = ConfigDict(extra="forbid", frozen=True)
279
+
280
+ requested: int = Field(ge=1, description="The N the rule asked for.")
281
+ n: int = Field(ge=1, description="The N that runs (1: a plain run).")
282
+ capped_by: CapReason | None = Field(default=None, description="Why n < requested.")
283
+ factor: float = Field(gt=0, description="The shared budget, in single-run budgets.")
284
+ limits: LedgerLimits = Field(description="The shared ledger's limits.")
285
+ slice: Budget = Field(description="Each candidate's budget (tries, tier, tokens).")
286
+ slice_cost: float | None = Field(default=None, description="Each candidate's cost cap.")
287
+
288
+ @property
289
+ def plain(self) -> bool:
290
+ """True when only one candidate fits: then it is a plain run of the rule."""
291
+ return self.n == 1
292
+
293
+
294
+ def plan_budget(
295
+ budget: Budget,
296
+ *,
297
+ requested: int,
298
+ cfg: SelectCfg | None = None,
299
+ max_cost: float | None = None,
300
+ variants: int | None = None,
301
+ ) -> BudgetPlan:
302
+ """N, the shared ledger and the slices for a rule with `budget` asking for `requested`.
303
+
304
+ N is capped at `cfg.max_n`, at `variants` (the hints there are), and at what the
305
+ ledger can give at least one try (and one token) each. The candidates start at the
306
+ rule's tier and keep its escalation rule; nothing escalates across candidates.
307
+ """
308
+ cfg = cfg if cfg is not None else SelectCfg()
309
+ factor = cfg.budget_factor
310
+ asked = max(1, requested)
311
+ n = cfg.effective_n(asked)
312
+ capped: CapReason | None = "max_n" if n < asked else None
313
+ if variants is not None and n > variants:
314
+ n, capped = max(1, variants), "variants"
315
+ tries = max(1, math.floor(budget.tries * factor + 1e-9))
316
+ tokens = max(1, math.floor(budget.tokens * factor + 1e-9)) if budget.tokens else None
317
+ cost = max_cost * factor if max_cost is not None and max_cost > 0 else None
318
+ fits = min(tries, tokens) if tokens is not None else tries
319
+ if n > fits:
320
+ n, capped = fits, "budget"
321
+ slice_budget = Budget(
322
+ tries=max(1, tries // n),
323
+ tier=budget.tier,
324
+ escalate=budget.escalate,
325
+ tokens=max(1, tokens // n) if tokens is not None else None,
326
+ )
327
+ return BudgetPlan(
328
+ requested=asked,
329
+ n=n,
330
+ capped_by=capped,
331
+ factor=factor,
332
+ limits=LedgerLimits(tries=tries, tokens=tokens, cost=cost),
333
+ slice=slice_budget,
334
+ slice_cost=cost / n if cost is not None else None,
335
+ )
336
+
337
+
338
+ # --- selection ----------------------------------------------------------------------------
339
+
340
+
341
+ class ReviewScore(BaseModel):
342
+ """The Critic's score of one tied candidate (from its `ReviewReport`), and its spend."""
343
+
344
+ model_config = ConfigDict(extra="forbid", frozen=True)
345
+
346
+ verdict: Verdict = Field(description="approve, or changes_requested.")
347
+ blocker: int = Field(default=0, ge=0)
348
+ major: int = Field(default=0, ge=0)
349
+ minor: int = Field(default=0, ge=0)
350
+ tokens: int | None = Field(default=None, ge=0, description="Tokens the review reported.")
351
+ cost: float | None = Field(default=None, ge=0, description="Cost the review reported.")
352
+
353
+ @classmethod
354
+ def from_report(
355
+ cls, report: ReviewReport, *, tokens: int | None = None, cost: float | None = None
356
+ ) -> ReviewScore:
357
+ """The score of a review (nits do not count)."""
358
+
359
+ def count(severity: str) -> int:
360
+ return sum(1 for c in report.comments if c.severity == severity)
361
+
362
+ return cls(
363
+ verdict=report.verdict,
364
+ blocker=count("blocker"),
365
+ major=count("major"),
366
+ minor=count("minor"),
367
+ tokens=tokens,
368
+ cost=cost,
369
+ )
370
+
371
+ def key(self) -> tuple[int, int, int, int]:
372
+ """Lower is better: approve first, then fewer blocker, major, minor comments."""
373
+ return (0 if self.verdict == "approve" else 1, self.blocker, self.major, self.minor)
374
+
375
+
376
+ @dataclass(frozen=True)
377
+ class Candidate:
378
+ """One finished (or not) candidate as the selection sees it."""
379
+
380
+ id: str
381
+ passed: bool
382
+ """The candidate's own run succeeded (the agent loop, or the plain executor)."""
383
+ checks: tuple[CheckResult, ...] = ()
384
+ """The final result of each of the rule's checks on the candidate's outputs."""
385
+ finished: bool = True
386
+ """False for a candidate that was cancelled, skipped or crashed: never eligible."""
387
+ workspace: Path | None = None
388
+ """Its workspace, while it exists (for a `MetricProvider` and the Critic)."""
389
+
390
+ @property
391
+ def all_passed(self) -> bool:
392
+ """Its run succeeded and every check result is ok."""
393
+ return self.passed and all(c.ok for c in self.checks)
394
+
395
+
396
+ @runtime_checkable
397
+ class MetricProvider(Protocol):
398
+ """Measures a candidate's workspace; optional (no provider: the area step is skipped)."""
399
+
400
+ def area(self, workspace: Path) -> float | None:
401
+ """The candidate's area (lower is better), or `None` when it cannot be had."""
402
+ ...
403
+
404
+
405
+ Tiebreak = Callable[[tuple[Candidate, ...]], Awaitable[Mapping[str, ReviewScore]]]
406
+ """The Critic: scores the tied candidates (candidate id -> score). In real use it runs
407
+ the review rule on each tied candidate's workspace; any async callable will do."""
408
+
409
+ SelectStep = Literal["none", "only", "passed", "errors", "warnings", "area", "critic", "lowest_id"]
410
+ """Which step of the key decided the winner (`none`: no candidate finished)."""
411
+
412
+ _STEPS: tuple[SelectStep, ...] = ("passed", "errors", "warnings", "area")
413
+
414
+
415
+ def error_count(checks: Sequence[CheckResult]) -> int:
416
+ """Error issues over `checks`; a failing check with no error issue counts as one."""
417
+ total = 0
418
+ for result in checks:
419
+ errors = sum(1 for i in result.issues if i.severity == "error")
420
+ total += errors if errors or result.ok else 1
421
+ return total
422
+
423
+
424
+ def warning_count(checks: Sequence[CheckResult]) -> int:
425
+ """Issues of severity `warning` over `checks` (`info` is ignored)."""
426
+ return sum(1 for r in checks for i in r.issues if i.severity == "warning")
427
+
428
+
429
+ class CandidateKey(BaseModel):
430
+ """A candidate's place in the selection: every component of its key."""
431
+
432
+ model_config = ConfigDict(extra="forbid", frozen=True)
433
+
434
+ id: str
435
+ eligible: bool = Field(description="It finished, so it took part.")
436
+ passed: bool = Field(default=False, description="Its run and every check passed.")
437
+ errors: int = Field(default=0, ge=0)
438
+ warnings: int = Field(default=0, ge=0)
439
+ area: float | None = Field(default=None, description="When a MetricProvider gave one.")
440
+ critic: ReviewScore | None = Field(default=None, description="When the Critic scored it.")
441
+ position: int | None = Field(default=None, ge=1, description="1 = the winner.")
442
+
443
+ def key(self, area_used: bool) -> tuple[int, int, int, tuple[int, float]]:
444
+ """Steps 1-4 as a tuple: lower is better."""
445
+ if not area_used:
446
+ area: tuple[int, float] = (0, 0.0)
447
+ elif self.area is None:
448
+ area = (1, 0.0)
449
+ else:
450
+ area = (0, self.area)
451
+ return (0 if self.passed else 1, self.errors, self.warnings, area)
452
+
453
+
454
+ class Selection(BaseModel):
455
+ """The outcome of `select_best`: the winner, every key, the deciding step, why."""
456
+
457
+ model_config = ConfigDict(extra="forbid", frozen=True)
458
+
459
+ schema_version: Literal[1] = Field(default=1, description="Schema version of this model.")
460
+ winner: str | None = Field(description="The chosen candidate; None if none finished.")
461
+ decided_by: SelectStep = Field(description="The step that set the winner apart.")
462
+ none_passed: bool = Field(
463
+ default=False, description="No candidate passed: the winner is only the least bad."
464
+ )
465
+ area_used: bool = Field(default=False, description="Whether the area step applied.")
466
+ critic_used: bool = Field(default=False, description="Whether the Critic was called.")
467
+ critic_note: str = Field(default="", description="Why the Critic was not used, if tied.")
468
+ order: tuple[str, ...] = Field(default=(), description="Eligible candidates, best first.")
469
+ keys: tuple[CandidateKey, ...] = Field(
470
+ default=(), description="Every candidate: the eligible in order, then the rest by id."
471
+ )
472
+
473
+ def key_of(self, candidate_id: str) -> CandidateKey | None:
474
+ """The key of `candidate_id`."""
475
+ return next((k for k in self.keys if k.id == candidate_id), None)
476
+
477
+ def explain(self) -> str:
478
+ """The choice and its reasons, one candidate per line."""
479
+ lines: list[str] = []
480
+ if self.winner is None:
481
+ lines.append("no winner: no candidate finished")
482
+ else:
483
+ lines.append(f"winner {self.winner}: {self._reason()}")
484
+ if self.none_passed:
485
+ lines.append("no candidate passed every check: the winner is the least bad one")
486
+ if self.critic_note:
487
+ lines.append(f"critic: {self.critic_note}")
488
+ for k in self.keys:
489
+ if not k.eligible:
490
+ lines.append(f"- {k.id}: did not finish, not eligible")
491
+ continue
492
+ parts = [
493
+ "passed" if k.passed else "failed",
494
+ _plural(k.errors, "error"),
495
+ _plural(k.warnings, "warning"),
496
+ ]
497
+ if self.area_used:
498
+ parts.append(f"area {k.area:g}" if k.area is not None else "no area")
499
+ if k.critic is not None:
500
+ c = k.critic
501
+ parts.append(
502
+ f"critic {c.verdict} ({c.blocker} blocker, {c.major} major, {c.minor} minor)"
503
+ )
504
+ lines.append(f"{k.position}. {k.id}: " + ", ".join(parts))
505
+ return "\n".join(lines)
506
+
507
+ def _reason(self) -> str:
508
+ if self.decided_by == "only" or len(self.order) < 2:
509
+ return "the only candidate that finished"
510
+ win, other = self.key_of(self.order[0]), self.key_of(self.order[1])
511
+ assert win is not None and other is not None
512
+ step = self.decided_by
513
+ if step == "passed":
514
+ return f"it passed every check and {other.id} did not"
515
+ if step == "errors":
516
+ return f"fewer errors than {other.id} ({win.errors} < {other.errors})"
517
+ if step == "warnings":
518
+ return f"fewer warnings than {other.id} ({win.warnings} < {other.warnings})"
519
+ if step == "area":
520
+ if other.area is None:
521
+ return f"it has an area and {other.id} has none"
522
+ return f"smaller area than {other.id} ({win.area:g} < {other.area:g})"
523
+ if step == "critic":
524
+ return f"tied with {other.id} on checks, warnings and area; the critic preferred it"
525
+ return f"tied with {other.id} on every step; the lowest id wins"
526
+
527
+
528
+ def _plural(n: int, word: str) -> str:
529
+ return f"{n} {word}{'' if n == 1 else 's'}"
530
+
531
+
532
+ def _area(metrics: MetricProvider | None, candidate: Candidate) -> float | None:
533
+ if metrics is None or candidate.workspace is None:
534
+ return None
535
+ try:
536
+ value = metrics.area(candidate.workspace)
537
+ except Exception: # a metric that cannot be had sorts the candidate after
538
+ return None
539
+ if value is None or not math.isfinite(value):
540
+ return None
541
+ return float(value)
542
+
543
+
544
+ def rank(
545
+ candidates: Sequence[Candidate], metrics: MetricProvider | None = None
546
+ ) -> tuple[tuple[CandidateKey, ...], bool]:
547
+ """Steps 1-4: the eligible candidates' keys, best first (ties by id), and whether
548
+ the area step applied. Pure apart from the metric provider."""
549
+ eligible = sorted((c for c in candidates if c.finished), key=lambda c: c.id)
550
+ keys = [
551
+ CandidateKey(
552
+ id=c.id,
553
+ eligible=True,
554
+ passed=c.all_passed,
555
+ errors=error_count(c.checks),
556
+ warnings=warning_count(c.checks),
557
+ area=_area(metrics, c),
558
+ )
559
+ for c in eligible
560
+ ]
561
+ area_used = any(k.area is not None for k in keys)
562
+ keys.sort(key=lambda k: (k.key(area_used), k.id))
563
+ return tuple(keys), area_used
564
+
565
+
566
+ async def select_best(
567
+ candidates: Sequence[Candidate],
568
+ metrics: MetricProvider | None = None,
569
+ tiebreak: Tiebreak | None = None,
570
+ *,
571
+ ledger: CostLedger | None = None,
572
+ ) -> Selection:
573
+ """Choose one candidate: deterministic checks first, the Critic only for exact ties.
574
+
575
+ The key (see the module docstring) orders every finished candidate; the order is
576
+ total (equal keys fall back to the id), so the winner does not depend on the order
577
+ the candidates are given or finished in. `tiebreak` is called only for the
578
+ candidates tied for first place after steps 1-4, so it can never override a
579
+ difference there; its cost is charged to `ledger` (one try per tied candidate, plus
580
+ what it reports), and it is skipped when the ledger cannot pay for it. Still tied,
581
+ no tiebreak, or skipped: the lowest id wins.
582
+ """
583
+ by_id = {c.id: c for c in candidates}
584
+ if len(by_id) != len(candidates):
585
+ raise ValueError("candidate ids must be unique")
586
+ keys, area_used = rank(candidates, metrics)
587
+ rest = tuple(
588
+ CandidateKey(id=c.id, eligible=False)
589
+ for c in sorted(by_id.values(), key=lambda c: c.id)
590
+ if not c.finished
591
+ )
592
+ if not keys:
593
+ return Selection(winner=None, decided_by="none", keys=rest)
594
+
595
+ top = keys[0].key(area_used)
596
+ tied = [k for k in keys if k.key(area_used) == top]
597
+ ordered = list(keys)
598
+ critic_used = False
599
+ note = ""
600
+ scores: Mapping[str, ReviewScore] = {}
601
+ if len(tied) > 1:
602
+ if tiebreak is None:
603
+ note = "no critic configured: the lowest id wins the tie"
604
+ elif ledger is not None and not ledger.can_afford(len(tied)):
605
+ note = (
606
+ f"the shared budget cannot pay for the critic ({len(tied)} tries needed, "
607
+ f"{ledger.remaining_tries} left, the {ledger.refuse(len(tied))} limit): "
608
+ "the lowest id wins the tie"
609
+ )
610
+ else:
611
+ critic_used = True
612
+ if ledger is not None:
613
+ ledger.reserve(CRITIC, len(tied))
614
+ try:
615
+ scores = dict(await tiebreak(tuple(by_id[k.id] for k in tied)))
616
+ except Exception as exc:
617
+ note = f"the critic failed ({type(exc).__name__}: {exc}): the lowest id wins"
618
+ scores = {}
619
+ if ledger is not None:
620
+ ledger.charge(
621
+ CRITIC,
622
+ tokens=sum(s.tokens or 0 for s in scores.values()) or None,
623
+ cost=sum(s.cost or 0.0 for s in scores.values()) or None,
624
+ )
625
+
626
+ def critic_key(k: CandidateKey) -> tuple[int, tuple[int, int, int, int], str]:
627
+ score = scores.get(k.id)
628
+ return (0, score.key(), k.id) if score is not None else (1, (0, 0, 0, 0), k.id)
629
+
630
+ tied_sorted = sorted(tied, key=critic_key)
631
+ ordered = [*tied_sorted, *keys[len(tied) :]]
632
+ ordered = [
633
+ k.model_copy(update={"position": i + 1, "critic": scores.get(k.id)})
634
+ for i, k in enumerate(ordered)
635
+ ]
636
+
637
+ decided: SelectStep
638
+ if len(ordered) == 1:
639
+ decided = "only"
640
+ else:
641
+ win, other = ordered[0], ordered[1]
642
+ wk, ok = win.key(area_used), other.key(area_used)
643
+ if wk != ok:
644
+ decided = next(_STEPS[i] for i in range(4) if wk[i] != ok[i])
645
+ elif win.critic is not None and (
646
+ other.critic is None or win.critic.key() < other.critic.key()
647
+ ):
648
+ decided = "critic"
649
+ else:
650
+ decided = "lowest_id"
651
+ return Selection(
652
+ winner=ordered[0].id,
653
+ decided_by=decided,
654
+ none_passed=not ordered[0].passed,
655
+ area_used=area_used,
656
+ critic_used=critic_used,
657
+ critic_note=note,
658
+ order=tuple(k.id for k in ordered),
659
+ keys=(*ordered, *rest),
660
+ )
661
+
662
+
663
+ # --- the executor ------------------------------------------------------------------------
664
+
665
+
666
+ @dataclass(frozen=True)
667
+ class CandidateEnv:
668
+ """What `inner_factory` gets to build one candidate's executor.
669
+
670
+ Build it to work in `workspace` (its store, runtime and checks rooted there), with
671
+ `budget`/`max_cost` as its caps, and without remembered stops (no state layout:
672
+ candidates share the instance id). Wrap its runtime with `guard`: every call then
673
+ reserves a try of the shared ledger first (none left: the candidate is cancelled),
674
+ has its token cap lowered to what the ledger has left, carries `variant`, `seed` and
675
+ the candidate id in `AgentTask.context`, and is charged what it reports.
676
+ """
677
+
678
+ candidate_id: str
679
+ variant: str
680
+ seed: str
681
+ workspace: Path
682
+ budget: Budget
683
+ max_cost: float | None
684
+ guard: Callable[[AgentRuntime], AgentRuntime]
685
+
686
+
687
+ @dataclass(frozen=True)
688
+ class CandidateParts:
689
+ """One candidate's executor, and the check runner for its workspace."""
690
+
691
+ executor: Executor
692
+ checks: CheckRunner | None = None
693
+
694
+
695
+ InnerFactory = Callable[[CandidateEnv], CandidateParts]
696
+
697
+
698
+ class _GuardedRuntime:
699
+ """An `AgentRuntime` that pays for every call from the shared ledger first."""
700
+
701
+ def __init__(
702
+ self, inner: AgentRuntime, ledger: CostLedger, account: str, context: Mapping[str, str]
703
+ ) -> None:
704
+ self.inner = inner
705
+ self.ledger = ledger
706
+ self.account = account
707
+ self.context = dict(context)
708
+ self.name: str = getattr(inner, "name", "runtime")
709
+
710
+ async def run_task(self, task: AgentTask) -> AgentResult:
711
+ budget = task.budget
712
+ left = self.ledger.remaining_tokens
713
+ hold: int | None = None
714
+ if left is not None:
715
+ hold = min(budget.tokens, left) if budget.tokens is not None else left
716
+ if hold >= 1:
717
+ budget = budget.model_copy(update={"tokens": hold})
718
+ if not self.ledger.reserve(self.account, tokens=hold):
719
+ raise LedgerExhausted(self.ledger.why_not(1, hold or 1) or "tries")
720
+ sent = task.model_copy(
721
+ update={"context": {**task.context, **self.context}, "budget": budget}
722
+ )
723
+ try:
724
+ result = await self.inner.run_task(sent)
725
+ except BaseException:
726
+ self.ledger.settle(self.account, held=hold)
727
+ raise
728
+ self.ledger.settle(self.account, held=hold, tokens=result.tokens, cost=result.cost)
729
+ return result
730
+
731
+
732
+ CandidateStatus = Literal["passed", "failed", "rejected", "cancelled", "skipped", "error"]
733
+
734
+
735
+ class CandidateResult(BaseModel):
736
+ """One candidate's outcome and spend."""
737
+
738
+ model_config = ConfigDict(extra="forbid", frozen=True)
739
+
740
+ id: str
741
+ variant: str
742
+ status: CandidateStatus
743
+ message: str = ""
744
+ errors: int = Field(default=0, ge=0)
745
+ warnings: int = Field(default=0, ge=0)
746
+ checks: tuple[CheckResult, ...] = ()
747
+ files: tuple[str, ...] = Field(default=(), description="Files it changed.")
748
+ spend: Spend = Field(default_factory=Spend)
749
+ agent: AgentResult | None = None
750
+
751
+
752
+ BestOfNStatus = Literal["done", "failed", "plain"]
753
+
754
+
755
+ class BestOfNResult(BaseModel):
756
+ """The outcome of one best-of-N run (the `best_of` payload of the rule's event)."""
757
+
758
+ model_config = ConfigDict(extra="forbid", frozen=True)
759
+
760
+ schema_version: Literal[1] = Field(default=1, description="Schema version of this model.")
761
+ status: BestOfNStatus = Field(description="done; failed; plain (only one fitted).")
762
+ instance: str
763
+ message: str = ""
764
+ plan: BudgetPlan
765
+ winner: str | None = None
766
+ candidates: tuple[CandidateResult, ...] = Field(default=(), description="By id.")
767
+ selection: Selection | None = None
768
+ spend: Spend = Field(default_factory=Spend, description="Total, Critic included.")
769
+ critic_spend: Spend = Field(default_factory=Spend)
770
+ limited_by: LedgerLimit | None = Field(
771
+ default=None, description="The shared limit that stopped a call, if any."
772
+ )
773
+ fanout: str | None = Field(default=None, description="The candidates' fan-out run id.")
774
+ branch: str | None = Field(default=None, description="The result branch, when created.")
775
+ commit: str | None = None
776
+
777
+
778
+ @dataclass
779
+ class _State:
780
+ id: str
781
+ variant: str
782
+ status: CandidateStatus = "skipped"
783
+ message: str = "not started"
784
+ outcome: ExecOutcome | None = None
785
+ checks: dict[str, CheckResult] = field(default_factory=dict)
786
+ guarded: bool = False
787
+
788
+
789
+ _SLUG_RE = re.compile(r"[^a-z0-9._-]+")
790
+
791
+
792
+ def _slug(instance_id: str) -> str:
793
+ slug = re.sub(r"\.{2,}", ".", _SLUG_RE.sub("-", instance_id.lower())).strip("-._")
794
+ return f"best-of-{slug}"[:80].rstrip("-._") if slug else "best-of"
795
+
796
+
797
+ def _synthetic(check_id: str, issues: Sequence[Issue]) -> CheckResult:
798
+ digest = hashlib.sha256("|".join(i.msg for i in issues).encode("utf-8")).hexdigest()
799
+ return CheckResult(
800
+ check_id=check_id,
801
+ status="fail" if issues else "pass",
802
+ issues=tuple(issues),
803
+ duration_s=0.0,
804
+ idempotency_key=f"{check_id}:{digest}",
805
+ )
806
+
807
+
808
+ def _noop_emit(_type: ExecEventType, _payload: dict[str, Any]) -> None:
809
+ return None
810
+
811
+
812
+ class BestOfNExecutor:
813
+ """An executor that runs an opted-in rule as N candidates and keeps the best.
814
+
815
+ A rule opts in with the instance parameter `best_of: <n>` or `select.rules` in the
816
+ profile (`cfg`); every other rule goes to `default`, unchanged. `inner_factory`
817
+ builds each candidate's executor for its workspace (`CandidateEnv`); `vcs` and
818
+ `root` are the fan-out's. `metrics` adds the area step, `tiebreak` the Critic.
819
+
820
+ It is a `CheckingExecutor`: for candidates it runs the rule's checks itself, in each
821
+ candidate's workspace; for a plain run it lets `default` do it, or runs them through
822
+ the scheduler's context. Wiring this into the claude-code runtime (a runtime per
823
+ workspace, the review rule as `tiebreak`, landing the result branch) is M2-11's job.
824
+ """
825
+
826
+ def __init__(
827
+ self,
828
+ inner_factory: InnerFactory,
829
+ vcs: FanoutVcs,
830
+ *,
831
+ root: Path,
832
+ default: Executor,
833
+ cfg: SelectCfg | None = None,
834
+ fanout: FanoutCfg | None = None,
835
+ max_cost: float | None = None,
836
+ metrics: MetricProvider | None = None,
837
+ tiebreak: Tiebreak | None = None,
838
+ variants: Mapping[str, str] | None = None,
839
+ join_checks: Sequence[JoinCheck] = (),
840
+ tmp_root: Path | None = None,
841
+ layout: StateLayout | None = None,
842
+ make_branch: bool = True,
843
+ ) -> None:
844
+ self.inner_factory = inner_factory
845
+ self.vcs = vcs
846
+ self.root = root
847
+ self.default = default
848
+ self.cfg = cfg if cfg is not None else SelectCfg()
849
+ self.fanout = fanout if fanout is not None else FanoutCfg()
850
+ self.max_cost = max_cost
851
+ self.metrics = metrics
852
+ self.tiebreak = tiebreak
853
+ self.variants = dict(variants if variants is not None else VARIANTS)
854
+ if not self.variants:
855
+ raise ValueError("best-of-N needs at least one variant")
856
+ self.join_checks = tuple(join_checks)
857
+ self.tmp_root = tmp_root
858
+ self.layout = layout
859
+ self.make_branch = make_branch
860
+ self.results: dict[str, BestOfNResult] = {}
861
+
862
+ # --- opting in ------------------------------------------------------------------
863
+
864
+ def requested_n(self, rule: RuleSpec, instance: RuleInstance) -> int | None:
865
+ """The N `instance` asks for, or `None` when its rule did not opt in.
866
+
867
+ Raises `ValueError` for a `best_of` parameter that is not a positive integer.
868
+ """
869
+ raw = instance.params.get(BEST_OF_PARAM)
870
+ if raw is not None:
871
+ try:
872
+ n = int(raw)
873
+ except ValueError:
874
+ raise ValueError(f"{BEST_OF_PARAM}={raw!r} is not a positive integer") from None
875
+ if n < 1:
876
+ raise ValueError(f"{BEST_OF_PARAM}={raw!r} is not a positive integer")
877
+ return n
878
+ return self.cfg.rules.get(rule.id)
879
+
880
+ def forget(self, instance_id: str) -> None:
881
+ """Pass a rewind on to the plain executor."""
882
+ if isinstance(self.default, RewindAware):
883
+ self.default.forget(instance_id)
884
+
885
+ # --- the executor protocols ----------------------------------------------------------
886
+
887
+ async def execute(self, rule: RuleSpec, instance: RuleInstance) -> ExecOutcome:
888
+ """Outside a scheduler: no journal, no check runner for a plain run."""
889
+ ctx = ExecContext(run_id="", run_check=None, emit=_noop_emit)
890
+ return await self.execute_checked(rule, instance, ctx)
891
+
892
+ async def execute_checked(
893
+ self, rule: RuleSpec, instance: RuleInstance, ctx: ExecContext
894
+ ) -> ExecOutcome:
895
+ try:
896
+ requested = self.requested_n(rule, instance)
897
+ except ValueError as exc:
898
+ return ExecOutcome(ok=False, failure_label="planning", message=str(exc))
899
+ if requested is None:
900
+ return await self._plain(rule, instance, ctx)
901
+ plan = plan_budget(
902
+ rule.budget,
903
+ requested=requested,
904
+ cfg=self.cfg,
905
+ max_cost=self.max_cost,
906
+ variants=len(self.variants),
907
+ )
908
+ if plan.plain:
909
+ outcome = await self._plain(rule, instance, ctx)
910
+ why = f" ({plan.capped_by})" if plan.capped_by else ""
911
+ result = BestOfNResult(
912
+ status="plain",
913
+ instance=instance.instance_id,
914
+ message=f"best-of-{plan.requested}: only one candidate fits{why}; a plain run",
915
+ plan=plan,
916
+ )
917
+ self.results[instance.instance_id] = result
918
+ return outcome.model_copy(
919
+ update={"payload": {**outcome.payload, "best_of": result.model_dump(mode="json")}}
920
+ )
921
+ return await _BestOfRun(self, rule, instance, ctx, plan).run()
922
+
923
+ async def _plain(self, rule: RuleSpec, instance: RuleInstance, ctx: ExecContext) -> ExecOutcome:
924
+ if isinstance(self.default, CheckingExecutor):
925
+ return await self.default.execute_checked(rule, instance, ctx)
926
+ outcome = await self.default.execute(rule, instance)
927
+ if outcome.ok and ctx.run_check is not None:
928
+ for check_id in rule.checks:
929
+ checked = await ctx.run_check(check_id)
930
+ if not checked.ok:
931
+ return ExecOutcome(
932
+ ok=False, failure_label="verification", message=f"check {check_id} failed"
933
+ )
934
+ return outcome
935
+
936
+
937
+ class _BestOfRun:
938
+ """One best-of-N run of one instance."""
939
+
940
+ def __init__(
941
+ self,
942
+ executor: BestOfNExecutor,
943
+ rule: RuleSpec,
944
+ instance: RuleInstance,
945
+ ctx: ExecContext,
946
+ plan: BudgetPlan,
947
+ ) -> None:
948
+ self.ex = executor
949
+ self.rule = rule
950
+ self.instance = instance
951
+ self.ctx = ctx
952
+ self.plan = plan
953
+ self.ledger = CostLedger(plan.limits)
954
+ self.run_id = new_run_id()
955
+ ids = list(executor.variants)[: plan.n]
956
+ self.states = {cid: _State(id=cid, variant=executor.variants[cid]) for cid in ids}
957
+ self.writes = tuple(r.path for r in instance.outputs if r.path is not None)
958
+ self.inputs = tuple(dict.fromkeys(r.path for r in instance.inputs if r.path is not None))
959
+ self.selection: Selection | None = None
960
+ self.sliced = rule.model_copy(update={"budget": plan.slice})
961
+
962
+ def seed(self, cid: str) -> str:
963
+ digest = hashlib.sha256(f"{self.instance.instance_id}\0{cid}".encode()).hexdigest()
964
+ return f"{cid}-{digest[:12]}"
965
+
966
+ # --- one candidate -------------------------------------------------------------------
967
+
968
+ def candidate_ctx(self, state: _State, checks: CheckRunner | None) -> ExecContext:
969
+ async def run_check(check_id: str) -> CheckResult:
970
+ assert checks is not None
971
+ checked = await checks.run(check_id, self.instance)
972
+ state.checks[check_id] = checked # the last result of each check counts
973
+ return checked
974
+
975
+ def emit(type_: ExecEventType, payload: dict[str, Any]) -> None:
976
+ if type_ == "agent_turn" and not state.guarded:
977
+ # An executor whose runtime is not guarded is charged after the fact.
978
+ self.ledger.charge(
979
+ state.id, tries=1, tokens=payload.get("tokens"), cost=payload.get("cost")
980
+ )
981
+ self.ctx.emit(type_, {**payload, "candidate": state.id, "best_of": self.run_id})
982
+
983
+ return ExecContext(
984
+ run_id=self.ctx.run_id, run_check=run_check if checks is not None else None, emit=emit
985
+ )
986
+
987
+ def branch_fn(self, cid: str) -> Callable[[BranchContext], Awaitable[BranchOutcome]]:
988
+ state = self.states[cid]
989
+
990
+ async def run(bctx: BranchContext) -> BranchOutcome:
991
+ if not self.ledger.can_afford(1):
992
+ self.ledger.refuse()
993
+ state.status, state.message = "skipped", "not started: the shared budget is spent"
994
+ return BranchOutcome(ok=False, message=state.message)
995
+ context = {"variant": state.variant, "seed": self.seed(cid), "candidate": cid}
996
+
997
+ def guard(runtime: AgentRuntime) -> AgentRuntime:
998
+ state.guarded = True
999
+ return _GuardedRuntime(runtime, self.ledger, cid, context)
1000
+
1001
+ env = CandidateEnv(
1002
+ candidate_id=cid,
1003
+ variant=state.variant,
1004
+ seed=context["seed"],
1005
+ workspace=bctx.path,
1006
+ budget=self.plan.slice,
1007
+ max_cost=self.plan.slice_cost,
1008
+ guard=guard,
1009
+ )
1010
+ state.status, state.message = "error", "did not finish"
1011
+ parts = self.ex.inner_factory(env)
1012
+ cctx = self.candidate_ctx(state, parts.checks)
1013
+ try:
1014
+ outcome = await self._run_inner(parts, cctx)
1015
+ except LedgerExhausted as stop:
1016
+ state.status = "cancelled"
1017
+ state.message = f"cancelled: the shared budget is spent ({stop.limit})"
1018
+ return BranchOutcome(ok=False, message=state.message)
1019
+ state.outcome = outcome
1020
+ missing = [
1021
+ Issue(file=p, rule="output.missing", msg=f"output {p} was not written")
1022
+ for p in self.writes
1023
+ if not (bctx.path / p).exists()
1024
+ ]
1025
+ if missing:
1026
+ state.checks[OUTPUTS_CHECK] = _synthetic(OUTPUTS_CHECK, missing)
1027
+ unchecked = [c for c in self.rule.checks if c not in state.checks]
1028
+ ok = outcome.ok and not unchecked and all(c.ok for c in state.checks.values())
1029
+ state.status = "passed" if ok else "failed"
1030
+ state.message = outcome.message or (
1031
+ f"checks not run: {', '.join(unchecked)}" if unchecked else ""
1032
+ )
1033
+ return BranchOutcome(ok=outcome.ok, message=state.message, agent=_agent_of(outcome))
1034
+
1035
+ return run
1036
+
1037
+ async def _run_inner(self, parts: CandidateParts, cctx: ExecContext) -> ExecOutcome:
1038
+ inner = parts.executor
1039
+ if isinstance(inner, CheckingExecutor):
1040
+ return await inner.execute_checked(self.sliced, self.instance, cctx)
1041
+ outcome = await inner.execute(self.sliced, self.instance)
1042
+ if outcome.ok and cctx.run_check is not None:
1043
+ for check_id in self.rule.checks:
1044
+ await cctx.run_check(check_id)
1045
+ return outcome
1046
+
1047
+ # --- choosing ----------------------------------------------------------------------
1048
+
1049
+ async def choose(
1050
+ self, results: tuple[BranchResult, ...], workspaces: Mapping[str, Path]
1051
+ ) -> str | None:
1052
+ branches = {r.id: r for r in results}
1053
+ candidates: list[Candidate] = []
1054
+ for cid, state in sorted(self.states.items()):
1055
+ branch = branches.get(cid)
1056
+ if branch is not None and branch.status == "rejected":
1057
+ state.status, state.message = "rejected", branch.message
1058
+ elif branch is not None and branch.status in ("error", "timeout"):
1059
+ state.status, state.message = "error", branch.message
1060
+ finished = state.status in ("passed", "failed", "rejected")
1061
+ candidates.append(
1062
+ Candidate(
1063
+ id=cid,
1064
+ passed=state.status == "passed",
1065
+ checks=tuple(state.checks[k] for k in sorted(state.checks)),
1066
+ finished=finished,
1067
+ workspace=workspaces.get(cid),
1068
+ )
1069
+ )
1070
+ self.selection = await select_best(
1071
+ candidates, self.ex.metrics, self.ex.tiebreak, ledger=self.ledger
1072
+ )
1073
+ winner = self.selection.winner
1074
+ if winner is None or self.selection.none_passed:
1075
+ return None
1076
+ return winner
1077
+
1078
+ # --- the run ----------------------------------------------------------------------
1079
+
1080
+ async def run(self) -> ExecOutcome:
1081
+ branches = [
1082
+ FanoutBranch(id=cid, run=self.branch_fn(cid), write_set=self.writes, inputs=self.inputs)
1083
+ for cid in self.states
1084
+ ]
1085
+ fan = await run_candidates(
1086
+ branches,
1087
+ choose=self.choose,
1088
+ root=self.ex.root,
1089
+ vcs=self.ex.vcs,
1090
+ name=_slug(self.instance.instance_id),
1091
+ run_id=self.run_id,
1092
+ join_checks=self.ex.join_checks,
1093
+ max_parallel=self.ex.fanout.effective_max_parallel(),
1094
+ branch_timeout_s=self.ex.fanout.branch_timeout_s,
1095
+ tmp_root=self.ex.tmp_root,
1096
+ layout=self.ex.layout,
1097
+ ctx=self.ctx,
1098
+ make_branch=self.ex.make_branch,
1099
+ )
1100
+ by_branch = {b.id: b for b in fan.branches}
1101
+ candidates: list[CandidateResult] = []
1102
+ for cid, state in sorted(self.states.items()):
1103
+ checks = tuple(state.checks[k] for k in sorted(state.checks))
1104
+ branch = by_branch.get(cid)
1105
+ cand = CandidateResult(
1106
+ id=cid,
1107
+ variant=state.variant,
1108
+ status=state.status,
1109
+ message=state.message,
1110
+ errors=error_count(checks),
1111
+ warnings=warning_count(checks),
1112
+ checks=checks,
1113
+ files=tuple(c.path for c in branch.files) if branch is not None else (),
1114
+ spend=self.ledger.spent(cid),
1115
+ agent=_agent_of(state.outcome) if state.outcome is not None else None,
1116
+ )
1117
+ candidates.append(cand)
1118
+ self.ctx.emit(
1119
+ "agent_turn",
1120
+ {
1121
+ "phase": "candidate",
1122
+ "best_of": self.run_id,
1123
+ "candidate": cid,
1124
+ "variant": state.variant,
1125
+ "status": state.status,
1126
+ "message": state.message,
1127
+ "errors": cand.errors,
1128
+ "warnings": cand.warnings,
1129
+ "failed_checks": [c.check_id for c in checks if not c.ok],
1130
+ "spend": cand.spend.model_dump(mode="json"),
1131
+ },
1132
+ )
1133
+
1134
+ selection = self.selection
1135
+ ok = fan.ok and selection is not None and fan.chosen == selection.winner
1136
+ if ok:
1137
+ assert selection is not None
1138
+ message = (
1139
+ f"best-of-{self.plan.n}: candidate {selection.winner} chosen by "
1140
+ f"{selection.decided_by}; result on branch {fan.branch or fan.commit}"
1141
+ )
1142
+ elif selection is None or selection.winner is None:
1143
+ message = f"best-of-{self.plan.n}: no candidate finished ({fan.message})"
1144
+ elif selection.none_passed:
1145
+ message = (
1146
+ f"best-of-{self.plan.n}: no candidate passed; the least bad is "
1147
+ f"{selection.winner}, nothing merged"
1148
+ )
1149
+ else:
1150
+ message = f"best-of-{self.plan.n}: {fan.message}"
1151
+ result = BestOfNResult(
1152
+ status="done" if ok else "failed",
1153
+ instance=self.instance.instance_id,
1154
+ message=message,
1155
+ plan=self.plan,
1156
+ winner=selection.winner if selection is not None else None,
1157
+ candidates=tuple(candidates),
1158
+ selection=selection,
1159
+ spend=self.ledger.total,
1160
+ critic_spend=self.ledger.spent(CRITIC),
1161
+ limited_by=self.ledger.limited_by,
1162
+ fanout=fan.run_id,
1163
+ branch=fan.branch if ok else None,
1164
+ commit=fan.commit if ok else None,
1165
+ )
1166
+ self.ex.results[self.instance.instance_id] = result
1167
+ self.ctx.emit(
1168
+ "tool_call",
1169
+ {
1170
+ "tool": "select",
1171
+ "event": "select",
1172
+ "best_of": self.run_id,
1173
+ "status": result.status,
1174
+ "winner": result.winner,
1175
+ "decided_by": selection.decided_by if selection is not None else "none",
1176
+ "critic_used": selection.critic_used if selection is not None else False,
1177
+ "spend": result.spend.model_dump(mode="json"),
1178
+ "limited_by": result.limited_by,
1179
+ "branch": result.branch,
1180
+ },
1181
+ )
1182
+ payload = {"best_of": result.model_dump(mode="json")}
1183
+ if ok:
1184
+ return ExecOutcome(ok=True, message=message, payload=payload)
1185
+ return ExecOutcome(
1186
+ ok=False, failure_label=self._failure_label(selection), message=message, payload=payload
1187
+ )
1188
+
1189
+ def _failure_label(self, selection: Selection | None) -> FailureLabel:
1190
+ if selection is not None and selection.winner is not None:
1191
+ outcome = self.states[selection.winner].outcome
1192
+ if outcome is not None and outcome.failure_label is not None:
1193
+ return outcome.failure_label
1194
+ return "verification"
1195
+
1196
+
1197
+ def _agent_of(outcome: ExecOutcome) -> AgentResult | None:
1198
+ """The `AgentResult` an agent rule loop put in its outcome's payload, if any."""
1199
+ raw = outcome.payload.get("result")
1200
+ if not isinstance(raw, Mapping):
1201
+ return None
1202
+ with contextlib.suppress(ValueError):
1203
+ return AgentResult.model_validate(dict(raw))
1204
+ return None
1205
+
1206
+
1207
+ __all__ = [
1208
+ "BEST_OF_PARAM",
1209
+ "CRITIC",
1210
+ "OUTPUTS_CHECK",
1211
+ "VARIANTS",
1212
+ "BestOfNExecutor",
1213
+ "BestOfNResult",
1214
+ "BestOfNStatus",
1215
+ "BudgetPlan",
1216
+ "Candidate",
1217
+ "CandidateEnv",
1218
+ "CandidateKey",
1219
+ "CandidateParts",
1220
+ "CandidateResult",
1221
+ "CandidateStatus",
1222
+ "CapReason",
1223
+ "CostLedger",
1224
+ "InnerFactory",
1225
+ "LedgerExhausted",
1226
+ "LedgerLimit",
1227
+ "LedgerLimits",
1228
+ "MetricProvider",
1229
+ "ReviewScore",
1230
+ "SelectStep",
1231
+ "Selection",
1232
+ "Spend",
1233
+ "Tiebreak",
1234
+ "error_count",
1235
+ "plan_budget",
1236
+ "rank",
1237
+ "select_best",
1238
+ "warning_count",
1239
+ ]