agentpace 0.11.1__tar.gz → 0.11.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. {agentpace-0.11.1 → agentpace-0.11.2}/CHANGELOG.md +13 -0
  2. {agentpace-0.11.1 → agentpace-0.11.2}/CITATION.cff +2 -2
  3. {agentpace-0.11.1 → agentpace-0.11.2}/PKG-INFO +3 -3
  4. {agentpace-0.11.1 → agentpace-0.11.2}/README.md +2 -2
  5. {agentpace-0.11.1 → agentpace-0.11.2}/docs/DESIGN.md +12 -0
  6. {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/__init__.py +1 -1
  7. {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/estimate.py +69 -8
  8. {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_estimate.py +122 -0
  9. agentpace-0.11.2/tests/test_frozen_method.py +225 -0
  10. {agentpace-0.11.1 → agentpace-0.11.2}/.gitattributes +0 -0
  11. {agentpace-0.11.1 → agentpace-0.11.2}/.github/ISSUE_TEMPLATE/config.yml +0 -0
  12. {agentpace-0.11.1 → agentpace-0.11.2}/.github/ISSUE_TEMPLATE/ingest-found-nothing.yml +0 -0
  13. {agentpace-0.11.1 → agentpace-0.11.2}/.github/ISSUE_TEMPLATE/remove-my-rows.yml +0 -0
  14. {agentpace-0.11.1 → agentpace-0.11.2}/.github/dependabot.yml +0 -0
  15. {agentpace-0.11.1 → agentpace-0.11.2}/.github/pull_request_template.md +0 -0
  16. {agentpace-0.11.1 → agentpace-0.11.2}/.github/workflows/ci.yml +0 -0
  17. {agentpace-0.11.1 → agentpace-0.11.2}/.github/workflows/publish.yml +0 -0
  18. {agentpace-0.11.1 → agentpace-0.11.2}/.github/workflows/validate-data.yml +0 -0
  19. {agentpace-0.11.1 → agentpace-0.11.2}/.gitignore +0 -0
  20. {agentpace-0.11.1 → agentpace-0.11.2}/CODE_OF_CONDUCT.md +0 -0
  21. {agentpace-0.11.1 → agentpace-0.11.2}/CONTRIBUTING.md +0 -0
  22. {agentpace-0.11.1 → agentpace-0.11.2}/LICENSE +0 -0
  23. {agentpace-0.11.1 → agentpace-0.11.2}/SECURITY.md +0 -0
  24. {agentpace-0.11.1 → agentpace-0.11.2}/data/LICENSE +0 -0
  25. {agentpace-0.11.1 → agentpace-0.11.2}/data/README.md +0 -0
  26. {agentpace-0.11.1 → agentpace-0.11.2}/data/runs/4546788e7930eccd-2026-W40-2026-W45.jsonl +0 -0
  27. {agentpace-0.11.1 → agentpace-0.11.2}/data/runs/4546788e7930eccd-2026-W44-2026-W45.jsonl +0 -0
  28. {agentpace-0.11.1 → agentpace-0.11.2}/docs/.nojekyll +0 -0
  29. {agentpace-0.11.1 → agentpace-0.11.2}/docs/assets/backtest-coverage.svg +0 -0
  30. {agentpace-0.11.1 → agentpace-0.11.2}/docs/assets/hero.webp +0 -0
  31. {agentpace-0.11.1 → agentpace-0.11.2}/docs/assets/run-ranges.svg +0 -0
  32. {agentpace-0.11.1 → agentpace-0.11.2}/docs/assets/social-preview.png +0 -0
  33. {agentpace-0.11.1 → agentpace-0.11.2}/docs/backtest.json +0 -0
  34. {agentpace-0.11.1 → agentpace-0.11.2}/docs/backtest.md +0 -0
  35. {agentpace-0.11.1 → agentpace-0.11.2}/docs/cli.md +0 -0
  36. {agentpace-0.11.1 → agentpace-0.11.2}/docs/index.html +0 -0
  37. {agentpace-0.11.1 → agentpace-0.11.2}/docs/releasing.md +0 -0
  38. {agentpace-0.11.1 → agentpace-0.11.2}/docs/robots.txt +0 -0
  39. {agentpace-0.11.1 → agentpace-0.11.2}/docs/sitemap.xml +0 -0
  40. {agentpace-0.11.1 → agentpace-0.11.2}/docs/verify-adapter.md +0 -0
  41. {agentpace-0.11.1 → agentpace-0.11.2}/examples/plan.json +0 -0
  42. {agentpace-0.11.1 → agentpace-0.11.2}/examples/quote-plan.json +0 -0
  43. {agentpace-0.11.1 → agentpace-0.11.2}/pyproject.toml +0 -0
  44. {agentpace-0.11.1 → agentpace-0.11.2}/schema/run.schema.json +0 -0
  45. {agentpace-0.11.1 → agentpace-0.11.2}/schema/turn.schema.json +0 -0
  46. {agentpace-0.11.1 → agentpace-0.11.2}/skills/agentpace/SKILL.md +0 -0
  47. {agentpace-0.11.1 → agentpace-0.11.2}/skills/agentpace/assets/portable-prompt.md +0 -0
  48. {agentpace-0.11.1 → agentpace-0.11.2}/skills/agentpace/references/method.md +0 -0
  49. {agentpace-0.11.1 → agentpace-0.11.2}/skills/pace-estimate/SKILL.md +0 -0
  50. {agentpace-0.11.1 → agentpace-0.11.2}/skills/pace-log/SKILL.md +0 -0
  51. {agentpace-0.11.1 → agentpace-0.11.2}/skills/pace-share/SKILL.md +0 -0
  52. {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/__main__.py +0 -0
  53. {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/adapters/__init__.py +0 -0
  54. {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/adapters/base.py +0 -0
  55. {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/adapters/claude_code.py +0 -0
  56. {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/adapters/cline.py +0 -0
  57. {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/adapters/codex.py +0 -0
  58. {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/adapters/gemini_cli.py +0 -0
  59. {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/adapters/opencode.py +0 -0
  60. {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/backtest.py +0 -0
  61. {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/cache.py +0 -0
  62. {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/charts.py +0 -0
  63. {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/claims.py +0 -0
  64. {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/classify.py +0 -0
  65. {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/cli.py +0 -0
  66. {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/forecasts.py +0 -0
  67. {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/ingest.py +0 -0
  68. {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/paths.py +0 -0
  69. {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/pseudonym.py +0 -0
  70. {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/py.typed +0 -0
  71. {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/render.py +0 -0
  72. {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/report.py +0 -0
  73. {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/schema.py +0 -0
  74. {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/share.py +0 -0
  75. {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/site.py +0 -0
  76. {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/turns.py +0 -0
  77. {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/validate.py +0 -0
  78. {agentpace-0.11.1 → agentpace-0.11.2}/tests/__init__.py +0 -0
  79. {agentpace-0.11.1 → agentpace-0.11.2}/tests/fixtures/__init__.py +0 -0
  80. {agentpace-0.11.1 → agentpace-0.11.2}/tests/fixtures/cline_tasks.py +0 -0
  81. {agentpace-0.11.1 → agentpace-0.11.2}/tests/fixtures/gemini_cli.py +0 -0
  82. {agentpace-0.11.1 → agentpace-0.11.2}/tests/fixtures/opencode.py +0 -0
  83. {agentpace-0.11.1 → agentpace-0.11.2}/tests/synthetic.py +0 -0
  84. {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_backtest.py +0 -0
  85. {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_claims.py +0 -0
  86. {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_classify.py +0 -0
  87. {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_cli.py +0 -0
  88. {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_cline.py +0 -0
  89. {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_codex.py +0 -0
  90. {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_docs.py +0 -0
  91. {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_forecasts.py +0 -0
  92. {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_gemini_cli.py +0 -0
  93. {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_ingest.py +0 -0
  94. {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_labels.py +0 -0
  95. {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_opencode.py +0 -0
  96. {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_outcomes.py +0 -0
  97. {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_review_capacity.py +0 -0
  98. {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_schema.py +0 -0
  99. {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_share_validate.py +0 -0
  100. {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_site.py +0 -0
  101. {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_skills.py +0 -0
  102. {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_turns.py +0 -0
@@ -4,6 +4,19 @@ This project follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) an
4
4
 
5
5
  ## Unreleased
6
6
 
7
+ ## 0.11.2 - 2026-11-09
8
+
9
+ ### Changed
10
+
11
+ - A cell warning now says which level had no recorded run (kind, risk or model), which of the task's levels were therefore not used, where the mean and the residuals came from, and which values are recorded under the parent. For example, a `feature` task marked `high` on `claude-opus-5-5` now reads: `cell feature x high: n=0. No recorded run has kind feature at risk high, so model claude-opus-5-5 was not used. Mean and residuals from feature (n=1353); interval widened by the spread between risk levels. Recorded risk levels under feature: normal (1353).`
12
+ - Thin cells say the mean is shrunk toward broader cells and name the residual pool.
13
+
14
+ ### Added
15
+
16
+ - A regression fixture, `tests/test_frozen_method.py`, pins the shipped method's numbers on synthetic rows, not only its name. Its values were generated with 0.11.1's code, and it passes on 0.11.1 unchanged.
17
+
18
+ No published number moved; the warning line in the two README examples changed.
19
+
7
20
  ## 0.11.1 - 2026-11-08
8
21
 
9
22
  ### Changed
@@ -8,8 +8,8 @@ authors:
8
8
  repository-code: https://github.com/mohamedzhioua/agentpace
9
9
  url: https://mohamedzhioua.github.io/agentpace/
10
10
  license: MIT
11
- version: 0.11.1
12
- date-released: 2026-11-08
11
+ version: 0.11.2
12
+ date-released: 2026-11-09
13
13
  keywords:
14
14
  - coding agents
15
15
  - estimation
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: agentpace
3
- Version: 0.11.1
3
+ Version: 0.11.2
4
4
  Summary: Stop estimating agent work in human weeks. agentpace measures how long coding agents actually take: P50/P80/P90 ranges from real Claude Code and Codex runs, the human part stated apart, never derived. Open CC0 dataset, zero-dependency Python CLI, agent skill.
5
5
  Project-URL: Homepage, https://mohamedzhioua.github.io/agentpace/
6
6
  Project-URL: Repository, https://github.com/mohamedzhioua/agentpace
@@ -63,7 +63,7 @@ Method: shrunk. 3383 runs, 47 not completed so excluded. 20000 simulations, seed
63
63
  Not completed, by harness: claude-code 13 of 3219 (0.4%; 3 failed, 10 unknown), codex 34 of 211 (16.1%; 33 failed, 1 unknown). These are recorded harness outcomes, not task acceptance, and they are left out of the fit.
64
64
 
65
65
  Warnings:
66
- - cell docs x normal x claude-opus-5-5: n=0, effective n=0.0, below 30. Pooled with all and interval widened.
66
+ - cell docs x normal x claude-opus-5-5: n=0. No recorded run has kind docs at risk normal with model claude-opus-5-5. Mean from docs x normal (n=21); residuals pooled from all; interval widened by the spread between models. Recorded models under docs x normal: claude-sonnet-5-5 (21).
67
67
  <!-- /claim -->
68
68
  ```
69
69
 
@@ -171,7 +171,7 @@ Method: shrunk. 3383 runs, 47 not completed so excluded. 20000 simulations, seed
171
171
  Not completed, by harness: claude-code 13 of 3219 (0.4%; 3 failed, 10 unknown), codex 34 of 211 (16.1%; 33 failed, 1 unknown). These are recorded harness outcomes, not task acceptance, and they are left out of the fit.
172
172
 
173
173
  Warnings:
174
- - cell tests x normal x claude-opus-5-5: n=2, effective n=2.0, below 30. Pooled with all and interval widened.
174
+ - cell tests x normal x claude-opus-5-5: n=2, effective n=2.0, below 30. Mean shrunk toward broader cells; residuals pooled from all; interval widened.
175
175
  <!-- /claim -->
176
176
  ```
177
177
 
@@ -36,7 +36,7 @@ Method: shrunk. 3383 runs, 47 not completed so excluded. 20000 simulations, seed
36
36
  Not completed, by harness: claude-code 13 of 3219 (0.4%; 3 failed, 10 unknown), codex 34 of 211 (16.1%; 33 failed, 1 unknown). These are recorded harness outcomes, not task acceptance, and they are left out of the fit.
37
37
 
38
38
  Warnings:
39
- - cell docs x normal x claude-opus-5-5: n=0, effective n=0.0, below 30. Pooled with all and interval widened.
39
+ - cell docs x normal x claude-opus-5-5: n=0. No recorded run has kind docs at risk normal with model claude-opus-5-5. Mean from docs x normal (n=21); residuals pooled from all; interval widened by the spread between models. Recorded models under docs x normal: claude-sonnet-5-5 (21).
40
40
  <!-- /claim -->
41
41
  ```
42
42
 
@@ -144,7 +144,7 @@ Method: shrunk. 3383 runs, 47 not completed so excluded. 20000 simulations, seed
144
144
  Not completed, by harness: claude-code 13 of 3219 (0.4%; 3 failed, 10 unknown), codex 34 of 211 (16.1%; 33 failed, 1 unknown). These are recorded harness outcomes, not task acceptance, and they are left out of the fit.
145
145
 
146
146
  Warnings:
147
- - cell tests x normal x claude-opus-5-5: n=2, effective n=2.0, below 30. Pooled with all and interval widened.
147
+ - cell tests x normal x claude-opus-5-5: n=2, effective n=2.0, below 30. Mean shrunk toward broader cells; residuals pooled from all; interval widened.
148
148
  <!-- /claim -->
149
149
  ```
150
150
 
@@ -495,6 +495,18 @@ Fable 5.1 ruled on 2026-11-08 on four questions users asked after the launch pos
495
495
  - **Declined:** retry bucketing, time to an accepted result, and new segmentation dimensions. The rows hold no attempt linkage, no acceptance event and no repository size.
496
496
  - **The 2026-W46 data change** takes the next patch number when that week has ended, and waits for none of this.
497
497
 
498
+ ### Ruling for 0.12
499
+
500
+ Fable 5.1 ruled on 2026-11-09, with an independent opinion from Codex (gpt-6-astra) on the same brief, both at the highest effort. The master checked every claim in the opinion before the ruling.
501
+
502
+ - **The finding.** In the rows, `risk` is the agent's role (review and research agents `low`, migration agents `high`, everything else `normal`; the classifier, section 4). A plan's `risk` was a judgment about the work. The lookup stops at the first unseen level, so a task marked `high` matched no runs, dropped the model, and for an Opus feature cut the P50 by a factor of 16, in the shorter direction. No backtest fold ever reaches that branch, because row risk follows the agent type.
503
+ - **Warnings say what was not reached (0.11.2).** Which level had no recorded run, which of the task's levels were therefore not used, where the mean and residuals came from, and what is recorded under the parent. No counterfactual is simulated. A synthetic fixture pins the shipped method's numbers, not only its name.
504
+ - **The portable prompt's table (0.11.3)** is by kind, recorded risk and model, with pooled rows that mirror the CLI's back-off and no range under 30 runs. Claim enforcement covers the prompt's cells.
505
+ - **Plan risk is read as the recorded category (0.12.0), at the plan boundary, not in the method.** A written risk is used only where the rows can record that kind at that risk (`review` low, `migration` high, `other` low for research or normal, all else normal). Any other value, and an omitted one, takes the kind's recorded risk, and a warning names the task. `--use-risk` looks the written values up as given, for labelled datasets. `fit`, `locate`, the backtest's queries and the frozen selection are unchanged, so there is no re-registration.
506
+ - **Declined:** reordering or extending the hierarchy, a model fallback at the root or a shared model effect (method changes under the frozen selection), relabelling unseen kinds, and a risk multiplier.
507
+ - **Cleanup (0.12.1, 0.12.2):** count messages agree in number, documentation corrections, and clean errors for a bad rows line and a contribution file another program holds open.
508
+ - The W46 data change takes the next patch when that week ends, and waits for nothing.
509
+
498
510
  ## 12. Sources
499
511
 
500
512
  Repos, audited at: `kiloloop/agent-estimate@8857979`, `MMMProd-Org/claude-eta@5448ae4` (the `mmmprod/claude-eta` URL redirects there), `ZhangHanDong/agent-estimation@71a4697`, `ecappa/web-dev-estimation@c61fb1c`, `MedAdemBHA/mythical-agent-month@b60165f`.
@@ -1,3 +1,3 @@
1
1
  """agentpace: real run times for coding agents, as P50/P80/P90 ranges."""
2
2
 
3
- __version__ = "0.11.1"
3
+ __version__ = "0.11.2"
@@ -147,6 +147,11 @@ class Located:
147
147
  label: str
148
148
  pooled_from: str
149
149
  thin: bool
150
+ # Where the walk stopped, for the warning only: the path level whose value had no node, the
151
+ # deepest node it did reach, and that node's run count.
152
+ missing: str | None = None
153
+ parent: str = "all"
154
+ parent_n: int = 0
150
155
 
151
156
 
152
157
  @dataclass(frozen=True)
@@ -242,7 +247,8 @@ class Model:
242
247
  mu, v = node.mu, node.v
243
248
  pool_node = node
244
249
  prefix: tuple[str, ...] = ()
245
- missing = False
250
+ missing: str | None = None
251
+ parent, parent_n = "all", 0
246
252
  for depth, name in enumerate(self.path, start=1):
247
253
  value = query.get(name)
248
254
  if value is None:
@@ -252,7 +258,7 @@ class Model:
252
258
  if child is None:
253
259
  # Never seen: the parent's mean, plus the spread between sibling cells.
254
260
  v += self.tau2.get(depth, 0.0)
255
- missing = True
261
+ missing, parent, parent_n = name, " x ".join(prefix[:-1]) or "all", node.n
256
262
  break
257
263
  node, mu, v = child, child.mu, child.v
258
264
  if child.n_eff >= POOL_MIN:
@@ -262,11 +268,14 @@ class Model:
262
268
  mu=mu,
263
269
  v=v,
264
270
  pool=pool_node.pool,
265
- n=0 if missing else node.n,
266
- n_eff=0.0 if missing else node.n_eff,
271
+ n=0 if missing is not None else node.n,
272
+ n_eff=0.0 if missing is not None else node.n_eff,
267
273
  label=" x ".join(prefix) or "all",
268
274
  pooled_from=" x ".join(pool_node.key) or "all",
269
- thin=missing or node.n_eff < POOL_MIN,
275
+ thin=missing is not None or node.n_eff < POOL_MIN,
276
+ missing=missing,
277
+ parent=parent,
278
+ parent_n=parent_n,
270
279
  )
271
280
 
272
281
  def throughput(self, cap: int) -> ThroughputBucket | None:
@@ -558,6 +567,49 @@ def _clamped_exp(value: float) -> float:
558
567
  return math.exp(min(value, 30.0))
559
568
 
560
569
 
570
+ _LEVEL = {"task_kind": "kind", "size": "size", "risk": "risk", "model": "model"}
571
+ _LEVELS = {"task_kind": "kinds", "size": "sizes", "risk": "risk levels", "model": "models"}
572
+ _PREFIX = {"task_kind": "kind", "size": "at size", "risk": "at risk", "model": "with model"}
573
+
574
+
575
+ def _and(items: Sequence[str]) -> str:
576
+ return items[0] if len(items) == 1 else f"{', '.join(items[:-1])} and {items[-1]}"
577
+
578
+
579
+ def _unseen_note(model: Model, task: Task, cell: Located) -> str:
580
+ """Say which level had no recorded run, what the task supplied below it, and what the range
581
+ leaned on instead. Reads the fitted nodes only: no draw, no counterfactual cell."""
582
+ assert cell.missing is not None
583
+ query = {"task_kind": task.kind, "risk": task.risk, "model": task.model, "size": task.size}
584
+ depth = model.path.index(cell.missing) + 1
585
+ reached = model.path[:depth]
586
+ has = " ".join(f"{_PREFIX[name]} {query[name]}" for name in reached)
587
+ supplied = [
588
+ f"{_LEVEL[name]} {query[name]}" for name in model.path[depth:] if query[name] is not None
589
+ ]
590
+ dropped = ""
591
+ if supplied:
592
+ dropped = f", so {_and(supplied)} {'was' if len(supplied) == 1 else 'were'} not used"
593
+ if cell.parent == cell.pooled_from:
594
+ source = f"Mean and residuals from {cell.parent} (n={cell.parent_n})"
595
+ else:
596
+ source = (
597
+ f"Mean from {cell.parent} (n={cell.parent_n}); residuals pooled from {cell.pooled_from}"
598
+ )
599
+ key = tuple(str(query[name]) for name in reached[:-1])
600
+ siblings = sorted(
601
+ (node for k, node in model.nodes.items() if len(k) == depth and k[:-1] == key),
602
+ key=lambda node: (-node.n, node.key[-1]),
603
+ )
604
+ listed = ", ".join(f"{node.key[-1]} ({node.n})" for node in siblings)
605
+ levels = _LEVELS[cell.missing]
606
+ under = f" under {cell.parent}" if cell.parent != "all" else ""
607
+ return (
608
+ f"cell {cell.label}: n=0. No recorded run has {has}{dropped}. {source}; interval widened "
609
+ f"by the spread between {levels}. Recorded {levels}{under}: {listed}."
610
+ )
611
+
612
+
561
613
  def _warnings(model: Model, tasks: Sequence[Task], located: Mapping[str, Located]) -> list[str]:
562
614
  notes: list[str] = []
563
615
  seen: set[str] = set()
@@ -565,10 +617,19 @@ def _warnings(model: Model, tasks: Sequence[Task], located: Mapping[str, Located
565
617
  cell = located[task.id]
566
618
  if cell.thin and cell.label not in seen:
567
619
  seen.add(cell.label)
568
- notes.append(
569
- f"cell {cell.label}: n={cell.n}, effective n={cell.n_eff:.1f}, below {POOL_MIN}. "
570
- f"Pooled with {cell.pooled_from} and interval widened."
620
+ below = (
621
+ f"cell {cell.label}: n={cell.n}, effective n={cell.n_eff:.1f}, below {POOL_MIN}."
571
622
  )
623
+ if cell.missing is not None:
624
+ notes.append(_unseen_note(model, task, cell))
625
+ elif cell.label == "all":
626
+ notes.append(f"{below} Interval widened.")
627
+ else:
628
+ mean = "shrunk toward" if model.method.shrink else "taken from"
629
+ notes.append(
630
+ f"{below} Mean {mean} broader cells; residuals pooled from "
631
+ f"{cell.pooled_from}; interval widened."
632
+ )
572
633
  if "size" not in model.path and any(task.size for task in tasks):
573
634
  notes.append(
574
635
  "size ignored: stored size is measured after a run, so it is left out unless you "
@@ -443,5 +443,127 @@ class PlanDocuments(unittest.TestCase):
443
443
  self.assertIsNone(tasks[0].model)
444
444
 
445
445
 
446
+ class WarningWording(unittest.TestCase):
447
+ """A cell warning says which level had no recorded run, what the task supplied below it,
448
+ where the mean and the residuals came from, and what is recorded under the parent."""
449
+
450
+ OPUS = "claude-opus-5-5"
451
+
452
+ @staticmethod
453
+ def rows():
454
+ extra = [
455
+ row(task_kind="migration", risk="high", duration_s=900 + i, batch=f"{0xA00 + i:012x}")
456
+ for i in range(3)
457
+ ]
458
+ extra += [
459
+ row(model="gpt-x", duration_s=1800 + i, batch=f"{0xB00 + i:012x}") for i in range(2)
460
+ ]
461
+ # feature 182 (claude-opus-5-5 180, gpt-x 2), review 180, docs 120, migration 3: 485
462
+ return dataset(sessions=60) + extra
463
+
464
+ @classmethod
465
+ def setUpClass(cls):
466
+ cls.shrunk = fit(cls.rows(), method="shrunk")
467
+ cls.backoff = fit(cls.rows(), method="backoff")
468
+
469
+ def warnings(self, task, model=None):
470
+ return est.estimate(model or self.shrunk, [task], 1, draws=200, seed=1).warnings
471
+
472
+ def test_unseen_kind_with_a_model(self):
473
+ self.assertEqual(
474
+ self.warnings(Task("a", "refactor", model=self.OPUS)),
475
+ (
476
+ "cell refactor: n=0. No recorded run has kind refactor, so risk normal and model "
477
+ "claude-opus-5-5 were not used. Mean and residuals from all (n=485); interval "
478
+ "widened by the spread between kinds. Recorded kinds: feature (182), "
479
+ "review (180), docs (120), migration (3).",
480
+ ),
481
+ )
482
+
483
+ def test_unseen_kind_without_a_model(self):
484
+ self.assertEqual(
485
+ self.warnings(Task("a", "refactor")),
486
+ (
487
+ "cell refactor: n=0. No recorded run has kind refactor, so risk normal was not "
488
+ "used. Mean and residuals from all (n=485); interval widened by the spread "
489
+ "between kinds. Recorded kinds: feature (182), review (180), docs (120), "
490
+ "migration (3).",
491
+ ),
492
+ )
493
+
494
+ def test_unseen_risk(self):
495
+ self.assertEqual(
496
+ self.warnings(Task("a", "feature", risk="high", model=self.OPUS)),
497
+ (
498
+ "cell feature x high: n=0. No recorded run has kind feature at risk high, so "
499
+ "model claude-opus-5-5 was not used. Mean and residuals from feature (n=182); "
500
+ "interval widened by the spread between risk levels. Recorded risk levels under "
501
+ "feature: normal (182).",
502
+ ),
503
+ )
504
+ self.assertEqual(
505
+ self.warnings(Task("a", "feature", risk="high")),
506
+ (
507
+ "cell feature x high: n=0. No recorded run has kind feature at risk high. Mean "
508
+ "and residuals from feature (n=182); interval widened by the spread between risk "
509
+ "levels. Recorded risk levels under feature: normal (182).",
510
+ ),
511
+ )
512
+
513
+ def test_unseen_model_lists_the_recorded_models(self):
514
+ self.assertEqual(
515
+ self.warnings(Task("a", "feature", model="claude-sonnet-5-5")),
516
+ (
517
+ "cell feature x normal x claude-sonnet-5-5: n=0. No recorded run has kind "
518
+ "feature at risk normal with model claude-sonnet-5-5. Mean and residuals from "
519
+ "feature x normal (n=182); interval widened by the spread between models. "
520
+ "Recorded models under feature x normal: claude-opus-5-5 (180), gpt-x (2).",
521
+ ),
522
+ )
523
+
524
+ def test_unseen_model_under_a_thin_parent_names_both_sources(self):
525
+ self.assertEqual(
526
+ self.warnings(Task("a", "migration", risk="high", model="claude-sonnet-5-5")),
527
+ (
528
+ "cell migration x high x claude-sonnet-5-5: n=0. No recorded run has kind "
529
+ "migration at risk high with model claude-sonnet-5-5. Mean from migration x high "
530
+ "(n=3); residuals pooled from all; interval widened by the spread between "
531
+ "models. Recorded models under migration x high: claude-opus-5-5 (3).",
532
+ ),
533
+ )
534
+
535
+ def test_a_thin_seen_cell_under_shrinkage(self):
536
+ self.assertEqual(
537
+ self.warnings(Task("a", "feature", model="gpt-x")),
538
+ (
539
+ "cell feature x normal x gpt-x: n=2, effective n=2.0, below 30. Mean shrunk "
540
+ "toward broader cells; residuals pooled from feature x normal; interval widened.",
541
+ ),
542
+ )
543
+
544
+ def test_a_thin_seen_cell_under_backoff(self):
545
+ self.assertEqual(
546
+ self.warnings(Task("a", "feature", model="gpt-x"), model=self.backoff),
547
+ (
548
+ "cell feature x normal x gpt-x: n=2, effective n=2.0, below 30. Mean taken from "
549
+ "broader cells; residuals pooled from feature x normal; interval widened.",
550
+ ),
551
+ )
552
+
553
+ def test_a_thin_global_cell(self):
554
+ model = fit(plain_rows(20, 60), method="global")
555
+ warnings = est.estimate(model, [Task("a", "feature")], 1, draws=200).warnings
556
+ self.assertEqual(
557
+ warnings[0], "cell all: n=20, effective n=20.0, below 30. Interval widened."
558
+ )
559
+ self.assertEqual(self.warnings(Task("a", "refactor"), fit(self.rows(), "global")), ())
560
+
561
+ def test_a_seen_cell_records_no_missing_level(self):
562
+ cell = self.shrunk.locate({"task_kind": "feature", "risk": "normal", "model": self.OPUS})
563
+ self.assertEqual((cell.missing, cell.parent, cell.parent_n), (None, "all", 0))
564
+ cell = self.shrunk.locate({"task_kind": "feature", "risk": "high", "model": self.OPUS})
565
+ self.assertEqual((cell.missing, cell.parent, cell.parent_n), ("risk", "feature", 182))
566
+
567
+
446
568
  if __name__ == "__main__":
447
569
  unittest.main()
@@ -0,0 +1,225 @@
1
+ """The shipped method's numbers, pinned on SYNTHETIC rows, not only its name.
2
+
3
+ The literals below were generated by running `main`'s `src/` at v0.11.1 (714adc9) on these rows.
4
+ A change to `fit`, `locate`, the random draws or the simulation moves them, and that is a method
5
+ change under the frozen selection. Nothing here reads the seed data, so a data release never
6
+ re-pins this file.
7
+ """
8
+
9
+ import math
10
+ import random
11
+ import unittest
12
+
13
+ from agentpace import estimate as est
14
+ from agentpace.estimate import Task
15
+ from tests.synthetic import dataset, row
16
+
17
+ REL = 1e-9
18
+ OPUS = "claude-opus-5-5"
19
+ SONNET = "claude-sonnet-5-5"
20
+
21
+
22
+ def rows():
23
+ """The synthetic dataset's three cells, a thin `migration` kind and a thin second model."""
24
+ migrations = [
25
+ row(
26
+ task_kind="migration",
27
+ risk="high",
28
+ size="L",
29
+ duration_s=seconds,
30
+ batch=f"{0xA00 + i:012x}",
31
+ )
32
+ for i, seconds in enumerate((900, 1500, 2400))
33
+ ]
34
+ other_model = [
35
+ row(model="gpt-x", duration_s=seconds, batch=f"{0xB00 + i:012x}")
36
+ for i, seconds in enumerate((1800, 2700))
37
+ ]
38
+ return dataset(seed=7, sessions=60) + migrations + other_model
39
+
40
+
41
+ QUERIES = {
42
+ "seen": {"task_kind": "feature", "risk": "normal", "model": OPUS},
43
+ "unseen_kind": {"task_kind": "refactor", "risk": "normal", "model": OPUS},
44
+ "unseen_risk": {"task_kind": "feature", "risk": "high", "model": OPUS},
45
+ "unseen_model": {"task_kind": "docs", "risk": "normal", "model": SONNET},
46
+ "thin_model": {"task_kind": "feature", "risk": "normal", "model": "gpt-x"},
47
+ "thin_branch": {"task_kind": "migration", "risk": "high", "model": OPUS},
48
+ "no_model": {"task_kind": "review", "risk": "low", "model": None},
49
+ }
50
+
51
+
52
+ def plan_tasks():
53
+ """Six tasks, one per cell above that a plan can reach, with dependencies."""
54
+ return [
55
+ Task("api", "feature", model=OPUS),
56
+ Task("harden", "feature", risk="high", model=OPUS, depends_on=("api",)),
57
+ Task("port", "migration", risk="high", model=OPUS, depends_on=("api",)),
58
+ Task("guide", "docs", model=SONNET, depends_on=("api",)),
59
+ Task("probe", "feature", model="gpt-x"),
60
+ Task("rework", "refactor", model=OPUS, depends_on=("port", "probe")),
61
+ ]
62
+
63
+
64
+ # Every task's risk is one the rows can record for its kind, so the plan boundary keeps it as
65
+ # written; omitted fields take the document's defaults.
66
+ CANONICAL_DOCUMENT = {
67
+ "agent_cap": 2,
68
+ "model": OPUS,
69
+ "tasks": [
70
+ {"id": "api", "kind": "feature", "risk": "normal"},
71
+ {"id": "port", "kind": "migration", "risk": "high", "depends_on": ["api"]},
72
+ {"id": "scan", "kind": "review", "risk": "low", "model": None},
73
+ {"id": "guide", "kind": "docs", "risk": "normal", "model": SONNET, "depends_on": ["api"]},
74
+ {"id": "probe", "kind": "feature", "risk": "normal", "model": "gpt-x"},
75
+ {"id": "rework", "kind": "refactor", "risk": "normal", "depends_on": ["port", "probe"]},
76
+ ],
77
+ }
78
+ CANONICAL_TASKS = [
79
+ Task("api", "feature", risk="normal", model=OPUS),
80
+ Task("port", "migration", risk="high", model=OPUS, depends_on=("api",)),
81
+ Task("scan", "review", risk="low", model=None),
82
+ Task("guide", "docs", risk="normal", model=SONNET, depends_on=("api",)),
83
+ Task("probe", "feature", risk="normal", model="gpt-x"),
84
+ Task("rework", "refactor", risk="normal", model=OPUS, depends_on=("port", "probe")),
85
+ ]
86
+
87
+ # --- generated literals: (mu, v, n, n_eff, label, pooled_from, thin) ---
88
+ LOCATED = {
89
+ "seen": (
90
+ 5.748561917464507,
91
+ 0.004430156024745457,
92
+ 180,
93
+ 180.0,
94
+ "feature x normal x claude-opus-5-5",
95
+ "feature x normal x claude-opus-5-5",
96
+ False,
97
+ ),
98
+ "unseen_kind": (5.021725817345783, 0.654926996567019, 0, 0.0, "refactor", "all", True),
99
+ "unseen_risk": (
100
+ 5.764844039026677,
101
+ 0.004447676968646228,
102
+ 0,
103
+ 0.0,
104
+ "feature x high",
105
+ "feature",
106
+ True,
107
+ ),
108
+ "unseen_model": (
109
+ 4.195219435035191,
110
+ 1.7053968486894486,
111
+ 0,
112
+ 0.0,
113
+ "docs x normal x claude-sonnet-5-5",
114
+ "docs x normal",
115
+ True,
116
+ ),
117
+ "thin_model": (
118
+ 7.329953035243061,
119
+ 0.3237615584003151,
120
+ 2,
121
+ 2.0,
122
+ "feature x normal x gpt-x",
123
+ "feature x normal",
124
+ True,
125
+ ),
126
+ "thin_branch": (
127
+ 7.208790485554894,
128
+ 0.23389289123850235,
129
+ 3,
130
+ 3.0,
131
+ "migration x high x claude-opus-5-5",
132
+ "all",
133
+ True,
134
+ ),
135
+ "no_model": (
136
+ 4.785607835027801,
137
+ 0.004496756296448909,
138
+ 180,
139
+ 180.0,
140
+ "review x low",
141
+ "review x low",
142
+ False,
143
+ ),
144
+ }
145
+ CELL_QUANTILES = {
146
+ "seen": (326.37855852487576, 678.9109715542419, 949.5685633561789),
147
+ "unseen_kind": (148.62660109580304, 410.3270975996236, 678.7296559946502),
148
+ "unseen_risk": (331.7379406557295, 689.9707766730423, 965.0769989527402),
149
+ "unseen_model": (62.435342338073596, 250.72378246646713, 472.295285463006),
150
+ "thin_model": (1580.3187981430283, 3695.0983044974687, 5539.908213468239),
151
+ "thin_branch": (1350.1751834448505, 3166.067208416465, 4825.334525711538),
152
+ "no_model": (122.95559172286664, 255.337383795307, 372.2208327109682),
153
+ }
154
+ PLAN = (3218.226765188425, 5954.69319603287, 8446.562927367682)
155
+ TASK_P50 = {
156
+ "api": 320.7033144630816,
157
+ "harden": 315.9384124292877,
158
+ "port": 1340.372829954017,
159
+ "guide": 65.31036298267271,
160
+ "probe": 1516.3577809135163,
161
+ "rework": 165.70645375745192,
162
+ }
163
+ # --- end of generated literals ---
164
+
165
+
166
+ class FrozenMethod(unittest.TestCase):
167
+ @classmethod
168
+ def setUpClass(cls):
169
+ cls.model = est.fit(rows(), method=est.DEFAULT_METHOD)
170
+
171
+ def assertClose(self, actual, expected):
172
+ self.assertTrue(
173
+ math.isclose(actual, expected, rel_tol=REL, abs_tol=0.0),
174
+ f"{actual!r} != {expected!r}",
175
+ )
176
+
177
+ def test_the_fixture_covers_every_query(self):
178
+ self.assertEqual(set(LOCATED), set(QUERIES))
179
+ self.assertEqual(set(CELL_QUANTILES), set(QUERIES))
180
+ self.assertEqual(set(TASK_P50), {task.id for task in plan_tasks()})
181
+
182
+ def test_located_cells(self):
183
+ for name, query in QUERIES.items():
184
+ with self.subTest(query=name):
185
+ cell = self.model.locate(query)
186
+ mu, v, n, n_eff, label, pooled_from, thin = LOCATED[name]
187
+ self.assertClose(cell.mu, mu)
188
+ self.assertClose(cell.v, v)
189
+ self.assertEqual(cell.n, n)
190
+ self.assertClose(cell.n_eff, n_eff)
191
+ self.assertEqual(cell.label, label)
192
+ self.assertEqual(cell.pooled_from, pooled_from)
193
+ self.assertIs(cell.thin, thin)
194
+
195
+ def test_cell_quantiles(self):
196
+ for name, query in QUERIES.items():
197
+ with self.subTest(query=name):
198
+ actual = est.cell_quantiles(self.model, query, 2000, random.Random(1))
199
+ for value, expected in zip(actual, CELL_QUANTILES[name], strict=True):
200
+ self.assertClose(value, expected)
201
+
202
+ def test_plan(self):
203
+ plan = est.estimate(self.model, plan_tasks(), agent_cap=2, draws=2000, seed=5)
204
+ for value, expected in zip((plan.p50, plan.p80, plan.p90), PLAN, strict=True):
205
+ self.assertClose(value, expected)
206
+ self.assertEqual([task.id for task in plan.tasks], [task.id for task in plan_tasks()])
207
+ for task in plan.tasks:
208
+ with self.subTest(task=task.id):
209
+ self.assertClose(task.p50, TASK_P50[task.id])
210
+
211
+ def test_a_canonical_plan_document_equals_its_tasks(self):
212
+ tasks, cap = est.tasks_from_json(CANONICAL_DOCUMENT)
213
+ self.assertEqual(cap, 2)
214
+
215
+ def fields(task):
216
+ return (task.id, task.kind, task.risk, task.size, task.model, task.depends_on)
217
+
218
+ self.assertEqual([fields(task) for task in tasks], [fields(t) for t in CANONICAL_TASKS])
219
+ through_document = est.estimate(self.model, tasks, cap, draws=2000, seed=5)
220
+ direct = est.estimate(self.model, CANONICAL_TASKS, 2, draws=2000, seed=5)
221
+ self.assertEqual(through_document, direct)
222
+
223
+
224
+ if __name__ == "__main__":
225
+ unittest.main()
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes