agentpace 0.11.1__tar.gz → 0.11.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {agentpace-0.11.1 → agentpace-0.11.2}/CHANGELOG.md +13 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/CITATION.cff +2 -2
- {agentpace-0.11.1 → agentpace-0.11.2}/PKG-INFO +3 -3
- {agentpace-0.11.1 → agentpace-0.11.2}/README.md +2 -2
- {agentpace-0.11.1 → agentpace-0.11.2}/docs/DESIGN.md +12 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/__init__.py +1 -1
- {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/estimate.py +69 -8
- {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_estimate.py +122 -0
- agentpace-0.11.2/tests/test_frozen_method.py +225 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/.gitattributes +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/.github/ISSUE_TEMPLATE/config.yml +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/.github/ISSUE_TEMPLATE/ingest-found-nothing.yml +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/.github/ISSUE_TEMPLATE/remove-my-rows.yml +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/.github/dependabot.yml +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/.github/pull_request_template.md +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/.github/workflows/ci.yml +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/.github/workflows/publish.yml +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/.github/workflows/validate-data.yml +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/.gitignore +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/CODE_OF_CONDUCT.md +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/CONTRIBUTING.md +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/LICENSE +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/SECURITY.md +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/data/LICENSE +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/data/README.md +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/data/runs/4546788e7930eccd-2026-W40-2026-W45.jsonl +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/data/runs/4546788e7930eccd-2026-W44-2026-W45.jsonl +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/docs/.nojekyll +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/docs/assets/backtest-coverage.svg +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/docs/assets/hero.webp +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/docs/assets/run-ranges.svg +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/docs/assets/social-preview.png +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/docs/backtest.json +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/docs/backtest.md +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/docs/cli.md +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/docs/index.html +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/docs/releasing.md +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/docs/robots.txt +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/docs/sitemap.xml +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/docs/verify-adapter.md +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/examples/plan.json +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/examples/quote-plan.json +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/pyproject.toml +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/schema/run.schema.json +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/schema/turn.schema.json +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/skills/agentpace/SKILL.md +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/skills/agentpace/assets/portable-prompt.md +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/skills/agentpace/references/method.md +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/skills/pace-estimate/SKILL.md +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/skills/pace-log/SKILL.md +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/skills/pace-share/SKILL.md +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/__main__.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/adapters/__init__.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/adapters/base.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/adapters/claude_code.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/adapters/cline.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/adapters/codex.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/adapters/gemini_cli.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/adapters/opencode.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/backtest.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/cache.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/charts.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/claims.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/classify.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/cli.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/forecasts.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/ingest.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/paths.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/pseudonym.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/py.typed +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/render.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/report.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/schema.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/share.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/site.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/turns.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/src/agentpace/validate.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/tests/__init__.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/tests/fixtures/__init__.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/tests/fixtures/cline_tasks.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/tests/fixtures/gemini_cli.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/tests/fixtures/opencode.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/tests/synthetic.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_backtest.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_claims.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_classify.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_cli.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_cline.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_codex.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_docs.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_forecasts.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_gemini_cli.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_ingest.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_labels.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_opencode.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_outcomes.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_review_capacity.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_schema.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_share_validate.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_site.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_skills.py +0 -0
- {agentpace-0.11.1 → agentpace-0.11.2}/tests/test_turns.py +0 -0
|
@@ -4,6 +4,19 @@ This project follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) an
|
|
|
4
4
|
|
|
5
5
|
## Unreleased
|
|
6
6
|
|
|
7
|
+
## 0.11.2 - 2026-11-09
|
|
8
|
+
|
|
9
|
+
### Changed
|
|
10
|
+
|
|
11
|
+
- A cell warning now says which level had no recorded run (kind, risk or model), which of the task's levels were therefore not used, where the mean and the residuals came from, and which values are recorded under the parent. For example, a `feature` task marked `high` on `claude-opus-5-5` now reads: `cell feature x high: n=0. No recorded run has kind feature at risk high, so model claude-opus-5-5 was not used. Mean and residuals from feature (n=1353); interval widened by the spread between risk levels. Recorded risk levels under feature: normal (1353).`
|
|
12
|
+
- Thin cells say the mean is shrunk toward broader cells and name the residual pool.
|
|
13
|
+
|
|
14
|
+
### Added
|
|
15
|
+
|
|
16
|
+
- A regression fixture, `tests/test_frozen_method.py`, pins the shipped method's numbers on synthetic rows, not only its name. Its values were generated with 0.11.1's code, and it passes on 0.11.1 unchanged.
|
|
17
|
+
|
|
18
|
+
No published number moved; the warning line in the two README examples changed.
|
|
19
|
+
|
|
7
20
|
## 0.11.1 - 2026-11-08
|
|
8
21
|
|
|
9
22
|
### Changed
|
|
@@ -8,8 +8,8 @@ authors:
|
|
|
8
8
|
repository-code: https://github.com/mohamedzhioua/agentpace
|
|
9
9
|
url: https://mohamedzhioua.github.io/agentpace/
|
|
10
10
|
license: MIT
|
|
11
|
-
version: 0.11.
|
|
12
|
-
date-released: 2026-11-
|
|
11
|
+
version: 0.11.2
|
|
12
|
+
date-released: 2026-11-09
|
|
13
13
|
keywords:
|
|
14
14
|
- coding agents
|
|
15
15
|
- estimation
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: agentpace
|
|
3
|
-
Version: 0.11.
|
|
3
|
+
Version: 0.11.2
|
|
4
4
|
Summary: Stop estimating agent work in human weeks. agentpace measures how long coding agents actually take: P50/P80/P90 ranges from real Claude Code and Codex runs, the human part stated apart, never derived. Open CC0 dataset, zero-dependency Python CLI, agent skill.
|
|
5
5
|
Project-URL: Homepage, https://mohamedzhioua.github.io/agentpace/
|
|
6
6
|
Project-URL: Repository, https://github.com/mohamedzhioua/agentpace
|
|
@@ -63,7 +63,7 @@ Method: shrunk. 3383 runs, 47 not completed so excluded. 20000 simulations, seed
|
|
|
63
63
|
Not completed, by harness: claude-code 13 of 3219 (0.4%; 3 failed, 10 unknown), codex 34 of 211 (16.1%; 33 failed, 1 unknown). These are recorded harness outcomes, not task acceptance, and they are left out of the fit.
|
|
64
64
|
|
|
65
65
|
Warnings:
|
|
66
|
-
- cell docs x normal x claude-opus-5-5: n=0
|
|
66
|
+
- cell docs x normal x claude-opus-5-5: n=0. No recorded run has kind docs at risk normal with model claude-opus-5-5. Mean from docs x normal (n=21); residuals pooled from all; interval widened by the spread between models. Recorded models under docs x normal: claude-sonnet-5-5 (21).
|
|
67
67
|
<!-- /claim -->
|
|
68
68
|
```
|
|
69
69
|
|
|
@@ -171,7 +171,7 @@ Method: shrunk. 3383 runs, 47 not completed so excluded. 20000 simulations, seed
|
|
|
171
171
|
Not completed, by harness: claude-code 13 of 3219 (0.4%; 3 failed, 10 unknown), codex 34 of 211 (16.1%; 33 failed, 1 unknown). These are recorded harness outcomes, not task acceptance, and they are left out of the fit.
|
|
172
172
|
|
|
173
173
|
Warnings:
|
|
174
|
-
- cell tests x normal x claude-opus-5-5: n=2, effective n=2.0, below 30.
|
|
174
|
+
- cell tests x normal x claude-opus-5-5: n=2, effective n=2.0, below 30. Mean shrunk toward broader cells; residuals pooled from all; interval widened.
|
|
175
175
|
<!-- /claim -->
|
|
176
176
|
```
|
|
177
177
|
|
|
@@ -36,7 +36,7 @@ Method: shrunk. 3383 runs, 47 not completed so excluded. 20000 simulations, seed
|
|
|
36
36
|
Not completed, by harness: claude-code 13 of 3219 (0.4%; 3 failed, 10 unknown), codex 34 of 211 (16.1%; 33 failed, 1 unknown). These are recorded harness outcomes, not task acceptance, and they are left out of the fit.
|
|
37
37
|
|
|
38
38
|
Warnings:
|
|
39
|
-
- cell docs x normal x claude-opus-5-5: n=0
|
|
39
|
+
- cell docs x normal x claude-opus-5-5: n=0. No recorded run has kind docs at risk normal with model claude-opus-5-5. Mean from docs x normal (n=21); residuals pooled from all; interval widened by the spread between models. Recorded models under docs x normal: claude-sonnet-5-5 (21).
|
|
40
40
|
<!-- /claim -->
|
|
41
41
|
```
|
|
42
42
|
|
|
@@ -144,7 +144,7 @@ Method: shrunk. 3383 runs, 47 not completed so excluded. 20000 simulations, seed
|
|
|
144
144
|
Not completed, by harness: claude-code 13 of 3219 (0.4%; 3 failed, 10 unknown), codex 34 of 211 (16.1%; 33 failed, 1 unknown). These are recorded harness outcomes, not task acceptance, and they are left out of the fit.
|
|
145
145
|
|
|
146
146
|
Warnings:
|
|
147
|
-
- cell tests x normal x claude-opus-5-5: n=2, effective n=2.0, below 30.
|
|
147
|
+
- cell tests x normal x claude-opus-5-5: n=2, effective n=2.0, below 30. Mean shrunk toward broader cells; residuals pooled from all; interval widened.
|
|
148
148
|
<!-- /claim -->
|
|
149
149
|
```
|
|
150
150
|
|
|
@@ -495,6 +495,18 @@ Fable 5.1 ruled on 2026-11-08 on four questions users asked after the launch pos
|
|
|
495
495
|
- **Declined:** retry bucketing, time to an accepted result, and new segmentation dimensions. The rows hold no attempt linkage, no acceptance event and no repository size.
|
|
496
496
|
- **The 2026-W46 data change** takes the next patch number when that week has ended, and waits for none of this.
|
|
497
497
|
|
|
498
|
+
### Ruling for 0.12
|
|
499
|
+
|
|
500
|
+
Fable 5.1 ruled on 2026-11-09, with an independent opinion from Codex (gpt-6-astra) on the same brief, both at the highest effort. The master checked every claim in the opinion before the ruling.
|
|
501
|
+
|
|
502
|
+
- **The finding.** In the rows, `risk` is the agent's role (review and research agents `low`, migration agents `high`, everything else `normal`; the classifier, section 4). A plan's `risk` was a judgment about the work. The lookup stops at the first unseen level, so a task marked `high` matched no runs, dropped the model, and for an Opus feature cut the P50 by a factor of 16, in the shorter direction. No backtest fold ever reaches that branch, because row risk follows the agent type.
|
|
503
|
+
- **Warnings say what was not reached (0.11.2).** Which level had no recorded run, which of the task's levels were therefore not used, where the mean and residuals came from, and what is recorded under the parent. No counterfactual is simulated. A synthetic fixture pins the shipped method's numbers, not only its name.
|
|
504
|
+
- **The portable prompt's table (0.11.3)** is by kind, recorded risk and model, with pooled rows that mirror the CLI's back-off and no range under 30 runs. Claim enforcement covers the prompt's cells.
|
|
505
|
+
- **Plan risk is read as the recorded category (0.12.0), at the plan boundary, not in the method.** A written risk is used only where the rows can record that kind at that risk (`review` low, `migration` high, `other` low for research or normal, all else normal). Any other value, and an omitted one, takes the kind's recorded risk, and a warning names the task. `--use-risk` looks the written values up as given, for labelled datasets. `fit`, `locate`, the backtest's queries and the frozen selection are unchanged, so there is no re-registration.
|
|
506
|
+
- **Declined:** reordering or extending the hierarchy, a model fallback at the root or a shared model effect (method changes under the frozen selection), relabelling unseen kinds, and a risk multiplier.
|
|
507
|
+
- **Cleanup (0.12.1, 0.12.2):** count messages agree in number, documentation corrections, and clean errors for a bad rows line and a contribution file another program holds open.
|
|
508
|
+
- The W46 data change takes the next patch when that week ends, and waits for nothing.
|
|
509
|
+
|
|
498
510
|
## 12. Sources
|
|
499
511
|
|
|
500
512
|
Repos, audited at: `kiloloop/agent-estimate@8857979`, `MMMProd-Org/claude-eta@5448ae4` (the `mmmprod/claude-eta` URL redirects there), `ZhangHanDong/agent-estimation@71a4697`, `ecappa/web-dev-estimation@c61fb1c`, `MedAdemBHA/mythical-agent-month@b60165f`.
|
|
@@ -147,6 +147,11 @@ class Located:
|
|
|
147
147
|
label: str
|
|
148
148
|
pooled_from: str
|
|
149
149
|
thin: bool
|
|
150
|
+
# Where the walk stopped, for the warning only: the path level whose value had no node, the
|
|
151
|
+
# deepest node it did reach, and that node's run count.
|
|
152
|
+
missing: str | None = None
|
|
153
|
+
parent: str = "all"
|
|
154
|
+
parent_n: int = 0
|
|
150
155
|
|
|
151
156
|
|
|
152
157
|
@dataclass(frozen=True)
|
|
@@ -242,7 +247,8 @@ class Model:
|
|
|
242
247
|
mu, v = node.mu, node.v
|
|
243
248
|
pool_node = node
|
|
244
249
|
prefix: tuple[str, ...] = ()
|
|
245
|
-
missing =
|
|
250
|
+
missing: str | None = None
|
|
251
|
+
parent, parent_n = "all", 0
|
|
246
252
|
for depth, name in enumerate(self.path, start=1):
|
|
247
253
|
value = query.get(name)
|
|
248
254
|
if value is None:
|
|
@@ -252,7 +258,7 @@ class Model:
|
|
|
252
258
|
if child is None:
|
|
253
259
|
# Never seen: the parent's mean, plus the spread between sibling cells.
|
|
254
260
|
v += self.tau2.get(depth, 0.0)
|
|
255
|
-
missing =
|
|
261
|
+
missing, parent, parent_n = name, " x ".join(prefix[:-1]) or "all", node.n
|
|
256
262
|
break
|
|
257
263
|
node, mu, v = child, child.mu, child.v
|
|
258
264
|
if child.n_eff >= POOL_MIN:
|
|
@@ -262,11 +268,14 @@ class Model:
|
|
|
262
268
|
mu=mu,
|
|
263
269
|
v=v,
|
|
264
270
|
pool=pool_node.pool,
|
|
265
|
-
n=0 if missing else node.n,
|
|
266
|
-
n_eff=0.0 if missing else node.n_eff,
|
|
271
|
+
n=0 if missing is not None else node.n,
|
|
272
|
+
n_eff=0.0 if missing is not None else node.n_eff,
|
|
267
273
|
label=" x ".join(prefix) or "all",
|
|
268
274
|
pooled_from=" x ".join(pool_node.key) or "all",
|
|
269
|
-
thin=missing or node.n_eff < POOL_MIN,
|
|
275
|
+
thin=missing is not None or node.n_eff < POOL_MIN,
|
|
276
|
+
missing=missing,
|
|
277
|
+
parent=parent,
|
|
278
|
+
parent_n=parent_n,
|
|
270
279
|
)
|
|
271
280
|
|
|
272
281
|
def throughput(self, cap: int) -> ThroughputBucket | None:
|
|
@@ -558,6 +567,49 @@ def _clamped_exp(value: float) -> float:
|
|
|
558
567
|
return math.exp(min(value, 30.0))
|
|
559
568
|
|
|
560
569
|
|
|
570
|
+
_LEVEL = {"task_kind": "kind", "size": "size", "risk": "risk", "model": "model"}
|
|
571
|
+
_LEVELS = {"task_kind": "kinds", "size": "sizes", "risk": "risk levels", "model": "models"}
|
|
572
|
+
_PREFIX = {"task_kind": "kind", "size": "at size", "risk": "at risk", "model": "with model"}
|
|
573
|
+
|
|
574
|
+
|
|
575
|
+
def _and(items: Sequence[str]) -> str:
|
|
576
|
+
return items[0] if len(items) == 1 else f"{', '.join(items[:-1])} and {items[-1]}"
|
|
577
|
+
|
|
578
|
+
|
|
579
|
+
def _unseen_note(model: Model, task: Task, cell: Located) -> str:
|
|
580
|
+
"""Say which level had no recorded run, what the task supplied below it, and what the range
|
|
581
|
+
leaned on instead. Reads the fitted nodes only: no draw, no counterfactual cell."""
|
|
582
|
+
assert cell.missing is not None
|
|
583
|
+
query = {"task_kind": task.kind, "risk": task.risk, "model": task.model, "size": task.size}
|
|
584
|
+
depth = model.path.index(cell.missing) + 1
|
|
585
|
+
reached = model.path[:depth]
|
|
586
|
+
has = " ".join(f"{_PREFIX[name]} {query[name]}" for name in reached)
|
|
587
|
+
supplied = [
|
|
588
|
+
f"{_LEVEL[name]} {query[name]}" for name in model.path[depth:] if query[name] is not None
|
|
589
|
+
]
|
|
590
|
+
dropped = ""
|
|
591
|
+
if supplied:
|
|
592
|
+
dropped = f", so {_and(supplied)} {'was' if len(supplied) == 1 else 'were'} not used"
|
|
593
|
+
if cell.parent == cell.pooled_from:
|
|
594
|
+
source = f"Mean and residuals from {cell.parent} (n={cell.parent_n})"
|
|
595
|
+
else:
|
|
596
|
+
source = (
|
|
597
|
+
f"Mean from {cell.parent} (n={cell.parent_n}); residuals pooled from {cell.pooled_from}"
|
|
598
|
+
)
|
|
599
|
+
key = tuple(str(query[name]) for name in reached[:-1])
|
|
600
|
+
siblings = sorted(
|
|
601
|
+
(node for k, node in model.nodes.items() if len(k) == depth and k[:-1] == key),
|
|
602
|
+
key=lambda node: (-node.n, node.key[-1]),
|
|
603
|
+
)
|
|
604
|
+
listed = ", ".join(f"{node.key[-1]} ({node.n})" for node in siblings)
|
|
605
|
+
levels = _LEVELS[cell.missing]
|
|
606
|
+
under = f" under {cell.parent}" if cell.parent != "all" else ""
|
|
607
|
+
return (
|
|
608
|
+
f"cell {cell.label}: n=0. No recorded run has {has}{dropped}. {source}; interval widened "
|
|
609
|
+
f"by the spread between {levels}. Recorded {levels}{under}: {listed}."
|
|
610
|
+
)
|
|
611
|
+
|
|
612
|
+
|
|
561
613
|
def _warnings(model: Model, tasks: Sequence[Task], located: Mapping[str, Located]) -> list[str]:
|
|
562
614
|
notes: list[str] = []
|
|
563
615
|
seen: set[str] = set()
|
|
@@ -565,10 +617,19 @@ def _warnings(model: Model, tasks: Sequence[Task], located: Mapping[str, Located
|
|
|
565
617
|
cell = located[task.id]
|
|
566
618
|
if cell.thin and cell.label not in seen:
|
|
567
619
|
seen.add(cell.label)
|
|
568
|
-
|
|
569
|
-
f"cell {cell.label}: n={cell.n}, effective n={cell.n_eff:.1f}, below {POOL_MIN}.
|
|
570
|
-
f"Pooled with {cell.pooled_from} and interval widened."
|
|
620
|
+
below = (
|
|
621
|
+
f"cell {cell.label}: n={cell.n}, effective n={cell.n_eff:.1f}, below {POOL_MIN}."
|
|
571
622
|
)
|
|
623
|
+
if cell.missing is not None:
|
|
624
|
+
notes.append(_unseen_note(model, task, cell))
|
|
625
|
+
elif cell.label == "all":
|
|
626
|
+
notes.append(f"{below} Interval widened.")
|
|
627
|
+
else:
|
|
628
|
+
mean = "shrunk toward" if model.method.shrink else "taken from"
|
|
629
|
+
notes.append(
|
|
630
|
+
f"{below} Mean {mean} broader cells; residuals pooled from "
|
|
631
|
+
f"{cell.pooled_from}; interval widened."
|
|
632
|
+
)
|
|
572
633
|
if "size" not in model.path and any(task.size for task in tasks):
|
|
573
634
|
notes.append(
|
|
574
635
|
"size ignored: stored size is measured after a run, so it is left out unless you "
|
|
@@ -443,5 +443,127 @@ class PlanDocuments(unittest.TestCase):
|
|
|
443
443
|
self.assertIsNone(tasks[0].model)
|
|
444
444
|
|
|
445
445
|
|
|
446
|
+
class WarningWording(unittest.TestCase):
|
|
447
|
+
"""A cell warning says which level had no recorded run, what the task supplied below it,
|
|
448
|
+
where the mean and the residuals came from, and what is recorded under the parent."""
|
|
449
|
+
|
|
450
|
+
OPUS = "claude-opus-5-5"
|
|
451
|
+
|
|
452
|
+
@staticmethod
|
|
453
|
+
def rows():
|
|
454
|
+
extra = [
|
|
455
|
+
row(task_kind="migration", risk="high", duration_s=900 + i, batch=f"{0xA00 + i:012x}")
|
|
456
|
+
for i in range(3)
|
|
457
|
+
]
|
|
458
|
+
extra += [
|
|
459
|
+
row(model="gpt-x", duration_s=1800 + i, batch=f"{0xB00 + i:012x}") for i in range(2)
|
|
460
|
+
]
|
|
461
|
+
# feature 182 (claude-opus-5-5 180, gpt-x 2), review 180, docs 120, migration 3: 485
|
|
462
|
+
return dataset(sessions=60) + extra
|
|
463
|
+
|
|
464
|
+
@classmethod
|
|
465
|
+
def setUpClass(cls):
|
|
466
|
+
cls.shrunk = fit(cls.rows(), method="shrunk")
|
|
467
|
+
cls.backoff = fit(cls.rows(), method="backoff")
|
|
468
|
+
|
|
469
|
+
def warnings(self, task, model=None):
|
|
470
|
+
return est.estimate(model or self.shrunk, [task], 1, draws=200, seed=1).warnings
|
|
471
|
+
|
|
472
|
+
def test_unseen_kind_with_a_model(self):
|
|
473
|
+
self.assertEqual(
|
|
474
|
+
self.warnings(Task("a", "refactor", model=self.OPUS)),
|
|
475
|
+
(
|
|
476
|
+
"cell refactor: n=0. No recorded run has kind refactor, so risk normal and model "
|
|
477
|
+
"claude-opus-5-5 were not used. Mean and residuals from all (n=485); interval "
|
|
478
|
+
"widened by the spread between kinds. Recorded kinds: feature (182), "
|
|
479
|
+
"review (180), docs (120), migration (3).",
|
|
480
|
+
),
|
|
481
|
+
)
|
|
482
|
+
|
|
483
|
+
def test_unseen_kind_without_a_model(self):
|
|
484
|
+
self.assertEqual(
|
|
485
|
+
self.warnings(Task("a", "refactor")),
|
|
486
|
+
(
|
|
487
|
+
"cell refactor: n=0. No recorded run has kind refactor, so risk normal was not "
|
|
488
|
+
"used. Mean and residuals from all (n=485); interval widened by the spread "
|
|
489
|
+
"between kinds. Recorded kinds: feature (182), review (180), docs (120), "
|
|
490
|
+
"migration (3).",
|
|
491
|
+
),
|
|
492
|
+
)
|
|
493
|
+
|
|
494
|
+
def test_unseen_risk(self):
|
|
495
|
+
self.assertEqual(
|
|
496
|
+
self.warnings(Task("a", "feature", risk="high", model=self.OPUS)),
|
|
497
|
+
(
|
|
498
|
+
"cell feature x high: n=0. No recorded run has kind feature at risk high, so "
|
|
499
|
+
"model claude-opus-5-5 was not used. Mean and residuals from feature (n=182); "
|
|
500
|
+
"interval widened by the spread between risk levels. Recorded risk levels under "
|
|
501
|
+
"feature: normal (182).",
|
|
502
|
+
),
|
|
503
|
+
)
|
|
504
|
+
self.assertEqual(
|
|
505
|
+
self.warnings(Task("a", "feature", risk="high")),
|
|
506
|
+
(
|
|
507
|
+
"cell feature x high: n=0. No recorded run has kind feature at risk high. Mean "
|
|
508
|
+
"and residuals from feature (n=182); interval widened by the spread between risk "
|
|
509
|
+
"levels. Recorded risk levels under feature: normal (182).",
|
|
510
|
+
),
|
|
511
|
+
)
|
|
512
|
+
|
|
513
|
+
def test_unseen_model_lists_the_recorded_models(self):
|
|
514
|
+
self.assertEqual(
|
|
515
|
+
self.warnings(Task("a", "feature", model="claude-sonnet-5-5")),
|
|
516
|
+
(
|
|
517
|
+
"cell feature x normal x claude-sonnet-5-5: n=0. No recorded run has kind "
|
|
518
|
+
"feature at risk normal with model claude-sonnet-5-5. Mean and residuals from "
|
|
519
|
+
"feature x normal (n=182); interval widened by the spread between models. "
|
|
520
|
+
"Recorded models under feature x normal: claude-opus-5-5 (180), gpt-x (2).",
|
|
521
|
+
),
|
|
522
|
+
)
|
|
523
|
+
|
|
524
|
+
def test_unseen_model_under_a_thin_parent_names_both_sources(self):
|
|
525
|
+
self.assertEqual(
|
|
526
|
+
self.warnings(Task("a", "migration", risk="high", model="claude-sonnet-5-5")),
|
|
527
|
+
(
|
|
528
|
+
"cell migration x high x claude-sonnet-5-5: n=0. No recorded run has kind "
|
|
529
|
+
"migration at risk high with model claude-sonnet-5-5. Mean from migration x high "
|
|
530
|
+
"(n=3); residuals pooled from all; interval widened by the spread between "
|
|
531
|
+
"models. Recorded models under migration x high: claude-opus-5-5 (3).",
|
|
532
|
+
),
|
|
533
|
+
)
|
|
534
|
+
|
|
535
|
+
def test_a_thin_seen_cell_under_shrinkage(self):
|
|
536
|
+
self.assertEqual(
|
|
537
|
+
self.warnings(Task("a", "feature", model="gpt-x")),
|
|
538
|
+
(
|
|
539
|
+
"cell feature x normal x gpt-x: n=2, effective n=2.0, below 30. Mean shrunk "
|
|
540
|
+
"toward broader cells; residuals pooled from feature x normal; interval widened.",
|
|
541
|
+
),
|
|
542
|
+
)
|
|
543
|
+
|
|
544
|
+
def test_a_thin_seen_cell_under_backoff(self):
|
|
545
|
+
self.assertEqual(
|
|
546
|
+
self.warnings(Task("a", "feature", model="gpt-x"), model=self.backoff),
|
|
547
|
+
(
|
|
548
|
+
"cell feature x normal x gpt-x: n=2, effective n=2.0, below 30. Mean taken from "
|
|
549
|
+
"broader cells; residuals pooled from feature x normal; interval widened.",
|
|
550
|
+
),
|
|
551
|
+
)
|
|
552
|
+
|
|
553
|
+
def test_a_thin_global_cell(self):
|
|
554
|
+
model = fit(plain_rows(20, 60), method="global")
|
|
555
|
+
warnings = est.estimate(model, [Task("a", "feature")], 1, draws=200).warnings
|
|
556
|
+
self.assertEqual(
|
|
557
|
+
warnings[0], "cell all: n=20, effective n=20.0, below 30. Interval widened."
|
|
558
|
+
)
|
|
559
|
+
self.assertEqual(self.warnings(Task("a", "refactor"), fit(self.rows(), "global")), ())
|
|
560
|
+
|
|
561
|
+
def test_a_seen_cell_records_no_missing_level(self):
|
|
562
|
+
cell = self.shrunk.locate({"task_kind": "feature", "risk": "normal", "model": self.OPUS})
|
|
563
|
+
self.assertEqual((cell.missing, cell.parent, cell.parent_n), (None, "all", 0))
|
|
564
|
+
cell = self.shrunk.locate({"task_kind": "feature", "risk": "high", "model": self.OPUS})
|
|
565
|
+
self.assertEqual((cell.missing, cell.parent, cell.parent_n), ("risk", "feature", 182))
|
|
566
|
+
|
|
567
|
+
|
|
446
568
|
if __name__ == "__main__":
|
|
447
569
|
unittest.main()
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
"""The shipped method's numbers, pinned on SYNTHETIC rows, not only its name.
|
|
2
|
+
|
|
3
|
+
The literals below were generated by running `main`'s `src/` at v0.11.1 (714adc9) on these rows.
|
|
4
|
+
A change to `fit`, `locate`, the random draws or the simulation moves them, and that is a method
|
|
5
|
+
change under the frozen selection. Nothing here reads the seed data, so a data release never
|
|
6
|
+
re-pins this file.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import math
|
|
10
|
+
import random
|
|
11
|
+
import unittest
|
|
12
|
+
|
|
13
|
+
from agentpace import estimate as est
|
|
14
|
+
from agentpace.estimate import Task
|
|
15
|
+
from tests.synthetic import dataset, row
|
|
16
|
+
|
|
17
|
+
REL = 1e-9
|
|
18
|
+
OPUS = "claude-opus-5-5"
|
|
19
|
+
SONNET = "claude-sonnet-5-5"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def rows():
|
|
23
|
+
"""The synthetic dataset's three cells, a thin `migration` kind and a thin second model."""
|
|
24
|
+
migrations = [
|
|
25
|
+
row(
|
|
26
|
+
task_kind="migration",
|
|
27
|
+
risk="high",
|
|
28
|
+
size="L",
|
|
29
|
+
duration_s=seconds,
|
|
30
|
+
batch=f"{0xA00 + i:012x}",
|
|
31
|
+
)
|
|
32
|
+
for i, seconds in enumerate((900, 1500, 2400))
|
|
33
|
+
]
|
|
34
|
+
other_model = [
|
|
35
|
+
row(model="gpt-x", duration_s=seconds, batch=f"{0xB00 + i:012x}")
|
|
36
|
+
for i, seconds in enumerate((1800, 2700))
|
|
37
|
+
]
|
|
38
|
+
return dataset(seed=7, sessions=60) + migrations + other_model
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
QUERIES = {
|
|
42
|
+
"seen": {"task_kind": "feature", "risk": "normal", "model": OPUS},
|
|
43
|
+
"unseen_kind": {"task_kind": "refactor", "risk": "normal", "model": OPUS},
|
|
44
|
+
"unseen_risk": {"task_kind": "feature", "risk": "high", "model": OPUS},
|
|
45
|
+
"unseen_model": {"task_kind": "docs", "risk": "normal", "model": SONNET},
|
|
46
|
+
"thin_model": {"task_kind": "feature", "risk": "normal", "model": "gpt-x"},
|
|
47
|
+
"thin_branch": {"task_kind": "migration", "risk": "high", "model": OPUS},
|
|
48
|
+
"no_model": {"task_kind": "review", "risk": "low", "model": None},
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def plan_tasks():
|
|
53
|
+
"""Six tasks, one per cell above that a plan can reach, with dependencies."""
|
|
54
|
+
return [
|
|
55
|
+
Task("api", "feature", model=OPUS),
|
|
56
|
+
Task("harden", "feature", risk="high", model=OPUS, depends_on=("api",)),
|
|
57
|
+
Task("port", "migration", risk="high", model=OPUS, depends_on=("api",)),
|
|
58
|
+
Task("guide", "docs", model=SONNET, depends_on=("api",)),
|
|
59
|
+
Task("probe", "feature", model="gpt-x"),
|
|
60
|
+
Task("rework", "refactor", model=OPUS, depends_on=("port", "probe")),
|
|
61
|
+
]
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
# Every task's risk is one the rows can record for its kind, so the plan boundary keeps it as
|
|
65
|
+
# written; omitted fields take the document's defaults.
|
|
66
|
+
CANONICAL_DOCUMENT = {
|
|
67
|
+
"agent_cap": 2,
|
|
68
|
+
"model": OPUS,
|
|
69
|
+
"tasks": [
|
|
70
|
+
{"id": "api", "kind": "feature", "risk": "normal"},
|
|
71
|
+
{"id": "port", "kind": "migration", "risk": "high", "depends_on": ["api"]},
|
|
72
|
+
{"id": "scan", "kind": "review", "risk": "low", "model": None},
|
|
73
|
+
{"id": "guide", "kind": "docs", "risk": "normal", "model": SONNET, "depends_on": ["api"]},
|
|
74
|
+
{"id": "probe", "kind": "feature", "risk": "normal", "model": "gpt-x"},
|
|
75
|
+
{"id": "rework", "kind": "refactor", "risk": "normal", "depends_on": ["port", "probe"]},
|
|
76
|
+
],
|
|
77
|
+
}
|
|
78
|
+
CANONICAL_TASKS = [
|
|
79
|
+
Task("api", "feature", risk="normal", model=OPUS),
|
|
80
|
+
Task("port", "migration", risk="high", model=OPUS, depends_on=("api",)),
|
|
81
|
+
Task("scan", "review", risk="low", model=None),
|
|
82
|
+
Task("guide", "docs", risk="normal", model=SONNET, depends_on=("api",)),
|
|
83
|
+
Task("probe", "feature", risk="normal", model="gpt-x"),
|
|
84
|
+
Task("rework", "refactor", risk="normal", model=OPUS, depends_on=("port", "probe")),
|
|
85
|
+
]
|
|
86
|
+
|
|
87
|
+
# --- generated literals: (mu, v, n, n_eff, label, pooled_from, thin) ---
|
|
88
|
+
LOCATED = {
|
|
89
|
+
"seen": (
|
|
90
|
+
5.748561917464507,
|
|
91
|
+
0.004430156024745457,
|
|
92
|
+
180,
|
|
93
|
+
180.0,
|
|
94
|
+
"feature x normal x claude-opus-5-5",
|
|
95
|
+
"feature x normal x claude-opus-5-5",
|
|
96
|
+
False,
|
|
97
|
+
),
|
|
98
|
+
"unseen_kind": (5.021725817345783, 0.654926996567019, 0, 0.0, "refactor", "all", True),
|
|
99
|
+
"unseen_risk": (
|
|
100
|
+
5.764844039026677,
|
|
101
|
+
0.004447676968646228,
|
|
102
|
+
0,
|
|
103
|
+
0.0,
|
|
104
|
+
"feature x high",
|
|
105
|
+
"feature",
|
|
106
|
+
True,
|
|
107
|
+
),
|
|
108
|
+
"unseen_model": (
|
|
109
|
+
4.195219435035191,
|
|
110
|
+
1.7053968486894486,
|
|
111
|
+
0,
|
|
112
|
+
0.0,
|
|
113
|
+
"docs x normal x claude-sonnet-5-5",
|
|
114
|
+
"docs x normal",
|
|
115
|
+
True,
|
|
116
|
+
),
|
|
117
|
+
"thin_model": (
|
|
118
|
+
7.329953035243061,
|
|
119
|
+
0.3237615584003151,
|
|
120
|
+
2,
|
|
121
|
+
2.0,
|
|
122
|
+
"feature x normal x gpt-x",
|
|
123
|
+
"feature x normal",
|
|
124
|
+
True,
|
|
125
|
+
),
|
|
126
|
+
"thin_branch": (
|
|
127
|
+
7.208790485554894,
|
|
128
|
+
0.23389289123850235,
|
|
129
|
+
3,
|
|
130
|
+
3.0,
|
|
131
|
+
"migration x high x claude-opus-5-5",
|
|
132
|
+
"all",
|
|
133
|
+
True,
|
|
134
|
+
),
|
|
135
|
+
"no_model": (
|
|
136
|
+
4.785607835027801,
|
|
137
|
+
0.004496756296448909,
|
|
138
|
+
180,
|
|
139
|
+
180.0,
|
|
140
|
+
"review x low",
|
|
141
|
+
"review x low",
|
|
142
|
+
False,
|
|
143
|
+
),
|
|
144
|
+
}
|
|
145
|
+
CELL_QUANTILES = {
|
|
146
|
+
"seen": (326.37855852487576, 678.9109715542419, 949.5685633561789),
|
|
147
|
+
"unseen_kind": (148.62660109580304, 410.3270975996236, 678.7296559946502),
|
|
148
|
+
"unseen_risk": (331.7379406557295, 689.9707766730423, 965.0769989527402),
|
|
149
|
+
"unseen_model": (62.435342338073596, 250.72378246646713, 472.295285463006),
|
|
150
|
+
"thin_model": (1580.3187981430283, 3695.0983044974687, 5539.908213468239),
|
|
151
|
+
"thin_branch": (1350.1751834448505, 3166.067208416465, 4825.334525711538),
|
|
152
|
+
"no_model": (122.95559172286664, 255.337383795307, 372.2208327109682),
|
|
153
|
+
}
|
|
154
|
+
PLAN = (3218.226765188425, 5954.69319603287, 8446.562927367682)
|
|
155
|
+
TASK_P50 = {
|
|
156
|
+
"api": 320.7033144630816,
|
|
157
|
+
"harden": 315.9384124292877,
|
|
158
|
+
"port": 1340.372829954017,
|
|
159
|
+
"guide": 65.31036298267271,
|
|
160
|
+
"probe": 1516.3577809135163,
|
|
161
|
+
"rework": 165.70645375745192,
|
|
162
|
+
}
|
|
163
|
+
# --- end of generated literals ---
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
class FrozenMethod(unittest.TestCase):
|
|
167
|
+
@classmethod
|
|
168
|
+
def setUpClass(cls):
|
|
169
|
+
cls.model = est.fit(rows(), method=est.DEFAULT_METHOD)
|
|
170
|
+
|
|
171
|
+
def assertClose(self, actual, expected):
|
|
172
|
+
self.assertTrue(
|
|
173
|
+
math.isclose(actual, expected, rel_tol=REL, abs_tol=0.0),
|
|
174
|
+
f"{actual!r} != {expected!r}",
|
|
175
|
+
)
|
|
176
|
+
|
|
177
|
+
def test_the_fixture_covers_every_query(self):
|
|
178
|
+
self.assertEqual(set(LOCATED), set(QUERIES))
|
|
179
|
+
self.assertEqual(set(CELL_QUANTILES), set(QUERIES))
|
|
180
|
+
self.assertEqual(set(TASK_P50), {task.id for task in plan_tasks()})
|
|
181
|
+
|
|
182
|
+
def test_located_cells(self):
|
|
183
|
+
for name, query in QUERIES.items():
|
|
184
|
+
with self.subTest(query=name):
|
|
185
|
+
cell = self.model.locate(query)
|
|
186
|
+
mu, v, n, n_eff, label, pooled_from, thin = LOCATED[name]
|
|
187
|
+
self.assertClose(cell.mu, mu)
|
|
188
|
+
self.assertClose(cell.v, v)
|
|
189
|
+
self.assertEqual(cell.n, n)
|
|
190
|
+
self.assertClose(cell.n_eff, n_eff)
|
|
191
|
+
self.assertEqual(cell.label, label)
|
|
192
|
+
self.assertEqual(cell.pooled_from, pooled_from)
|
|
193
|
+
self.assertIs(cell.thin, thin)
|
|
194
|
+
|
|
195
|
+
def test_cell_quantiles(self):
|
|
196
|
+
for name, query in QUERIES.items():
|
|
197
|
+
with self.subTest(query=name):
|
|
198
|
+
actual = est.cell_quantiles(self.model, query, 2000, random.Random(1))
|
|
199
|
+
for value, expected in zip(actual, CELL_QUANTILES[name], strict=True):
|
|
200
|
+
self.assertClose(value, expected)
|
|
201
|
+
|
|
202
|
+
def test_plan(self):
|
|
203
|
+
plan = est.estimate(self.model, plan_tasks(), agent_cap=2, draws=2000, seed=5)
|
|
204
|
+
for value, expected in zip((plan.p50, plan.p80, plan.p90), PLAN, strict=True):
|
|
205
|
+
self.assertClose(value, expected)
|
|
206
|
+
self.assertEqual([task.id for task in plan.tasks], [task.id for task in plan_tasks()])
|
|
207
|
+
for task in plan.tasks:
|
|
208
|
+
with self.subTest(task=task.id):
|
|
209
|
+
self.assertClose(task.p50, TASK_P50[task.id])
|
|
210
|
+
|
|
211
|
+
def test_a_canonical_plan_document_equals_its_tasks(self):
|
|
212
|
+
tasks, cap = est.tasks_from_json(CANONICAL_DOCUMENT)
|
|
213
|
+
self.assertEqual(cap, 2)
|
|
214
|
+
|
|
215
|
+
def fields(task):
|
|
216
|
+
return (task.id, task.kind, task.risk, task.size, task.model, task.depends_on)
|
|
217
|
+
|
|
218
|
+
self.assertEqual([fields(task) for task in tasks], [fields(t) for t in CANONICAL_TASKS])
|
|
219
|
+
through_document = est.estimate(self.model, tasks, cap, draws=2000, seed=5)
|
|
220
|
+
direct = est.estimate(self.model, CANONICAL_TASKS, 2, draws=2000, seed=5)
|
|
221
|
+
self.assertEqual(through_document, direct)
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
if __name__ == "__main__":
|
|
225
|
+
unittest.main()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|