aimpg 0.4.2__tar.gz → 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {aimpg-0.4.2 → aimpg-0.5.0}/PKG-INFO +12 -1
- {aimpg-0.4.2 → aimpg-0.5.0}/README.md +11 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/aimpg/cli.py +27 -0
- aimpg-0.5.0/aimpg/defaults.py +158 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/aimpg/replay/picker.py +15 -6
- {aimpg-0.4.2 → aimpg-0.5.0}/aimpg/replay/verify.py +11 -1
- aimpg-0.5.0/aimpg/sci.py +150 -0
- aimpg-0.5.0/aimpg/scoreboard.py +621 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/pyproject.toml +1 -1
- {aimpg-0.4.2 → aimpg-0.5.0}/tests/conftest.py +7 -0
- aimpg-0.5.0/tests/test_defaults.py +94 -0
- aimpg-0.5.0/tests/test_sci.py +63 -0
- aimpg-0.5.0/tests/test_scoreboard.py +201 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/uv.lock +1 -1
- {aimpg-0.4.2 → aimpg-0.5.0}/.github/workflows/publish.yml +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/.github/workflows/test.yml +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/.gitignore +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/LICENSE +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/TODOS.md +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/aimpg/__init__.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/aimpg/attribution.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/aimpg/coach.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/aimpg/codex_logs.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/aimpg/cost.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/aimpg/cursor_usage.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/aimpg/durable.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/aimpg/energy.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/aimpg/equivalence.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/aimpg/equivalences.json +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/aimpg/factors.json +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/aimpg/gitkept.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/aimpg/ledger.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/aimpg/live.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/aimpg/logs.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/aimpg/model.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/aimpg/prices.json +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/aimpg/receipt.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/aimpg/replay/__init__.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/aimpg/replay/cli.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/aimpg/replay/fake_agent.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/aimpg/replay/hints.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/aimpg/replay/proxy.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/aimpg/replay/run.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/aimpg/replay/sandbox.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/aimpg/replay/select.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/aimpg/replay/setups.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/aimpg/replay/stats.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/aimpg/replay/workspace.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/aimpg/share.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/docs/STRATEGY.md +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/docs/designs/aimpg-design.md +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/docs/designs/scoreboard-brief.md +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/evals/attribution_eval.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/evals/label.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/spikes/agent/RESULTS.md +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/spikes/agent/allowlist_proxy.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/spikes/agent/make_profile.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/spikes/egress/RESULTS.md +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/spikes/egress/allowlist_proxy.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/spikes/egress/replay.sb +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/tests/gitrepo.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/tests/test_attribution.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/tests/test_coach.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/tests/test_codex_logs.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/tests/test_cost_and_equivalence.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/tests/test_cursor_usage.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/tests/test_durable.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/tests/test_energy.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/tests/test_eval_scoring.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/tests/test_gitkept.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/tests/test_live.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/tests/test_logs.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/tests/test_perf.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/tests/test_replay_cli.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/tests/test_replay_e2e.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/tests/test_replay_hints.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/tests/test_replay_picker.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/tests/test_replay_proxy.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/tests/test_replay_sandbox.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/tests/test_replay_stats.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/tests/test_sandbox_required.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/tests/test_share.py +0 -0
- {aimpg-0.4.2 → aimpg-0.5.0}/tests/test_verify.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: aimpg
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.0
|
|
4
4
|
Summary: Real-world energy per solved task for AI coding agents
|
|
5
5
|
Project-URL: Homepage, https://github.com/kumarganduri/aimpg
|
|
6
6
|
Project-URL: Issues, https://github.com/kumarganduri/aimpg/issues
|
|
@@ -90,6 +90,8 @@ Use sonnet: it solved 100% of tasks (best: 100%) at 3.1x lower cost per solved t
|
|
|
90
90
|
|
|
91
91
|
It re-does your real past commits with each model in a locked sandbox, and **your own tests decide** what counts as solved.
|
|
92
92
|
|
|
93
|
+
**Make it the team's default:** `aimpg defaults --repo .` reads that evidence and proposes the repo's default model for Claude Code; `--apply` writes it to `.claude/settings.json` (it asks first and keeps a backup). Commit it, and everyone on the repo starts with the model your own tests showed is good enough; anyone can still switch with `/model`.
|
|
94
|
+
|
|
93
95
|
### 4. Verify any efficiency claim on your own code
|
|
94
96
|
Token savers, prompts, hooks, CLAUDE.md files, models, even other agents: one command reruns your real commits both ways in a locked sandbox, and your own tests judge them.
|
|
95
97
|
```bash
|
|
@@ -115,6 +117,8 @@ That's the real result on the author's Legwork repo: no saving shown, far from t
|
|
|
115
117
|
|
|
116
118
|
Every run writes a **record** (`~/.aimpg/verify/*.record.json`) holding versions, both setups, each run's tokens, $ and outcome, and the verdict, but never code, diffs or commit messages. Repo and commits are hashed unless you add `--public`. Anyone can recheck the math and the record's consistency for free with `aimpg verify --check record.json` (it can't prove the runs happened), and rerun a public record on their own machine with `aimpg verify --rerun record.json --repo <clone>`, which first shows every command, hook and prompt the record would run and asks before running them. Private records keep only fingerprints (SHA-256) of your prompts, CLAUDE.md, settings and commands, never the text. Records also list every redone attempt and excluded commit.
|
|
117
119
|
|
|
120
|
+
**Share it:** `aimpg submit <record>` publishes a stripped-down copy on the [public scoreboard](https://kumarganduri.github.io/aimpg-scoreboard) (no code, prompts, commit ids or repo name). It shows the exact file first and asks, then opens a pull request. Results only count once enough repos and people back them, and only independent reruns make the headline.
|
|
121
|
+
|
|
118
122
|
Same model: the energy test decides (it must hold at every corner of the energy ranges). Different models or agents: cost per solved task decides, because model sizes are secret. A challenger that solves more than 10 points fewer tasks is never "supported". Codex runs are measured from its own logs and priced from OpenAI's published prices. Codex logs in with an OpenAI **API key** (a ChatGPT subscription login can't be used inside the sandbox); the key is written to the run's throwaway folder and deleted when the run ends. Other commands are judged on solve rate and time only.
|
|
119
123
|
|
|
120
124
|
### 5. Show the cost of each pull request
|
|
@@ -131,6 +135,13 @@ aimpg export --csv ai-energy.csv --no-subjects --no-authors
|
|
|
131
135
|
```
|
|
132
136
|
One row per AI-assisted commit: date, repo, energy range, CO₂ range and cost. Teams can combine everyone's files. The privacy switches drop commit messages and author emails.
|
|
133
137
|
|
|
138
|
+
For a standard figure, `aimpg sci --json sci.json` reports your carbon intensity in the Green Software Foundation's [SCI for AI](https://github.com/Green-Software-Foundation/sci-ai/blob/main/SPEC.md) format (Consumer SCI). It uses the spec's units: per million tokens, per AI-assisted commit (a "workflow execution" for agentic AI), and per working change. The JSON lists the boundary, every input with its source, and what's left out. Embodied hardware emissions come from Google's measured per-prompt figures; grid intensity is location-based, because the spec doesn't let offsets or renewable certificates lower the score.
|
|
139
|
+
```
|
|
140
|
+
per million tokens 2.18–21 g CO₂e
|
|
141
|
+
per AI-assisted commit 19.3–185 g CO₂e
|
|
142
|
+
per working change available from 2026-10-13
|
|
143
|
+
```
|
|
144
|
+
|
|
134
145
|
## Privacy
|
|
135
146
|
|
|
136
147
|
`report`, `pr` and `export` run entirely on your machine with no network calls (except `--fetch` for `git fetch`, and `pr --post`, which talks to GitHub). Your code and prompts are never uploaded. Replays send only the agent's own work to the model API, from a sandbox that can't read your home folder.
|
|
@@ -70,6 +70,8 @@ Use sonnet: it solved 100% of tasks (best: 100%) at 3.1x lower cost per solved t
|
|
|
70
70
|
|
|
71
71
|
It re-does your real past commits with each model in a locked sandbox, and **your own tests decide** what counts as solved.
|
|
72
72
|
|
|
73
|
+
**Make it the team's default:** `aimpg defaults --repo .` reads that evidence and proposes the repo's default model for Claude Code; `--apply` writes it to `.claude/settings.json` (it asks first and keeps a backup). Commit it, and everyone on the repo starts with the model your own tests showed is good enough; anyone can still switch with `/model`.
|
|
74
|
+
|
|
73
75
|
### 4. Verify any efficiency claim on your own code
|
|
74
76
|
Token savers, prompts, hooks, CLAUDE.md files, models, even other agents: one command reruns your real commits both ways in a locked sandbox, and your own tests judge them.
|
|
75
77
|
```bash
|
|
@@ -95,6 +97,8 @@ That's the real result on the author's Legwork repo: no saving shown, far from t
|
|
|
95
97
|
|
|
96
98
|
Every run writes a **record** (`~/.aimpg/verify/*.record.json`) holding versions, both setups, each run's tokens, $ and outcome, and the verdict, but never code, diffs or commit messages. Repo and commits are hashed unless you add `--public`. Anyone can recheck the math and the record's consistency for free with `aimpg verify --check record.json` (it can't prove the runs happened), and rerun a public record on their own machine with `aimpg verify --rerun record.json --repo <clone>`, which first shows every command, hook and prompt the record would run and asks before running them. Private records keep only fingerprints (SHA-256) of your prompts, CLAUDE.md, settings and commands, never the text. Records also list every redone attempt and excluded commit.
|
|
97
99
|
|
|
100
|
+
**Share it:** `aimpg submit <record>` publishes a stripped-down copy on the [public scoreboard](https://kumarganduri.github.io/aimpg-scoreboard) (no code, prompts, commit ids or repo name). It shows the exact file first and asks, then opens a pull request. Results only count once enough repos and people back them, and only independent reruns make the headline.
|
|
101
|
+
|
|
98
102
|
Same model: the energy test decides (it must hold at every corner of the energy ranges). Different models or agents: cost per solved task decides, because model sizes are secret. A challenger that solves more than 10 points fewer tasks is never "supported". Codex runs are measured from its own logs and priced from OpenAI's published prices. Codex logs in with an OpenAI **API key** (a ChatGPT subscription login can't be used inside the sandbox); the key is written to the run's throwaway folder and deleted when the run ends. Other commands are judged on solve rate and time only.
|
|
99
103
|
|
|
100
104
|
### 5. Show the cost of each pull request
|
|
@@ -111,6 +115,13 @@ aimpg export --csv ai-energy.csv --no-subjects --no-authors
|
|
|
111
115
|
```
|
|
112
116
|
One row per AI-assisted commit: date, repo, energy range, CO₂ range and cost. Teams can combine everyone's files. The privacy switches drop commit messages and author emails.
|
|
113
117
|
|
|
118
|
+
For a standard figure, `aimpg sci --json sci.json` reports your carbon intensity in the Green Software Foundation's [SCI for AI](https://github.com/Green-Software-Foundation/sci-ai/blob/main/SPEC.md) format (Consumer SCI). It uses the spec's units: per million tokens, per AI-assisted commit (a "workflow execution" for agentic AI), and per working change. The JSON lists the boundary, every input with its source, and what's left out. Embodied hardware emissions come from Google's measured per-prompt figures; grid intensity is location-based, because the spec doesn't let offsets or renewable certificates lower the score.
|
|
119
|
+
```
|
|
120
|
+
per million tokens 2.18–21 g CO₂e
|
|
121
|
+
per AI-assisted commit 19.3–185 g CO₂e
|
|
122
|
+
per working change available from 2026-10-13
|
|
123
|
+
```
|
|
124
|
+
|
|
114
125
|
## Privacy
|
|
115
126
|
|
|
116
127
|
`report`, `pr` and `export` run entirely on your machine with no network calls (except `--fetch` for `git fetch`, and `pr --post`, which talks to GitHub). Your code and prompts are never uploaded. Replays send only the agent's own work to the model API, from a sandbox that can't read your home folder.
|
|
@@ -20,6 +20,8 @@ from aimpg.gitkept import DAY
|
|
|
20
20
|
from aimpg.logs import DEFAULT_ROOT, iter_log_files, parse_logs
|
|
21
21
|
from aimpg.receipt import render
|
|
22
22
|
from aimpg.replay import cli as replay_cli
|
|
23
|
+
from aimpg import defaults as defaults_mod
|
|
24
|
+
from aimpg import scoreboard as scoreboard_mod
|
|
23
25
|
from aimpg.replay import verify as verify_mod
|
|
24
26
|
|
|
25
27
|
|
|
@@ -65,6 +67,11 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
65
67
|
export.add_argument("--no-subjects", action="store_true", help="leave commit messages out (privacy)")
|
|
66
68
|
export.add_argument("--no-authors", action="store_true", help="leave author emails out (privacy)")
|
|
67
69
|
|
|
70
|
+
sci_p = sub.add_parser("sci", help="carbon intensity in the GSF SCI for AI format (per token, commit, working change)")
|
|
71
|
+
_common(sci_p, 30)
|
|
72
|
+
sci_p.add_argument("--grid", type=float, default=None, help="grid intensity in g CO2e/kWh (default: world average)")
|
|
73
|
+
sci_p.add_argument("--json", type=Path, default=None, help="also write the full disclosure as JSON")
|
|
74
|
+
|
|
68
75
|
live = sub.add_parser("live", help="live status line + post-commit note inside Claude Code")
|
|
69
76
|
live.add_argument("action", choices=("install", "uninstall"))
|
|
70
77
|
sub.add_parser("statusline", help=argparse.SUPPRESS) # called by Claude Code
|
|
@@ -73,11 +80,17 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
73
80
|
|
|
74
81
|
replay_cli.add_parser(sub)
|
|
75
82
|
verify_mod.add_parser(sub)
|
|
83
|
+
scoreboard_mod.add_parsers(sub)
|
|
84
|
+
defaults_mod.add_parser(sub)
|
|
76
85
|
args = parser.parse_args(argv)
|
|
77
86
|
if args.command == "replay":
|
|
78
87
|
return replay_cli.main(args)
|
|
79
88
|
if args.command == "verify":
|
|
80
89
|
return verify_mod.main(args)
|
|
90
|
+
if args.command in ("submit", "scoreboard"):
|
|
91
|
+
return scoreboard_mod.main(args)
|
|
92
|
+
if args.command == "defaults":
|
|
93
|
+
return defaults_mod.main(args)
|
|
81
94
|
if args.command in ("statusline", "hook", "live"):
|
|
82
95
|
from aimpg import live as live_mod
|
|
83
96
|
|
|
@@ -140,6 +153,20 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
140
153
|
from aimpg import durable, ledger
|
|
141
154
|
from aimpg.receipt import task_energy
|
|
142
155
|
|
|
156
|
+
if args.command == "sci":
|
|
157
|
+
from aimpg import sci
|
|
158
|
+
|
|
159
|
+
ledger.record(task_energy(attribution.tasks))
|
|
160
|
+
judged = durable.judge(ledger.load(), now)
|
|
161
|
+
in_window = [r for r in parsed.requests if r.ts >= since]
|
|
162
|
+
result = sci.compute(in_window, len(attribution.tasks), judged,
|
|
163
|
+
grid=args.grid if args.grid is not None else sci.default_grid(), period=(since, now))
|
|
164
|
+
sys.stdout.write(sci.render(result))
|
|
165
|
+
if args.json:
|
|
166
|
+
sci.write(result, args.json)
|
|
167
|
+
print(f"\nFull disclosure (boundary, inputs, sources, ranges): {args.json}")
|
|
168
|
+
return 0
|
|
169
|
+
|
|
143
170
|
ledger.record(task_energy(attribution.tasks))
|
|
144
171
|
judged = durable.judge(ledger.load(), now)
|
|
145
172
|
sys.stdout.write(render(parsed, attribution, since, now, judged))
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
"""`aimpg defaults`: turn a repo's replay evidence into a Claude Code setting the whole team gets.
|
|
2
|
+
|
|
3
|
+
aimpg defaults --repo . what the evidence says, and the setting it supports
|
|
4
|
+
aimpg defaults --repo . --apply write it to <repo>/.claude/settings.json (asks; keeps a backup)
|
|
5
|
+
|
|
6
|
+
Evidence, newest first, for this repo only:
|
|
7
|
+
* model-picker runs (`aimpg replay models`): the cheapest model per solved
|
|
8
|
+
task that solves within 10 points of the best;
|
|
9
|
+
* `aimpg verify` records comparing two Claude models: a SUPPORTED cheaper challenger.
|
|
10
|
+
|
|
11
|
+
The setting is the project's default model. It's committed with the repo, so
|
|
12
|
+
everyone on the team starts with it; anyone can still switch with /model, and
|
|
13
|
+
a personal choice goes in .claude/settings.local.json instead. Evidence from
|
|
14
|
+
fewer than 20 runs per model is marked tentative, and nothing is written
|
|
15
|
+
without a yes.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import json
|
|
21
|
+
import shutil
|
|
22
|
+
import time
|
|
23
|
+
from dataclasses import dataclass
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
|
|
26
|
+
from aimpg.replay import picker
|
|
27
|
+
from aimpg.replay.run import load_results
|
|
28
|
+
|
|
29
|
+
REPLAY_STATE = Path.home() / ".aimpg" / "replay"
|
|
30
|
+
VERIFY_STATE = Path.home() / ".aimpg" / "verify"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass
|
|
34
|
+
class Evidence:
|
|
35
|
+
source: str # file it came from
|
|
36
|
+
model: str # model id it supports as the default
|
|
37
|
+
summary: str
|
|
38
|
+
runs_per_model: int
|
|
39
|
+
when: float
|
|
40
|
+
|
|
41
|
+
@property
|
|
42
|
+
def tentative(self) -> bool:
|
|
43
|
+
return self.runs_per_model < picker.MIN_RUNS
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def gather(repo: Path) -> list[Evidence]:
|
|
47
|
+
repo = repo.resolve()
|
|
48
|
+
found: list[Evidence] = []
|
|
49
|
+
folder = REPLAY_STATE / repo.name
|
|
50
|
+
for path in sorted(folder.glob("models-*.jsonl")) if folder.is_dir() else []:
|
|
51
|
+
records, _ = load_results(path)
|
|
52
|
+
records = [r for r in records if Path(r.repo).resolve() == repo]
|
|
53
|
+
if not records:
|
|
54
|
+
continue
|
|
55
|
+
table = picker.rows(records)
|
|
56
|
+
choice = picker.pick(table)
|
|
57
|
+
if choice is None:
|
|
58
|
+
continue
|
|
59
|
+
found.append(Evidence(str(path), choice.model, picker.recommend(table), min(r.runs for r in table), path.stat().st_mtime))
|
|
60
|
+
found += _from_verify(repo)
|
|
61
|
+
return sorted(found, key=lambda e: e.when, reverse=True)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _from_verify(repo: Path) -> list[Evidence]:
|
|
65
|
+
from aimpg.scoreboard import repo_id
|
|
66
|
+
|
|
67
|
+
rid = repo_id(str(repo))
|
|
68
|
+
out = []
|
|
69
|
+
for path in sorted(VERIFY_STATE.glob("*.record.json")) if VERIFY_STATE.is_dir() else []:
|
|
70
|
+
try:
|
|
71
|
+
rec = json.loads(path.read_text())
|
|
72
|
+
except ValueError:
|
|
73
|
+
continue
|
|
74
|
+
b, c, v = rec.get("baseline", {}), rec.get("challenger", {}), rec.get("verdict", {})
|
|
75
|
+
if rid is None or rec.get("repo_id") != rid or not (b.get("agent", "claude") == c.get("agent", "claude") == "claude"):
|
|
76
|
+
continue
|
|
77
|
+
models = v.get("challenger", {}).get("models") or []
|
|
78
|
+
if v.get("answer") != "SUPPORTED" or len(models) != 1 or models == v.get("baseline", {}).get("models"):
|
|
79
|
+
continue
|
|
80
|
+
ratio = v.get("cost_ratio")
|
|
81
|
+
summary = (f"verify: {c['name']} vs {b['name']} → SUPPORTED"
|
|
82
|
+
+ (f", {1 - ratio:.0%} lower cost per solved task" if ratio else ""))
|
|
83
|
+
runs = min(v.get("baseline", {}).get("runs", 0), v.get("challenger", {}).get("runs", 0))
|
|
84
|
+
out.append(Evidence(str(path), models[0], summary, runs, path.stat().st_mtime))
|
|
85
|
+
return out
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def settings_path(repo: Path) -> Path:
|
|
89
|
+
return repo / ".claude" / "settings.json"
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def current_model(repo: Path) -> str | None:
|
|
93
|
+
try:
|
|
94
|
+
return json.loads(settings_path(repo).read_text()).get("model")
|
|
95
|
+
except (OSError, ValueError, AttributeError):
|
|
96
|
+
return None
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def apply(repo: Path, model: str) -> Path | None:
|
|
100
|
+
"""Merge {"model": model} into the project settings. Returns the backup path, if a file existed."""
|
|
101
|
+
path = settings_path(repo)
|
|
102
|
+
data, backup = {}, None
|
|
103
|
+
if path.exists():
|
|
104
|
+
data = json.loads(path.read_text())
|
|
105
|
+
if not isinstance(data, dict):
|
|
106
|
+
raise ValueError(f"{path} isn't a JSON object")
|
|
107
|
+
backup = path.with_name(f"settings.json.aimpg-backup-{time.strftime('%Y%m%d-%H%M%S')}")
|
|
108
|
+
shutil.copy2(path, backup)
|
|
109
|
+
data["model"] = model
|
|
110
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
111
|
+
path.write_text(json.dumps(data, indent=2) + "\n")
|
|
112
|
+
return backup
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def add_parser(sub) -> None:
|
|
116
|
+
d = sub.add_parser("defaults", help="turn replay evidence into the repo's default Claude Code model (asks first)")
|
|
117
|
+
d.add_argument("--repo", type=Path, default=Path("."))
|
|
118
|
+
d.add_argument("--apply", action="store_true", help="write it to <repo>/.claude/settings.json")
|
|
119
|
+
d.add_argument("--yes", action="store_true", help="skip the confirmation")
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def main(args) -> int:
|
|
123
|
+
repo = args.repo.expanduser().resolve()
|
|
124
|
+
evidence = gather(repo)
|
|
125
|
+
now = current_model(repo)
|
|
126
|
+
print(f"Repo: {repo}")
|
|
127
|
+
print(f"Current team default model: {now or 'none set (each person uses their own)'}")
|
|
128
|
+
if not evidence:
|
|
129
|
+
print("\nNo replay evidence for this repo yet. Find out which model is good enough on your own commits:")
|
|
130
|
+
print(f" aimpg replay select {repo}")
|
|
131
|
+
print(f" aimpg replay models {repo} --cap 10 --task-mode tests")
|
|
132
|
+
return 0
|
|
133
|
+
print("\nEvidence (newest first):")
|
|
134
|
+
for e in evidence:
|
|
135
|
+
print(f" • {e.summary}" + (" (tentative)" if e.tentative and "Tentative" not in e.summary else ""))
|
|
136
|
+
print(f" from {e.source}")
|
|
137
|
+
top = evidence[0]
|
|
138
|
+
if now == top.model:
|
|
139
|
+
print(f"\nThe default already matches the evidence: {now}.")
|
|
140
|
+
return 0
|
|
141
|
+
print(f"\nProposed: {settings_path(repo)}")
|
|
142
|
+
print(f' "model": "{top.model}"' + (f' (was "{now}")' if now else ""))
|
|
143
|
+
print("Commit it and everyone on the repo starts with this model; anyone can still switch with /model.")
|
|
144
|
+
if top.tentative:
|
|
145
|
+
print(f"Tentative: fewer than {picker.MIN_RUNS} runs per model. A bigger run makes this safer to roll out.")
|
|
146
|
+
if not args.apply:
|
|
147
|
+
print("\nTo write it: aimpg defaults --apply" + (f" --repo {args.repo}" if args.repo != Path(".") else ""))
|
|
148
|
+
return 0
|
|
149
|
+
if not args.yes and input("Write it? [y/N] ").strip().lower() != "y":
|
|
150
|
+
print("Stopped. Nothing was changed.")
|
|
151
|
+
return 1
|
|
152
|
+
try:
|
|
153
|
+
backup = apply(repo, top.model)
|
|
154
|
+
except ValueError as exc:
|
|
155
|
+
print(f"Can't update the settings file: {exc}")
|
|
156
|
+
return 1
|
|
157
|
+
print(f"Written. {'Backup: ' + str(backup) if backup else ''}\nReview and commit {settings_path(repo).relative_to(repo)}.")
|
|
158
|
+
return 0
|
|
@@ -61,17 +61,26 @@ def rows(records: list[Record]) -> list[ModelRow]:
|
|
|
61
61
|
return sorted(out.values(), key=lambda x: (x.usd_per_solved is None, x.usd_per_solved or 0))
|
|
62
62
|
|
|
63
63
|
|
|
64
|
+
def pick(table: list[ModelRow]) -> ModelRow | None:
|
|
65
|
+
"""The cheapest model per solved task whose solve rate is within 10 points of the best."""
|
|
66
|
+
solved = [r for r in table if r.solved and r.usd_per_solved is not None]
|
|
67
|
+
if not solved:
|
|
68
|
+
return None
|
|
69
|
+
best = max(r.solve_rate for r in table)
|
|
70
|
+
return min((r for r in solved if r.solve_rate >= best - GOOD_ENOUGH), key=lambda r: r.usd_per_solved, default=None)
|
|
71
|
+
|
|
72
|
+
|
|
64
73
|
def recommend(table: list[ModelRow]) -> str:
|
|
65
74
|
solved = [r for r in table if r.solved]
|
|
66
|
-
|
|
75
|
+
choice = pick(table)
|
|
76
|
+
if choice is None:
|
|
67
77
|
return "No model solved any task: nothing to recommend yet."
|
|
68
78
|
best = max(r.solve_rate for r in table)
|
|
69
|
-
|
|
70
|
-
pick = min(good, key=lambda r: r.usd_per_solved)
|
|
79
|
+
pick_ = choice
|
|
71
80
|
priciest = max(solved, key=lambda r: r.usd_per_solved)
|
|
72
|
-
text = f"Use {
|
|
73
|
-
if priciest is not
|
|
74
|
-
ratio = priciest.usd_per_solved /
|
|
81
|
+
text = f"Use {pick_.setup.split('@')[-1]}: it solved {pick_.solve_rate:.0%} of tasks (best: {best:.0%})"
|
|
82
|
+
if priciest is not pick_:
|
|
83
|
+
ratio = priciest.usd_per_solved / pick_.usd_per_solved
|
|
75
84
|
text += f" at {ratio:.1f}x lower cost per solved task than {priciest.setup.split('@')[-1]}"
|
|
76
85
|
text += "."
|
|
77
86
|
if min(r.runs for r in table) < MIN_RUNS:
|
|
@@ -211,6 +211,12 @@ def _spec(setup: S.Setup, public: bool) -> dict:
|
|
|
211
211
|
return spec
|
|
212
212
|
|
|
213
213
|
|
|
214
|
+
def _repo_id(repo: str) -> str | None:
|
|
215
|
+
from aimpg.scoreboard import repo_id
|
|
216
|
+
|
|
217
|
+
return repo_id(repo)
|
|
218
|
+
|
|
219
|
+
|
|
214
220
|
def build_record(records: list[Record], *, claim: str, baseline: S.Setup, challenger: S.Setup, repo: str,
|
|
215
221
|
task_mode: str, public: bool, extra: dict | None = None,
|
|
216
222
|
attempts: list[Record] | None = None, excluded: set[str] | None = None) -> dict:
|
|
@@ -233,6 +239,7 @@ def build_record(records: list[Record], *, claim: str, baseline: S.Setup, challe
|
|
|
233
239
|
"claim": claim,
|
|
234
240
|
"public": public,
|
|
235
241
|
"repo": _remote(repo) if public else hide(os.path.realpath(repo)),
|
|
242
|
+
"repo_id": _repo_id(repo), # keyed with a local secret: lets the scoreboard count repos
|
|
236
243
|
"task_mode": task_mode,
|
|
237
244
|
"versions": _versions(),
|
|
238
245
|
"baseline": _spec(baseline, public),
|
|
@@ -518,9 +525,12 @@ def _run(args) -> int:
|
|
|
518
525
|
if batch.stopped:
|
|
519
526
|
print(f"STOPPED EARLY: an account can't make calls ({batch.stopped}). Fix it, then rerun with --resume {results}")
|
|
520
527
|
records, excluded = runner.load_results(results)
|
|
528
|
+
# A rerun of a scoreboard upload names it by the upload's SHA-256 (its file name there)
|
|
529
|
+
rerun_of = hashlib.sha256(args.rerun.read_bytes()).hexdigest() if prior and prior.get("aimpg_upload") else None
|
|
521
530
|
record = build_record(records, claim=claim, baseline=baseline, challenger=challenger, repo=str(repo),
|
|
522
531
|
task_mode=task_mode, public=args.public if not prior else True,
|
|
523
|
-
attempts=runner.all_attempts(results), excluded=excluded
|
|
532
|
+
attempts=runner.all_attempts(results), excluded=excluded,
|
|
533
|
+
extra={"rerun_of": rerun_of} if rerun_of else None)
|
|
524
534
|
out = results.with_suffix(".record.json")
|
|
525
535
|
out.write_text(json.dumps(record, indent=1))
|
|
526
536
|
print()
|
aimpg-0.5.0/aimpg/sci.py
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
"""SCI for AI (Consumer) for AI coding: the Green Software Foundation's carbon
|
|
2
|
+
intensity score, computed from your agent logs.
|
|
3
|
+
|
|
4
|
+
SCI = (O + M) / R O = E × I (operational), M = embodied, R = functional unit
|
|
5
|
+
|
|
6
|
+
Spec: GSF "Software Carbon Intensity for AI" (ratified; extends ISO/IEC 21031:2024),
|
|
7
|
+
https://github.com/Green-Software-Foundation/sci-ai/blob/main/SPEC.md
|
|
8
|
+
* Consumer boundary: the Operation and Monitoring stage (API and inference,
|
|
9
|
+
orchestration, monitoring, ...). Provider training is the Provider SCI, not this.
|
|
10
|
+
* Functional units: "Per Token" for LLMs and "Per Workflow Execution" for
|
|
11
|
+
agentic AI. A coding agent's workflow ends in a commit, so aimpg reports
|
|
12
|
+
per AI-assisted commit and, the unit that matters, per working change
|
|
13
|
+
(a commit still standing after 30 days).
|
|
14
|
+
* Offsets, RECs, PPAs and other market-based measures may not reduce the
|
|
15
|
+
score, so I is a location-based grid average.
|
|
16
|
+
|
|
17
|
+
E is aimpg's serving-energy range (full stack: accelerators, host, idle
|
|
18
|
+
machines, data-centre overhead). M is amortized hardware emissions per kWh
|
|
19
|
+
served, from Google's measured Scope 1+3 per prompt. Every number is a
|
|
20
|
+
low–high range because model sizes and data-centre locations aren't disclosed.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import json
|
|
26
|
+
import time
|
|
27
|
+
from pathlib import Path
|
|
28
|
+
|
|
29
|
+
from aimpg.energy import load_factors, total_wh
|
|
30
|
+
from aimpg.equivalence import load as load_equivalences
|
|
31
|
+
|
|
32
|
+
SPEC_URL = "https://github.com/Green-Software-Foundation/sci-ai/blob/main/SPEC.md"
|
|
33
|
+
# Google, arXiv 2508.15734, Table 2 (median Gemini Apps prompt, May 2025):
|
|
34
|
+
# Scope 1+3 (accelerators + host CPU & DRAM, amortized) 0.010 g per 0.24 Wh
|
|
35
|
+
# (comprehensive) and 0.007 g per 0.10 Wh (narrow method) → ~42–70 g per kWh served.
|
|
36
|
+
EMBODIED_G_PER_KWH = (40.0, 70.0)
|
|
37
|
+
EMBODIED_CITATION = ("Google, 'Measuring the environmental impact of delivering AI at Google scale' (arXiv 2508.15734), "
|
|
38
|
+
"Table 2: Scope 1+3 emissions 0.010 gCO2e per 0.24 Wh median prompt (≈42 g/kWh) and 0.007 g per "
|
|
39
|
+
"0.10 Wh with the narrower method (≈70 g/kWh)")
|
|
40
|
+
INCLUDED = ["model inference via API (accelerators, host CPU and memory, idle provisioned capacity, data-centre overhead)",
|
|
41
|
+
"every model request the agents made, including sub-agents, tool loops and retries",
|
|
42
|
+
"embodied (manufacturing) emissions of the serving hardware, amortized per kWh served"]
|
|
43
|
+
EXCLUDED = ["the developer's own computer and network (client side): not in the agents' logs",
|
|
44
|
+
"provider training (that is the Provider SCI, reported by the model provider)",
|
|
45
|
+
"orchestration, monitoring and storage outside the model API, where the provider doesn't disclose them",
|
|
46
|
+
"offsets, RECs, PPAs and other market-based measures (excluded by the spec)"]
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def compute(requests, n_commits: int, judged, *, grid: float, period: tuple[float, float]) -> dict:
|
|
50
|
+
"""requests: in-window model requests; judged: durable.Judged (or None)."""
|
|
51
|
+
e = total_wh(requests) # Wh
|
|
52
|
+
tokens = sum(r.usage.ctx_len + r.usage.output for r in requests)
|
|
53
|
+
lo_g = e.low / 1000 * (grid + EMBODIED_G_PER_KWH[0])
|
|
54
|
+
hi_g = e.high / 1000 * (grid + EMBODIED_G_PER_KWH[1])
|
|
55
|
+
|
|
56
|
+
def per(r: float | None) -> list[float] | None:
|
|
57
|
+
return [round(lo_g / r, 6), round(hi_g / r, 6)] if r else None
|
|
58
|
+
|
|
59
|
+
working = None
|
|
60
|
+
if judged is not None and judged.durable:
|
|
61
|
+
wh_lo = sum(c.get("wh_low", 0) for c in judged.judged)
|
|
62
|
+
wh_hi = sum(c.get("wh_high", 0) for c in judged.judged)
|
|
63
|
+
n = len(judged.durable)
|
|
64
|
+
working = {"value_g": [round(wh_lo / 1000 * (grid + EMBODIED_G_PER_KWH[0]) / n, 3),
|
|
65
|
+
round(wh_hi / 1000 * (grid + EMBODIED_G_PER_KWH[1]) / n, 3)],
|
|
66
|
+
"working_changes": n, "judged_commits": len(judged.judged),
|
|
67
|
+
"note": "carbon of all judged AI-assisted commits (kept or not) divided by the ones still standing after 30 days"}
|
|
68
|
+
first = judged.first_judgeable() if judged is not None else None
|
|
69
|
+
return {
|
|
70
|
+
"standard": "Software Carbon Intensity for AI (GSF), Consumer SCI",
|
|
71
|
+
"specification": SPEC_URL,
|
|
72
|
+
"tool": "aimpg",
|
|
73
|
+
"generated": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
74
|
+
"period": {"from": _day(period[0]), "to": _day(period[1])},
|
|
75
|
+
"formula": "SCI = (O + M) / R, O = E × I",
|
|
76
|
+
"boundary": {"persona": "consumer", "lifecycle_stage": "Operation and Monitoring",
|
|
77
|
+
"included": INCLUDED, "excluded": EXCLUDED},
|
|
78
|
+
"inputs": {
|
|
79
|
+
"E_kwh": [round(e.low / 1000, 4), round(e.high / 1000, 4)],
|
|
80
|
+
"E_method": f"aimpg energy model, factors version {load_factors().get('version')} (method: https://github.com/kumarganduri/aimpg#how-energy-is-estimated)",
|
|
81
|
+
"I_g_per_kwh": grid,
|
|
82
|
+
"I_method": "location-based grid average (no market-based measures)",
|
|
83
|
+
"M_g_per_kwh_served": list(EMBODIED_G_PER_KWH),
|
|
84
|
+
"M_method": EMBODIED_CITATION,
|
|
85
|
+
"requests": len(requests),
|
|
86
|
+
"tokens": tokens,
|
|
87
|
+
"tokens_definition": "input (fresh, cache write and cache read) plus output tokens of every request",
|
|
88
|
+
},
|
|
89
|
+
"total_g": [round(lo_g, 1), round(hi_g, 1)],
|
|
90
|
+
"sci": {
|
|
91
|
+
"per_million_tokens_g": per(tokens / 1e6 if tokens else None),
|
|
92
|
+
"per_ai_assisted_commit_g": per(n_commits),
|
|
93
|
+
"per_working_change_g": working,
|
|
94
|
+
},
|
|
95
|
+
"functional_units": {
|
|
96
|
+
"per_million_tokens": "the spec's suggested unit for LLMs (per token), scaled",
|
|
97
|
+
"per_ai_assisted_commit": "per workflow execution (the spec's unit for agentic AI): one commit the agent helped make",
|
|
98
|
+
"per_working_change": "per workflow execution that lasted: committed, on the main branch and not reverted or largely rewritten 30 days later",
|
|
99
|
+
},
|
|
100
|
+
"working_change_available_from": _day(first + 30 * 86400) if working is None and first is not None else None,
|
|
101
|
+
"uncertainty": "Every value is a low–high range: model sizes and data-centre locations aren't disclosed, "
|
|
102
|
+
"so energy is bracketed by assumptions (aimpg/factors.json); the range is the result, not noise.",
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _day(ts: float) -> str:
|
|
107
|
+
return time.strftime("%Y-%m-%d", time.gmtime(ts))
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _g(v: list[float] | None) -> str:
|
|
111
|
+
if not v:
|
|
112
|
+
return "—"
|
|
113
|
+
lo, hi = v
|
|
114
|
+
unit, k = ("kg", 1000) if hi >= 1000 else ("g", 1)
|
|
115
|
+
return f"{lo / k:,.3g}–{hi / k:,.3g} {unit} CO₂e"
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def render(s: dict) -> str:
|
|
119
|
+
sci, inp = s["sci"], s["inputs"]
|
|
120
|
+
lines = [
|
|
121
|
+
f"SCI FOR AI (Consumer) · {s['period']['from']} → {s['period']['to']}",
|
|
122
|
+
f" Standard: GSF Software Carbon Intensity for AI ({s['specification']})",
|
|
123
|
+
f" SCI = (O + M) / R, O = E × I",
|
|
124
|
+
"",
|
|
125
|
+
f" E (serving energy) {inp['E_kwh'][0]:,.2f}–{inp['E_kwh'][1]:,.2f} kWh over {inp['requests']:,} model requests",
|
|
126
|
+
f" I (grid, location-based) {inp['I_g_per_kwh']:.0f} g CO₂e/kWh",
|
|
127
|
+
f" M (embodied hardware) {inp['M_g_per_kwh_served'][0]:.0f}–{inp['M_g_per_kwh_served'][1]:.0f} g CO₂e per kWh served",
|
|
128
|
+
f" Total (O + M) {_g(s['total_g'])}",
|
|
129
|
+
"",
|
|
130
|
+
" Per functional unit (R):",
|
|
131
|
+
f" per million tokens {_g(sci['per_million_tokens_g'])}",
|
|
132
|
+
f" per AI-assisted commit {_g(sci['per_ai_assisted_commit_g'])}",
|
|
133
|
+
]
|
|
134
|
+
w = sci["per_working_change_g"]
|
|
135
|
+
if w:
|
|
136
|
+
lines.append(f" per working change {_g(w['value_g'])} ({w['working_changes']} of {w['judged_commits']} judged commits lasted)")
|
|
137
|
+
else:
|
|
138
|
+
when = s.get("working_change_available_from")
|
|
139
|
+
lines.append(f" per working change available from {when}" if when else " per working change not available yet")
|
|
140
|
+
lines += ["", " Not included: " + "; ".join(x.split(":")[0] for x in s["boundary"]["excluded"]) + ".",
|
|
141
|
+
" Offsets and renewable-energy certificates never reduce this score (the spec excludes them)."]
|
|
142
|
+
return "\n".join(lines) + "\n"
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def default_grid() -> float:
|
|
146
|
+
return float(load_equivalences()["co2_g_per_kwh"]["value"])
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def write(s: dict, path: Path) -> None:
|
|
150
|
+
path.write_text(json.dumps(s, indent=2) + "\n")
|