aimpg 0.4.3__tar.gz → 0.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- aimpg-0.6.0/.github/workflows/test.yml +72 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/PKG-INFO +36 -2
- {aimpg-0.4.3 → aimpg-0.6.0}/README.md +35 -1
- aimpg-0.6.0/action.yml +69 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/aimpg/cli.py +72 -0
- aimpg-0.6.0/aimpg/defaults.py +158 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/aimpg/replay/picker.py +15 -6
- aimpg-0.6.0/aimpg/sci.py +150 -0
- aimpg-0.6.0/aimpg/team.py +200 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/docs/STRATEGY.md +1 -1
- {aimpg-0.4.3 → aimpg-0.6.0}/pyproject.toml +1 -1
- {aimpg-0.4.3 → aimpg-0.6.0}/tests/conftest.py +4 -0
- aimpg-0.6.0/tests/test_defaults.py +94 -0
- aimpg-0.6.0/tests/test_sci.py +63 -0
- aimpg-0.6.0/tests/test_team.py +146 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/uv.lock +1 -1
- aimpg-0.4.3/.github/workflows/test.yml +0 -34
- {aimpg-0.4.3 → aimpg-0.6.0}/.github/workflows/publish.yml +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/.gitignore +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/LICENSE +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/TODOS.md +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/aimpg/__init__.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/aimpg/attribution.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/aimpg/coach.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/aimpg/codex_logs.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/aimpg/cost.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/aimpg/cursor_usage.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/aimpg/durable.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/aimpg/energy.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/aimpg/equivalence.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/aimpg/equivalences.json +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/aimpg/factors.json +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/aimpg/gitkept.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/aimpg/ledger.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/aimpg/live.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/aimpg/logs.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/aimpg/model.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/aimpg/prices.json +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/aimpg/receipt.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/aimpg/replay/__init__.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/aimpg/replay/cli.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/aimpg/replay/fake_agent.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/aimpg/replay/hints.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/aimpg/replay/proxy.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/aimpg/replay/run.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/aimpg/replay/sandbox.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/aimpg/replay/select.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/aimpg/replay/setups.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/aimpg/replay/stats.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/aimpg/replay/verify.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/aimpg/replay/workspace.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/aimpg/scoreboard.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/aimpg/share.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/docs/designs/aimpg-design.md +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/docs/designs/scoreboard-brief.md +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/evals/attribution_eval.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/evals/label.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/spikes/agent/RESULTS.md +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/spikes/agent/allowlist_proxy.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/spikes/agent/make_profile.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/spikes/egress/RESULTS.md +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/spikes/egress/allowlist_proxy.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/spikes/egress/replay.sb +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/tests/gitrepo.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/tests/test_attribution.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/tests/test_coach.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/tests/test_codex_logs.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/tests/test_cost_and_equivalence.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/tests/test_cursor_usage.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/tests/test_durable.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/tests/test_energy.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/tests/test_eval_scoring.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/tests/test_gitkept.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/tests/test_live.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/tests/test_logs.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/tests/test_perf.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/tests/test_replay_cli.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/tests/test_replay_e2e.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/tests/test_replay_hints.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/tests/test_replay_picker.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/tests/test_replay_proxy.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/tests/test_replay_sandbox.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/tests/test_replay_stats.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/tests/test_sandbox_required.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/tests/test_scoreboard.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/tests/test_share.py +0 -0
- {aimpg-0.4.3 → aimpg-0.6.0}/tests/test_verify.py +0 -0
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
name: test
|
|
2
|
+
on:
|
|
3
|
+
push:
|
|
4
|
+
branches: [main, "linux-*"]
|
|
5
|
+
pull_request:
|
|
6
|
+
jobs:
|
|
7
|
+
test:
|
|
8
|
+
strategy:
|
|
9
|
+
matrix:
|
|
10
|
+
os: [ubuntu-latest, macos-latest]
|
|
11
|
+
python: ["3.10", "3.13"]
|
|
12
|
+
runs-on: ${{ matrix.os }}
|
|
13
|
+
steps:
|
|
14
|
+
- uses: actions/checkout@v4
|
|
15
|
+
- uses: astral-sh/setup-uv@v6
|
|
16
|
+
- run: git config --global user.email ci@example.com && git config --global user.name CI
|
|
17
|
+
- run: uv run --python ${{ matrix.python }} --group dev pytest -q
|
|
18
|
+
linux-sandbox:
|
|
19
|
+
# Replays on Linux: bubblewrap + the AppArmor rule Ubuntu 24.04 needs for bwrap only
|
|
20
|
+
runs-on: ubuntu-24.04
|
|
21
|
+
steps:
|
|
22
|
+
- uses: actions/checkout@v4
|
|
23
|
+
- uses: astral-sh/setup-uv@v6
|
|
24
|
+
- run: sudo apt-get update -q && sudo apt-get install -y -q bubblewrap
|
|
25
|
+
- name: allow user namespaces for bwrap only
|
|
26
|
+
run: |
|
|
27
|
+
printf 'abi <abi/4.0>,\ninclude <tunables/global>\n\nprofile bwrap /usr/bin/bwrap flags=(unconfined) {\n userns,\n include if exists <local/bwrap>\n}\n' | sudo tee /etc/apparmor.d/bwrap
|
|
28
|
+
sudo apparmor_parser -r /etc/apparmor.d/bwrap
|
|
29
|
+
- run: git config --global user.email ci@example.com && git config --global user.name CI
|
|
30
|
+
- name: an SSH agent for the sandbox to fail to reach
|
|
31
|
+
run: ssh-agent -a /tmp/aimpg-ci-agent.sock && echo "SSH_AUTH_SOCK=/tmp/aimpg-ci-agent.sock" >> "$GITHUB_ENV"
|
|
32
|
+
- run: uv run --group dev pytest -q -rs tests/test_sandbox_required.py tests/test_replay_sandbox.py tests/test_replay_e2e.py
|
|
33
|
+
env:
|
|
34
|
+
AIMPG_REQUIRE_SANDBOX: "1"
|
|
35
|
+
action-smoke:
|
|
36
|
+
# The GitHub Action (action.yml) on a throwaway repo with aimpg notes
|
|
37
|
+
runs-on: ubuntu-latest
|
|
38
|
+
steps:
|
|
39
|
+
- uses: actions/checkout@v4
|
|
40
|
+
- name: a repo with a remote, a feature branch and notes from five people
|
|
41
|
+
run: |
|
|
42
|
+
set -euo pipefail
|
|
43
|
+
git config --global init.defaultBranch main
|
|
44
|
+
git init -q --bare /tmp/origin.git
|
|
45
|
+
git clone -q /tmp/origin.git /tmp/demo && cd /tmp/demo
|
|
46
|
+
git config user.email ci@x && git config user.name ci
|
|
47
|
+
for i in 1 2 3 4 5; do
|
|
48
|
+
echo $i > f$i && git add f$i
|
|
49
|
+
GIT_AUTHOR_DATE="@$(( $(date +%s) - (40 - i) * 86400 )) +0000" GIT_COMMITTER_DATE="@$(( $(date +%s) - (40 - i) * 86400 )) +0000" \
|
|
50
|
+
git -c user.email=p$i@x commit -q -m "feat: f$i"
|
|
51
|
+
git notes --ref=aimpg add -m "{\"aimpg_note\": 1, \"usd\": $i.0, \"wh\": [10, 100], \"requests\": 3, \"tier\": \"exact\"}" HEAD
|
|
52
|
+
done
|
|
53
|
+
git push -q origin main refs/notes/aimpg
|
|
54
|
+
git checkout -q -b feature && echo x > x && git add x && git commit -q -m "feat: x"
|
|
55
|
+
git notes --ref=aimpg add -m '{"aimpg_note": 1, "usd": 4.1, "wh": [200, 1900], "requests": 9, "tier": "exact"}' HEAD
|
|
56
|
+
git push -q origin refs/notes/aimpg # as aimpg notes push would
|
|
57
|
+
- id: pr
|
|
58
|
+
uses: ./
|
|
59
|
+
with:
|
|
60
|
+
mode: pr
|
|
61
|
+
path: /tmp/demo
|
|
62
|
+
- id: rollup
|
|
63
|
+
uses: ./
|
|
64
|
+
with:
|
|
65
|
+
mode: rollup
|
|
66
|
+
path: /tmp/demo
|
|
67
|
+
- name: check the outputs
|
|
68
|
+
run: |
|
|
69
|
+
grep -q 'AI cost of this pull request' "${{ steps.pr.outputs.markdown-file }}"
|
|
70
|
+
grep -q 'Cost per working change' "${{ steps.rollup.outputs.markdown-file }}"
|
|
71
|
+
grep -q '\$15.00' "${{ steps.rollup.outputs.markdown-file }}"
|
|
72
|
+
cat "${{ steps.rollup.outputs.markdown-file }}"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: aimpg
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6.0
|
|
4
4
|
Summary: Real-world energy per solved task for AI coding agents
|
|
5
5
|
Project-URL: Homepage, https://github.com/kumarganduri/aimpg
|
|
6
6
|
Project-URL: Issues, https://github.com/kumarganduri/aimpg/issues
|
|
@@ -90,6 +90,8 @@ Use sonnet: it solved 100% of tasks (best: 100%) at 3.1x lower cost per solved t
|
|
|
90
90
|
|
|
91
91
|
It re-does your real past commits with each model in a locked sandbox, and **your own tests decide** what counts as solved.
|
|
92
92
|
|
|
93
|
+
**Make it the team's default:** `aimpg defaults --repo .` reads that evidence and proposes the repo's default model for Claude Code; `--apply` writes it to `.claude/settings.json` (it asks first and keeps a backup). Commit it, and everyone on the repo starts with the model your own tests showed is good enough; anyone can still switch with `/model`.
|
|
94
|
+
|
|
93
95
|
### 4. Verify any efficiency claim on your own code
|
|
94
96
|
Token savers, prompts, hooks, CLAUDE.md files, models, even other agents: one command reruns your real commits both ways in a locked sandbox, and your own tests judge them.
|
|
95
97
|
```bash
|
|
@@ -127,12 +129,44 @@ aimpg pr --base main --post # add it as a comment on the open PR (uses gh)
|
|
|
127
129
|
> ### ⚡ AI energy for this pull request
|
|
128
130
|
> **73.4 Wh – 564.0 Wh** ≈ 4–33 full phone charges · 34–258 g CO₂ · **$5.76** API-equivalent
|
|
129
131
|
|
|
130
|
-
### 6.
|
|
132
|
+
### 6. Team numbers, without a server
|
|
133
|
+
Each developer who opts in shares a small note on **their own** commits (cost, energy range, request count; never prompts or code) as git notes that travel with the repo:
|
|
134
|
+
```bash
|
|
135
|
+
aimpg notes push # writes notes on your AI-assisted commits and pushes refs/notes/aimpg
|
|
136
|
+
```
|
|
137
|
+
Then add the GitHub Action to the repo:
|
|
138
|
+
```yaml
|
|
139
|
+
# .github/workflows/aimpg.yml
|
|
140
|
+
on:
|
|
141
|
+
pull_request:
|
|
142
|
+
schedule: [{cron: "0 8 * * 1"}] # weekly team rollup
|
|
143
|
+
permissions: {contents: read, pull-requests: write}
|
|
144
|
+
jobs:
|
|
145
|
+
aimpg:
|
|
146
|
+
runs-on: ubuntu-latest
|
|
147
|
+
steps:
|
|
148
|
+
- uses: actions/checkout@v4
|
|
149
|
+
with: {fetch-depth: 0}
|
|
150
|
+
- uses: kumarganduri/aimpg@v0.6.0
|
|
151
|
+
with:
|
|
152
|
+
mode: ${{ github.event_name == 'pull_request' && 'pr' || 'rollup' }}
|
|
153
|
+
comment: "true"
|
|
154
|
+
```
|
|
155
|
+
On pull requests it posts the PR's AI cost ("$4.10 · 0.20–1.90 kWh · 9 model requests"). Weekly, it writes the team's **cost per working change**, waste ratio and trend to the run summary. Working changes are judged from the repo's own history. Team numbers appear only once at least 5 people share notes (`min-group` lowers it if your team agrees), and never per person. Anyone with read access to the repo can read the notes.
|
|
156
|
+
|
|
157
|
+
### 7. Report for a team or a sustainability report
|
|
131
158
|
```bash
|
|
132
159
|
aimpg export --csv ai-energy.csv --no-subjects --no-authors
|
|
133
160
|
```
|
|
134
161
|
One row per AI-assisted commit: date, repo, energy range, CO₂ range and cost. Teams can combine everyone's files. The privacy switches drop commit messages and author emails.
|
|
135
162
|
|
|
163
|
+
For a standard figure, `aimpg sci --json sci.json` reports your carbon intensity in the Green Software Foundation's [SCI for AI](https://github.com/Green-Software-Foundation/sci-ai/blob/main/SPEC.md) format (Consumer SCI). It uses the spec's units: per million tokens, per AI-assisted commit (a "workflow execution" for agentic AI), and per working change. The JSON lists the boundary, every input with its source, and what's left out. Embodied hardware emissions come from Google's measured per-prompt figures; grid intensity is location-based, because the spec doesn't let offsets or renewable certificates lower the score.
|
|
164
|
+
```
|
|
165
|
+
per million tokens 2.18–21 g CO₂e
|
|
166
|
+
per AI-assisted commit 19.3–185 g CO₂e
|
|
167
|
+
per working change available from 2026-10-13
|
|
168
|
+
```
|
|
169
|
+
|
|
136
170
|
## Privacy
|
|
137
171
|
|
|
138
172
|
`report`, `pr` and `export` run entirely on your machine with no network calls (except `--fetch` for `git fetch`, and `pr --post`, which talks to GitHub). Your code and prompts are never uploaded. Replays send only the agent's own work to the model API, from a sandbox that can't read your home folder.
|
|
@@ -70,6 +70,8 @@ Use sonnet: it solved 100% of tasks (best: 100%) at 3.1x lower cost per solved t
|
|
|
70
70
|
|
|
71
71
|
It re-does your real past commits with each model in a locked sandbox, and **your own tests decide** what counts as solved.
|
|
72
72
|
|
|
73
|
+
**Make it the team's default:** `aimpg defaults --repo .` reads that evidence and proposes the repo's default model for Claude Code; `--apply` writes it to `.claude/settings.json` (it asks first and keeps a backup). Commit it, and everyone on the repo starts with the model your own tests showed is good enough; anyone can still switch with `/model`.
|
|
74
|
+
|
|
73
75
|
### 4. Verify any efficiency claim on your own code
|
|
74
76
|
Token savers, prompts, hooks, CLAUDE.md files, models, even other agents: one command reruns your real commits both ways in a locked sandbox, and your own tests judge them.
|
|
75
77
|
```bash
|
|
@@ -107,12 +109,44 @@ aimpg pr --base main --post # add it as a comment on the open PR (uses gh)
|
|
|
107
109
|
> ### ⚡ AI energy for this pull request
|
|
108
110
|
> **73.4 Wh – 564.0 Wh** ≈ 4–33 full phone charges · 34–258 g CO₂ · **$5.76** API-equivalent
|
|
109
111
|
|
|
110
|
-
### 6.
|
|
112
|
+
### 6. Team numbers, without a server
|
|
113
|
+
Each developer who opts in shares a small note on **their own** commits (cost, energy range, request count; never prompts or code) as git notes that travel with the repo:
|
|
114
|
+
```bash
|
|
115
|
+
aimpg notes push # writes notes on your AI-assisted commits and pushes refs/notes/aimpg
|
|
116
|
+
```
|
|
117
|
+
Then add the GitHub Action to the repo:
|
|
118
|
+
```yaml
|
|
119
|
+
# .github/workflows/aimpg.yml
|
|
120
|
+
on:
|
|
121
|
+
pull_request:
|
|
122
|
+
schedule: [{cron: "0 8 * * 1"}] # weekly team rollup
|
|
123
|
+
permissions: {contents: read, pull-requests: write}
|
|
124
|
+
jobs:
|
|
125
|
+
aimpg:
|
|
126
|
+
runs-on: ubuntu-latest
|
|
127
|
+
steps:
|
|
128
|
+
- uses: actions/checkout@v4
|
|
129
|
+
with: {fetch-depth: 0}
|
|
130
|
+
- uses: kumarganduri/aimpg@v0.6.0
|
|
131
|
+
with:
|
|
132
|
+
mode: ${{ github.event_name == 'pull_request' && 'pr' || 'rollup' }}
|
|
133
|
+
comment: "true"
|
|
134
|
+
```
|
|
135
|
+
On pull requests it posts the PR's AI cost ("$4.10 · 0.20–1.90 kWh · 9 model requests"). Weekly, it writes the team's **cost per working change**, waste ratio and trend to the run summary. Working changes are judged from the repo's own history. Team numbers appear only once at least 5 people share notes (`min-group` lowers it if your team agrees), and never per person. Anyone with read access to the repo can read the notes.
|
|
136
|
+
|
|
137
|
+
### 7. Report for a team or a sustainability report
|
|
111
138
|
```bash
|
|
112
139
|
aimpg export --csv ai-energy.csv --no-subjects --no-authors
|
|
113
140
|
```
|
|
114
141
|
One row per AI-assisted commit: date, repo, energy range, CO₂ range and cost. Teams can combine everyone's files. The privacy switches drop commit messages and author emails.
|
|
115
142
|
|
|
143
|
+
For a standard figure, `aimpg sci --json sci.json` reports your carbon intensity in the Green Software Foundation's [SCI for AI](https://github.com/Green-Software-Foundation/sci-ai/blob/main/SPEC.md) format (Consumer SCI). It uses the spec's units: per million tokens, per AI-assisted commit (a "workflow execution" for agentic AI), and per working change. The JSON lists the boundary, every input with its source, and what's left out. Embodied hardware emissions come from Google's measured per-prompt figures; grid intensity is location-based, because the spec doesn't let offsets or renewable certificates lower the score.
|
|
144
|
+
```
|
|
145
|
+
per million tokens 2.18–21 g CO₂e
|
|
146
|
+
per AI-assisted commit 19.3–185 g CO₂e
|
|
147
|
+
per working change available from 2026-10-13
|
|
148
|
+
```
|
|
149
|
+
|
|
116
150
|
## Privacy
|
|
117
151
|
|
|
118
152
|
`report`, `pr` and `export` run entirely on your machine with no network calls (except `--fetch` for `git fetch`, and `pr --post`, which talks to GitHub). Your code and prompts are never uploaded. Replays send only the agent's own work to the model API, from a sandbox that can't read your home folder.
|
aimpg-0.6.0/action.yml
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
name: aimpg
|
|
2
|
+
description: AI cost and energy of pull requests and of your team (cost per working change), from aimpg git notes
|
|
3
|
+
author: Kumar Ganduri
|
|
4
|
+
branding:
|
|
5
|
+
icon: zap
|
|
6
|
+
color: green
|
|
7
|
+
inputs:
|
|
8
|
+
mode:
|
|
9
|
+
description: "pr: AI cost of this pull request · rollup: team cost per working change and waste ratio"
|
|
10
|
+
default: pr
|
|
11
|
+
comment:
|
|
12
|
+
description: "pr mode: also post or update a comment on the pull request (needs permissions: pull-requests: write)"
|
|
13
|
+
default: "false"
|
|
14
|
+
days:
|
|
15
|
+
description: "rollup mode: period in days"
|
|
16
|
+
default: "90"
|
|
17
|
+
min-group:
|
|
18
|
+
description: "rollup mode: people who must share notes before team numbers show"
|
|
19
|
+
default: "5"
|
|
20
|
+
path:
|
|
21
|
+
description: "repo folder (checked out with fetch-depth: 0)"
|
|
22
|
+
default: "."
|
|
23
|
+
outputs:
|
|
24
|
+
markdown-file:
|
|
25
|
+
description: path of the generated markdown
|
|
26
|
+
value: ${{ steps.run.outputs.file }}
|
|
27
|
+
runs:
|
|
28
|
+
using: composite
|
|
29
|
+
steps:
|
|
30
|
+
- uses: astral-sh/setup-uv@v6
|
|
31
|
+
- name: fetch aimpg notes
|
|
32
|
+
shell: bash
|
|
33
|
+
working-directory: ${{ inputs.path }}
|
|
34
|
+
run: |
|
|
35
|
+
git fetch --quiet origin '+refs/notes/aimpg:refs/notes/aimpg' || echo "No aimpg notes on origin yet (developers add them with aimpg notes push)"
|
|
36
|
+
- name: aimpg
|
|
37
|
+
id: run
|
|
38
|
+
shell: bash
|
|
39
|
+
working-directory: ${{ inputs.path }}
|
|
40
|
+
env:
|
|
41
|
+
MODE: ${{ inputs.mode }}
|
|
42
|
+
DAYS: ${{ inputs.days }}
|
|
43
|
+
MIN_GROUP: ${{ inputs.min-group }}
|
|
44
|
+
BASE_REF: ${{ github.base_ref }}
|
|
45
|
+
ACTION_PATH: ${{ github.action_path }}
|
|
46
|
+
run: |
|
|
47
|
+
set -euo pipefail
|
|
48
|
+
out="$RUNNER_TEMP/aimpg-$MODE.md"
|
|
49
|
+
case "$MODE" in
|
|
50
|
+
pr)
|
|
51
|
+
base="origin/${BASE_REF:-main}"
|
|
52
|
+
uvx --quiet --from "$ACTION_PATH" aimpg team pr --repo . --base "$base" > "$out" ;;
|
|
53
|
+
rollup)
|
|
54
|
+
uvx --quiet --from "$ACTION_PATH" aimpg team rollup --repo . --days "$DAYS" --min-group "$MIN_GROUP" > "$out" ;;
|
|
55
|
+
*) echo "::error::mode must be pr or rollup"; exit 1 ;;
|
|
56
|
+
esac
|
|
57
|
+
cat "$out" >> "$GITHUB_STEP_SUMMARY"
|
|
58
|
+
echo "file=$out" >> "$GITHUB_OUTPUT"
|
|
59
|
+
- name: comment on the pull request
|
|
60
|
+
if: inputs.mode == 'pr' && inputs.comment == 'true' && github.event.pull_request.number
|
|
61
|
+
shell: bash
|
|
62
|
+
working-directory: ${{ inputs.path }}
|
|
63
|
+
env:
|
|
64
|
+
GH_TOKEN: ${{ github.token }}
|
|
65
|
+
PR: ${{ github.event.pull_request.number }}
|
|
66
|
+
FILE: ${{ steps.run.outputs.file }}
|
|
67
|
+
run: |
|
|
68
|
+
gh pr comment "$PR" --body-file "$FILE" --edit-last --create-if-none \
|
|
69
|
+
|| echo "::warning::Couldn't comment (pull requests from forks get a read-only token); the summary above has the numbers."
|
|
@@ -20,6 +20,7 @@ from aimpg.gitkept import DAY
|
|
|
20
20
|
from aimpg.logs import DEFAULT_ROOT, iter_log_files, parse_logs
|
|
21
21
|
from aimpg.receipt import render
|
|
22
22
|
from aimpg.replay import cli as replay_cli
|
|
23
|
+
from aimpg import defaults as defaults_mod
|
|
23
24
|
from aimpg import scoreboard as scoreboard_mod
|
|
24
25
|
from aimpg.replay import verify as verify_mod
|
|
25
26
|
|
|
@@ -66,6 +67,27 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
66
67
|
export.add_argument("--no-subjects", action="store_true", help="leave commit messages out (privacy)")
|
|
67
68
|
export.add_argument("--no-authors", action="store_true", help="leave author emails out (privacy)")
|
|
68
69
|
|
|
70
|
+
sci_p = sub.add_parser("sci", help="carbon intensity in the GSF SCI for AI format (per token, commit, working change)")
|
|
71
|
+
_common(sci_p, 30)
|
|
72
|
+
sci_p.add_argument("--grid", type=float, default=None, help="grid intensity in g CO2e/kWh (default: world average)")
|
|
73
|
+
sci_p.add_argument("--json", type=Path, default=None, help="also write the full disclosure as JSON")
|
|
74
|
+
|
|
75
|
+
notes_p = sub.add_parser("notes", help="share your AI cost per commit with your team, as git notes (metadata only)")
|
|
76
|
+
notes_p.add_argument("action", choices=("push", "write"), help="push: write notes on your commits and push them; write: local only")
|
|
77
|
+
_common(notes_p, 30)
|
|
78
|
+
notes_p.add_argument("--repo", type=Path, default=Path("."), help="repo to note (default: here)")
|
|
79
|
+
notes_p.add_argument("--remote", default="origin")
|
|
80
|
+
|
|
81
|
+
team_p = sub.add_parser("team", help="team numbers from aimpg git notes (used by the GitHub Action)")
|
|
82
|
+
team_sub = team_p.add_subparsers(dest="team_command", required=True)
|
|
83
|
+
tpr = team_sub.add_parser("pr", help="AI cost of this branch's commits")
|
|
84
|
+
tpr.add_argument("--repo", type=Path, default=Path("."))
|
|
85
|
+
tpr.add_argument("--base", default="origin/main", help="base ref (default origin/main)")
|
|
86
|
+
trl = team_sub.add_parser("rollup", help="cost per working change and waste ratio for the team")
|
|
87
|
+
trl.add_argument("--repo", type=Path, default=Path("."))
|
|
88
|
+
trl.add_argument("--days", type=int, default=90)
|
|
89
|
+
trl.add_argument("--min-group", type=int, default=5, help="people needed before numbers show (default 5)")
|
|
90
|
+
|
|
69
91
|
live = sub.add_parser("live", help="live status line + post-commit note inside Claude Code")
|
|
70
92
|
live.add_argument("action", choices=("install", "uninstall"))
|
|
71
93
|
sub.add_parser("statusline", help=argparse.SUPPRESS) # called by Claude Code
|
|
@@ -75,6 +97,7 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
75
97
|
replay_cli.add_parser(sub)
|
|
76
98
|
verify_mod.add_parser(sub)
|
|
77
99
|
scoreboard_mod.add_parsers(sub)
|
|
100
|
+
defaults_mod.add_parser(sub)
|
|
78
101
|
args = parser.parse_args(argv)
|
|
79
102
|
if args.command == "replay":
|
|
80
103
|
return replay_cli.main(args)
|
|
@@ -82,6 +105,8 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
82
105
|
return verify_mod.main(args)
|
|
83
106
|
if args.command in ("submit", "scoreboard"):
|
|
84
107
|
return scoreboard_mod.main(args)
|
|
108
|
+
if args.command == "defaults":
|
|
109
|
+
return defaults_mod.main(args)
|
|
85
110
|
if args.command in ("statusline", "hook", "live"):
|
|
86
111
|
from aimpg import live as live_mod
|
|
87
112
|
|
|
@@ -101,6 +126,21 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
101
126
|
print(live_mod.uninstall())
|
|
102
127
|
return 0
|
|
103
128
|
|
|
129
|
+
if args.command == "team":
|
|
130
|
+
from aimpg import team
|
|
131
|
+
|
|
132
|
+
repo = str(args.repo.resolve())
|
|
133
|
+
if args.team_command == "pr":
|
|
134
|
+
try:
|
|
135
|
+
shas = share.branch_commits(args.repo, args.base)
|
|
136
|
+
except RuntimeError as exc:
|
|
137
|
+
print(f"Can't list this branch's commits: {exc}")
|
|
138
|
+
return 1
|
|
139
|
+
sys.stdout.write(team.pr_markdown(team.pr_summary(repo, shas)))
|
|
140
|
+
else:
|
|
141
|
+
sys.stdout.write(team.rollup_markdown(team.rollup(repo, days=args.days, min_group=args.min_group)))
|
|
142
|
+
return 0
|
|
143
|
+
|
|
104
144
|
if args.days <= 0:
|
|
105
145
|
parser.error("--days must be positive")
|
|
106
146
|
from aimpg import codex_logs
|
|
@@ -144,6 +184,38 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
144
184
|
from aimpg import durable, ledger
|
|
145
185
|
from aimpg.receipt import task_energy
|
|
146
186
|
|
|
187
|
+
if args.command == "notes":
|
|
188
|
+
from aimpg import team
|
|
189
|
+
from aimpg.gitkept import GitError, repo_root
|
|
190
|
+
|
|
191
|
+
repo = repo_root(str(args.repo.resolve()))
|
|
192
|
+
if repo is None:
|
|
193
|
+
print(f"{args.repo} is not a git repo.")
|
|
194
|
+
return 1
|
|
195
|
+
try:
|
|
196
|
+
written, _ = team.write_notes(repo, task_energy(attribution.tasks))
|
|
197
|
+
print(f"Wrote {written} notes on your AI-assisted commits in {repo} ({team.FULL_REF}; metadata only).")
|
|
198
|
+
if args.action == "push":
|
|
199
|
+
print(team.push(repo, args.remote) + ". Your team's GitHub Action can now include them.")
|
|
200
|
+
except GitError as exc:
|
|
201
|
+
print(f"notes: {exc}")
|
|
202
|
+
return 1
|
|
203
|
+
return 0
|
|
204
|
+
|
|
205
|
+
if args.command == "sci":
|
|
206
|
+
from aimpg import sci
|
|
207
|
+
|
|
208
|
+
ledger.record(task_energy(attribution.tasks))
|
|
209
|
+
judged = durable.judge(ledger.load(), now)
|
|
210
|
+
in_window = [r for r in parsed.requests if r.ts >= since]
|
|
211
|
+
result = sci.compute(in_window, len(attribution.tasks), judged,
|
|
212
|
+
grid=args.grid if args.grid is not None else sci.default_grid(), period=(since, now))
|
|
213
|
+
sys.stdout.write(sci.render(result))
|
|
214
|
+
if args.json:
|
|
215
|
+
sci.write(result, args.json)
|
|
216
|
+
print(f"\nFull disclosure (boundary, inputs, sources, ranges): {args.json}")
|
|
217
|
+
return 0
|
|
218
|
+
|
|
147
219
|
ledger.record(task_energy(attribution.tasks))
|
|
148
220
|
judged = durable.judge(ledger.load(), now)
|
|
149
221
|
sys.stdout.write(render(parsed, attribution, since, now, judged))
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
"""`aimpg defaults`: turn a repo's replay evidence into a Claude Code setting the whole team gets.
|
|
2
|
+
|
|
3
|
+
aimpg defaults --repo . what the evidence says, and the setting it supports
|
|
4
|
+
aimpg defaults --repo . --apply write it to <repo>/.claude/settings.json (asks; keeps a backup)
|
|
5
|
+
|
|
6
|
+
Evidence, newest first, for this repo only:
|
|
7
|
+
* model-picker runs (`aimpg replay models`): the cheapest model per solved
|
|
8
|
+
task that solves within 10 points of the best;
|
|
9
|
+
* `aimpg verify` records comparing two Claude models: a SUPPORTED cheaper challenger.
|
|
10
|
+
|
|
11
|
+
The setting is the project's default model. It's committed with the repo, so
|
|
12
|
+
everyone on the team starts with it; anyone can still switch with /model, and
|
|
13
|
+
a personal choice goes in .claude/settings.local.json instead. Evidence from
|
|
14
|
+
fewer than 20 runs per model is marked tentative, and nothing is written
|
|
15
|
+
without a yes.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import json
|
|
21
|
+
import shutil
|
|
22
|
+
import time
|
|
23
|
+
from dataclasses import dataclass
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
|
|
26
|
+
from aimpg.replay import picker
|
|
27
|
+
from aimpg.replay.run import load_results
|
|
28
|
+
|
|
29
|
+
REPLAY_STATE = Path.home() / ".aimpg" / "replay"
|
|
30
|
+
VERIFY_STATE = Path.home() / ".aimpg" / "verify"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass
|
|
34
|
+
class Evidence:
|
|
35
|
+
source: str # file it came from
|
|
36
|
+
model: str # model id it supports as the default
|
|
37
|
+
summary: str
|
|
38
|
+
runs_per_model: int
|
|
39
|
+
when: float
|
|
40
|
+
|
|
41
|
+
@property
|
|
42
|
+
def tentative(self) -> bool:
|
|
43
|
+
return self.runs_per_model < picker.MIN_RUNS
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def gather(repo: Path) -> list[Evidence]:
|
|
47
|
+
repo = repo.resolve()
|
|
48
|
+
found: list[Evidence] = []
|
|
49
|
+
folder = REPLAY_STATE / repo.name
|
|
50
|
+
for path in sorted(folder.glob("models-*.jsonl")) if folder.is_dir() else []:
|
|
51
|
+
records, _ = load_results(path)
|
|
52
|
+
records = [r for r in records if Path(r.repo).resolve() == repo]
|
|
53
|
+
if not records:
|
|
54
|
+
continue
|
|
55
|
+
table = picker.rows(records)
|
|
56
|
+
choice = picker.pick(table)
|
|
57
|
+
if choice is None:
|
|
58
|
+
continue
|
|
59
|
+
found.append(Evidence(str(path), choice.model, picker.recommend(table), min(r.runs for r in table), path.stat().st_mtime))
|
|
60
|
+
found += _from_verify(repo)
|
|
61
|
+
return sorted(found, key=lambda e: e.when, reverse=True)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _from_verify(repo: Path) -> list[Evidence]:
|
|
65
|
+
from aimpg.scoreboard import repo_id
|
|
66
|
+
|
|
67
|
+
rid = repo_id(str(repo))
|
|
68
|
+
out = []
|
|
69
|
+
for path in sorted(VERIFY_STATE.glob("*.record.json")) if VERIFY_STATE.is_dir() else []:
|
|
70
|
+
try:
|
|
71
|
+
rec = json.loads(path.read_text())
|
|
72
|
+
except ValueError:
|
|
73
|
+
continue
|
|
74
|
+
b, c, v = rec.get("baseline", {}), rec.get("challenger", {}), rec.get("verdict", {})
|
|
75
|
+
if rid is None or rec.get("repo_id") != rid or not (b.get("agent", "claude") == c.get("agent", "claude") == "claude"):
|
|
76
|
+
continue
|
|
77
|
+
models = v.get("challenger", {}).get("models") or []
|
|
78
|
+
if v.get("answer") != "SUPPORTED" or len(models) != 1 or models == v.get("baseline", {}).get("models"):
|
|
79
|
+
continue
|
|
80
|
+
ratio = v.get("cost_ratio")
|
|
81
|
+
summary = (f"verify: {c['name']} vs {b['name']} → SUPPORTED"
|
|
82
|
+
+ (f", {1 - ratio:.0%} lower cost per solved task" if ratio else ""))
|
|
83
|
+
runs = min(v.get("baseline", {}).get("runs", 0), v.get("challenger", {}).get("runs", 0))
|
|
84
|
+
out.append(Evidence(str(path), models[0], summary, runs, path.stat().st_mtime))
|
|
85
|
+
return out
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def settings_path(repo: Path) -> Path:
|
|
89
|
+
return repo / ".claude" / "settings.json"
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def current_model(repo: Path) -> str | None:
|
|
93
|
+
try:
|
|
94
|
+
return json.loads(settings_path(repo).read_text()).get("model")
|
|
95
|
+
except (OSError, ValueError, AttributeError):
|
|
96
|
+
return None
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def apply(repo: Path, model: str) -> Path | None:
|
|
100
|
+
"""Merge {"model": model} into the project settings. Returns the backup path, if a file existed."""
|
|
101
|
+
path = settings_path(repo)
|
|
102
|
+
data, backup = {}, None
|
|
103
|
+
if path.exists():
|
|
104
|
+
data = json.loads(path.read_text())
|
|
105
|
+
if not isinstance(data, dict):
|
|
106
|
+
raise ValueError(f"{path} isn't a JSON object")
|
|
107
|
+
backup = path.with_name(f"settings.json.aimpg-backup-{time.strftime('%Y%m%d-%H%M%S')}")
|
|
108
|
+
shutil.copy2(path, backup)
|
|
109
|
+
data["model"] = model
|
|
110
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
111
|
+
path.write_text(json.dumps(data, indent=2) + "\n")
|
|
112
|
+
return backup
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def add_parser(sub) -> None:
|
|
116
|
+
d = sub.add_parser("defaults", help="turn replay evidence into the repo's default Claude Code model (asks first)")
|
|
117
|
+
d.add_argument("--repo", type=Path, default=Path("."))
|
|
118
|
+
d.add_argument("--apply", action="store_true", help="write it to <repo>/.claude/settings.json")
|
|
119
|
+
d.add_argument("--yes", action="store_true", help="skip the confirmation")
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def main(args) -> int:
|
|
123
|
+
repo = args.repo.expanduser().resolve()
|
|
124
|
+
evidence = gather(repo)
|
|
125
|
+
now = current_model(repo)
|
|
126
|
+
print(f"Repo: {repo}")
|
|
127
|
+
print(f"Current team default model: {now or 'none set (each person uses their own)'}")
|
|
128
|
+
if not evidence:
|
|
129
|
+
print("\nNo replay evidence for this repo yet. Find out which model is good enough on your own commits:")
|
|
130
|
+
print(f" aimpg replay select {repo}")
|
|
131
|
+
print(f" aimpg replay models {repo} --cap 10 --task-mode tests")
|
|
132
|
+
return 0
|
|
133
|
+
print("\nEvidence (newest first):")
|
|
134
|
+
for e in evidence:
|
|
135
|
+
print(f" • {e.summary}" + (" (tentative)" if e.tentative and "Tentative" not in e.summary else ""))
|
|
136
|
+
print(f" from {e.source}")
|
|
137
|
+
top = evidence[0]
|
|
138
|
+
if now == top.model:
|
|
139
|
+
print(f"\nThe default already matches the evidence: {now}.")
|
|
140
|
+
return 0
|
|
141
|
+
print(f"\nProposed: {settings_path(repo)}")
|
|
142
|
+
print(f' "model": "{top.model}"' + (f' (was "{now}")' if now else ""))
|
|
143
|
+
print("Commit it and everyone on the repo starts with this model; anyone can still switch with /model.")
|
|
144
|
+
if top.tentative:
|
|
145
|
+
print(f"Tentative: fewer than {picker.MIN_RUNS} runs per model. A bigger run makes this safer to roll out.")
|
|
146
|
+
if not args.apply:
|
|
147
|
+
print("\nTo write it: aimpg defaults --apply" + (f" --repo {args.repo}" if args.repo != Path(".") else ""))
|
|
148
|
+
return 0
|
|
149
|
+
if not args.yes and input("Write it? [y/N] ").strip().lower() != "y":
|
|
150
|
+
print("Stopped. Nothing was changed.")
|
|
151
|
+
return 1
|
|
152
|
+
try:
|
|
153
|
+
backup = apply(repo, top.model)
|
|
154
|
+
except ValueError as exc:
|
|
155
|
+
print(f"Can't update the settings file: {exc}")
|
|
156
|
+
return 1
|
|
157
|
+
print(f"Written. {'Backup: ' + str(backup) if backup else ''}\nReview and commit {settings_path(repo).relative_to(repo)}.")
|
|
158
|
+
return 0
|
|
@@ -61,17 +61,26 @@ def rows(records: list[Record]) -> list[ModelRow]:
|
|
|
61
61
|
return sorted(out.values(), key=lambda x: (x.usd_per_solved is None, x.usd_per_solved or 0))
|
|
62
62
|
|
|
63
63
|
|
|
64
|
+
def pick(table: list[ModelRow]) -> ModelRow | None:
|
|
65
|
+
"""The cheapest model per solved task whose solve rate is within 10 points of the best."""
|
|
66
|
+
solved = [r for r in table if r.solved and r.usd_per_solved is not None]
|
|
67
|
+
if not solved:
|
|
68
|
+
return None
|
|
69
|
+
best = max(r.solve_rate for r in table)
|
|
70
|
+
return min((r for r in solved if r.solve_rate >= best - GOOD_ENOUGH), key=lambda r: r.usd_per_solved, default=None)
|
|
71
|
+
|
|
72
|
+
|
|
64
73
|
def recommend(table: list[ModelRow]) -> str:
|
|
65
74
|
solved = [r for r in table if r.solved]
|
|
66
|
-
|
|
75
|
+
choice = pick(table)
|
|
76
|
+
if choice is None:
|
|
67
77
|
return "No model solved any task: nothing to recommend yet."
|
|
68
78
|
best = max(r.solve_rate for r in table)
|
|
69
|
-
|
|
70
|
-
pick = min(good, key=lambda r: r.usd_per_solved)
|
|
79
|
+
pick_ = choice
|
|
71
80
|
priciest = max(solved, key=lambda r: r.usd_per_solved)
|
|
72
|
-
text = f"Use {
|
|
73
|
-
if priciest is not
|
|
74
|
-
ratio = priciest.usd_per_solved /
|
|
81
|
+
text = f"Use {pick_.setup.split('@')[-1]}: it solved {pick_.solve_rate:.0%} of tasks (best: {best:.0%})"
|
|
82
|
+
if priciest is not pick_:
|
|
83
|
+
ratio = priciest.usd_per_solved / pick_.usd_per_solved
|
|
75
84
|
text += f" at {ratio:.1f}x lower cost per solved task than {priciest.setup.split('@')[-1]}"
|
|
76
85
|
text += "."
|
|
77
86
|
if min(r.runs for r in table) < MIN_RUNS:
|