turntax 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +125 -15
- package/dist/cli.js +120 -11
- package/dist/core/advice.js +5 -3
- package/dist/core/audit.js +40 -6
- package/dist/core/classify.js +48 -7
- package/dist/core/diff.js +63 -0
- package/dist/core/files.js +26 -5
- package/dist/core/markdown.js +10 -0
- package/dist/core/mcp.js +227 -0
- package/dist/core/render.js +219 -18
- package/dist/core/tools.js +250 -0
- package/dist/core/user.js +80 -0
- package/dist/node/discover.js +45 -0
- package/dist/node/git.js +46 -0
- package/dist/node/mcp.js +212 -0
- package/package.json +14 -3
package/README.md
CHANGED
|
@@ -1,19 +1,48 @@
|
|
|
1
1
|
# turntax
|
|
2
2
|
|
|
3
|
-
Coding agents re-send their instruction files with every turn. `turntax` measures that fixed per-turn cost for a repository,
|
|
3
|
+
Coding agents re-send their instruction files with every turn. `turntax` measures that fixed per-turn cost for a repository, for every agent that reads it, and shows where to cut it.
|
|
4
4
|
|
|
5
5
|
```bash
|
|
6
6
|
npx turntax
|
|
7
7
|
```
|
|
8
8
|
|
|
9
|
-
|
|
9
|
+
Try it in the browser, with nothing uploaded: https://torutdev.github.io/turntax/
|
|
10
|
+
|
|
11
|
+
## Supported agents
|
|
12
|
+
|
|
13
|
+
Each agent's documented loading rules decide what is sent on every turn, what loads only when the agent works in a folder or touches a matching file, and what is ignored.
|
|
14
|
+
|
|
15
|
+
| Agent | Files it reads from the repository |
|
|
16
|
+
| --- | --- |
|
|
17
|
+
| Aider | files listed under `read:` in `.aider.conf.yml` (`CONVENTIONS.md` only when listed or passed with `--read`) |
|
|
18
|
+
| Amp | `AGENTS.md` (or `AGENT.md` / `CLAUDE.md`), nested `AGENTS.md`, @-mentioned files |
|
|
19
|
+
| Claude Code | `CLAUDE.md`, `CLAUDE.local.md`, `.claude/rules`, skills, commands, subagents, `AGENTS.md` when there is no `CLAUDE.md` |
|
|
20
|
+
| Cline | `.clinerules/`, `.cline/rules/` |
|
|
21
|
+
| Continue | `.continue/rules/` |
|
|
22
|
+
| Cursor | `.cursor/rules/*.mdc`, `.cursorrules`, `AGENTS.md`, `CLAUDE.md` |
|
|
23
|
+
| Gemini CLI | `GEMINI.md` |
|
|
24
|
+
| GitHub Copilot | `.github/copilot-instructions.md`, `.github/instructions/`, `AGENTS.md` |
|
|
25
|
+
| Goose | `.goosehints`, `AGENTS.md` |
|
|
26
|
+
| JetBrains Junie | `.junie/AGENTS.md`, or `AGENTS.md` with `.junie/rules/`, or `.junie/guidelines` |
|
|
27
|
+
| Kiro | `.kiro/steering/`, `AGENTS.md` |
|
|
28
|
+
| OpenAI Codex | `AGENTS.md`, `AGENTS.override.md`, `.agents/skills` |
|
|
29
|
+
| OpenCode | `AGENTS.md`, or `CLAUDE.md` when there is none |
|
|
30
|
+
| Roo Code | `.roo/rules/`, `.roo/rules-{mode}/`, `.roorules`, `AGENTS.md` |
|
|
31
|
+
| Windsurf (Devin Desktop) | `.devin/rules/`, `.windsurf/rules/`, `.windsurfrules`, `AGENTS.md` |
|
|
32
|
+
| Zed | the first of `.rules`, `.cursorrules`, `.windsurfrules`, `.clinerules`, `.github/copilot-instructions.md`, `AGENT.md`, `AGENTS.md`, `CLAUDE.md`, `GEMINI.md` |
|
|
33
|
+
|
|
34
|
+
Agents that load exactly the same files share one row in the report:
|
|
10
35
|
|
|
11
36
|
```text
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
37
|
+
always o200k/turn on-demand model tokens/turn counted by 100 turns cached agents
|
|
38
|
+
4 272 53 (no selected model) Claude Code
|
|
39
|
+
1 63 0 GPT-6.1 Sol 63 estimate (o200k_base proxy) $0.01 $0.0006 OpenAI Codex, Roo Code, Kiro, JetBrains Junie, Goose, OpenCode
|
|
40
|
+
1 17 0 (no selected model) Gemini CLI
|
|
41
|
+
4 297 37 GPT-6.1 Sol 297 estimate (o200k_base proxy) $0.06 $0.0030 Cursor
|
|
42
|
+
2 74 8 GPT-6.1 Sol 74 estimate (o200k_base proxy) $0.01 $0.0007 GitHub Copilot
|
|
43
|
+
2 75 0 GPT-6.1 Sol 75 estimate (o200k_base proxy) $0.01 $0.0008 Windsurf (Devin Desktop)
|
|
44
|
+
1 11 0 GPT-6.1 Sol 11 estimate (o200k_base proxy) $0.0022 $0.0001 Zed
|
|
45
|
+
1 63 19 GPT-6.1 Sol 63 estimate (o200k_base proxy) $0.01 $0.0006 Amp
|
|
17
46
|
|
|
18
47
|
Advice (savings are estimates in o200k tokens per turn)
|
|
19
48
|
HIGH duplicate CLAUDE.md, AGENTS.md [saves ~48 tokens/turn]
|
|
@@ -24,7 +53,7 @@ Advice (savings are estimates in o200k tokens per turn)
|
|
|
24
53
|
Section "API package" is mostly about packages/api/. Move it to packages/api/AGENTS.md ...
|
|
25
54
|
```
|
|
26
55
|
|
|
27
|
-
(
|
|
56
|
+
(Excerpt of the output for [`test/fixtures/mixed`](test/fixtures/mixed) with `--models gpt-6.1-sol`.)
|
|
28
57
|
|
|
29
58
|
## What it counts, and how
|
|
30
59
|
|
|
@@ -39,7 +68,7 @@ Claude's tokenizer is not public, and Anthropic's docs note that newer Claude mo
|
|
|
39
68
|
|
|
40
69
|
Cost per N turns = tokens per turn × N × input price. The "cached" column assumes every turn reads the context from the prompt cache. Prices come from a dated table with a source link per model (`turntax --list-models`); pass `--prices my-prices.json` to use your own.
|
|
41
70
|
|
|
42
|
-
See [docs/how-it-counts.md](docs/how-it-counts.md) for the loading rules per
|
|
71
|
+
See [docs/how-it-counts.md](docs/how-it-counts.md) for the loading rules per agent and their sources.
|
|
43
72
|
|
|
44
73
|
## Advice
|
|
45
74
|
|
|
@@ -54,16 +83,97 @@ All checks are deterministic. Each one shows an estimated saving in o200k tokens
|
|
|
54
83
|
- **stale-reference**: referenced files or `npm run` scripts that no longer exist.
|
|
55
84
|
- **non-english**: non-English sections. The token multiplier is measured only against an English version you supply (`--english CLAUDE.md=CLAUDE.en.md`) or, if you opt in, a Claude translation (`--translate`).
|
|
56
85
|
|
|
86
|
+
## MCP servers
|
|
87
|
+
|
|
88
|
+
Every MCP server an agent starts adds its tool definitions (names, descriptions, input schemas) to the context. `turntax` reads the agents' MCP configs and lists the servers each agent starts:
|
|
89
|
+
|
|
90
|
+
| Agent | Project config | User config |
|
|
91
|
+
| --- | --- | --- |
|
|
92
|
+
| Claude Code | `.mcp.json` | `~/.claude.json` (user and per-project servers) |
|
|
93
|
+
| Cursor | `.cursor/mcp.json` | `~/.cursor/mcp.json` |
|
|
94
|
+
| GitHub Copilot (VS Code) | `.vscode/mcp.json` | |
|
|
95
|
+
| Gemini CLI | `.gemini/settings.json` | `~/.gemini/settings.json` |
|
|
96
|
+
| Kiro | `.kiro/settings/mcp.json` | `~/.kiro/settings/mcp.json` |
|
|
97
|
+
| Roo Code | `.roo/mcp.json` | |
|
|
98
|
+
| OpenAI Codex | `.codex/config.toml` (trusted projects) | `~/.codex/config.toml` |
|
|
99
|
+
| OpenCode | `opencode.json` | `~/.config/opencode/opencode.json` |
|
|
100
|
+
| Zed | `.zed/settings.json` | |
|
|
101
|
+
| Windsurf (Devin Desktop) | | `~/.config/devin/mcp_config.json` |
|
|
102
|
+
|
|
103
|
+
Counting the tool definitions needs the servers running, so it is opt-in:
|
|
104
|
+
|
|
105
|
+
```bash
|
|
106
|
+
npx turntax --mcp
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
`--mcp` starts each stdio server with the command from its config (in the repository folder) and connects to streamable HTTP servers, asks for `tools/list`, and counts the definitions as JSON with o200k. A server several agents share is started once. Environment variables in the config (`${VAR}`, `${VAR:-default}`, `${env:VAR}`) are expanded; servers that need a VS Code `${input:...}` prompt, legacy SSE servers and servers that fail or time out (`--mcp-timeout`, default 30 s) are listed with the reason and not counted.
|
|
110
|
+
|
|
111
|
+
Claude Code defers MCP tool definitions by default (tool search): only the tool names are in context until a tool is used, so its row counts the names and shows the full size separately. A server with `alwaysLoad: true` is counted in full.
|
|
112
|
+
|
|
113
|
+
MCP tokens are an o200k estimate and are not included in the model cost columns.
|
|
114
|
+
|
|
115
|
+
## User-level files
|
|
116
|
+
|
|
117
|
+
Agents also read instruction files from your home directory in every project. `--include-user` adds them in a separate `user/turn` column and a `total`:
|
|
118
|
+
|
|
119
|
+
```bash
|
|
120
|
+
npx turntax --include-user
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
It reads only the documented locations: `~/.claude/` (CLAUDE.md, rules, skills, commands, agents), `~/.codex/AGENTS.md`, `~/.agents/skills`, `~/.gemini/GEMINI.md`, Windsurf's `global_rules.md`, Cline's global rules folders, `~/.roo/rules*`, `~/.kiro/steering`, `~/.config/goose/.goosehints`, Amp's `~/.config/AGENTS.md` and `~/.config/amp/AGENTS.md`, and `~/.config/opencode/AGENTS.md` (OpenCode falls back to `~/.claude/CLAUDE.md`), plus the user MCP configs above. Contradictions and duplicates between user-level and repository files are reported. User-level files never count toward the badge or `--max-tokens`, so the result does not depend on who runs it.
|
|
124
|
+
|
|
125
|
+
## Comparing with a baseline
|
|
126
|
+
|
|
127
|
+
```bash
|
|
128
|
+
npx turntax --compare origin/main # instruction files at a git ref, read with git show
|
|
129
|
+
npx turntax --json --out before.json # or save a report ...
|
|
130
|
+
npx turntax --baseline before.json # ... and compare with it later
|
|
131
|
+
npx turntax --compare origin/main --max-increase 200 # exit 1 when any agent grows by more than 200 tokens
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
The comparison covers repository instruction files (o200k), per agent and per file, and lists MCP servers that were added or removed. It never changes the working tree.
|
|
135
|
+
|
|
136
|
+
## GitHub Action
|
|
137
|
+
|
|
138
|
+
The repository is also an Action. On pull requests it compares with the base branch, posts one comment (updated on every push) and writes the report to the job summary:
|
|
139
|
+
|
|
140
|
+
```yaml
|
|
141
|
+
name: turntax
|
|
142
|
+
on: pull_request
|
|
143
|
+
permissions:
|
|
144
|
+
contents: read
|
|
145
|
+
pull-requests: write
|
|
146
|
+
jobs:
|
|
147
|
+
turntax:
|
|
148
|
+
runs-on: ubuntu-latest
|
|
149
|
+
steps:
|
|
150
|
+
- uses: actions/checkout@v4
|
|
151
|
+
- uses: TorutDev/turntax@v0.3.0
|
|
152
|
+
with:
|
|
153
|
+
max-increase: 500 # optional: fail when an agent grows by more than 500 tokens per turn
|
|
154
|
+
# max-tokens: 8000 # optional: absolute budget
|
|
155
|
+
# args: --tools claude-code,cursor
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
The comment opens with a table of the change per agent and per file, and folds the full report below it. Pull requests from forks get a read-only token, so the comment step is skipped there; the job summary still has the report.
|
|
159
|
+
|
|
57
160
|
## Options
|
|
58
161
|
|
|
59
162
|
```text
|
|
60
163
|
turntax [path]
|
|
61
|
-
--tools
|
|
164
|
+
--tools <ids> agents to report (default: every agent that reads a file); see --help
|
|
62
165
|
--models <ids> see --list-models (default: claude-sonnet-5.5, gpt-6.1-sol, gemini-3.8-flash)
|
|
63
166
|
--turns <n> cost projection (default 100)
|
|
64
167
|
--json | --md | --badge output format (default: terminal table)
|
|
168
|
+
--pr-comment Markdown for a pull request comment (change first, full report folded)
|
|
65
169
|
--out <file> write the output to a file
|
|
66
170
|
--max-tokens <n> exit 1 when any tool loads more than n o200k tokens per turn (CI gate)
|
|
171
|
+
--include-user add user-level files and MCP servers from your home directory
|
|
172
|
+
--mcp start the configured MCP servers and count their tool definitions
|
|
173
|
+
--mcp-timeout <s> seconds to wait for each MCP server (default 30)
|
|
174
|
+
--baseline <file.json> compare with an earlier --json report
|
|
175
|
+
--compare <git-ref> compare with the instruction files at a git ref
|
|
176
|
+
--max-increase <n> exit 1 when any tool grows by more than n tokens per turn vs the baseline
|
|
67
177
|
--claude-api exact Claude counts (ANTHROPIC_API_KEY)
|
|
68
178
|
--gemini-api Gemini counts (GEMINI_API_KEY)
|
|
69
179
|
--english <file=en> measure a language multiplier against your English version
|
|
@@ -76,12 +186,12 @@ turntax [path]
|
|
|
76
186
|
|
|
77
187
|
Offline by default: no network calls, no telemetry. File contents are sent only when you pass `--claude-api`, `--gemini-api` or `--translate`, and only to that vendor's API. Keys are read from environment variables and sent only in request headers.
|
|
78
188
|
|
|
189
|
+
`--mcp` runs the commands in the MCP configs it finds and connects to the configured HTTP servers, as the agents themselves would. Run it only on repositories whose configs you trust. `turntax` prints each command before starting it. `--include-user` reads files in your home directory and never sends them anywhere.
|
|
190
|
+
|
|
79
191
|
## Web page
|
|
80
192
|
|
|
81
193
|
[`web/`](web) is a static page that runs the same engine in the browser with the offline tokenizer only. Drop a repository folder or paste files; nothing is uploaded.
|
|
82
194
|
|
|
83
|
-
Try it: https://torutdev.github.io/turntax/
|
|
84
|
-
|
|
85
195
|
```bash
|
|
86
196
|
npm run build:web
|
|
87
197
|
```
|
|
@@ -94,12 +204,12 @@ npx turntax --badge
|
|
|
94
204
|
|
|
95
205
|
[-297%20tokens%2Fturn-brightgreen)](https://github.com/TorutDev/turntax)
|
|
96
206
|
|
|
97
|
-
The badge shows the largest always-loaded total among the tools found, counted with o200k_base.
|
|
207
|
+
The badge shows the largest always-loaded total among the tools found, counted with o200k_base: repository files plus measured project MCP tools, never user-level files.
|
|
98
208
|
|
|
99
209
|
## Limits
|
|
100
210
|
|
|
101
|
-
- Assumes the agent starts at the repository root.
|
|
102
|
-
- Counts instruction files
|
|
211
|
+
- Assumes the agent starts at the repository root. User-level files are counted only with `--include-user`; rules kept in an app's settings or a team dashboard (Cursor user rules, Copilot organization instructions) are not counted.
|
|
212
|
+
- Counts instruction files and, with `--mcp`, MCP tool definitions; not the agent's own system prompt or built-in tools.
|
|
103
213
|
- Loading rules change. Each rule cites the vendor page and the date it was checked; please open an issue when one is out of date.
|
|
104
214
|
|
|
105
215
|
## Contributing
|
package/dist/cli.js
CHANGED
|
@@ -1,13 +1,19 @@
|
|
|
1
1
|
import { readFileSync, writeFileSync } from 'node:fs';
|
|
2
|
-
import {
|
|
2
|
+
import { homedir } from 'node:os';
|
|
3
|
+
import { basename, relative, resolve } from 'node:path';
|
|
3
4
|
import { parseArgs } from 'node:util';
|
|
4
5
|
import { hasNonEnglish } from './core/advice.js';
|
|
5
6
|
import { audit } from './core/audit.js';
|
|
6
7
|
import { classify } from './core/classify.js';
|
|
8
|
+
import { asDiffInput, diffReports, maxIncrease } from './core/diff.js';
|
|
9
|
+
import { parseMcpConfigs, serverKey, toolDefinitionsText, toolNamesText } from './core/mcp.js';
|
|
7
10
|
import { DEFAULT_MODEL_IDS, MODELS, validateModels } from './core/prices.js';
|
|
8
|
-
import { badgeMarkdown, headlineTokens, renderMarkdown, renderText } from './core/render.js';
|
|
11
|
+
import { badgeMarkdown, headlineTokens, renderMarkdown, renderPrComment, renderText } from './core/render.js';
|
|
12
|
+
import { countO200k } from './core/tokenize.js';
|
|
9
13
|
import { TOOL_IDS } from './core/tools.js';
|
|
10
|
-
import { discover, existsIn, packageScripts } from './node/discover.js';
|
|
14
|
+
import { discover, discoverMcpConfigs, discoverUser, existsIn, packageScripts } from './node/discover.js';
|
|
15
|
+
import { filesAtRef } from './node/git.js';
|
|
16
|
+
import { measureServer } from './node/mcp.js';
|
|
11
17
|
import { anthropicCounter, geminiCounter, translateToEnglish } from './node/remote.js';
|
|
12
18
|
const VERSION = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')).version;
|
|
13
19
|
const HELP = `turntax ${VERSION}
|
|
@@ -22,9 +28,20 @@ Options:
|
|
|
22
28
|
--turns <n> Turns for the cost projection (default: 100)
|
|
23
29
|
--json Print the full report as JSON
|
|
24
30
|
--md Print a Markdown report
|
|
31
|
+
--pr-comment Print a pull request comment: the change first, the full report folded
|
|
25
32
|
--out <file> Write the report to a file instead of stdout
|
|
26
33
|
--badge Print a shields.io badge snippet
|
|
27
34
|
--max-tokens <n> Exit with code 1 if any tool loads more than n o200k tokens per turn
|
|
35
|
+
(repository files plus measured project MCP tools)
|
|
36
|
+
--include-user Also count user-level files and MCP servers in your home directory
|
|
37
|
+
(~/.claude/CLAUDE.md, ~/.codex/AGENTS.md, ~/.gemini/GEMINI.md, ...)
|
|
38
|
+
--mcp Start the configured MCP servers and count their tool definitions
|
|
39
|
+
(runs the commands in .mcp.json and the other agents' MCP configs)
|
|
40
|
+
--mcp-timeout <s> Seconds to wait for each MCP server (default: 30)
|
|
41
|
+
--baseline <file.json> Compare with an earlier --json report
|
|
42
|
+
--compare <git-ref> Compare with the instruction files at a git ref (for example origin/main)
|
|
43
|
+
--max-increase <n> Exit with code 1 if any tool's per-turn tokens grew by more than n
|
|
44
|
+
compared with --baseline or --compare
|
|
28
45
|
--claude-api Count Claude tokens with Anthropic's count_tokens (needs ANTHROPIC_API_KEY;
|
|
29
46
|
sends file contents to api.anthropic.com)
|
|
30
47
|
--gemini-api Count Gemini tokens with Google's countTokens (needs GEMINI_API_KEY;
|
|
@@ -37,7 +54,7 @@ Options:
|
|
|
37
54
|
-h, --help Show help
|
|
38
55
|
-v, --version Show version
|
|
39
56
|
|
|
40
|
-
Without --claude-api
|
|
57
|
+
Without --claude-api, --gemini-api or --mcp nothing leaves your machine.
|
|
41
58
|
`;
|
|
42
59
|
class UsageError extends Error {
|
|
43
60
|
}
|
|
@@ -68,6 +85,7 @@ export async function main(argv, env = process.env) {
|
|
|
68
85
|
turns: { type: 'string' },
|
|
69
86
|
json: { type: 'boolean' },
|
|
70
87
|
md: { type: 'boolean' },
|
|
88
|
+
'pr-comment': { type: 'boolean' },
|
|
71
89
|
out: { type: 'string' },
|
|
72
90
|
badge: { type: 'boolean' },
|
|
73
91
|
'max-tokens': { type: 'string' },
|
|
@@ -77,6 +95,12 @@ export async function main(argv, env = process.env) {
|
|
|
77
95
|
translate: { type: 'boolean' },
|
|
78
96
|
prices: { type: 'string' },
|
|
79
97
|
'section-tokens': { type: 'string' },
|
|
98
|
+
'include-user': { type: 'boolean' },
|
|
99
|
+
mcp: { type: 'boolean' },
|
|
100
|
+
'mcp-timeout': { type: 'string' },
|
|
101
|
+
baseline: { type: 'string' },
|
|
102
|
+
compare: { type: 'string' },
|
|
103
|
+
'max-increase': { type: 'string' },
|
|
80
104
|
help: { type: 'boolean', short: 'h' },
|
|
81
105
|
version: { type: 'boolean', short: 'v' },
|
|
82
106
|
},
|
|
@@ -121,6 +145,27 @@ export async function main(argv, env = process.env) {
|
|
|
121
145
|
throw new UsageError(`Unknown tool "${t}". Use: ${TOOL_IDS.join(', ')}`);
|
|
122
146
|
const root = resolve(positionals[0] ?? '.');
|
|
123
147
|
const files = await discover(root);
|
|
148
|
+
if (o.baseline && o.compare)
|
|
149
|
+
throw new UsageError('Use --baseline or --compare, not both.');
|
|
150
|
+
if (o['max-increase'] !== undefined && !o.baseline && !o.compare)
|
|
151
|
+
throw new UsageError('--max-increase needs --baseline or --compare.');
|
|
152
|
+
const maxIncreaseLimit = o['max-increase'] === undefined ? undefined : nonNegativeInt('max-increase', o['max-increase']);
|
|
153
|
+
let userFiles = [];
|
|
154
|
+
let mcpFiles = discoverMcpConfigs(root);
|
|
155
|
+
if (o['include-user']) {
|
|
156
|
+
const user = discoverUser(env.HOME || env.USERPROFILE || homedir());
|
|
157
|
+
userFiles = user.files;
|
|
158
|
+
mcpFiles = [...mcpFiles, ...user.mcpConfigs];
|
|
159
|
+
}
|
|
160
|
+
const parsedMcp = parseMcpConfigs(mcpFiles, root);
|
|
161
|
+
for (const p of parsedMcp.problems)
|
|
162
|
+
notice(`${p.path} ${p.message}`);
|
|
163
|
+
const servers = parsedMcp.servers;
|
|
164
|
+
let measured;
|
|
165
|
+
if (o.mcp) {
|
|
166
|
+
const timeoutMs = positiveInt('mcp-timeout', o['mcp-timeout'], 30) * 1000;
|
|
167
|
+
measured = await measureAll(servers.filter((s) => !s.disabled && (!tools || tools.includes(s.tool))), root, timeoutMs, env);
|
|
168
|
+
}
|
|
124
169
|
const counters = {};
|
|
125
170
|
if (o['claude-api']) {
|
|
126
171
|
const key = env.ANTHROPIC_API_KEY;
|
|
@@ -167,6 +212,8 @@ export async function main(argv, env = process.env) {
|
|
|
167
212
|
models,
|
|
168
213
|
counters,
|
|
169
214
|
turns: positiveInt('turns', o.turns, 100),
|
|
215
|
+
userFiles,
|
|
216
|
+
mcp: { servers, measured },
|
|
170
217
|
advice: {
|
|
171
218
|
exists: existsIn(root),
|
|
172
219
|
packageScripts: packageScripts(root),
|
|
@@ -174,23 +221,51 @@ export async function main(argv, env = process.env) {
|
|
|
174
221
|
sectionTokens: positiveInt('section-tokens', o['section-tokens'], 600),
|
|
175
222
|
},
|
|
176
223
|
});
|
|
224
|
+
if (o.baseline) {
|
|
225
|
+
let data;
|
|
226
|
+
try {
|
|
227
|
+
data = JSON.parse(readFileSync(o.baseline, 'utf8'));
|
|
228
|
+
}
|
|
229
|
+
catch (e) {
|
|
230
|
+
throw new UsageError(`Could not read --baseline ${o.baseline}: ${e.message}`);
|
|
231
|
+
}
|
|
232
|
+
report.diff = diffReports(asDiffInput(data), report, basename(o.baseline));
|
|
233
|
+
}
|
|
234
|
+
else if (o.compare) {
|
|
235
|
+
const base = await filesAtRef(root, o.compare).catch((e) => {
|
|
236
|
+
throw new UsageError(e.message);
|
|
237
|
+
});
|
|
238
|
+
const before = await audit(base.files, {
|
|
239
|
+
tools,
|
|
240
|
+
models: [],
|
|
241
|
+
mcp: { servers: parseMcpConfigs(base.mcpConfigs, root).servers },
|
|
242
|
+
});
|
|
243
|
+
report.diff = diffReports(before, report, o.compare);
|
|
244
|
+
}
|
|
177
245
|
const output = o.json
|
|
178
246
|
? JSON.stringify({ root, ...report }, null, 2) + '\n'
|
|
179
|
-
: o
|
|
180
|
-
?
|
|
181
|
-
: o.
|
|
182
|
-
?
|
|
183
|
-
:
|
|
247
|
+
: o['pr-comment']
|
|
248
|
+
? renderPrComment(report)
|
|
249
|
+
: o.md
|
|
250
|
+
? renderMarkdown(report)
|
|
251
|
+
: o.badge
|
|
252
|
+
? badgeMarkdown(report) + '\n'
|
|
253
|
+
: renderText(report, root) + '\n';
|
|
184
254
|
if (o.out)
|
|
185
255
|
writeFileSync(o.out, output);
|
|
186
256
|
else
|
|
187
257
|
process.stdout.write(output);
|
|
258
|
+
let code = 0;
|
|
188
259
|
const max = o['max-tokens'] === undefined ? undefined : positiveInt('max-tokens', o['max-tokens'], 0);
|
|
189
260
|
if (max !== undefined && headlineTokens(report) > max) {
|
|
190
261
|
process.stderr.write(`turntax: ${headlineTokens(report)} o200k tokens per turn exceeds --max-tokens ${max}\n`);
|
|
191
|
-
|
|
262
|
+
code = 1;
|
|
192
263
|
}
|
|
193
|
-
|
|
264
|
+
if (maxIncreaseLimit !== undefined && report.diff && maxIncrease(report.diff) > maxIncreaseLimit) {
|
|
265
|
+
process.stderr.write(`turntax: per-turn tokens grew by ${maxIncrease(report.diff)} vs ${report.diff.base}, more than --max-increase ${maxIncreaseLimit}\n`);
|
|
266
|
+
code = 1;
|
|
267
|
+
}
|
|
268
|
+
return code;
|
|
194
269
|
}
|
|
195
270
|
catch (e) {
|
|
196
271
|
if (e instanceof UsageError) {
|
|
@@ -201,6 +276,40 @@ export async function main(argv, env = process.env) {
|
|
|
201
276
|
return 1;
|
|
202
277
|
}
|
|
203
278
|
}
|
|
279
|
+
function nonNegativeInt(name, value) {
|
|
280
|
+
const n = Number(value);
|
|
281
|
+
if (!Number.isInteger(n) || n < 0)
|
|
282
|
+
throw new UsageError(`--${name} must be a non-negative integer`);
|
|
283
|
+
return n;
|
|
284
|
+
}
|
|
285
|
+
/** Starts each distinct server once (several agents often share one) and counts its tool definitions. */
|
|
286
|
+
async function measureAll(servers, root, timeoutMs, env) {
|
|
287
|
+
const unique = new Map();
|
|
288
|
+
for (const s of servers)
|
|
289
|
+
if (!unique.has(serverKey(s)))
|
|
290
|
+
unique.set(serverKey(s), s);
|
|
291
|
+
const out = new Map();
|
|
292
|
+
const queue = [...unique.entries()];
|
|
293
|
+
for (const [, s] of queue)
|
|
294
|
+
notice(s.transport === 'stdio'
|
|
295
|
+
? `Starting MCP server "${s.name}" from ${s.configPath}: ${[s.command, ...(s.args ?? [])].join(' ')}`
|
|
296
|
+
: `Connecting to MCP server "${s.name}" from ${s.configPath}: ${s.url}`);
|
|
297
|
+
const worker = async () => {
|
|
298
|
+
for (let next = queue.shift(); next; next = queue.shift()) {
|
|
299
|
+
const [key, s] = next;
|
|
300
|
+
const r = await measureServer(s, { root, timeoutMs, env });
|
|
301
|
+
if (r.tools) {
|
|
302
|
+
out.set(key, { tools: r.tools.length, o200k: countO200k(toolDefinitionsText(r.tools)), namesO200k: countO200k(toolNamesText(r.tools)) });
|
|
303
|
+
}
|
|
304
|
+
else {
|
|
305
|
+
out.set(key, { error: r.error ?? 'failed' });
|
|
306
|
+
notice(`MCP server "${s.name}" was not measured: ${r.error}`);
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
};
|
|
310
|
+
await Promise.all(Array.from({ length: Math.min(4, queue.length) }, worker));
|
|
311
|
+
return out;
|
|
312
|
+
}
|
|
204
313
|
function notice(message) {
|
|
205
314
|
process.stderr.write(`turntax: ${message}\n`);
|
|
206
315
|
}
|
package/dist/core/advice.js
CHANGED
|
@@ -3,14 +3,16 @@ import { countO200k } from './tokenize.js';
|
|
|
3
3
|
import { TOOLS } from './tools.js';
|
|
4
4
|
const DEFAULT_SECTION_TOKENS = 600;
|
|
5
5
|
export function buildAdvice(units, files, opts = {}) {
|
|
6
|
+
// User-level files take part in checks across files; moving sections or checking paths only makes sense in the repository.
|
|
7
|
+
const repo = units.filter((u) => !u.scope);
|
|
6
8
|
const advice = [
|
|
7
9
|
...misconfigured(units),
|
|
8
10
|
...sizeLimits(units, files),
|
|
9
11
|
...duplicates(units),
|
|
10
12
|
...contradictions(units),
|
|
11
|
-
...sections(
|
|
13
|
+
...sections(repo, opts.sectionTokens ?? DEFAULT_SECTION_TOKENS),
|
|
12
14
|
...nonEnglish(units, opts.english),
|
|
13
|
-
...staleReferences(
|
|
15
|
+
...staleReferences(repo, opts),
|
|
14
16
|
];
|
|
15
17
|
const order = { high: 0, medium: 1, low: 2 };
|
|
16
18
|
return advice.sort((a, b) => order[a.severity] - order[b.severity] || (b.savingTokens ?? 0) - (a.savingTokens ?? 0));
|
|
@@ -396,7 +398,7 @@ function nonEnglish(units, english) {
|
|
|
396
398
|
/** True for instruction files that live in a package subdirectory rather than at the root or in a root tool folder. */
|
|
397
399
|
function isNested(path) {
|
|
398
400
|
const dir = dirname(path);
|
|
399
|
-
return dir !== '' && !/^\.(claude|github|cursor|windsurf|devin|agents)(\/|$)/.test(dir);
|
|
401
|
+
return dir !== '' && !/^\.(claude|github|cursor|windsurf|devin|agents|clinerules|cline|roo|kiro|junie|continue)(\/|$)/.test(dir);
|
|
400
402
|
}
|
|
401
403
|
function staleReferences(units, opts) {
|
|
402
404
|
const out = [];
|
package/dist/core/audit.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import { buildAdvice } from './advice.js';
|
|
2
2
|
import { classify, detectTools } from './classify.js';
|
|
3
|
+
import { defersTools, serverKey } from './mcp.js';
|
|
3
4
|
import { countO200k, offlineCounter } from './tokenize.js';
|
|
4
5
|
import { TOOLS } from './tools.js';
|
|
6
|
+
import { classifyUser } from './user.js';
|
|
5
7
|
export function modelsForTool(tool, models) {
|
|
6
8
|
const spec = TOOLS.find((t) => t.id === tool);
|
|
7
9
|
return spec.families.length ? models.filter((m) => spec.families.includes(m.family)) : models;
|
|
@@ -15,9 +17,12 @@ function counterFor(model, opts) {
|
|
|
15
17
|
}
|
|
16
18
|
export async function audit(files, opts) {
|
|
17
19
|
const turns = opts.turns ?? 100;
|
|
18
|
-
const allUnits = classify(files);
|
|
19
|
-
const
|
|
20
|
+
const allUnits = [...classify(files), ...classifyUser(opts.userFiles ?? [])];
|
|
21
|
+
const servers = opts.mcp?.servers ?? [];
|
|
22
|
+
const toolIds = opts.tools ??
|
|
23
|
+
[...new Set([...detectTools(allUnits), ...servers.filter((s) => !s.disabled).map((s) => s.tool)])].sort((a, b) => TOOLS.findIndex((t) => t.id === a) - TOOLS.findIndex((t) => t.id === b));
|
|
20
24
|
const units = allUnits.filter((u) => toolIds.includes(u.tool));
|
|
25
|
+
const mcp = servers.filter((s) => toolIds.includes(s.tool)).map((s) => mcpRow(s, opts.mcp?.measured));
|
|
21
26
|
// Count each distinct text once per model, only for the tools that run that model.
|
|
22
27
|
const counts = new Map();
|
|
23
28
|
for (const model of opts.models) {
|
|
@@ -39,6 +44,8 @@ export async function audit(files, opts) {
|
|
|
39
44
|
};
|
|
40
45
|
if (u.importedFrom)
|
|
41
46
|
row.importedFrom = u.importedFrom;
|
|
47
|
+
if (u.scope)
|
|
48
|
+
row.scope = u.scope;
|
|
42
49
|
if (u.mode !== 'ignored')
|
|
43
50
|
for (const m of modelsForTool(u.tool, opts.models)) {
|
|
44
51
|
const c = counts.get(m.id)?.get(u.text);
|
|
@@ -51,21 +58,48 @@ export async function audit(files, opts) {
|
|
|
51
58
|
const spec = TOOLS.find((t) => t.id === id);
|
|
52
59
|
const rows = files_.filter((r) => r.tool === id);
|
|
53
60
|
const always = rows.filter((r) => r.mode === 'always');
|
|
54
|
-
const
|
|
61
|
+
const repoAlways = always.filter((r) => !r.scope);
|
|
62
|
+
const userAlways = always.filter((r) => r.scope === 'user');
|
|
63
|
+
const onDemand = rows.filter((r) => r.mode === 'on-demand' && !r.scope);
|
|
55
64
|
const models = modelsForTool(id, opts.models).map((m) => modelCost(m, always, turns));
|
|
65
|
+
const toolMcp = mcp.filter((r) => r.tool === id && r.status !== 'disabled');
|
|
66
|
+
const mcpTokens = (scope) => sum(toolMcp.filter((r) => r.scope === scope).map((r) => r.o200k ?? 0));
|
|
67
|
+
const perTurnO200k = sum(repoAlways.map((r) => r.o200k));
|
|
68
|
+
const userPerTurnO200k = sum(userAlways.map((r) => r.o200k)) + mcpTokens('user');
|
|
69
|
+
const mcpPerTurnO200k = mcpTokens('project');
|
|
56
70
|
return {
|
|
57
71
|
tool: id,
|
|
58
72
|
name: spec.name,
|
|
59
|
-
alwaysFiles: new Set(
|
|
73
|
+
alwaysFiles: new Set(repoAlways.map((r) => r.path)).size,
|
|
60
74
|
onDemandFiles: new Set(onDemand.map((r) => r.path)).size,
|
|
61
|
-
perTurnO200k
|
|
75
|
+
perTurnO200k,
|
|
62
76
|
onDemandO200k: sum(onDemand.map((r) => r.o200k)),
|
|
77
|
+
userAlwaysFiles: new Set(userAlways.map((r) => r.path)).size,
|
|
78
|
+
userPerTurnO200k,
|
|
79
|
+
mcpServers: toolMcp.length,
|
|
80
|
+
mcpPerTurnO200k,
|
|
81
|
+
mcpMeasured: toolMcp.every((r) => r.status === 'measured'),
|
|
82
|
+
totalPerTurnO200k: perTurnO200k + mcpPerTurnO200k + userPerTurnO200k,
|
|
63
83
|
models,
|
|
64
84
|
notes: spec.notes,
|
|
65
85
|
sources: spec.sources,
|
|
66
86
|
};
|
|
67
87
|
});
|
|
68
|
-
|
|
88
|
+
const adviceFiles = [...files, ...(opts.userFiles ?? [])];
|
|
89
|
+
return { turns, tools, files: files_, advice: buildAdvice(units, adviceFiles, opts.advice), models: opts.models, mcp };
|
|
90
|
+
}
|
|
91
|
+
function mcpRow(s, measured) {
|
|
92
|
+
const row = { name: s.name, tool: s.tool, configPath: s.configPath, scope: s.scope, transport: s.transport, status: 'not-measured' };
|
|
93
|
+
if (s.disabled)
|
|
94
|
+
return { ...row, status: 'disabled' };
|
|
95
|
+
const m = measured?.get(serverKey(s));
|
|
96
|
+
if (!m)
|
|
97
|
+
return row;
|
|
98
|
+
if ('error' in m)
|
|
99
|
+
return { ...row, status: 'failed', error: m.error };
|
|
100
|
+
if (defersTools(s))
|
|
101
|
+
return { ...row, status: 'measured', tools: m.tools, o200k: m.namesO200k, deferredO200k: m.o200k };
|
|
102
|
+
return { ...row, status: 'measured', tools: m.tools, o200k: m.o200k };
|
|
69
103
|
}
|
|
70
104
|
function modelCost(model, always, turns) {
|
|
71
105
|
const counts = always.map((r) => r.counts[model.id]);
|
package/dist/core/classify.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { basename, dirname, findImports, firstParagraph, parseFrontmatter, resolvePath, stripHtmlComments } from './markdown.js';
|
|
1
|
+
import { basename, configList, dirname, findImports, firstParagraph, parseFrontmatter, resolvePath, stripHtmlComments } from './markdown.js';
|
|
2
2
|
import { TOOLS } from './tools.js';
|
|
3
3
|
/** Works out which files each tool loads, when, and with what text. */
|
|
4
4
|
export function classify(input, toolIds) {
|
|
@@ -48,6 +48,28 @@ function applyRule(tool, rule, files, byPath) {
|
|
|
48
48
|
return files
|
|
49
49
|
.filter((f) => rule.pattern.test(f.path))
|
|
50
50
|
.map((f) => unit(tool.id, f.path, 'ignored', rule.reason, f.text, { problem: rule.reason }));
|
|
51
|
+
case 'pattern':
|
|
52
|
+
return files
|
|
53
|
+
.filter((f) => rule.pattern.test(f.path))
|
|
54
|
+
.map((f) => unit(tool.id, f.path, rule.mode, rule.trigger, f.text, { notes: rule.note ? [rule.note] : [] }));
|
|
55
|
+
case 'precedence': {
|
|
56
|
+
const inGroup = (g) => files.filter((f) => g.patterns.some((p) => p.test(f.path)));
|
|
57
|
+
const winner = rule.groups.find((g) => inGroup(g).length > 0);
|
|
58
|
+
if (!winner)
|
|
59
|
+
return [];
|
|
60
|
+
return rule.groups.flatMap((g) => inGroup(g).map((f) => g === winner
|
|
61
|
+
? unit(tool.id, f.path, 'always', 'project instructions', f.text, { notes: rule.note ? [rule.note] : [] })
|
|
62
|
+
: unit(tool.id, f.path, 'ignored', `${winner.label} takes precedence`, f.text)));
|
|
63
|
+
}
|
|
64
|
+
case 'config-list': {
|
|
65
|
+
const config = byPath.get(rule.configPath);
|
|
66
|
+
if (!config)
|
|
67
|
+
return [];
|
|
68
|
+
return configList(config, rule.key)
|
|
69
|
+
.map((p) => byPath.get(p))
|
|
70
|
+
.filter((f) => f !== undefined)
|
|
71
|
+
.map((f) => unit(tool.id, f.path, 'always', rule.trigger, f.text));
|
|
72
|
+
}
|
|
51
73
|
case 'frontmatter':
|
|
52
74
|
return files
|
|
53
75
|
.filter((f) => rule.pattern.test(f.path))
|
|
@@ -111,19 +133,27 @@ function applyHierarchical(tool, rule, files, byPath) {
|
|
|
111
133
|
conditionFails = 'CLAUDE.md exists at the root, so Claude Code reads it instead of AGENTS.md (default setting).';
|
|
112
134
|
if (rule.condition === 'copilot-no-agents-md' && files.some((f) => f.path === 'AGENTS.md'))
|
|
113
135
|
conditionFails = 'A root AGENTS.md exists, which Copilot uses instead.';
|
|
114
|
-
|
|
136
|
+
// Every matching file is read, except where the tool reads only one file per directory.
|
|
137
|
+
const firstPerLevel = new Map();
|
|
138
|
+
const read = [];
|
|
115
139
|
const out = [];
|
|
116
140
|
for (const name of rule.names) {
|
|
117
141
|
for (const f of matches.filter((m) => basename(m.path) === name)) {
|
|
118
142
|
const level = levelOf(f.path, rule.sameLevelDirs);
|
|
119
|
-
|
|
120
|
-
|
|
143
|
+
// The docs do not say when subdirectory copies load, so they are not counted either way.
|
|
144
|
+
if (level !== '' && rule.nested === 'skip')
|
|
145
|
+
continue;
|
|
146
|
+
const first = firstPerLevel.get(level);
|
|
147
|
+
if (rule.onePerDirectory && first) {
|
|
148
|
+
out.push(unit(tool.id, f.path, 'ignored', `${basename(first.path)} takes precedence`, f.text));
|
|
121
149
|
continue;
|
|
122
150
|
}
|
|
123
|
-
|
|
151
|
+
if (!first)
|
|
152
|
+
firstPerLevel.set(level, f);
|
|
153
|
+
read.push(f);
|
|
124
154
|
}
|
|
125
155
|
}
|
|
126
|
-
for (const f of
|
|
156
|
+
for (const f of read) {
|
|
127
157
|
if (conditionFails) {
|
|
128
158
|
out.push(unit(tool.id, f.path, 'ignored', conditionFails, f.text));
|
|
129
159
|
continue;
|
|
@@ -186,7 +216,18 @@ function expandImports(tool, parent, rule, byPath, depth, seen = new Set([parent
|
|
|
186
216
|
continue;
|
|
187
217
|
seen.add(resolved);
|
|
188
218
|
const notes = [];
|
|
189
|
-
|
|
219
|
+
let mode = parent.mode;
|
|
220
|
+
let trigger = parent.trigger;
|
|
221
|
+
let text = f.text;
|
|
222
|
+
if (rule.importGlobs) {
|
|
223
|
+
const { frontmatter, body } = parseFrontmatter(f.text);
|
|
224
|
+
if (frontmatter.globs !== undefined && String(frontmatter.globs) !== '') {
|
|
225
|
+
mode = 'on-demand';
|
|
226
|
+
trigger = `globs: ${Array.isArray(frontmatter.globs) ? frontmatter.globs.join(', ') : String(frontmatter.globs)}`;
|
|
227
|
+
text = body;
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
const child = unit(tool, f.path, mode, trigger, transform(text, rule, notes), {
|
|
190
231
|
importedFrom: parent.path,
|
|
191
232
|
notes,
|
|
192
233
|
});
|