turntax 0.0.0-stage → 0.1.0

package/dist/core/render.js

@@ -0,0 +1,143 @@
+export const METHOD_LABEL = {
+    'offline-exact': 'exact (offline tokenizer)',
+    'offline-proxy': 'estimate (o200k_base proxy)',
+    api: 'vendor API count',
+    unavailable: 'not counted',
+    'none-loaded': 'nothing always loaded',
+};
+export function fmtTokens(n) {
+    return n.toLocaleString('en-US');
+}
+export function fmtUsd(n) {
+    if (n === null)
+        return 'n/a';
+    if (n === 0)
+        return '$0';
+    if (n < 0.01)
+        return `$${n.toFixed(4)}`;
+    return `$${n.toFixed(2)}`;
+}
+function table(rows, indent = '') {
+    const widths = rows[0].map((_, c) => Math.max(...rows.map((r) => (r[c] ?? '').length)));
+    return rows.map((r) => indent + r.map((cell, c) => cell.padEnd(widths[c])).join('  ').trimEnd()).join('\n');
+}
+/** Highest always-loaded o200k total across the reported tools. */
+export function headlineTokens(report) {
+    return Math.max(0, ...report.tools.map((t) => t.perTurnO200k));
+}
+export function renderText(report, root) {
+    const out = [];
+    out.push(`turntax: per-turn context tax${root ? ` for ${root}` : ''}`);
+    out.push('');
+    if (!report.tools.length) {
+        out.push('No agent instruction files found.');
+        return out.join('\n');
+    }
+    const rows = [
+        ['tool', 'always', 'o200k/turn', 'on-demand', 'model', 'tokens/turn', 'counted by', `${report.turns} turns`, 'cached'],
+    ];
+    for (const t of report.tools) {
+        const head = [t.name, String(t.alwaysFiles), fmtTokens(t.perTurnO200k), fmtTokens(t.onDemandO200k)];
+        const blank = head.map(() => '');
+        if (!t.models.length)
+            rows.push([...head, '(no selected model)', '', '', '', '']);
+        t.models.forEach((m, i) => rows.push([
+            ...(i === 0 ? head : blank),
+            m.label,
+            ...(m.perTurn === null
+                ? ['-', m.reason ?? METHOD_LABEL[m.method], '-', '-']
+                : [fmtTokens(m.perTurn), METHOD_LABEL[m.method], fmtUsd(m.uncached), fmtUsd(m.cached)]),
+        ]));
+    }
+    out.push(table(rows));
+    out.push('');
+    out.push('Files');
+    const fileRows = [['tool', 'mode', 'file', 'o200k', 'loads']];
+    for (const f of report.files) {
+        const name = f.part === 'full' ? f.path : `${f.path} (${f.part})`;
+        fileRows.push([f.tool, f.mode, name + (f.importedFrom ? ` <- @${f.importedFrom}` : ''), fmtTokens(f.o200k), f.trigger]);
+    }
+    out.push(table(fileRows, '  '));
+    const notes = [...new Set(report.files.flatMap((f) => f.notes.map((n) => `${f.path}: ${n}`)))];
+    if (notes.length) {
+        out.push('');
+        out.push('Notes');
+        for (const n of notes)
+            out.push(`  - ${n}`);
+    }
+    out.push('');
+    out.push('Advice (savings are estimates in o200k tokens per turn)');
+    if (!report.advice.length)
+        out.push('  No issues found.');
+    for (const a of report.advice) {
+        const saving = a.savingTokens === null ? '' : ` [saves ~${fmtTokens(a.savingTokens)} tokens/turn]`;
+        const where = a.paths.join(', ') + (a.line ? `:${a.line}` : '');
+        out.push(`  ${a.severity.toUpperCase()} ${a.kind} ${where}${saving}`);
+        out.push(`    ${a.message}${a.savingNote ? ` (${a.savingNote})` : ''}`);
+    }
+    out.push('');
+    out.push(footer(report));
+    return out.join('\n');
+}
+function footer(report) {
+    const lines = [
+        'o200k = OpenAI o200k_base, counted offline. It is a reference for every file, not a Claude or Gemini count.',
+        'Claude and Gemini tokenizers are not public: their counts come only from the vendor APIs (--claude-api, --gemini-api).',
+        `Cost = tokens/turn x ${report.turns} turns x input price. "Cached" assumes every turn reads the context from the prompt cache.`,
+        'Prices (USD per 1M input tokens):',
+    ];
+    for (const m of report.models) {
+        lines.push(`  ${m.label}: ${m.inputPerMTok ?? 'n/a'} input, ${m.cachedInputPerMTok ?? 'n/a'} cached` +
+            `${m.priceNote ? ` (${m.priceNote})` : ''}; ${m.source}, checked ${m.checkedAt}`);
+    }
+    return lines.join('\n');
+}
+export function renderMarkdown(report) {
+    const out = ['# Agent context tax report', ''];
+    out.push(`Every agent re-sends its always-loaded instruction files with each turn. Costs are for ${report.turns} turns.`, '');
+    out.push('## Per tool', '');
+    out.push('| Tool | Model | Tokens / turn | Counted by | Uncached | Cached |', '| --- | --- | ---: | --- | ---: | ---: |');
+    for (const t of report.tools) {
+        if (!t.models.length)
+            out.push(`| ${t.name} | - | ${fmtTokens(t.perTurnO200k)} (o200k) | reference | - | - |`);
+        for (const m of t.models)
+            out.push(m.perTurn === null
+                ? `| ${t.name} | ${m.label} | - | ${m.reason ?? ''} | - | - |`
+                : `| ${t.name} | ${m.label} | ${fmtTokens(m.perTurn)} | ${METHOD_LABEL[m.method]} | ${fmtUsd(m.uncached)} | ${fmtUsd(m.cached)} |`);
+    }
+    out.push('', '## Files', '', '| Tool | Mode | File | o200k tokens | Loads |', '| --- | --- | --- | ---: | --- |');
+    for (const f of report.files) {
+        const name = f.part === 'full' ? f.path : `${f.path} (${f.part})`;
+        out.push(`| ${f.tool} | ${f.mode} | \`${name}\` | ${fmtTokens(f.o200k)} | ${escapeCell(f.trigger)} |`);
+    }
+    out.push('', '## Advice', '');
+    if (!report.advice.length)
+        out.push('No issues found.');
+    for (const a of report.advice) {
+        const saving = a.savingTokens === null ? '' : ` Saves about ${fmtTokens(a.savingTokens)} tokens per turn.`;
+        out.push(`- **${a.severity}** \`${a.kind}\` ${a.paths.map((p) => `\`${p}\``).join(', ')}${a.line ? ` line ${a.line}` : ''}: ${a.message}${saving}${a.savingNote ? ` _${a.savingNote}_` : ''}`);
+    }
+    out.push('', '## Method', '');
+    out.push('- o200k = OpenAI `o200k_base`, counted offline; a reference, not a Claude or Gemini count.');
+    out.push('- Claude counts come only from Anthropic `count_tokens`; Gemini counts only from Google `countTokens`.');
+    out.push('- "Cached" assumes every turn reads the context from the prompt cache.');
+    for (const m of report.models)
+        out.push(`- ${m.label}: $${m.inputPerMTok ?? 'n/a'} input / $${m.cachedInputPerMTok ?? 'n/a'} cached per 1M tokens ([source](${m.source}), checked ${m.checkedAt}).`);
+    out.push('', badgeMarkdown(report));
+    return out.join('\n') + '\n';
+}
+function escapeCell(s) {
+    return s.replace(/\|/g, '\\|');
+}
+function shieldsEscape(s) {
+    return encodeURIComponent(s.replace(/-/g, '--').replace(/_/g, '__'));
+}
+export function badgeUrl(report) {
+    const n = headlineTokens(report);
+    const value = n >= 1000 ? `${(n / 1000).toFixed(1)}k tokens/turn` : `${n} tokens/turn`;
+    const color = n < 2000 ? 'brightgreen' : n < 5000 ? 'yellow' : n < 10000 ? 'orange' : 'red';
+    return `https://img.shields.io/badge/${shieldsEscape('context tax (o200k)')}-${shieldsEscape(value)}-${color}`;
+}
+export function badgeMarkdown(report) {
+    return `[![context tax (o200k)](${badgeUrl(report)})](https://github.com/TorutDev/turntax)`;
+}

package/dist/core/tokenize.js

@@ -0,0 +1,21 @@
+import { countTokens } from 'gpt-tokenizer/encoding/o200k_base';
+const PLAIN_TEXT = { disallowedSpecial: new Set() };
+/** OpenAI o200k_base token count. Special-token strings are counted as ordinary text. */
+export function countO200k(text) {
+    return text ? countTokens(text, PLAIN_TEXT) : 0;
+}
+/** Counts offline where a public tokenizer exists; reports the rest as unavailable. */
+export const offlineCounter = {
+    async count(model, texts) {
+        if (model.offline === 'none') {
+            const reason = model.family === 'claude'
+                ? 'needs --claude-api and ANTHROPIC_API_KEY'
+                : model.family === 'gemini'
+                    ? 'needs --gemini-api and GEMINI_API_KEY'
+                    : 'no offline tokenizer';
+            return texts.map(() => ({ tokens: null, method: 'unavailable', reason }));
+        }
+        const method = model.offline === 'o200k-exact' ? 'offline-exact' : 'offline-proxy';
+        return texts.map((t) => ({ tokens: countO200k(t), method }));
+    },
+};

package/dist/core/tools.js

@@ -0,0 +1,301 @@
+const str = (v) => v === undefined ? '' : Array.isArray(v) ? v.join(', ') : String(v);
+const has = (fm, key) => str(fm[key]).trim() !== '';
+export const TOOLS = [
+    {
+        id: 'claude-code',
+        name: 'Claude Code',
+        families: ['claude'],
+        sources: [
+            'https://code.claude.com/docs/en/memory',
+            'https://code.claude.com/docs/en/skills',
+            'https://code.claude.com/docs/en/sub-agents',
+        ],
+        checkedAt: '2026-10-10',
+        rules: [
+            {
+                kind: 'hierarchical',
+                names: ['CLAUDE.md', 'CLAUDE.local.md'],
+                sameLevelDirs: ['.claude'],
+                nested: 'on-demand',
+                nestedTrigger: 'when Claude reads or edits a file in',
+                stripHtmlComments: true,
+                importDepth: 4,
+            },
+            {
+                kind: 'hierarchical',
+                names: ['AGENTS.md'],
+                sameLevelDirs: ['.claude'],
+                nested: 'on-demand',
+                nestedTrigger: 'when Claude reads a file in',
+                importDepth: 4,
+                condition: 'claude-reads-agents-md',
+                note: 'Read only because no CLAUDE.md or CLAUDE.local.md exists at the root (default setting).',
+            },
+            {
+                kind: 'frontmatter',
+                pattern: /^(?:.*\/)?\.claude\/rules\/.+\.md$/,
+                rootPrefixes: ['.claude/rules/'],
+                decide: (fm) => has(fm, 'paths')
+                    ? { mode: 'on-demand', trigger: `paths: ${str(fm.paths)}` }
+                    : { mode: 'always', trigger: 'rule without paths' },
+            },
+            {
+                kind: 'skill',
+                pattern: /^(?:.*\/)?\.claude\/skills\/[^/]+\/SKILL\.md$/,
+                rootPrefix: '.claude/skills/',
+                descriptionCap: 1536,
+                hideFlag: 'disable-model-invocation',
+                bodyTrigger: 'when the skill is invoked',
+            },
+            {
+                kind: 'skill',
+                pattern: /^(?:.*\/)?\.claude\/commands\/.+\.md$/,
+                rootPrefix: '.claude/commands/',
+                descriptionCap: 1536,
+                hideFlag: 'disable-model-invocation',
+                bodyTrigger: 'when the command is invoked',
+            },
+            {
+                kind: 'skill',
+                pattern: /^(?:.*\/)?\.claude\/agents\/[^/]+\.md$/,
+                rootPrefix: '.claude/agents/',
+                bodyTrigger: "as the subagent's own system prompt when it runs",
+            },
+        ],
+        limits: [
+            {
+                pattern: /(^|\/)CLAUDE(\.local)?\.md$/,
+                unit: 'lines',
+                max: 200,
+                severity: 'advice',
+                text: 'Claude Code docs recommend keeping each CLAUDE.md under 200 lines.',
+                source: 'https://code.claude.com/docs/en/memory',
+            },
+        ],
+        notes: [
+            'User (~/.claude/CLAUDE.md), managed policy and auto-memory files live outside the repository and are not counted.',
+            'With the "claude-md-and-agents-md" setting Claude Code reads AGENTS.md as well; the default reads it only when no CLAUDE.md exists.',
+        ],
+    },
+    {
+        id: 'codex',
+        name: 'OpenAI Codex',
+        families: ['openai'],
+        sources: [
+            'https://learn.chatgpt.com/docs/agent-configuration/agents-md',
+            'https://learn.chatgpt.com/docs/build-skills',
+        ],
+        checkedAt: '2026-10-10',
+        rules: [
+            {
+                kind: 'hierarchical',
+                names: ['AGENTS.override.md', 'AGENTS.md'],
+                onePerDirectory: true,
+                nested: 'on-demand',
+                nestedTrigger: 'only when Codex is launched in',
+                maxBytes: 32 * 1024,
+            },
+            {
+                kind: 'skill',
+                pattern: /^(?:.*\/)?\.agents\/skills\/[^/]+\/SKILL\.md$/,
+                rootPrefix: '.agents/skills/',
+                bodyTrigger: 'when the skill is selected',
+            },
+        ],
+        limits: [
+            {
+                pattern: /(^|\/)AGENTS(\.override)?\.md$/,
+                unit: 'bytes',
+                max: 32 * 1024,
+                severity: 'truncate',
+                text: 'Codex stops reading project docs at project_doc_max_bytes (default 32 KiB); the rest is dropped.',
+                source: 'https://learn.chatgpt.com/docs/agent-configuration/agents-md',
+            },
+        ],
+        notes: ['Global ~/.codex/AGENTS.md is outside the repository and not counted.'],
+    },
+    {
+        id: 'gemini-cli',
+        name: 'Gemini CLI',
+        families: ['gemini'],
+        sources: ['https://geminicli.com/docs/cli/gemini-md/'],
+        checkedAt: '2026-10-10',
+        rules: [
+            {
+                kind: 'hierarchical',
+                names: ['GEMINI.md'],
+                nested: 'on-demand',
+                nestedTrigger: 'when a tool accesses',
+                // The docs describe @file.md imports but not a depth limit; deeper chains are rare.
+                importDepth: 5,
+            },
+        ],
+        limits: [],
+        notes: [
+            'Only GEMINI.md is read by default; context.fileName can add AGENTS.md (not modelled).',
+            'Global ~/.gemini/GEMINI.md is outside the repository and not counted.',
+        ],
+    },
+    {
+        id: 'cursor',
+        name: 'Cursor',
+        families: [],
+        sources: ['https://cursor.com/docs/context/rules', 'https://cursor.com/help/customization/rules'],
+        checkedAt: '2026-10-10',
+        rules: [
+            {
+                kind: 'frontmatter',
+                pattern: /^(?:.*\/)?\.cursor\/rules\/.+\.mdc$/,
+                rootPrefixes: ['.cursor/rules/'],
+                decide: (fm) => {
+                    if (fm.alwaysApply === true || str(fm.alwaysApply) === 'true')
+                        return { mode: 'always', trigger: 'alwaysApply: true' };
+                    if (has(fm, 'globs'))
+                        return { mode: 'on-demand', trigger: `globs: ${str(fm.globs)}` };
+                    if (has(fm, 'description'))
+                        return {
+                            mode: 'on-demand',
+                            trigger: 'agent decides from description',
+                            descriptionOnly: true,
+                            note: 'Description assumed to be in context so the agent can decide (inferred, not documented).',
+                        };
+                    return { mode: 'manual', trigger: '@-mention only' };
+                },
+            },
+            {
+                kind: 'ignored',
+                pattern: /^(?:.*\/)?\.cursor\/rules\/.+\.md$/,
+                reason: 'Cursor ignores plain .md files in .cursor/rules; rename to .mdc.',
+            },
+            { kind: 'root-file', path: '.cursorrules', note: 'Legacy file; Cursor plans to deprecate it.' },
+            {
+                kind: 'hierarchical',
+                names: ['AGENTS.md'],
+                nested: 'on-demand',
+                nestedTrigger: 'when working in',
+            },
+            {
+                kind: 'hierarchical',
+                names: ['CLAUDE.md'],
+                nested: 'on-demand',
+                nestedTrigger: 'when working in',
+            },
+        ],
+        limits: [
+            {
+                pattern: /\.mdc$/,
+                unit: 'lines',
+                max: 500,
+                severity: 'advice',
+                text: 'Cursor docs recommend keeping each rule under 500 lines.',
+                source: 'https://cursor.com/docs/context/rules',
+            },
+        ],
+        notes: ['User and Team rules are configured in Cursor settings, not in the repository, and are not counted.'],
+    },
+    {
+        id: 'copilot',
+        name: 'GitHub Copilot',
+        families: [],
+        sources: [
+            'https://docs.github.com/en/copilot/how-tos/configure-custom-instructions/add-repository-instructions',
+            'https://code.visualstudio.com/docs/copilot/customization/custom-instructions',
+        ],
+        checkedAt: '2026-10-10',
+        rules: [
+            { kind: 'root-file', path: '.github/copilot-instructions.md' },
+            {
+                kind: 'frontmatter',
+                pattern: /^\.github\/instructions\/.+\.instructions\.md$/,
+                rootPrefixes: ['.github/instructions/'],
+                decide: (fm) => {
+                    if (has(fm, 'applyTo'))
+                        return { mode: 'on-demand', trigger: `applyTo: ${str(fm.applyTo)}` };
+                    if (has(fm, 'description'))
+                        return {
+                            mode: 'on-demand',
+                            trigger: 'agent decides from description (VS Code)',
+                            descriptionOnly: true,
+                            note: 'Description assumed to be in context so the agent can decide (inferred, not documented).',
+                        };
+                    return { mode: 'manual', trigger: 'attached manually' };
+                },
+            },
+            {
+                kind: 'hierarchical',
+                names: ['AGENTS.md'],
+                nested: 'on-demand',
+                nestedTrigger: 'nearest AGENTS.md wins when working in',
+            },
+            {
+                kind: 'hierarchical',
+                names: ['CLAUDE.md', 'GEMINI.md'],
+                onePerDirectory: true,
+                nested: 'ignored',
+                nestedTrigger: '',
+                condition: 'copilot-no-agents-md',
+                note: 'Copilot docs allow a single root CLAUDE.md or GEMINI.md as an alternative to AGENTS.md. When both exist the docs do not say which wins; CLAUDE.md is assumed.',
+            },
+        ],
+        limits: [],
+        notes: ['Support for AGENTS.md and CLAUDE.md differs by Copilot surface and VS Code settings.'],
+    },
+    {
+        id: 'windsurf',
+        name: 'Windsurf (Devin Desktop)',
+        families: [],
+        sources: ['https://docs.devin.ai/desktop/cascade/memories'],
+        checkedAt: '2026-10-10',
+        rules: [
+            {
+                kind: 'frontmatter',
+                pattern: /^(?:.*\/)?\.(?:devin|windsurf)\/rules\/.+\.md$/,
+                rootPrefixes: ['.devin/rules/', '.windsurf/rules/'],
+                decide: (fm) => {
+                    switch (str(fm.trigger).trim()) {
+                        case 'always_on':
+                            return { mode: 'always', trigger: 'trigger: always_on' };
+                        case 'model_decision':
+                            return { mode: 'on-demand', trigger: 'trigger: model_decision', descriptionOnly: true };
+                        case 'glob':
+                            return { mode: 'on-demand', trigger: `trigger: glob ${str(fm.globs)}`.trim() };
+                        case 'manual':
+                            return { mode: 'manual', trigger: 'trigger: manual (@rule-name)' };
+                        default:
+                            return {
+                                mode: 'manual',
+                                trigger: 'no trigger set',
+                                note: 'Docs do not state the default when trigger is missing; not counted.',
+                                problem: 'Rule has no trigger frontmatter; set always_on, model_decision, glob or manual.',
+                            };
+                    }
+                },
+            },
+            { kind: 'root-file', path: '.windsurfrules', note: 'Legacy single-file rules, still read.' },
+            {
+                kind: 'hierarchical',
+                names: ['AGENTS.md'],
+                nested: 'on-demand',
+                nestedTrigger: 'auto-glob for files in',
+            },
+        ],
+        limits: [
+            {
+                pattern: /(^|\/)(\.(devin|windsurf)\/rules\/.+\.md|\.windsurfrules)$/,
+                unit: 'chars',
+                max: 12000,
+                severity: 'limit',
+                text: 'Windsurf limits workspace rule files to 12,000 characters each.',
+                source: 'https://docs.devin.ai/desktop/cascade/memories',
+            },
+        ],
+        notes: ['Global rules (~/.codeium/windsurf/memories/global_rules.md) are outside the repository and not counted.'],
+    },
+];
+export const TOOL_IDS = TOOLS.map((t) => t.id);
+export function getTool(id) {
+    const tool = TOOLS.find((t) => t.id === id);
+    if (!tool)
+        throw new Error(`Unknown tool: ${id}`);
+    return tool;
+}

package/dist/core/types.js

@@ -0,0 +1 @@
+export {};

package/dist/node/discover.js

@@ -0,0 +1,63 @@
+import { existsSync, lstatSync, readdirSync, readFileSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+import { MAX_DEPTH, addImports, isInstructionFile, isSkippedDir } from '../core/files.js';
+const MAX_BYTES = 2 * 1024 * 1024;
+function readText(abs) {
+    try {
+        if (statSync(abs).size > MAX_BYTES)
+            return null;
+        return readFileSync(abs, 'utf8');
+    }
+    catch {
+        return null;
+    }
+}
+/** Finds agent instruction files under `root`, plus any in-repo files they @import. Read-only. */
+export async function discover(root) {
+    const found = new Map();
+    const walk = (rel, depth) => {
+        if (depth > MAX_DEPTH)
+            return;
+        let entries;
+        try {
+            entries = readdirSync(join(root, rel), { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        for (const e of entries) {
+            const path = rel ? `${rel}/${e.name}` : e.name;
+            if (e.isDirectory()) {
+                if (isSkippedDir(path))
+                    continue;
+                walk(path, depth + 1);
+            }
+            else if ((e.isFile() || e.isSymbolicLink()) && isInstructionFile(path)) {
+                const text = readText(join(root, path));
+                if (text !== null)
+                    found.set(path, { path, text });
+            }
+        }
+    };
+    walk('', 0);
+    // Pull in @imported files (for example a CLAUDE.md that contains @docs/agents.md).
+    await addImports(found, (rel) => {
+        const abs = join(root, rel);
+        return existsSync(abs) && !lstatSync(abs).isDirectory() ? readText(abs) : null;
+    });
+    return [...found.values()].sort((a, b) => a.path.localeCompare(b.path));
+}
+export function existsIn(root) {
+    return (path) => existsSync(join(root, path));
+}
+export function packageScripts(root) {
+    const text = readText(join(root, 'package.json'));
+    if (text === null)
+        return undefined;
+    try {
+        return new Set(Object.keys(JSON.parse(text).scripts ?? {}));
+    }
+    catch {
+        return undefined;
+    }
+}

package/dist/node/remote.js

@@ -0,0 +1,100 @@
+const ANTHROPIC_URL = 'https://api.anthropic.com/v1/messages/count_tokens';
+const ANTHROPIC_MESSAGES_URL = 'https://api.anthropic.com/v1/messages';
+const GEMINI_BASE = 'https://generativelanguage.googleapis.com/v1beta/models';
+const PROBE = [{ role: 'user', content: '.' }];
+async function postJson(fetchFn, url, headers, body) {
+    const res = await fetchFn(url, {
+        method: 'POST',
+        headers: { 'content-type': 'application/json', ...headers },
+        body: JSON.stringify(body),
+    });
+    if (!res.ok) {
+        // Only the status goes into the message: response bodies can echo request content.
+        throw new Error(`HTTP ${res.status}`);
+    }
+    return res.json();
+}
+async function mapLimited(items, limit, fn) {
+    const results = new Array(items.length);
+    let next = 0;
+    const workers = Array.from({ length: Math.min(limit, items.length) }, async () => {
+        while (next < items.length) {
+            const i = next++;
+            results[i] = await fn(items[i]);
+        }
+    });
+    await Promise.all(workers);
+    return results;
+}
+/**
+ * Exact Claude counts from Anthropic's count_tokens endpoint.
+ * Each text is sent as a system prompt next to a one-character user message; the same request
+ * without the system prompt is subtracted, so the result is the cost the text adds to a turn.
+ */
+export function anthropicCounter(apiKey, fetchFn = fetch) {
+    const headers = { 'x-api-key': apiKey, 'anthropic-version': '2023-06-01' };
+    const baselines = new Map();
+    return {
+        async count(model, texts) {
+            if (!model.apiModel)
+                return texts.map(() => ({ tokens: null, method: 'unavailable', reason: 'no API model id' }));
+            const apiModel = model.apiModel;
+            const baseline = () => {
+                if (!baselines.has(apiModel))
+                    baselines.set(apiModel, postJson(fetchFn, ANTHROPIC_URL, headers, { model: apiModel, messages: PROBE }).then((r) => r.input_tokens));
+                return baselines.get(apiModel);
+            };
+            return mapLimited(texts, 4, async (text) => {
+                if (!text.trim())
+                    return { tokens: 0, method: 'api' };
+                try {
+                    const [base, full] = await Promise.all([
+                        baseline(),
+                        postJson(fetchFn, ANTHROPIC_URL, headers, { model: apiModel, system: text, messages: PROBE }),
+                    ]);
+                    return { tokens: Math.max(0, full.input_tokens - base), method: 'api' };
+                }
+                catch (e) {
+                    return { tokens: null, method: 'unavailable', reason: `Anthropic count_tokens failed: ${e.message}` };
+                }
+            });
+        },
+    };
+}
+/** Gemini counts from Google's models.countTokens endpoint. */
+export function geminiCounter(apiKey, fetchFn = fetch) {
+    const headers = { 'x-goog-api-key': apiKey };
+    return {
+        async count(model, texts) {
+            if (!model.apiModel)
+                return texts.map(() => ({ tokens: null, method: 'unavailable', reason: 'no API model id' }));
+            const url = `${GEMINI_BASE}/${encodeURIComponent(model.apiModel)}:countTokens`;
+            return mapLimited(texts, 4, async (text) => {
+                if (!text.trim())
+                    return { tokens: 0, method: 'api' };
+                try {
+                    const r = await postJson(fetchFn, url, headers, { contents: [{ role: 'user', parts: [{ text }] }] });
+                    return { tokens: r.totalTokens, method: 'api' };
+                }
+                catch (e) {
+                    return { tokens: null, method: 'unavailable', reason: `Gemini countTokens failed: ${e.message}` };
+                }
+            });
+        },
+    };
+}
+/** Opt-in English translation through Claude, used only to measure a language token multiplier. */
+export async function translateToEnglish(apiKey, text, fetchFn = fetch) {
+    const r = await postJson(fetchFn, ANTHROPIC_MESSAGES_URL, { 'x-api-key': apiKey, 'anthropic-version': '2023-06-01' }, {
+        model: 'claude-haiku-5-5',
+        max_tokens: 8000,
+        system: 'Translate the user text into natural English. Keep markdown structure, code, paths and commands unchanged. Output only the translation.',
+        messages: [{ role: 'user', content: text }],
+    });
+    if (r.stop_reason === 'max_tokens')
+        throw new Error('translation was cut off (file too long)');
+    return (r.content ?? [])
+        .filter((b) => b.type === 'text')
+        .map((b) => b.text)
+        .join('');
+}