tokc 0.1.0__tar.gz

tokc-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Sylvain Mestre
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

tokc-0.1.0/PKG-INFO

@@ -0,0 +1,299 @@
+Metadata-Version: 2.4
+Name: tokc
+Version: 0.1.0
+Summary: Measure and shrink the token cost of Claude Code skills.
+Author: Sylvain Mestre
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/Shult/token_consumer
+Project-URL: Repository, https://github.com/Shult/token_consumer
+Project-URL: Issues, https://github.com/Shult/token_consumer/issues
+Project-URL: Changelog, https://github.com/Shult/token_consumer/blob/main/CHANGELOG.md
+Keywords: claude,claude-code,skills,tokens,cost,llm
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Quality Assurance
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: fast
+Requires-Dist: tiktoken>=0.7; extra == "fast"
+Provides-Extra: exact
+Requires-Dist: anthropic>=0.40; extra == "exact"
+Provides-Extra: yaml
+Requires-Dist: pyyaml>=6; extra == "yaml"
+Provides-Extra: dev
+Requires-Dist: pytest>=8; extra == "dev"
+Requires-Dist: tiktoken>=0.7; extra == "dev"
+Requires-Dist: pyyaml>=6; extra == "dev"
+Dynamic: license-file
+
+# tokc — token consumption analyzer for Claude Code skills
+
+[![CI](https://github.com/Shult/token_consumer/actions/workflows/ci.yml/badge.svg)](https://github.com/Shult/token_consumer/actions/workflows/ci.yml)
+[![PyPI](https://img.shields.io/pypi/v/tokc.svg)](https://pypi.org/project/tokc/)
+[![Python](https://img.shields.io/pypi/pyversions/tokc.svg)](https://pypi.org/project/tokc/)
+[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
+
+Measures what a skill actually costs you, tier by tier, and tells you where to cut.
+
+```
+  29 skill(s)   tokenizer: o200k_base x1.180 (default)
+
+  SKILL                   T1/sess   T2/use      T3  ALWAYS-ON TAX                  $/mo
+------------------------------------------------------------------------------
+  project-artifact            241    5,415    8.5k  ████████████████████████ !   $2.99
+  receipts                    159    4,865     20k  ████████████████         !   $2.40
+  hook-development            140    4,548     16k  ██████████████               $2.20
+  ...
+------------------------------------------------------------------------------
+  TOTAL                     2,787   99,352    294k                               $45.65
+
+  Always-on tax
+    2,787 tokens ride in the system prompt of every request, in every session,
+    whether or not any skill fires.
+    -> 2.9M billed token-equivalents/month at 200 sessions x 40 requests
+
+  Listing budget  (characters, not tokens)
+    ████████████████████████████████████████  10,577 / 8,000 chars  (132%)
+    ! 2,577 chars over budget -- 6 skill(s) ship as `- name`, description omitted
+      demoted, least-used first: plugin-structure, project-artifact, receipts, ...
+```
+
+## Why this exists
+
+Existing tools estimate skill cost by asking Claude — which costs tokens and gives
+different numbers each run — or by running `tiktoken` over `SKILL.md`, which
+**undercounts Claude by 15-20%** and, worse, reports a single number for something
+that is not a single cost.
+
+A skill is loaded in three tiers, billed at wildly different frequencies:
+
+| Tier | What | Paid |
+|---|---|---|
+| **T1** | frontmatter `name` + `description` (+ `whenToUse`, which the listing appends) | in the system prompt of **every request, every session** |
+| **T2** | the `SKILL.md` body | once per invocation |
+| **T3** | `references/`, `scripts/`, … | only when Claude actually reads the file |
+
+The number that matters is almost never the one people optimize. A 250-token
+description is re-sent on every request of every session — with prompt caching that
+is a **×5.15 multiplier per session** (1.25× cache write, then 0.10× per read).
+A 6,000-token body that fires five times a month costs less than a 150-token
+description that ships 200 times. `tokc` computes both and ranks accordingly.
+
+## Install
+
+```bash
+pip install tokc              # or: uvx tokc scan
+```
+
+Zero hard dependencies. Three optional extras, none required:
+
+```bash
+pip install "tokc[fast]"      # tiktoken — much better offline counts
+pip install "tokc[exact]"     # anthropic — enables `--exact` and `tokc calibrate`
+pip install "tokc[yaml]"      # pyyaml — frontmatter edge cases
+```
+
+Without them `tokc` falls back to a pure-python estimator and a minimal YAML
+parser. Python 3.10+, Linux / macOS / Windows.
+
+From source:
+
+```bash
+git clone https://github.com/Shult/token_consumer && cd token_consumer
+pip install -e ".[dev]"       # tiktoken + pyyaml + pytest
+```
+
+## What leaves your machine
+
+Nothing, unless you ask for it. Worth stating plainly, because `tokc` reads
+files that are more personal than the skills themselves:
+
+**What it reads locally.** Your skill directories; `settings.json` for
+`skillOverrides` and listing settings; `~/.claude.json` for the per-skill usage
+counts that drive the listing order; your installed Claude Code build, to read
+its own defaults out of the settings schema. `tokc setup` additionally scans
+transcripts under `~/.claude`. From those it keeps four things and nothing else:
+the session count, distinct `requestId`s, the *names* of invoked skills (from
+`Skill` / `SlashCommand` tool-use blocks) and the recorded `skill_listing`
+attachment. Prompts, responses and tool output are never retained, and what is
+collected never leaves the summary figures written to your config file.
+
+**What it sends.** Only `--exact` and `tokc calibrate` open a network
+connection. They post skill text — frontmatter and body — to Anthropic's
+`count_tokens` endpoint, using your own credentials, and get a token count back.
+Every other command, and every default, is fully offline. `--no-config` ignores
+saved answers entirely.
+
+**What it writes.** `./.tokc.json` or `~/.claude/tokc.json` (`tokc setup`), a
+baseline at `./.tokc-baseline.json` (`tokc snapshot`), and the HTML file you
+name (`tokc report -o`). Nothing else on disk is modified — `tokc` never edits
+your skills, it only tells you what to cut.
+
+Both config files record local paths and usage figures, so keep them out of
+version control:
+
+```gitignore
+.tokc.json
+.tokc-baseline.json
+```
+
+## Commands
+
+```bash
+tokc setup                   # detect your harness, measure your usage, save both
+tokc scan                    # every skill, ranked by always-on tax
+tokc audit <name|path>       # one skill: tiers, body heatmap, findings
+tokc budget                  # simulate the system-prompt listing budget
+tokc snapshot                # record a baseline
+tokc diff                    # what changed since the baseline
+tokc watch <path>            # live token counter while you edit
+tokc report -o out.html      # self-contained HTML report
+tokc calibrate               # measure the offline estimator against the real API
+```
+
+With no path, `tokc` scans the same roots Claude Code does: `~/.claude/skills`,
+`<project>/.claude/skills`, and installed plugin skills.
+
+### `tokc setup` — stop guessing
+
+The numbers below only mean something if they match *your* harness and *your*
+usage. `tokc setup` works both out and writes them to `./.tokc.json` (or
+`~/.claude/tokc.json` with `--user`), which every other command then reads.
+
+```bash
+tokc setup            # detect, measure, confirm
+tokc setup -y         # accept everything detected
+tokc setup --harness other    # a harness tokc cannot inspect: ask, don't assume
+```
+
+Four things, in descending order of trust:
+
+1. **A listing your harness actually sent.** Transcripts record the
+   `skill_listing` attachment verbatim, so we compare our reconstruction line by
+   line against the real thing — and recover the characters spent by skills
+   bundled *inside* the harness, which never appear on disk but still eat the
+   budget first.
+2. **The installed build's own settings schema.** `skillListingMaxDescChars` and
+   `skillListingBudgetFraction` defaults are read out of the install (the VS Code
+   extension's schema, or the CLI binary), not hard-coded — so an upgrade that
+   changes them changes `tokc` too.
+3. **Your transcripts.** Sessions, API requests per session (counted by distinct
+   `requestId`, not by message) and per-skill invocations over the last 30 days,
+   replacing the 200 / 40 / 20 defaults.
+4. **You.** For a harness that cannot be probed — Claude Desktop, claude.ai, a
+   third-party client — nothing is assumed. `tokc` says it does not know and asks
+   whether the harness caps the listing at all.
+
+Precedence runs weakest to strongest: probed defaults → `~/.claude/tokc.json` →
+`./.tokc.json` → the live `settings.json` chain → `SLASH_COMMAND_TOOL_CHAR_BUDGET`
+→ command-line flags. `--no-config` ignores the saved answers entirely.
+
+### Tuning the cost model
+
+Every projection is driven by explicit, overridable assumptions:
+
+```bash
+tokc scan --sessions 400 --requests 60 --invocations 10 --model claude-sonnet-5
+tokc scan --no-cache         # price without prompt caching (worst case)
+```
+
+### Accuracy
+
+Three counting backends, picked automatically:
+
+1. **`--exact`** — Anthropic's `count_tokens` endpoint. Exact. Needs credentials
+   (`ANTHROPIC_API_KEY`, or an `ant auth login` profile).
+2. **tiktoken** — `o200k_base` × a calibration factor (default 1.18, the midpoint
+   of Anthropic's stated 15-20% undercount). Offline.
+3. **heuristic** — pure python, no dependencies. Roughly ±10%.
+
+`tokc calibrate` measures backend 2 against backend 1 **on your own skills** and
+stores the corrected factor, so subsequent offline runs are accurate for your corpus.
+
+## What it flags
+
+Findings carry the tier they belong to and a savings estimate at the right rate —
+a T1 win is multiplied by sessions, a T2 win by invocations.
+
+**Tier 1 — the every-session tax**
+- `T1-DESC-MISSING` / `T1-DESC-BLOAT` / `T1-DESC-TRUNCATED` — over
+  `skillListingMaxDescChars` the harness cuts mid-sentence and appends an
+  ellipsis, silently losing any trigger keywords at the end. The cap applies to
+  `description` and `whenToUse` **together**, since the listing shows the pair.
+- `T1-HIDDEN` / `T1-NAME-ONLY` — `skillOverrides` in `settings.json` took this
+  skill out of the listing, or left it there without its description. Reported so
+  a zero is not mistaken for a saving.
+- `T1-NO-TRIGGER` — nothing says *when* to use the skill, so you pay the tax and
+  never get the benefit. (Recognises English and French trigger phrasing.)
+- `T1-FILLER` — "this skill is used to", "in order to", …
+
+**Tier 2 — the per-invocation cost**
+- `T2-BODY-LARGE`, `T2-SECTION-HOG` — one section dominating the body is the
+  extraction candidate.
+- `T2-CODE-BLOCK`, `T2-BIG-TABLE` — reference material paid on every invocation.
+- `T2-REPEAT` — lines repeated 3+ times (code fences excluded).
+- `T2-HTML-COMMENT` — invisible when rendered, fully billed to the model.
+- `T2-NO-STRUCTURE`, `T2-WHITESPACE`.
+
+**Tier 3 — on-demand files**
+- `T3-UNREFERENCED` — a bundled file `SKILL.md` never points at is unreachable.
+- `T3-HUGE` — reading it will force compaction mid-task.
+
+## CI
+
+```bash
+tokc budget --strict     # exit 1 if the listing overflows and skills get dropped
+tokc audit my-skill --strict   # exit 1 on any HIGH finding
+tokc diff --strict       # exit 1 if the always-on tax grew
+tokc scan --json         # machine-readable
+```
+
+## How the listing budget actually works
+
+Worth stating precisely, because it is easy to get wrong in three ways at once:
+
+1. **The budget is a character count, not a token count.** It is derived from the
+   context window — `window(tokens) × charsPerToken × fraction`, so
+   `200 000 × 4 × 0.01 = 8 000` by default — but what it caps is the *length of
+   the assembled listing text*. A model the harness does not recognise converts at
+   3 chars/token instead of 4, and the budget shrinks to 6 000.
+2. **Nothing is dropped.** An entry that does not fit ships as `- name` with no
+   description. The skill stays listed and stays invocable by name; what it loses
+   is the text the model selects on.
+3. **The order is neither arbitrary nor "largest first".** Entries are ranked by
+   `usageCount × max(0.5^(days_since_use / 7), 0.1)` — recorded in
+   `~/.claude.json` — and the best-ranked keep their descriptions, taken greedily,
+   so a small entry can still fit after a large one was refused. Skills bundled
+   inside the harness are exempt and consume the budget before any of yours.
+
+`tokc` reproduces all three, reads the real usage ranking, and `tokc setup`
+checks the result against a listing your harness genuinely sent.
+
+## Caveats
+
+- The **T1 multiplier** assumes 40 requests/session and a 5-minute cache TTL.
+  Override with `--requests`, or let `tokc setup` measure it; use `--no-cache` for
+  the uncached worst case.
+- **T3 is excluded from projections.** Reference files load unpredictably, and
+  counting them would penalise exactly the progressive-disclosure structure this
+  tool tells you to adopt. They are reported, not projected.
+- **Skills bundled in the harness are invisible on disk.** Without `tokc setup`
+  the budget simulation ignores them and is optimistic by a few thousand
+  characters. Setup recovers the figure from a real listing.
+- **Only Claude Code is modelled.** The CLI and the VS Code extension share one
+  binary and one set of rules, read from the install. Claude Desktop, claude.ai
+  and third-party clients assemble the system prompt elsewhere; `tokc` will not
+  pretend to know their policy, and `tokc setup --harness other` asks instead.
+- Offline counts are estimates until you run `tokc calibrate`.
+
+## Tests
+
+```bash
+python -m pytest -q       # 37 tests
+```

tokc-0.1.0/README.md

@@ -0,0 +1,265 @@
+# tokc — token consumption analyzer for Claude Code skills
+
+[![CI](https://github.com/Shult/token_consumer/actions/workflows/ci.yml/badge.svg)](https://github.com/Shult/token_consumer/actions/workflows/ci.yml)
+[![PyPI](https://img.shields.io/pypi/v/tokc.svg)](https://pypi.org/project/tokc/)
+[![Python](https://img.shields.io/pypi/pyversions/tokc.svg)](https://pypi.org/project/tokc/)
+[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
+
+Measures what a skill actually costs you, tier by tier, and tells you where to cut.
+
+```
+  29 skill(s)   tokenizer: o200k_base x1.180 (default)
+
+  SKILL                   T1/sess   T2/use      T3  ALWAYS-ON TAX                  $/mo
+------------------------------------------------------------------------------
+  project-artifact            241    5,415    8.5k  ████████████████████████ !   $2.99
+  receipts                    159    4,865     20k  ████████████████         !   $2.40
+  hook-development            140    4,548     16k  ██████████████               $2.20
+  ...
+------------------------------------------------------------------------------
+  TOTAL                     2,787   99,352    294k                               $45.65
+
+  Always-on tax
+    2,787 tokens ride in the system prompt of every request, in every session,
+    whether or not any skill fires.
+    -> 2.9M billed token-equivalents/month at 200 sessions x 40 requests
+
+  Listing budget  (characters, not tokens)
+    ████████████████████████████████████████  10,577 / 8,000 chars  (132%)
+    ! 2,577 chars over budget -- 6 skill(s) ship as `- name`, description omitted
+      demoted, least-used first: plugin-structure, project-artifact, receipts, ...
+```
+
+## Why this exists
+
+Existing tools estimate skill cost by asking Claude — which costs tokens and gives
+different numbers each run — or by running `tiktoken` over `SKILL.md`, which
+**undercounts Claude by 15-20%** and, worse, reports a single number for something
+that is not a single cost.
+
+A skill is loaded in three tiers, billed at wildly different frequencies:
+
+| Tier | What | Paid |
+|---|---|---|
+| **T1** | frontmatter `name` + `description` (+ `whenToUse`, which the listing appends) | in the system prompt of **every request, every session** |
+| **T2** | the `SKILL.md` body | once per invocation |
+| **T3** | `references/`, `scripts/`, … | only when Claude actually reads the file |
+
+The number that matters is almost never the one people optimize. A 250-token
+description is re-sent on every request of every session — with prompt caching that
+is a **×5.15 multiplier per session** (1.25× cache write, then 0.10× per read).
+A 6,000-token body that fires five times a month costs less than a 150-token
+description that ships 200 times. `tokc` computes both and ranks accordingly.
+
+## Install
+
+```bash
+pip install tokc              # or: uvx tokc scan
+```
+
+Zero hard dependencies. Three optional extras, none required:
+
+```bash
+pip install "tokc[fast]"      # tiktoken — much better offline counts
+pip install "tokc[exact]"     # anthropic — enables `--exact` and `tokc calibrate`
+pip install "tokc[yaml]"      # pyyaml — frontmatter edge cases
+```
+
+Without them `tokc` falls back to a pure-python estimator and a minimal YAML
+parser. Python 3.10+, Linux / macOS / Windows.
+
+From source:
+
+```bash
+git clone https://github.com/Shult/token_consumer && cd token_consumer
+pip install -e ".[dev]"       # tiktoken + pyyaml + pytest
+```
+
+## What leaves your machine
+
+Nothing, unless you ask for it. Worth stating plainly, because `tokc` reads
+files that are more personal than the skills themselves:
+
+**What it reads locally.** Your skill directories; `settings.json` for
+`skillOverrides` and listing settings; `~/.claude.json` for the per-skill usage
+counts that drive the listing order; your installed Claude Code build, to read
+its own defaults out of the settings schema. `tokc setup` additionally scans
+transcripts under `~/.claude`. From those it keeps four things and nothing else:
+the session count, distinct `requestId`s, the *names* of invoked skills (from
+`Skill` / `SlashCommand` tool-use blocks) and the recorded `skill_listing`
+attachment. Prompts, responses and tool output are never retained, and what is
+collected never leaves the summary figures written to your config file.
+
+**What it sends.** Only `--exact` and `tokc calibrate` open a network
+connection. They post skill text — frontmatter and body — to Anthropic's
+`count_tokens` endpoint, using your own credentials, and get a token count back.
+Every other command, and every default, is fully offline. `--no-config` ignores
+saved answers entirely.
+
+**What it writes.** `./.tokc.json` or `~/.claude/tokc.json` (`tokc setup`), a
+baseline at `./.tokc-baseline.json` (`tokc snapshot`), and the HTML file you
+name (`tokc report -o`). Nothing else on disk is modified — `tokc` never edits
+your skills, it only tells you what to cut.
+
+Both config files record local paths and usage figures, so keep them out of
+version control:
+
+```gitignore
+.tokc.json
+.tokc-baseline.json
+```
+
+## Commands
+
+```bash
+tokc setup                   # detect your harness, measure your usage, save both
+tokc scan                    # every skill, ranked by always-on tax
+tokc audit <name|path>       # one skill: tiers, body heatmap, findings
+tokc budget                  # simulate the system-prompt listing budget
+tokc snapshot                # record a baseline
+tokc diff                    # what changed since the baseline
+tokc watch <path>            # live token counter while you edit
+tokc report -o out.html      # self-contained HTML report
+tokc calibrate               # measure the offline estimator against the real API
+```
+
+With no path, `tokc` scans the same roots Claude Code does: `~/.claude/skills`,
+`<project>/.claude/skills`, and installed plugin skills.
+
+### `tokc setup` — stop guessing
+
+The numbers below only mean something if they match *your* harness and *your*
+usage. `tokc setup` works both out and writes them to `./.tokc.json` (or
+`~/.claude/tokc.json` with `--user`), which every other command then reads.
+
+```bash
+tokc setup            # detect, measure, confirm
+tokc setup -y         # accept everything detected
+tokc setup --harness other    # a harness tokc cannot inspect: ask, don't assume
+```
+
+Four things, in descending order of trust:
+
+1. **A listing your harness actually sent.** Transcripts record the
+   `skill_listing` attachment verbatim, so we compare our reconstruction line by
+   line against the real thing — and recover the characters spent by skills
+   bundled *inside* the harness, which never appear on disk but still eat the
+   budget first.
+2. **The installed build's own settings schema.** `skillListingMaxDescChars` and
+   `skillListingBudgetFraction` defaults are read out of the install (the VS Code
+   extension's schema, or the CLI binary), not hard-coded — so an upgrade that
+   changes them changes `tokc` too.
+3. **Your transcripts.** Sessions, API requests per session (counted by distinct
+   `requestId`, not by message) and per-skill invocations over the last 30 days,
+   replacing the 200 / 40 / 20 defaults.
+4. **You.** For a harness that cannot be probed — Claude Desktop, claude.ai, a
+   third-party client — nothing is assumed. `tokc` says it does not know and asks
+   whether the harness caps the listing at all.
+
+Precedence runs weakest to strongest: probed defaults → `~/.claude/tokc.json` →
+`./.tokc.json` → the live `settings.json` chain → `SLASH_COMMAND_TOOL_CHAR_BUDGET`
+→ command-line flags. `--no-config` ignores the saved answers entirely.
+
+### Tuning the cost model
+
+Every projection is driven by explicit, overridable assumptions:
+
+```bash
+tokc scan --sessions 400 --requests 60 --invocations 10 --model claude-sonnet-5
+tokc scan --no-cache         # price without prompt caching (worst case)
+```
+
+### Accuracy
+
+Three counting backends, picked automatically:
+
+1. **`--exact`** — Anthropic's `count_tokens` endpoint. Exact. Needs credentials
+   (`ANTHROPIC_API_KEY`, or an `ant auth login` profile).
+2. **tiktoken** — `o200k_base` × a calibration factor (default 1.18, the midpoint
+   of Anthropic's stated 15-20% undercount). Offline.
+3. **heuristic** — pure python, no dependencies. Roughly ±10%.
+
+`tokc calibrate` measures backend 2 against backend 1 **on your own skills** and
+stores the corrected factor, so subsequent offline runs are accurate for your corpus.
+
+## What it flags
+
+Findings carry the tier they belong to and a savings estimate at the right rate —
+a T1 win is multiplied by sessions, a T2 win by invocations.
+
+**Tier 1 — the every-session tax**
+- `T1-DESC-MISSING` / `T1-DESC-BLOAT` / `T1-DESC-TRUNCATED` — over
+  `skillListingMaxDescChars` the harness cuts mid-sentence and appends an
+  ellipsis, silently losing any trigger keywords at the end. The cap applies to
+  `description` and `whenToUse` **together**, since the listing shows the pair.
+- `T1-HIDDEN` / `T1-NAME-ONLY` — `skillOverrides` in `settings.json` took this
+  skill out of the listing, or left it there without its description. Reported so
+  a zero is not mistaken for a saving.
+- `T1-NO-TRIGGER` — nothing says *when* to use the skill, so you pay the tax and
+  never get the benefit. (Recognises English and French trigger phrasing.)
+- `T1-FILLER` — "this skill is used to", "in order to", …
+
+**Tier 2 — the per-invocation cost**
+- `T2-BODY-LARGE`, `T2-SECTION-HOG` — one section dominating the body is the
+  extraction candidate.
+- `T2-CODE-BLOCK`, `T2-BIG-TABLE` — reference material paid on every invocation.
+- `T2-REPEAT` — lines repeated 3+ times (code fences excluded).
+- `T2-HTML-COMMENT` — invisible when rendered, fully billed to the model.
+- `T2-NO-STRUCTURE`, `T2-WHITESPACE`.
+
+**Tier 3 — on-demand files**
+- `T3-UNREFERENCED` — a bundled file `SKILL.md` never points at is unreachable.
+- `T3-HUGE` — reading it will force compaction mid-task.
+
+## CI
+
+```bash
+tokc budget --strict     # exit 1 if the listing overflows and skills get dropped
+tokc audit my-skill --strict   # exit 1 on any HIGH finding
+tokc diff --strict       # exit 1 if the always-on tax grew
+tokc scan --json         # machine-readable
+```
+
+## How the listing budget actually works
+
+Worth stating precisely, because it is easy to get wrong in three ways at once:
+
+1. **The budget is a character count, not a token count.** It is derived from the
+   context window — `window(tokens) × charsPerToken × fraction`, so
+   `200 000 × 4 × 0.01 = 8 000` by default — but what it caps is the *length of
+   the assembled listing text*. A model the harness does not recognise converts at
+   3 chars/token instead of 4, and the budget shrinks to 6 000.
+2. **Nothing is dropped.** An entry that does not fit ships as `- name` with no
+   description. The skill stays listed and stays invocable by name; what it loses
+   is the text the model selects on.
+3. **The order is neither arbitrary nor "largest first".** Entries are ranked by
+   `usageCount × max(0.5^(days_since_use / 7), 0.1)` — recorded in
+   `~/.claude.json` — and the best-ranked keep their descriptions, taken greedily,
+   so a small entry can still fit after a large one was refused. Skills bundled
+   inside the harness are exempt and consume the budget before any of yours.
+
+`tokc` reproduces all three, reads the real usage ranking, and `tokc setup`
+checks the result against a listing your harness genuinely sent.
+
+## Caveats
+
+- The **T1 multiplier** assumes 40 requests/session and a 5-minute cache TTL.
+  Override with `--requests`, or let `tokc setup` measure it; use `--no-cache` for
+  the uncached worst case.
+- **T3 is excluded from projections.** Reference files load unpredictably, and
+  counting them would penalise exactly the progressive-disclosure structure this
+  tool tells you to adopt. They are reported, not projected.
+- **Skills bundled in the harness are invisible on disk.** Without `tokc setup`
+  the budget simulation ignores them and is optimistic by a few thousand
+  characters. Setup recovers the figure from a real listing.
+- **Only Claude Code is modelled.** The CLI and the VS Code extension share one
+  binary and one set of rules, read from the install. Claude Desktop, claude.ai
+  and third-party clients assemble the system prompt elsewhere; `tokc` will not
+  pretend to know their policy, and `tokc setup --harness other` asks instead.
+- Offline counts are estimates until you run `tokc calibrate`.
+
+## Tests
+
+```bash
+python -m pytest -q       # 37 tests
+```

tokc-0.1.0/pyproject.toml

@@ -0,0 +1,49 @@
+[build-system]
+# 77+ for PEP 639 (`license` as an SPDX string, `license-files`).
+requires = ["setuptools>=77"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "tokc"
+version = "0.1.0"
+description = "Measure and shrink the token cost of Claude Code skills."
+readme = "README.md"
+requires-python = ">=3.10"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{ name = "Sylvain Mestre" }]
+keywords = ["claude", "claude-code", "skills", "tokens", "cost", "llm"]
+classifiers = [
+    "Environment :: Console",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Software Development :: Quality Assurance",
+]
+dependencies = []
+
+[project.urls]
+Homepage = "https://github.com/Shult/token_consumer"
+Repository = "https://github.com/Shult/token_consumer"
+Issues = "https://github.com/Shult/token_consumer/issues"
+Changelog = "https://github.com/Shult/token_consumer/blob/main/CHANGELOG.md"
+
+[project.optional-dependencies]
+# tiktoken makes offline counts far better than the pure-python fallback.
+fast = ["tiktoken>=0.7"]
+# anthropic enables `--exact` and `tokc calibrate`.
+exact = ["anthropic>=0.40"]
+yaml = ["pyyaml>=6"]
+dev = ["pytest>=8", "tiktoken>=0.7", "pyyaml>=6"]
+
+[project.scripts]
+tokc = "tokc.cli:main"
+
+[tool.setuptools]
+packages = ["tokc"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

tokc-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+