skillcost 0.0.3__py3-none-any.whl

skillcost/__init__.py

@@ -0,0 +1,320 @@
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+import urllib.request
+from dataclasses import asdict, dataclass
+from pathlib import Path
+
+import tiktoken
+import yaml
+
+# ---------------------------------------------------------------------------
+# Token counting
+# ---------------------------------------------------------------------------
+
+_ENC: tiktoken.Encoding | None = None
+
+
+def _get_enc() -> tiktoken.Encoding:
+    global _ENC
+    if _ENC is None:
+        _ENC = tiktoken.get_encoding("cl100k_base")
+    return _ENC
+
+
+def count_tokens(text: str) -> int:
+    return len(_get_enc().encode(text))
+
+
+# ---------------------------------------------------------------------------
+# Data models
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class TargetData:
+    text: str
+    resident_cost: int
+    ref_links: list[tuple[Path, int]]
+    local_links: list[tuple[Path, int]]
+    http_links: list[str]
+
+
+@dataclass
+class CostReport:
+    resident: int
+    baseline: int
+    maximum: int
+    files: dict[str, int]  # str(Path) -> token count, all crawled files (excl. target)
+    urls: dict[str, int]  # url -> token count
+    ref_link_count: int
+    local_link_count: int
+    http_link_count: int
+
+
+# ---------------------------------------------------------------------------
+# Parsing helpers
+# ---------------------------------------------------------------------------
+
+_REF_LINK_RE = re.compile(r"@\{([^}]+)\}")
+_MD_LINK_RE = re.compile(r"\[[^\]]*\]\(([^)]+)\)")
+_HTTP_RE = re.compile(r"^https?://")
+
+
+def _line_of(text: str, pos: int) -> int:
+    return text.count("\n", 0, pos) + 1
+
+
+def _parse_md_links(
+    text: str, base_dir: Path
+) -> tuple[list[tuple[Path, int]], list[tuple[Path, int]], list[str]]:
+    ref_links: list[tuple[Path, int]] = []
+    local_links: list[tuple[Path, int]] = []
+    http_links: list[str] = []
+
+    for m in _REF_LINK_RE.finditer(text):
+        ref_links.append(((base_dir / m.group(1)).resolve(), _line_of(text, m.start())))
+
+    for m in _MD_LINK_RE.finditer(text):
+        target = m.group(1)
+        if _HTTP_RE.match(target):
+            http_links.append(target)
+        else:
+            target = target.split("#")[0]
+            if not target:
+                continue
+            local_links.append(
+                ((base_dir / target).resolve(), _line_of(text, m.start()))
+            )
+
+    return ref_links, local_links, http_links
+
+
+def _parse_frontmatter(text: str) -> dict | None:
+    if not text.startswith("---"):
+        return None
+    end = text.find("\n---", 3)
+    if end == -1:
+        return None
+    return yaml.safe_load(text[3:end].strip())
+
+
+# ---------------------------------------------------------------------------
+# Target loading
+# ---------------------------------------------------------------------------
+
+
+def load_target(path: Path) -> TargetData:
+    try:
+        text = path.read_text(encoding="utf-8")
+    except OSError as e:
+        print(f"ERROR: cannot read {path}: {e}", file=sys.stderr)
+        sys.exit(1)
+
+    resident_cost = 0
+    ref_links: list[tuple[Path, int]] = []
+    local_links: list[tuple[Path, int]] = []
+    http_links: list[str] = []
+
+    if path.name == "SKILL.md":
+        try:
+            fm = _parse_frontmatter(text)
+            if fm:
+                name = str(fm.get("name", ""))
+                description = str(fm.get("description", ""))
+                resident_cost = count_tokens(name) + count_tokens(description)
+        except Exception as e:
+            print(
+                f"WARNING: failed to parse frontmatter in {path}: {e}", file=sys.stderr
+            )
+
+    if path.suffix == ".md":
+        ref_links, local_links, http_links = _parse_md_links(text, path.parent)
+
+    return TargetData(
+        text=text,
+        resident_cost=resident_cost,
+        ref_links=ref_links,
+        local_links=local_links,
+        http_links=http_links,
+    )
+
+
+# ---------------------------------------------------------------------------
+# Cost accumulation
+# ---------------------------------------------------------------------------
+
+
+def _crawl_files(
+    initial_queue: list[tuple[Path, int, Path]], visited: dict[Path, int]
+) -> int:
+    queue = list(initial_queue)
+    total = 0
+    while queue:
+        path, line_no, source = queue.pop(0)
+        rp = path.resolve()
+        if rp in visited:
+            continue
+        if not rp.is_file():
+            print(
+                f"WARNING: {rp} is not a file or does not exist"
+                f" (referenced from {source}:{line_no})",
+                file=sys.stderr,
+            )
+            visited[rp] = 0
+            continue
+        try:
+            text = rp.read_text(encoding="utf-8")
+        except OSError as e:
+            print(
+                f"WARNING: cannot read {rp}: {e} (referenced from {source}:{line_no})",
+                file=sys.stderr,
+            )
+            visited[rp] = 0
+            continue
+        tokens = count_tokens(text)
+        visited[rp] = tokens
+        total += tokens
+        if rp.suffix == ".md":
+            ref_links, local_links, _ = _parse_md_links(text, rp.parent)
+            for p, ln in ref_links + local_links:
+                if p.resolve() not in visited:
+                    queue.append((p, ln, rp))
+    return total
+
+
+def _fetch_url_costs(http_links: list[str], url_costs: dict[str, int]) -> int:
+    total = 0
+    for url in http_links:
+        if url in url_costs:
+            continue
+        try:
+            with urllib.request.urlopen(url, timeout=10) as resp:
+                body = resp.read().decode("utf-8", errors="replace")
+            tokens = count_tokens(body)
+            url_costs[url] = tokens
+            total += tokens
+        except Exception as e:
+            print(f"WARNING: failed to fetch {url}: {e}", file=sys.stderr)
+            url_costs[url] = 0
+    return total
+
+
+def compute_costs(target_path: Path, target_data: TargetData) -> CostReport:
+    target_resolved = target_path.resolve()
+    target_tokens = count_tokens(target_data.text)
+
+    # Baseline: target + resident + ref_links (flat, no recursion)
+    ref_token_map: dict[Path, int] = {}
+    for p, _ln in target_data.ref_links:
+        rp = p.resolve()
+        if rp in ref_token_map or rp == target_resolved:
+            continue
+        try:
+            ref_token_map[rp] = count_tokens(p.read_text(encoding="utf-8"))
+        except OSError as e:
+            print(f"WARNING: cannot read ref link {p}: {e}", file=sys.stderr)
+            ref_token_map[rp] = 0
+
+    baseline = target_tokens + target_data.resident_cost + sum(ref_token_map.values())
+
+    # Pre-populate visited with target + ref_links so they aren't double-counted
+    visited: dict[Path, int] = {target_resolved: target_tokens, **ref_token_map}
+
+    # Crawl all local files recursively (ref_links already visited → not re-added)
+    seen_paths: set[Path] = set()
+    initial_queue: list[tuple[Path, int, Path]] = []
+    for p, ln in target_data.ref_links + target_data.local_links:
+        rp = p.resolve()
+        if rp not in seen_paths:
+            seen_paths.add(rp)
+            initial_queue.append((rp, ln, target_path))
+    file_crawl_total = _crawl_files(initial_queue, visited)
+
+    # Files breakdown: everything visited except the target itself
+    files: dict[str, int] = {
+        str(p): t for p, t in visited.items() if p != target_resolved
+    }
+
+    url_costs: dict[str, int] = {}
+    url_total = _fetch_url_costs(target_data.http_links, url_costs)
+
+    maximum = baseline + file_crawl_total + url_total
+
+    return CostReport(
+        resident=target_data.resident_cost,
+        baseline=baseline,
+        maximum=maximum,
+        files=files,
+        urls=url_costs,
+        ref_link_count=len(target_data.ref_links),
+        local_link_count=len(target_data.local_links),
+        http_link_count=len(target_data.http_links),
+    )
+
+
+# ---------------------------------------------------------------------------
+# Output formatting
+# ---------------------------------------------------------------------------
+
+
+def format_report(report: CostReport, fmt: str) -> str:
+    if fmt == "json":
+        return json.dumps(asdict(report), indent=2)
+    if fmt == "yaml":
+        return yaml.dump(asdict(report), sort_keys=False)
+
+    # Human-readable text
+    lines: list[str] = []
+    lines.append("Token Cost Report")
+    lines.append("=================")
+    lines.append(f"  Resident : {report.resident:,}")
+    lines.append(f"  Baseline : {report.baseline:,}")
+    lines.append(f"  Maximum  : {report.maximum:,}")
+
+    if report.files:
+        lines.append(f"\nFiles ({len(report.files)})")
+        for path_str, tokens in report.files.items():
+            lines.append(f"  {path_str:<60} {tokens:>8,}")
+
+    if report.urls:
+        lines.append(f"\nURLs ({len(report.urls)})")
+        for url, tokens in report.urls.items():
+            lines.append(f"  {url:<60} {tokens:>8,}")
+
+    lines.append("\nLinks")
+    lines.append(f"  @-directives : {report.ref_link_count}")
+    lines.append(f"  Local links  : {report.local_link_count}")
+    lines.append(f"  HTTP links   : {report.http_link_count}")
+
+    return "\n".join(lines)
+
+
+# ---------------------------------------------------------------------------
+# CLI entry point
+# ---------------------------------------------------------------------------
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Estimate the token cost of a text file and its linked resources."
+    )
+    parser.add_argument("target", help="Path to a plain-text file to analyse")
+    fmt_group = parser.add_mutually_exclusive_group()
+    fmt_group.add_argument("--json", action="store_true", help="Output as JSON")
+    fmt_group.add_argument("--yaml", action="store_true", help="Output as YAML")
+    args = parser.parse_args()
+
+    target_path = Path(args.target).resolve()
+    target_data = load_target(target_path)
+    report = compute_costs(target_path, target_data)
+
+    fmt = "json" if args.json else "yaml" if args.yaml else "text"
+    print(format_report(report, fmt))
+
+
+if __name__ == "__main__":
+    main()

skillcost/__main__.py

@@ -0,0 +1,3 @@
+from skillcost import main
+
+main()

skillcost-0.0.3.dist-info/METADATA

@@ -0,0 +1,80 @@
+Metadata-Version: 2.4
+Name: skillcost
+Version: 0.0.3
+Summary: Estimates the token cost of agent skills and their linked resources
+Requires-Python: >=3.12
+Requires-Dist: pyyaml>=6.0.3
+Requires-Dist: tiktoken>=0.12.0
+Description-Content-Type: text/markdown
+
+# skillcost
+
+Estimates the token cost of agent skills and their linked resources.
+
+Given a target file (typically a `SKILL.md`), `skillcost` counts tokens and crawls all linked files
+and URLs to produce three cost figures:
+
+| Metric       | Description                                                                     |
+| ------------ | ------------------------------------------------------------------------------- |
+| **Resident** | Tokens loaded on every prompt, just because the skill is installed.             |
+| **Baseline** | Tokens loaded when the skill is invoked — the skill file plus any `@`-includes. |
+| **Maximum**  | Tokens loaded if the agent follows every link in the skill file.                |
+
+This also works on `CLAUDE.md` or any other plain-text UTF-8 file.
+
+## Installation
+
+Requires Python 3.12+ and [uv](https://github.com/astral-sh/uv).
+
+```sh
+uv sync
+```
+
+## Usage
+
+```sh
+uv run python -m main <path/to/file>
+```
+
+Output defaults to human-readable text. Pass `--json` or `--yaml` for machine-readable output.
+
+```sh
+uv run python -m main SKILL.md
+uv run python -m main SKILL.md --json
+uv run python -m main SKILL.md --yaml
+```
+
+### Example output
+
+```
+Token Cost Report
+=================
+  Resident : 97
+  Baseline : 1,940
+  Maximum  : 18,390
+
+Files (24)
+  /path/to/reference/conventions.md                                       926
+  /path/to/reference/utilities.md                                       1,751
+  ...
+
+Links
+  @-directives : 0
+  Local links  : 17
+  HTTP links   : 0
+```
+
+## How links are counted
+
+`skillcost` recognizes two link syntaxes in Markdown files:
+
+- **`@{relative/path}`** — an include directive; the file is always fetched when the skill is
+  invoked, so it counts toward the **baseline** cost.
+- **`[text](relative/path)`** — a standard Markdown link; followed recursively and counted toward
+  the **maximum** cost only.
+
+HTTP links (`https://...`) are fetched and counted toward the maximum cost only.
+
+## Contributing
+
+See [CONTRIBUTING.md](CONTRIBUTING.md).

skillcost-0.0.3.dist-info/RECORD

@@ -0,0 +1,6 @@
+skillcost/__init__.py,sha256=r4moz-gPU2Z2AeEz9qeM6dtjuD0ntAqRFKm9_lq6z8M,10160
+skillcost/__main__.py,sha256=B2dbxzFEPolhdyA6HsXPdJm61iIfMje39O-2nAW1dcU,35
+skillcost-0.0.3.dist-info/METADATA,sha256=3QclmxCm4d-CSFVmww0gF5BE4HuYznH0RCrGdcxCALA,2285
+skillcost-0.0.3.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+skillcost-0.0.3.dist-info/entry_points.txt,sha256=2dLjNS5yR0In9TcvC_91vT8oLLEso2AVGNC6Bq1zUOs,45
+skillcost-0.0.3.dist-info/RECORD,,

skillcost-0.0.3.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.29.0
+Root-Is-Purelib: true
+Tag: py3-none-any

skillcost-0.0.3.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+skillcost = skillcost:main