tokenledger-llm 0.1.0__py3-none-any.whl

tokenledger/__init__.py

@@ -0,0 +1,22 @@
+"""tokenledger — measure the real cost structure of multi-agent LLM systems."""
+from .parse import Request, load_requests
+from .account import Metrics, account, what_if_remove_step
+from .adapters import (
+    from_anthropic_usage,
+    from_openai_usage,
+    from_records,
+    load_provider,
+)
+
+__all__ = [
+    "Request",
+    "load_requests",
+    "Metrics",
+    "account",
+    "what_if_remove_step",
+    "from_anthropic_usage",
+    "from_openai_usage",
+    "from_records",
+    "load_provider",
+]
+__version__ = "0.1.0"

tokenledger/account.py

@@ -0,0 +1,87 @@
+"""The accounting math: where the tokens actually go.
+
+Core claims this quantifies (see docs/METHODOLOGY.md):
+  1. input cost tracks (prefix size x number of requests), dominated by cache re-reads;
+  2. cost is superlinear in step count, because each step's priming AND its tool
+     result are re-carried into every later step of the turn.
+"""
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass
+from typing import Dict, List
+
+from .parse import Request
+
+
+@dataclass
+class Metrics:
+    n_requests: int
+    n_turns: int
+    steps_per_turn: Dict[int, int]
+    total_input: int
+    total_output: int
+    total_cache_read: int
+    cache_read_pct: float
+    priming_total: int
+    priming_pct: float
+    conversation_total: int
+    conversation_pct: float
+    mean_prefix: float
+    max_prefix: int
+    prefix_reread_cost: int  # tokens spent re-reading already-seen (cached) context
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+def _pct(part: int, whole: int) -> float:
+    return (part / whole) if whole else 0.0
+
+
+def account(requests: List[Request]) -> Metrics:
+    if not requests:
+        raise ValueError("no requests to account")
+
+    total_input = sum(r.input_tokens for r in requests)
+    total_output = sum(r.output_tokens for r in requests)
+    total_cache = sum(r.cache_read_tokens for r in requests)
+    priming_total = sum(r.priming_tokens for r in requests)
+    conversation_total = sum(r.conversation_tokens for r in requests)
+
+    turns: Dict[int, List[Request]] = {}
+    for r in requests:
+        turns.setdefault(r.turn, []).append(r)
+    steps_per_turn = {t: len(rs) for t, rs in sorted(turns.items())}
+
+    prefixes = [r.input_tokens for r in requests]
+    return Metrics(
+        n_requests=len(requests),
+        n_turns=len(turns),
+        steps_per_turn=steps_per_turn,
+        total_input=total_input,
+        total_output=total_output,
+        total_cache_read=total_cache,
+        cache_read_pct=_pct(total_cache, total_input),
+        priming_total=priming_total,
+        priming_pct=_pct(priming_total, total_input),
+        conversation_total=conversation_total,
+        conversation_pct=_pct(conversation_total, total_input),
+        mean_prefix=sum(prefixes) / len(prefixes),
+        max_prefix=max(prefixes),
+        prefix_reread_cost=total_cache,
+    )
+
+
+def what_if_remove_step(requests: List[Request], turn: int, step: int) -> int:
+    """Estimated input tokens saved by removing (turn, step).
+
+    Saving = the removed request's own input, PLUS its tool result, which would
+    have been re-carried into every later step of the same turn. This is why
+    removing an EARLY step saves more than a late one: the superlinear effect.
+    """
+    same_turn = [r for r in requests if r.turn == turn]
+    target = next((r for r in same_turn if r.step == step), None)
+    if target is None:
+        raise KeyError(f"no step {step} in turn {turn}")
+    later_steps = [r for r in same_turn if r.step > step]
+    return target.input_tokens + target.tool_result_tokens * len(later_steps)

tokenledger/adapters.py

@@ -0,0 +1,177 @@
+"""Adapters: turn real provider usage logs into normalized Request records.
+
+The point of tokenledger is that you measure *your own* system, so it has to
+read the logs you already have. Providers report token usage differently, and
+one difference matters for correctness:
+
+  * Anthropic Messages API: `usage.input_tokens` is the FRESH (uncached) input
+    only. Cached context is reported SEPARATELY in `cache_read_input_tokens`
+    and `cache_creation_input_tokens`. So the *total* context sent on a request
+    is the sum of all three — a naive reader who takes `input_tokens` alone
+    undercounts the prefix, which is exactly the cost this tool is about.
+
+  * OpenAI Chat Completions: `usage.prompt_tokens` is ALREADY the total input
+    (it includes cached tokens); the cached portion is broken out under
+    `usage.prompt_tokens_details.cached_tokens`.
+
+These adapters normalize both into Request.input_tokens = total context sent.
+
+Turn/step structure usually is NOT in a raw usage record. The superlinearity
+metric (what_if_remove_step, steps_per_turn) depends on it, so you supply it:
+pass turn_key/step_key (field names present in each record) or turn_fn/step_fn
+(callables over the record). If you supply neither, every record is treated as
+its own step within a single turn (step = file order), which still gives you
+correct totals, cache-read % and priming %, but a flat step structure.
+"""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Callable, Iterable, List, Optional, Union
+
+from .parse import Request
+
+
+def _usage(record: dict, usage_key: Optional[str]) -> dict:
+    """Return the sub-dict holding token counts, or the record itself."""
+    if usage_key and isinstance(record.get(usage_key), dict):
+        return record[usage_key]
+    if usage_key is None and isinstance(record.get("usage"), dict):
+        return record["usage"]
+    return record
+
+
+def _int(d: dict, key: str, default: int = 0) -> int:
+    v = d.get(key, default)
+    return int(v) if v is not None else default
+
+
+def from_anthropic_usage(
+    record: dict, *, turn: int, step: int, label: str = "", usage_key: Optional[str] = None
+) -> Request:
+    """One Anthropic Messages API record -> Request.
+
+    total input = input_tokens + cache_creation_input_tokens + cache_read_input_tokens
+    (because Anthropic's `input_tokens` excludes cached context).
+    """
+    u = _usage(record, usage_key)
+    fresh = _int(u, "input_tokens")
+    cache_creation = _int(u, "cache_creation_input_tokens")
+    cache_read = _int(u, "cache_read_input_tokens")
+    total_input = fresh + cache_creation + cache_read
+    return Request(
+        turn=turn,
+        step=step,
+        input_tokens=total_input,
+        cache_read_tokens=cache_read,
+        output_tokens=_int(u, "output_tokens"),
+        label=label or str(record.get("label", "")),
+    )
+
+
+def from_openai_usage(
+    record: dict, *, turn: int, step: int, label: str = "", usage_key: Optional[str] = None
+) -> Request:
+    """One OpenAI Chat Completions record -> Request.
+
+    `prompt_tokens` is already the total input; the cached portion is in
+    `prompt_tokens_details.cached_tokens`.
+    """
+    u = _usage(record, usage_key)
+    total_input = _int(u, "prompt_tokens")
+    details = u.get("prompt_tokens_details") or {}
+    cache_read = _int(details, "cached_tokens") if isinstance(details, dict) else 0
+    return Request(
+        turn=turn,
+        step=step,
+        input_tokens=total_input,
+        cache_read_tokens=cache_read,
+        output_tokens=_int(u, "completion_tokens"),
+        label=label or str(record.get("label", "")),
+    )
+
+
+_CONVERTERS = {
+    "anthropic": from_anthropic_usage,
+    "openai": from_openai_usage,
+}
+
+
+def _turn_step_resolver(
+    turn_key: Optional[str],
+    step_key: Optional[str],
+    turn_fn: Optional[Callable[[dict], int]],
+    step_fn: Optional[Callable[[dict], int]],
+):
+    def resolve(record: dict, idx: int):
+        if turn_fn is not None:
+            turn = int(turn_fn(record))
+        elif turn_key is not None:
+            turn = int(record[turn_key])
+        else:
+            turn = 1
+        if step_fn is not None:
+            step = int(step_fn(record))
+        elif step_key is not None:
+            step = int(record[step_key])
+        else:
+            step = idx
+        return turn, step
+
+    return resolve
+
+
+def from_records(
+    records: Iterable[dict],
+    fmt: str,
+    *,
+    turn_key: Optional[str] = None,
+    step_key: Optional[str] = None,
+    turn_fn: Optional[Callable[[dict], int]] = None,
+    step_fn: Optional[Callable[[dict], int]] = None,
+    usage_key: Optional[str] = None,
+) -> List[Request]:
+    """Convert an iterable of provider usage dicts into Request records."""
+    if fmt not in _CONVERTERS:
+        raise ValueError(f"unknown format {fmt!r}; expected one of {sorted(_CONVERTERS)}")
+    convert = _CONVERTERS[fmt]
+    resolve = _turn_step_resolver(turn_key, step_key, turn_fn, step_fn)
+    out: List[Request] = []
+    for idx, record in enumerate(records):
+        turn, step = resolve(record, idx)
+        out.append(convert(record, turn=turn, step=step, usage_key=usage_key))
+    return out
+
+
+def load_provider(
+    path: Union[str, Path],
+    fmt: str,
+    *,
+    turn_key: Optional[str] = None,
+    step_key: Optional[str] = None,
+    turn_fn: Optional[Callable[[dict], int]] = None,
+    step_fn: Optional[Callable[[dict], int]] = None,
+    usage_key: Optional[str] = None,
+) -> List[Request]:
+    """Load a JSONL usage log in a provider format into Request records.
+
+    Blank lines and lines starting with '#' are skipped. Each remaining line is
+    one JSON object carrying a usage record (either at top level or under a
+    `usage` key).
+    """
+    records: List[dict] = []
+    with open(path, "r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line or line.startswith("#"):
+                continue
+            records.append(json.loads(line))
+    return from_records(
+        records,
+        fmt,
+        turn_key=turn_key,
+        step_key=step_key,
+        turn_fn=turn_fn,
+        step_fn=step_fn,
+        usage_key=usage_key,
+    )

tokenledger/parse.py

@@ -0,0 +1,59 @@
+"""Ingest request logs into a normalized Request model.
+
+A log is JSONL: one JSON object per line, one object per LLM request (one step
+within a turn). Unknown keys are ignored; missing optional fields default to 0.
+Map a provider's own field names with the `mapping` argument.
+"""
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+from typing import List, Optional, Union
+
+
+@dataclass
+class Request:
+    """One LLM request = one step within a conversation turn."""
+    turn: int                      # which turn (main-loop iteration) this belongs to
+    step: int                      # step index within the turn (0-based)
+    input_tokens: int              # total input tokens sent on this request (the context)
+    cache_read_tokens: int = 0     # input tokens served from cache (re-read prefix)
+    priming_tokens: int = 0        # fixed re-sent block: system prompt, tool schemas, agreements
+    conversation_tokens: int = 0   # re-carried prior messages + tool results
+    tool_result_tokens: int = 0    # tokens this step emitted as tool output (carried forward)
+    output_tokens: int = 0
+    label: str = ""
+
+    @property
+    def fresh_tokens(self) -> int:
+        """Input tokens that were NOT served from cache on this request."""
+        return max(self.input_tokens - self.cache_read_tokens, 0)
+
+
+_FIELDS = {
+    "turn", "step", "input_tokens", "cache_read_tokens", "priming_tokens",
+    "conversation_tokens", "tool_result_tokens", "output_tokens", "label",
+}
+
+
+def _coerce(obj: dict, mapping: Optional[dict]) -> Request:
+    if mapping:
+        obj = {ours: obj.get(theirs) for ours, theirs in mapping.items()}
+    kwargs = {k: v for k, v in obj.items() if k in _FIELDS and v is not None}
+    for required in ("turn", "step", "input_tokens"):
+        if required not in kwargs:
+            raise ValueError(f"record missing required field '{required}': {obj}")
+    return Request(**kwargs)
+
+
+def load_requests(path: Union[str, Path], mapping: Optional[dict] = None) -> List[Request]:
+    """Load a JSONL request log. Blank lines and lines starting with '#' are skipped."""
+    out: List[Request] = []
+    with open(path, "r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line or line.startswith("#"):
+                continue
+            out.append(_coerce(json.loads(line), mapping))
+    return out

tokenledger/report.py

@@ -0,0 +1,84 @@
+"""CLI: `tokenledger report <log.jsonl>` → human or JSON cost-structure report."""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from typing import Optional, Sequence
+
+from .account import Metrics, account
+from .adapters import load_provider
+from .parse import load_requests
+
+
+def format_report(m: Metrics) -> str:
+    lines = [
+        "tokenledger — cost structure report",
+        "=" * 42,
+        f"requests (steps): {m.n_requests}",
+        f"turns:            {m.n_turns}",
+        f"steps per turn:   {m.steps_per_turn}",
+        "",
+        f"total input tokens:  {m.total_input:,}",
+        f"total output tokens: {m.total_output:,}",
+        "",
+        f"cache-read:               {m.total_cache_read:,} ({m.cache_read_pct:.1%} of input)",
+        f"priming re-sent:          {m.priming_total:,} ({m.priming_pct:.1%} of input)",
+        f"re-carried conversation:  {m.conversation_total:,} ({m.conversation_pct:.1%} of input)",
+        "",
+        f"mean prefix / request: {m.mean_prefix:,.0f}    max prefix: {m.max_prefix:,}",
+        f"prefix re-read cost:   {m.prefix_reread_cost:,} tokens",
+        "",
+        "takeaway: cost tracks (prefix size x requests), dominated by cache re-reads —",
+        "not prompt size. Cut steps and reset context to move the bill.",
+    ]
+    return "\n".join(lines)
+
+
+def main(argv: Optional[Sequence[str]] = None) -> int:
+    p = argparse.ArgumentParser(
+        prog="tokenledger",
+        description="Measure the cost structure of multi-agent LLM systems from request logs.",
+    )
+    sub = p.add_subparsers(dest="cmd")
+    rep = sub.add_parser("report", help="print a cost-structure report for a JSONL log")
+    rep.add_argument("logfile", help="path to a JSONL request log")
+    rep.add_argument("--json", action="store_true", help="emit JSON instead of text")
+    rep.add_argument(
+        "--format",
+        choices=["native", "anthropic", "openai"],
+        default="native",
+        help="log format: 'native' tokenledger fields (default), or a provider usage log",
+    )
+    rep.add_argument("--turn-key", help="record field naming the turn (provider formats)")
+    rep.add_argument("--step-key", help="record field naming the step (provider formats)")
+    rep.add_argument(
+        "--usage-key",
+        help="record field holding the usage sub-object (default: 'usage' if present, else top level)",
+    )
+    args = p.parse_args(argv)
+
+    if args.cmd != "report":
+        p.print_help()
+        return 1
+
+    if args.format == "native":
+        requests = load_requests(args.logfile)
+    else:
+        requests = load_provider(
+            args.logfile,
+            args.format,
+            turn_key=args.turn_key,
+            step_key=args.step_key,
+            usage_key=args.usage_key,
+        )
+    metrics = account(requests)
+    if args.json:
+        print(json.dumps(metrics.to_dict(), indent=2))
+    else:
+        print(format_report(metrics))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tokenledger_llm-0.1.0.dist-info/METADATA

@@ -0,0 +1,125 @@
+Metadata-Version: 2.4
+Name: tokenledger-llm
+Version: 0.1.0
+Summary: Measure the real cost structure of multi-agent / multi-model LLM systems from request logs.
+Author: Pallav Pokhriyal
+License: MIT
+Keywords: llm,agents,cost,tokens,observability,mlops
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: license-file
+
+# tokenledger
+
+**Measure where the tokens actually go in a multi-agent / multi-model LLM system.**
+
+Most teams reason about LLM cost as if it scales with *prompt size*. In a multi-step agent it
+doesn't — it scales with **(context size × number of requests)**, because almost all input is a
+**cache re-read of a growing prefix**, and it is **superlinear in step count**, because each step's
+priming *and* its tool result get re-carried into every later step. `tokenledger` reads your request
+logs and shows you exactly that, so you optimize the right thing.
+
+> Status: **v0.1 — early scaffold, feedback very welcome.** Author: Pallav Pokhriyal.
+> Zero dependencies, zero telemetry. It reads logs you already have.
+
+## Install
+```bash
+pip install tokenledger-llm     # on PyPI; the import name and CLI are both `tokenledger`
+# or from source:
+pip install -e .
+```
+
+## Quickstart
+```bash
+tokenledger report examples/sample_logs/sample.jsonl
+# or JSON:
+tokenledger report examples/sample_logs/sample.jsonl --json
+```
+
+Example output:
+```
+tokenledger — cost structure report
+==========================================
+requests (steps): 4
+turns:            2
+steps per turn:   {1: 3, 2: 1}
+
+total input tokens:  89,532
+total output tokens: 505
+
+cache-read:               88,432 (98.8% of input)
+priming re-sent:          80,464 (89.9% of input)
+re-carried conversation:  7,968 (8.9% of input)
+
+mean prefix / request: 22,383    max prefix: 24,000
+prefix re-read cost:   88,432 tokens
+
+takeaway: cost tracks (prefix size x requests), dominated by cache re-reads —
+not prompt size. Cut steps and reset context to move the bill.
+```
+
+## Log format
+JSONL, one request (one step within a turn) per line. Required: `turn`, `step`, `input_tokens`.
+Everything else is optional and defaults to 0:
+
+| field | meaning |
+|---|---|
+| `turn` / `step` | which turn, and the step index within it |
+| `input_tokens` | total input sent on this request (the context) |
+| `cache_read_tokens` | input served from cache (the re-read prefix) |
+| `priming_tokens` | fixed re-sent block: system prompt, tool schemas |
+| `conversation_tokens` | re-carried prior messages + tool results |
+| `tool_result_tokens` | tokens this step emitted as tool output (carried forward) |
+| `output_tokens` | completion tokens |
+
+Have a different log shape? Map it:
+```python
+from tokenledger import load_requests, account
+reqs = load_requests("mylog.jsonl", mapping={"input_tokens": "prompt_tok", "turn": "iteration"})
+print(account(reqs).cache_read_pct)
+```
+
+## Provider usage logs (no reshaping)
+
+Point it straight at Anthropic or OpenAI usage records:
+```bash
+tokenledger report anthropic_usage.jsonl --format anthropic --turn-key turn --step-key step
+tokenledger report openai_usage.jsonl   --format openai   --turn-key turn --step-key step
+```
+Each line is one request whose token counts live at top level or under a `usage` key.
+One correctness detail the adapters handle for you:
+
+- **Anthropic** reports `usage.input_tokens` as the *fresh* (uncached) input only; cached
+  context is reported separately in `cache_read_input_tokens` / `cache_creation_input_tokens`.
+  The adapter sums all three so `input_tokens` is the *true* per-request context — taking
+  `input_tokens` alone silently undercounts the prefix, which is the whole thing being measured.
+- **OpenAI** reports `usage.prompt_tokens` as the total input already, with the cached portion
+  in `prompt_tokens_details.cached_tokens`.
+
+If your records don't carry turn/step, omit the keys — each line becomes its own step in a single
+turn (totals and percentages stay correct; the per-turn step structure just goes flat). In Python:
+```python
+from tokenledger import load_provider, account
+reqs = load_provider("anthropic_usage.jsonl", "anthropic", turn_key="turn", step_key="step")
+print(account(reqs).cache_read_pct)
+```
+
+## What-if: which step to cut
+```python
+from tokenledger import load_requests, what_if_remove_step
+reqs = load_requests("examples/sample_logs/sample.jsonl")
+print(what_if_remove_step(reqs, turn=1, step=0))  # tokens saved by removing an early step
+```
+Removing an **early** step saves more than a late one — it drops that step's own cost *and* its
+tool result from every later step. That's the superlinearity, made concrete.
+
+## Why it exists
+See [`docs/METHODOLOGY.md`](docs/METHODOLOGY.md) for the measured findings this operationalizes.
+
+## Contributing
+Issues and PRs welcome — especially log-format adapters for different providers/harnesses. Keep
+contributions genuinely useful; no telemetry, no secrets in sample data (synthetic only).
+
+## License
+MIT — see [LICENSE](LICENSE).

tokenledger_llm-0.1.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+tokenledger/__init__.py,sha256=LKaBz16YZd7L3pqNL_hOg81QMzNaTtbPEa-6alVEk_g,519
+tokenledger/account.py,sha256=-o386ERFKPShfHRfY2MW3nUWPcTl_EsiQ8XGICXqFPQ,3056
+tokenledger/adapters.py,sha256=npU3mTUghugXKYkG57FBLAAWuyBXs5pbIDfU_LMbzlg,6180
+tokenledger/parse.py,sha256=DZrMGJwWFowVqy9iK7nwUjeag_dPjbRefpemXZRKFRw,2429
+tokenledger/report.py,sha256=4oXlACfIaDcR6GjwEsmuLU1_PZyPT5cvOhU17_Ok_Hk,3017
+tokenledger_llm-0.1.0.dist-info/licenses/LICENSE,sha256=cr7GSixCcUQiaPQc2oX2OMvcx2qVQZ7XvscNZxxRVpA,1073
+tokenledger_llm-0.1.0.dist-info/METADATA,sha256=QoxdFzYD-T_DTjf_y3T9OmZ72cPqNcy1ltx5-mpUwGU,5128
+tokenledger_llm-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+tokenledger_llm-0.1.0.dist-info/entry_points.txt,sha256=89SvmLxIdpGkB-X27TThW-orGPj_A8CVFlv5hSwdFrc,56
+tokenledger_llm-0.1.0.dist-info/top_level.txt,sha256=xZRn06uJDCJV_gIxBUfJeJGmN0xyWD2UG87zXqzhPIA,12
+tokenledger_llm-0.1.0.dist-info/RECORD,,

tokenledger_llm-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

tokenledger_llm-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+tokenledger = tokenledger.report:main

tokenledger_llm-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Pallav Pokhriyal
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

tokenledger_llm-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+tokenledger