tokencur 0.3.0__py3-none-any.whl

tokencur/__init__.py

@@ -0,0 +1,7 @@
+"""tokencur — the CUR for your tokens.
+
+Converts multi-provider AI/LLM usage data into FOCUS-conformant cost
+datasets, with unit economics and savings recommendations.
+"""
+
+__version__ = "0.3.0"  # the only place the version lives (see pyproject)

tokencur/__main__.py

@@ -0,0 +1,6 @@
+"""``python -m tokencur``: the same CLI as the ``tokencur`` command."""
+
+from tokencur.cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tokencur/cli.py

@@ -0,0 +1,349 @@
+"""Command-line interface: ``tokencur <command>``.
+
+One argparse parser for every command, installed as the ``tokencur``
+console script and reachable as ``python -m tokencur``. The older
+per-module entry points (``python -m tokencur.report`` and friends)
+keep working: each delegates here.
+
+Periods follow billing convention: ``--since`` is the first UTC day
+included, ``--until`` the first day excluded, so ``--since 2026-09-01
+--until 2026-10-01`` is exactly September.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sqlite3
+import sys
+from collections.abc import Sequence
+from datetime import date
+from pathlib import Path
+
+from tokencur import __version__, doctor, ledger, observatory, outcomes, prices
+from tokencur.export import export_csv
+from tokencur.focus import undated_count, unpriced_models
+from tokencur.ingest import claude_code, runpod
+from tokencur.pricing import ConfigError, load_discounts
+from tokencur.recommend import recommendations, render
+from tokencur.records import UsageRecord, in_period
+from tokencur.report import summarize
+from tokencur.sources import load_records
+
+_NOTHING = "no usage in known log locations or the ledger"
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="tokencur",
+        description="The CUR for your tokens: AI coding-agent usage as FOCUS cost data.",
+        epilog="Figures are API-equivalent list prices (showback), not a bill: "
+        "see the README's 'Money concepts'. With no command, runs `report`.",
+    )
+    parser.add_argument(
+        "--version", action="version", version=f"%(prog)s {__version__}"
+    )
+    commands = parser.add_subparsers(dest="command", metavar="<command>")
+
+    period = argparse.ArgumentParser(add_help=False)
+    period.add_argument(
+        "--since", type=_day, metavar="YYYY-MM-DD", help="first UTC day included"
+    )
+    period.add_argument(
+        "--until", type=_day, metavar="YYYY-MM-DD", help="first UTC day excluded"
+    )
+    root_help = "read these Claude Code logs instead; never stored in the ledger"
+    contract = argparse.ArgumentParser(add_help=False)
+    contract.add_argument(
+        "--discounts",
+        type=Path,
+        metavar="FILE",
+        help='negotiated discounts: {"discounts": {"Anthropic": 0.15}} (15%% off list)',
+    )
+
+    report = commands.add_parser(
+        "report", parents=[period, contract], help="cost summary in the terminal"
+    )
+    report.add_argument("root", nargs="?", type=Path, help=root_help)
+    report.add_argument(
+        "--currency", type=_currency, metavar="CODE", help="also show totals in CODE"
+    )
+    report.add_argument(
+        "--fx-rate",
+        type=_positive,
+        metavar="RATE",
+        help="units of --currency per USD; you give the rate, nothing is fetched",
+    )
+    report.set_defaults(handler=_report)
+
+    export = commands.add_parser(
+        "export", parents=[period, contract], help="write a FOCUS 1.2 CSV"
+    )
+    export.add_argument("output", type=Path, help="CSV file to write")
+    export.add_argument("root", nargs="?", type=Path, help=root_help)
+    export.set_defaults(handler=_export)
+
+    recommend = commands.add_parser(
+        "recommend", parents=[period], help="avoided cost and what-if headroom"
+    )
+    recommend.set_defaults(handler=_recommend)
+
+    unit = commands.add_parser(
+        "outcomes",
+        parents=[period],
+        help="usage value per commit, repository by repository",
+    )
+    unit.add_argument(
+        "repos",
+        nargs="*",
+        type=Path,
+        metavar="REPO",
+        help="git repositories to measure (default: every one the agents worked in)",
+    )
+    unit.add_argument(
+        "--all-authors",
+        action="store_true",
+        help="count every author's commits, not only your git user.email's",
+    )
+    unit.set_defaults(handler=_outcomes)
+
+    obs = commands.add_parser(
+        "observatory", help="render the static spend dashboard (aggregates only)"
+    )
+    obs.add_argument(
+        "outdir",
+        nargs="?",
+        type=Path,
+        default=observatory.DEFAULT_OUTPUT,
+        help=f"default: {observatory.DEFAULT_OUTPUT}",
+    )
+    obs.set_defaults(handler=_observatory)
+
+    card = commands.add_parser("prices", help="render the static price card")
+    card.add_argument(
+        "outdir",
+        nargs="?",
+        type=Path,
+        default=prices.DEFAULT_OUTPUT,
+        help=f"default: {prices.DEFAULT_OUTPUT}",
+    )
+    card.set_defaults(handler=_prices)
+
+    keep = commands.add_parser(
+        "import",
+        help="keep a provider's billing export (billed cost) in the ledger",
+    )
+    keep.add_argument("provider", choices=["runpod"])
+    keep.add_argument("files", nargs="+", type=Path, help="exports to import")
+    keep.set_defaults(handler=_import)
+
+    check = commands.add_parser(
+        "doctor",
+        help="check log formats, the ledger and pricing (read-only)",
+    )
+    check.set_defaults(handler=_doctor)
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    if args.command is None:
+        args = parser.parse_args(["report"])
+    since, until = getattr(args, "since", None), getattr(args, "until", None)
+    if since and until and since >= until:
+        parser.error("--since must be an earlier day than --until")
+    if (getattr(args, "currency", None) is None) != (
+        getattr(args, "fx_rate", None) is None
+    ):
+        parser.error("--currency and --fx-rate go together")
+    try:
+        return args.handler(args)
+    except (ledger.LedgerError, ConfigError, outcomes.OutcomesError) as exc:
+        _fail(str(exc))
+    except sqlite3.DatabaseError as exc:
+        _fail(f"the ledger looks damaged ({exc}); run `tokencur doctor`")
+    return 1
+
+
+def _currency(text: str) -> str:
+    if len(text) != 3 or not text.isalpha() or not text.isupper():
+        raise argparse.ArgumentTypeError(f"not a 3-letter currency code: {text!r}")
+    return text
+
+
+def _positive(text: str) -> float:
+    try:
+        value = float(text)
+    except ValueError:
+        raise argparse.ArgumentTypeError(f"not a number: {text!r}") from None
+    if not value > 0:
+        raise argparse.ArgumentTypeError(f"must be above 0: {text!r}")
+    return value
+
+
+def _charges(args: argparse.Namespace) -> list:
+    """Billed charges in the ledger for the command's period. An explicit
+    log root is an ad hoc look at those logs alone, so it gets none."""
+    if getattr(args, "root", None) is not None:
+        return []
+    return in_period(
+        ledger.read_charges(), args.since, args.until, when=lambda c: c.period_start
+    )
+
+
+def _discounts(args: argparse.Namespace) -> dict[str, float] | None:
+    return load_discounts(args.discounts) if args.discounts else None
+
+
+def _day(text: str) -> date:
+    try:
+        return date.fromisoformat(text)
+    except ValueError:
+        raise argparse.ArgumentTypeError(f"not a YYYY-MM-DD date: {text!r}") from None
+
+
+def _records(args: argparse.Namespace) -> list[UsageRecord] | None:
+    """The command's records, or None after printing why there are none."""
+    root = getattr(args, "root", None)
+    if root is not None:
+        if not root.exists():
+            return _fail(f"{root} does not exist")
+        records = list(claude_code.iter_usage_records(root))
+    else:
+        records = load_records()
+    if not records:
+        return _fail(_NOTHING)
+    return in_period(records, args.since, args.until)
+
+
+def _period(args: argparse.Namespace) -> str | None:
+    since, until = args.since, args.until
+    if since and until:
+        return f"{since} to {until} (UTC, end excluded)"
+    if since:
+        return f"since {since} (UTC)"
+    if until:
+        return f"before {until} (UTC)"
+    return None
+
+
+def _fail(message: str) -> None:
+    print(f"error: {message}", file=sys.stderr)
+
+
+def _report(args: argparse.Namespace) -> int:
+    discounts = _discounts(args)  # a bad file fails before any scan
+    records = _records(args)
+    if records is None:
+        return 1
+    if not records:
+        _fail(f"no usage {_period(args)}")
+        return 1
+    fx = (args.currency, args.fx_rate) if args.currency else None
+    print(
+        summarize(
+            records,
+            period=_period(args),
+            discounts=discounts,
+            fx=fx,
+            charges=_charges(args),
+        )
+    )
+    return 0
+
+
+def _export(args: argparse.Namespace) -> int:
+    discounts = _discounts(args)  # a bad file fails before any scan
+    records = _records(args)
+    if records is None:
+        return 1
+    rows = export_csv(records, args.output, discounts, _charges(args))
+    print(f"wrote {rows} FOCUS charge rows to {args.output}", file=sys.stderr)
+    skipped = unpriced_models(records)
+    if skipped:
+        pairs = ", ".join(f"{m} x{n}" for m, n in sorted(skipped.items()))
+        print(f"skipped unpriced usage: {pairs}", file=sys.stderr)
+    undated = undated_count(records)
+    if undated:
+        print(
+            f"skipped undated usage: {undated} records (no parseable timestamp)",
+            file=sys.stderr,
+        )
+    return 0
+
+
+def _recommend(args: argparse.Namespace) -> int:
+    records = _records(args)
+    if not records:
+        if records is not None:
+            _fail(f"no usage {_period(args)}")
+        return 1
+    print(render(recommendations(records)))
+    return 0
+
+
+def _outcomes(args: argparse.Namespace) -> int:
+    records = _records(args)
+    if records is None:
+        return 1
+    result = outcomes.outcomes(
+        records,
+        repos=args.repos,
+        since=args.since,
+        until=args.until,
+        all_authors=args.all_authors,
+        period=_period(args),
+    )
+    print(outcomes.render(result))
+    return 0
+
+
+def _observatory(args: argparse.Namespace) -> int:
+    records = load_records()
+    if not records:
+        _fail(_NOTHING)
+        return 1
+    snap = observatory.snapshot(
+        records, observatory.load_subscriptions(), ledger.read_charges()
+    )
+    observatory.write_site(snap, args.outdir)
+    print(f"observatory written to {args.outdir} ({len(records)} records aggregated)")
+    return 0
+
+
+def _prices(args: argparse.Namespace) -> int:
+    changes = prices.price_changes()
+    prices.write_site(args.outdir, changes)
+    moved = sum(len(c.changed) for c in changes)
+    gained = sum(len(c.added) for c in changes if not c.introduced)
+    print(
+        f"prices page written to {args.outdir} "
+        f"({len(changes)} events: {moved} rate moves, {gained} models added)"
+    )
+    return 0
+
+
+def _doctor(args: argparse.Namespace) -> int:
+    diagnosis = doctor.diagnose()
+    print(doctor.render(diagnosis))
+    return 1 if diagnosis.problems else 0
+
+
+def _import(args: argparse.Namespace) -> int:
+    charges = []
+    for path in args.files:
+        if not path.exists():
+            _fail(f"{path} does not exist")
+            return 1
+        charges.extend(runpod.iter_charges(path))
+    if not charges:
+        _fail("no billed charges in those files")
+        return 1
+    added = ledger.record_charges(charges)
+    total = sum(c.amount_usd for c in charges)
+    providers = ", ".join(sorted({c.provider for c in charges}))
+    print(
+        f"{len(charges)} billed charges ({added} new), ${total:,.2f} from "
+        f"{providers} — {ledger.default_path()}"
+    )
+    return 0

tokencur/dashboard.py

@@ -0,0 +1,177 @@
+"""Local cost dashboard over the FOCUS dataset.
+
+Usage:
+    pip install -e ".[dashboard]"
+    streamlit run src/tokencur/dashboard.py
+
+Loads the full usage history (every known local source, through the
+ledger — see ``tokencur.sources``), normalizes to FOCUS charge rows and
+lets DuckDB run the analytics — the same SQL any FinOps analyst would
+write over a FOCUS dataset. Every view exposes its SQL and its table
+(the table doubles as the accessibility fallback for chart colors).
+"""
+
+from __future__ import annotations
+
+import duckdb
+import pandas as pd
+import streamlit as st
+
+from tokencur.focus import to_focus_rows, unpriced_models
+from tokencur.recommend import recommendations
+from tokencur.sources import load_records
+
+# Fixed service→color mapping (color follows the entity, never the
+# rank). Both palettes validated for their surface; see the dataviz
+# palette notes in the commit message.
+SERVICE_ORDER = ["Claude Code", "Codex CLI", "Kimi Code"]
+SERVICE_COLORS = {
+    "dark": {"Claude Code": "#3987e5", "Codex CLI": "#199e70", "Kimi Code": "#c98500"},
+    "light": {"Claude Code": "#2a78d6", "Codex CLI": "#1baf7a", "Kimi Code": "#eda100"},
+}
+SINGLE_HUE = {"dark": "#3987e5", "light": "#2a78d6"}  # magnitude charts
+
+
+def _mode() -> str:
+    try:
+        return st.context.theme.type or "dark"
+    except Exception:
+        return "dark"
+
+
+@st.cache_data(show_spinner="Scanning local usage logs…")
+def load() -> tuple[pd.DataFrame, dict[str, int], list]:
+    records = load_records()
+    return pd.DataFrame(to_focus_rows(records)), unpriced_models(records), records
+
+
+def view(title: str, sql: str, frame: pd.DataFrame) -> pd.DataFrame:
+    """Run one DuckDB query and render its SQL + table under the title."""
+    result = duckdb.sql(sql.replace("FOCUS", "frame")).df()
+    st.subheader(title)
+    with st.expander("SQL + tabla"):
+        st.code(sql, language="sql")
+        st.dataframe(result, use_container_width=True)
+    return result
+
+
+st.set_page_config(page_title="tokencur", page_icon="🧾", layout="wide")
+st.title("tokencur — local AI spend, FOCUS-shaped")
+
+focus, unpriced, records = load()
+if focus.empty:
+    st.warning("No usage found in any known local source.")
+    st.stop()
+
+mode = _mode()
+total = focus["BilledCost"].sum()
+days = focus["ChargePeriodStart"].str[:10].nunique()
+
+k1, k2, k3, k4 = st.columns(4)
+k1.metric("API-equiv. usage value (showback)", f"${total:,.2f}")
+k2.metric("Charge rows", f"{len(focus):,}")
+k3.metric("Days covered", days)
+k4.metric("Avg cost / day", f"${total / max(days, 1):,.2f}")
+if unpriced:
+    st.caption(
+        "Unpriced usage (excluded): "
+        + ", ".join(f"{m} ×{n}" for m, n in sorted(unpriced.items()))
+    )
+
+daily = view(
+    "Daily cost by service",
+    """
+    SELECT substr(ChargePeriodStart, 1, 10) AS day,
+           ServiceName,
+           round(sum(BilledCost), 4) AS cost_usd
+    FROM FOCUS
+    GROUP BY 1, 2
+    ORDER BY 1
+    """,
+    focus,
+)
+pivot = daily.pivot(index="day", columns="ServiceName", values="cost_usd")
+services = [s for s in SERVICE_ORDER if s in pivot.columns]
+st.line_chart(
+    pivot[services],
+    color=[SERVICE_COLORS[mode][s] for s in services],
+    height=320,
+)
+
+by_model = view(
+    "Cost by model",
+    """
+    SELECT regexp_replace(SkuId, '/[^/]+$', '') AS model,
+           round(sum(BilledCost), 2) AS cost_usd,
+           sum(ConsumedQuantity)::BIGINT AS tokens
+    FROM FOCUS
+    GROUP BY 1
+    ORDER BY 2 DESC
+    """,
+    focus,
+)
+st.bar_chart(
+    by_model.set_index("model")["cost_usd"],
+    color=SINGLE_HUE[mode],
+    horizontal=True,
+    height=320,
+)
+
+by_bucket = view(
+    "Where the money goes (token type)",
+    """
+    SELECT regexp_extract(SkuId, '[^/]+$') AS token_type,
+           round(sum(BilledCost), 2) AS cost_usd
+    FROM FOCUS
+    GROUP BY 1
+    ORDER BY 2 DESC
+    """,
+    focus,
+)
+st.bar_chart(
+    by_bucket.set_index("token_type")["cost_usd"],
+    color=SINGLE_HUE[mode],
+    height=260,
+)
+
+view(
+    "Unit economics by model",
+    """
+    SELECT regexp_replace(SkuId, '/[^/]+$', '') AS model,
+           round(sum(BilledCost), 2) AS cost_usd,
+           round(sum(BilledCost) / count(DISTINCT substr(ChargePeriodStart, 1, 10)), 3)
+               AS usd_per_active_day,
+           round(1e6 * sum(BilledCost) / sum(ConsumedQuantity), 3)
+               AS usd_per_mtok_effective
+    FROM FOCUS
+    GROUP BY 1
+    ORDER BY 2 DESC
+    """,
+    focus,
+)
+
+st.subheader("Recommendations")
+recs = recommendations(records)
+achieved = sum(r.savings_usd for r in recs if r.kind == "achieved")
+potential = sum(r.savings_usd for r in recs if r.kind == "potential")
+r1, r2 = st.columns(2)
+r1.metric("Avoided by caching (counterfactual)", f"${achieved:,.2f}")
+r2.metric("Right-sizing headroom (what-if)", f"${potential:,.2f}")
+st.dataframe(
+    pd.DataFrame(
+        {
+            "kind": r.kind,
+            "recommendation": r.title,
+            "savings USD": round(r.savings_usd, 2),
+            "% of baseline": round(r.savings_pct, 1),
+            "detail": r.detail,
+        }
+        for r in recs
+    ),
+    use_container_width=True,
+)
+
+st.caption(
+    "Costs are API-equivalent list prices (showback) over local agent logs. "
+    "Source: tokencur FOCUS export."
+)