codex-usage-tracking 0.3.0__py3-none-any.whl

codex_usage_tracker/context.py

@@ -0,0 +1,410 @@
+"""Lazy raw-context loading for one aggregate usage record."""
+
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+from typing import Any
+
+from codex_usage_tracker.paths import DEFAULT_DB_PATH
+from codex_usage_tracker.store import query_usage_record
+
+DEFAULT_CONTEXT_CHARS = 20_000
+DEFAULT_CONTEXT_ENTRIES = 80
+
+_SECRET_PATTERNS: tuple[tuple[re.Pattern[str], str], ...] = (
+    (re.compile(r"sk-[A-Za-z0-9_-]{10,}"), "[REDACTED_OPENAI_KEY]"),
+    (re.compile(r"github" r"_pat_[A-Za-z0-9_]{20,}"), "[REDACTED_GITHUB_TOKEN]"),
+    (re.compile(r"gh[pousr]_[A-Za-z0-9_]{20,}"), "[REDACTED_GITHUB_TOKEN]"),
+    (re.compile(r"\bA(?:KI|SI)A[0-9A-Z]{16}\b"), "[REDACTED_AWS_ACCESS_KEY]"),
+    (
+        re.compile(r"(?i)\baws_secret_access_key\s*[:=]\s*(['\"]?)[A-Za-z0-9/+=]{30,}\1"),
+        "aws_secret_access_key=[REDACTED_AWS_SECRET]",
+    ),
+    (
+        re.compile(r"(?i)\bAuthorization\s*[:=]\s*Bearer\s+[A-Za-z0-9._~+/-]+=*"),
+        "Authorization: Bearer [REDACTED_BEARER_TOKEN]",
+    ),
+    (re.compile(r"(?i)\bBearer\s+[A-Za-z0-9._~+/-]+=*"), "Bearer [REDACTED_BEARER_TOKEN]"),
+    (
+        re.compile(r"\bxox(?:a|b|p|r|s)-[A-Za-z0-9-]{10,}\b"),
+        "[REDACTED_SLACK_TOKEN]",
+    ),
+    (re.compile(r"\bxapp-[A-Za-z0-9-]{10,}\b"), "[REDACTED_SLACK_TOKEN]"),
+    (
+        re.compile(r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b"),
+        "[REDACTED_JWT]",
+    ),
+    (
+        re.compile(
+            r"-----BEGIN [A-Z ]*PRIVATE KEY-----.*?-----END [A-Z ]*PRIVATE KEY-----",
+            re.S,
+        ),
+        "[REDACTED_PRIVATE_KEY]",
+    ),
+    (
+        re.compile(
+            r"(?i)\b([A-Z0-9_ -]*(?:password|api[_-]?key|token|secret|credential|private[_-]?key)[A-Z0-9_ -]*)\s*[:=]\s*"
+            r"(['\"]?)[^'\"\s,;}]+\2"
+        ),
+        r"\1=[REDACTED_SECRET]",
+    ),
+)
+
+_OUTPUT_OMITTED = (
+    "Tool output omitted by default. Reload with include_tool_output=true to inspect "
+    "redacted, size-limited output."
+)
+
+
+def load_call_context(
+    record_id: str,
+    db_path: Path = DEFAULT_DB_PATH,
+    max_chars: int = DEFAULT_CONTEXT_CHARS,
+    max_entries: int = DEFAULT_CONTEXT_ENTRIES,
+    include_tool_output: bool = False,
+) -> dict[str, Any]:
+    """Load logged turn context for one model call from the source JSONL file.
+
+    This intentionally reads raw transcript-like data only on demand. The returned
+    context is not written back to SQLite or embedded in dashboard HTML.
+    """
+
+    row = query_usage_record(db_path=db_path, record_id=record_id)
+    if row is None:
+        raise ValueError(f"No usage record found for record_id: {record_id}")
+
+    source_file = Path(str(row.get("source_file") or ""))
+    if not source_file.exists():
+        raise FileNotFoundError(f"Source log not found: {source_file}")
+
+    line_number = _positive_int(row.get("line_number"))
+    if line_number is None:
+        raise ValueError(f"Usage record has no valid source line: {record_id}")
+
+    target_turn_id = _optional_str(row.get("turn_id"))
+    entries, omitted = _read_context_entries(
+        path=source_file,
+        token_line=line_number,
+        target_turn_id=target_turn_id,
+        max_chars=max(1_000, max_chars),
+        max_entries=max(1, max_entries),
+        include_tool_output=include_tool_output,
+    )
+    return {
+        "schema": "codex-usage-tracker-context-v1",
+        "loaded_on_demand": True,
+        "raw_context_persisted": False,
+        "include_tool_output": include_tool_output,
+        "record": {
+            "record_id": row.get("record_id"),
+            "session_id": row.get("session_id"),
+            "thread_name": row.get("thread_name"),
+            "turn_id": row.get("turn_id"),
+            "event_timestamp": row.get("event_timestamp"),
+            "model": row.get("model"),
+            "effort": row.get("effort"),
+            "parent_session_id": row.get("parent_session_id"),
+            "parent_thread_name": row.get("parent_thread_name"),
+            "total_tokens": row.get("total_tokens"),
+            "cumulative_total_tokens": row.get("cumulative_total_tokens"),
+        },
+        "source": {
+            "file": str(source_file),
+            "line_number": line_number,
+        },
+        "entries": entries,
+        "omitted": omitted,
+    }
+
+
+def _read_context_entries(
+    path: Path,
+    token_line: int,
+    target_turn_id: str | None,
+    max_chars: int,
+    max_entries: int,
+    include_tool_output: bool,
+) -> tuple[list[dict[str, Any]], dict[str, Any]]:
+    candidates: list[dict[str, Any]] = []
+    omitted_parse_errors = 0
+    current_turn_id: str | None = None
+    collecting = target_turn_id is None
+
+    with path.open(encoding="utf-8") as handle:
+        for line_number, line in enumerate(handle, 1):
+            if line_number > token_line:
+                break
+            try:
+                envelope = json.loads(line)
+            except json.JSONDecodeError:
+                omitted_parse_errors += 1
+                continue
+            if not isinstance(envelope, dict):
+                continue
+            entry_type = _optional_str(envelope.get("type")) or "unknown"
+            payload = envelope.get("payload") if isinstance(envelope.get("payload"), dict) else {}
+            timestamp = _optional_str(envelope.get("timestamp"))
+
+            if entry_type == "turn_context":
+                current_turn_id = _optional_str(payload.get("turn_id"))
+                collecting = target_turn_id is None or current_turn_id == target_turn_id
+                if collecting:
+                    candidates = []
+                    candidates.append(
+                        _context_entry(
+                            line_number,
+                            timestamp,
+                            entry_type,
+                            "Turn context",
+                            _summarize_turn_context(payload),
+                        )
+                    )
+                continue
+
+            if not collecting:
+                continue
+
+            summarized = _summarize_payload(
+                entry_type=entry_type,
+                payload=payload,
+                include_tool_output=include_tool_output,
+            )
+            if summarized is not None:
+                candidates.append(
+                    _context_entry(
+                        line_number,
+                        timestamp,
+                        entry_type,
+                        summarized["label"],
+                        summarized["text"],
+                    )
+                )
+
+            if (
+                line_number >= token_line
+                and entry_type == "event_msg"
+                and payload.get("type") == "token_count"
+            ):
+                break
+
+    limited, omitted = _limit_entries(candidates, max_chars=max_chars, max_entries=max_entries)
+    omitted["parse_errors"] = omitted_parse_errors
+    omitted["target_turn_id"] = target_turn_id
+    return limited, omitted
+
+
+def _summarize_payload(
+    entry_type: str,
+    payload: dict[str, Any],
+    include_tool_output: bool,
+) -> dict[str, str] | None:
+    if entry_type == "response_item":
+        return _summarize_response_item(payload, include_tool_output=include_tool_output)
+    if entry_type == "event_msg":
+        return _summarize_event_msg(payload, include_tool_output=include_tool_output)
+    if entry_type == "compacted":
+        message = _optional_str(payload.get("message")) or "Compaction event"
+        return {"label": "Compaction", "text": message}
+    return None
+
+
+def _summarize_turn_context(payload: dict[str, Any]) -> str:
+    fields = [
+        ("turn_id", payload.get("turn_id")),
+        ("cwd", payload.get("cwd")),
+        ("model", payload.get("model")),
+        ("effort", payload.get("effort")),
+        ("current_date", payload.get("current_date")),
+        ("timezone", payload.get("timezone")),
+    ]
+    lines = [f"{key}: {value}" for key, value in fields if value not in (None, "")]
+    summary = _optional_str(payload.get("summary"))
+    if summary:
+        lines.append(f"summary: {summary}")
+    return "\n".join(lines) if lines else "Turn context"
+
+
+def _summarize_response_item(
+    payload: dict[str, Any],
+    include_tool_output: bool,
+) -> dict[str, str] | None:
+    item_type = _optional_str(payload.get("type")) or "response_item"
+    role = _optional_str(payload.get("role"))
+    name = _optional_str(payload.get("name"))
+    label_bits = [item_type]
+    if role:
+        label_bits.append(role)
+    if name:
+        label_bits.append(name)
+    label = " / ".join(label_bits)
+
+    content_text = _content_text(payload.get("content"))
+    if content_text:
+        return {"label": label, "text": content_text}
+
+    if "arguments" in payload:
+        return {
+            "label": label,
+            "text": f"Tool call arguments:\n{_jsonish(payload.get('arguments'))}",
+        }
+
+    if "input" in payload:
+        return {
+            "label": label,
+            "text": f"Tool input:\n{_jsonish(payload.get('input'))}",
+        }
+
+    if "output" in payload:
+        output = _optional_str(payload.get("output")) or _jsonish(payload.get("output"))
+        return {
+            "label": label,
+            "text": output if include_tool_output else _OUTPUT_OMITTED,
+        }
+
+    summary = _content_text(payload.get("summary"))
+    if summary:
+        return {"label": label, "text": summary}
+
+    action = payload.get("action")
+    if isinstance(action, dict):
+        return {"label": label, "text": f"Action:\n{_jsonish(action)}"}
+
+    return None
+
+
+def _summarize_event_msg(
+    payload: dict[str, Any],
+    include_tool_output: bool,
+) -> dict[str, str] | None:
+    event_type = _optional_str(payload.get("type")) or "event_msg"
+    if event_type == "token_count":
+        info = payload.get("info") if isinstance(payload.get("info"), dict) else {}
+        return {"label": "Token count", "text": _jsonish(_token_count_summary(info))}
+
+    if "message" in payload:
+        return {"label": event_type, "text": _optional_str(payload.get("message")) or ""}
+
+    output_fields = [field for field in ("stdout", "stderr", "result") if field in payload]
+    if output_fields:
+        if not include_tool_output:
+            return {"label": event_type, "text": _OUTPUT_OMITTED}
+        text = "\n".join(f"{field}:\n{_jsonish(payload.get(field))}" for field in output_fields)
+        return {"label": event_type, "text": text}
+
+    compact = {
+        key: payload.get(key)
+        for key in ("call_id", "turn_id", "phase", "status", "duration_ms")
+        if key in payload
+    }
+    return {"label": event_type, "text": _jsonish(compact)} if compact else None
+
+
+def _token_count_summary(info: dict[str, Any]) -> dict[str, Any]:
+    return {
+        "last_token_usage": info.get("last_token_usage"),
+        "total_token_usage": info.get("total_token_usage"),
+        "model_context_window": info.get("model_context_window"),
+    }
+
+
+def _context_entry(
+    line_number: int,
+    timestamp: str | None,
+    entry_type: str,
+    label: str,
+    text: str,
+) -> dict[str, Any]:
+    return {
+        "line_number": line_number,
+        "timestamp": timestamp,
+        "type": entry_type,
+        "label": label,
+        "text": _redact(text),
+        "truncated": False,
+    }
+
+
+def _limit_entries(
+    entries: list[dict[str, Any]],
+    max_chars: int,
+    max_entries: int,
+) -> tuple[list[dict[str, Any]], dict[str, Any]]:
+    limited_reversed: list[dict[str, Any]] = []
+    remaining = max_chars
+    omitted_entries = 0
+    omitted_chars = 0
+    selected = entries[-max_entries:]
+
+    for entry in reversed(selected):
+        text = str(entry.get("text") or "")
+        if remaining <= 0:
+            omitted_entries += 1
+            omitted_chars += len(text)
+            continue
+        if len(text) > remaining:
+            entry = dict(entry)
+            entry["text"] = text[:remaining] + "\n[TRUNCATED]"
+            entry["truncated"] = True
+            omitted_chars += len(text) - remaining
+            remaining = 0
+        else:
+            remaining -= len(text)
+        limited_reversed.append(entry)
+
+    limited = list(reversed(limited_reversed))
+    return limited, {
+        "older_entries": max(0, len(entries) - max_entries),
+        "over_budget_entries": omitted_entries,
+        "over_budget_chars": omitted_chars,
+        "max_chars": max_chars,
+        "max_entries": max_entries,
+        "returned_entries": len(limited),
+    }
+
+
+def _content_text(value: object) -> str:
+    if value is None:
+        return ""
+    if isinstance(value, str):
+        return value
+    if isinstance(value, list):
+        pieces: list[str] = []
+        for item in value:
+            if isinstance(item, str):
+                pieces.append(item)
+            elif isinstance(item, dict):
+                text = item.get("text") or item.get("content")
+                if isinstance(text, str):
+                    pieces.append(text)
+        return "\n".join(piece for piece in pieces if piece)
+    return _jsonish(value)
+
+
+def _jsonish(value: object) -> str:
+    if isinstance(value, str):
+        return value
+    try:
+        return json.dumps(value, ensure_ascii=True, indent=2, sort_keys=True)
+    except TypeError:
+        return str(value)
+
+
+def _redact(text: str) -> str:
+    redacted = text
+    for pattern, replacement in _SECRET_PATTERNS:
+        redacted = pattern.sub(replacement, redacted)
+    return redacted
+
+
+def _positive_int(value: object) -> int | None:
+    try:
+        number = int(value)  # type: ignore[arg-type]
+    except (TypeError, ValueError):
+        return None
+    return number if number > 0 else None
+
+
+def _optional_str(value: object) -> str | None:
+    return value if isinstance(value, str) and value else None

codex_usage_tracker/costing.py

@@ -0,0 +1,176 @@
+"""Cost estimation and pricing coverage calculations for aggregate usage rows."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from codex_usage_tracker.paths import DEFAULT_PRICING_PATH
+from codex_usage_tracker.pricing_config import PricingConfig, load_pricing_config
+
+
+def summarize_pricing_coverage(
+    rows: list[dict[str, Any]],
+    pricing: PricingConfig | None = None,
+    *,
+    model_field: str = "group_key",
+) -> dict[str, Any]:
+    """Summarize which aggregate model rows have usable local pricing."""
+
+    config = pricing or load_pricing_config()
+    coverage_rows: list[dict[str, Any]] = []
+    totals = {
+        "model_count": 0,
+        "priced_model_count": 0,
+        "unpriced_model_count": 0,
+        "total_tokens": 0.0,
+        "priced_tokens": 0.0,
+        "unpriced_tokens": 0.0,
+        "estimated_cost_usd": 0.0,
+    }
+
+    for row in rows:
+        model = row.get(model_field)
+        priced_as = config.priced_as(model)
+        copy = dict(row)
+        copy["model"] = model
+        copy["priced"] = priced_as is not None
+        copy["priced_as"] = priced_as
+        copy["pricing_estimated"] = config.is_estimated_model(model)
+        copy["estimated_cost_usd"] = estimate_cost_usd(copy, config, model=model)
+        total_tokens = _number(copy.get("total_tokens"))
+        totals["model_count"] += 1
+        totals["total_tokens"] += total_tokens
+        if priced_as:
+            totals["priced_model_count"] += 1
+            totals["priced_tokens"] += total_tokens
+        else:
+            totals["unpriced_model_count"] += 1
+            totals["unpriced_tokens"] += total_tokens
+        if isinstance(copy["estimated_cost_usd"], int | float):
+            totals["estimated_cost_usd"] += float(copy["estimated_cost_usd"])
+        coverage_rows.append(copy)
+
+    total_tokens = totals["total_tokens"]
+    totals["priced_token_ratio"] = (
+        totals["priced_tokens"] / total_tokens if total_tokens else 0.0
+    )
+    coverage_rows.sort(
+        key=lambda row: (
+            0 if row.get("priced") is False else 1,
+            -_number(row.get("total_tokens")),
+        )
+    )
+    return {
+        "schema": "codex-usage-tracker-pricing-coverage-v1",
+        **totals,
+        "pricing_loaded": config.loaded and not config.error,
+        "pricing_path": str(config.path),
+        "pricing_source": config.source,
+        "rows": coverage_rows,
+    }
+
+
+def annotate_rows_with_efficiency(
+    rows: list[dict[str, Any]],
+    pricing: PricingConfig | None = None,
+    *,
+    model_field: str = "model",
+    pricing_path: Path = DEFAULT_PRICING_PATH,
+) -> list[dict[str, Any]]:
+    """Return copied rows with local cost estimates and efficiency flags."""
+
+    config = pricing or load_pricing_config(pricing_path)
+    annotated: list[dict[str, Any]] = []
+    for row in rows:
+        copy = dict(row)
+        model = copy.get(model_field)
+        cost = estimate_cost_usd(copy, config, model=model)
+        savings = estimate_cache_savings_usd(copy, config, model=model)
+        copy["estimated_cost_usd"] = cost
+        copy["estimated_cache_savings_usd"] = savings
+        copy["pricing_model"] = config.priced_as(model)
+        copy["pricing_estimated"] = config.is_estimated_model(model)
+        copy["efficiency_flags"] = efficiency_flags(copy)
+        annotated.append(copy)
+    return annotated
+
+
+def estimate_cost_usd(
+    row: dict[str, Any], pricing: PricingConfig, *, model: object | None = None
+) -> float | None:
+    """Estimate call cost from aggregate tokens and local model rates."""
+
+    rates = pricing.rates_for(model if model is not None else row.get("model"))
+    if not rates:
+        return None
+
+    input_rate = rates.get("input_per_million")
+    cached_rate = rates.get("cached_input_per_million", input_rate)
+    output_rate = rates.get("output_per_million")
+    if input_rate is None or cached_rate is None or output_rate is None:
+        return None
+
+    cached_input = _number(row.get("cached_input_tokens"))
+    uncached_input = _number(row.get("uncached_input_tokens"))
+    if uncached_input <= 0:
+        uncached_input = max(_number(row.get("input_tokens")) - cached_input, 0.0)
+    output_tokens = _number(row.get("output_tokens"))
+
+    return (
+        (uncached_input * input_rate)
+        + (cached_input * cached_rate)
+        + (output_tokens * output_rate)
+    ) / 1_000_000
+
+
+def estimate_cache_savings_usd(
+    row: dict[str, Any], pricing: PricingConfig, *, model: object | None = None
+) -> float | None:
+    """Estimate local cache savings when cached input has a lower configured rate."""
+
+    rates = pricing.rates_for(model if model is not None else row.get("model"))
+    if not rates:
+        return None
+    input_rate = rates.get("input_per_million")
+    cached_rate = rates.get("cached_input_per_million")
+    if input_rate is None or cached_rate is None or cached_rate >= input_rate:
+        return None
+    return (_number(row.get("cached_input_tokens")) * (input_rate - cached_rate)) / 1_000_000
+
+
+def efficiency_flags(row: dict[str, Any]) -> list[str]:
+    """Generate aggregate-only signals worth reviewing."""
+
+    flags: list[str] = []
+    total_tokens = _number(row.get("total_tokens"))
+    output_tokens = _number(row.get("output_tokens"))
+    input_tokens = _number(row.get("input_tokens"))
+    context = _number(row.get("context_window_percent"))
+    cache = _number(row.get("cache_ratio"))
+    reasoning = _number(row.get("reasoning_output_ratio"))
+    cost = row.get("estimated_cost_usd")
+
+    if context >= 0.8:
+        flags.append("high context use")
+    elif context >= 0.5:
+        flags.append("elevated context use")
+    if reasoning >= 0.75 and output_tokens >= 100:
+        flags.append("high reasoning share")
+    if input_tokens >= 10_000 and cache < 0.1:
+        flags.append("low cache reuse")
+    if total_tokens >= 20_000 and output_tokens <= 100:
+        flags.append("expensive low-output call")
+    if isinstance(cost, int | float) and cost >= 1:
+        flags.append("high estimated cost")
+    return flags
+
+
+def _number(value: object) -> float:
+    if isinstance(value, bool):
+        return float(int(value))
+    if isinstance(value, int | float):
+        return float(value)
+    if isinstance(value, str) and value.strip():
+        return float(value)
+    return 0.0