cost-per-task 0.4.0__tar.gz

cost_per_task-0.4.0/.github/workflows/ci.yml

@@ -0,0 +1,23 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+        python: ["3.11", "3.12", "3.13"]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+      - run: python -m pip install --upgrade pip
+      - run: pip install -e . pytest
+      - run: pytest -q

cost_per_task-0.4.0/.github/workflows/publish-pypi.yml

@@ -0,0 +1,183 @@
+name: Publish cost-per-task to PyPI
+
+# Release flow, mirroring OptimNow/cloud-finops-skills:
+#   1. bump the version in pyproject.toml and src/cost_per_task/__init__.py,
+#      update CHANGELOG.md, merge to main
+#   2. git tag vX.Y.Z && git push origin vX.Y.Z
+#   3. this workflow tests the tagged commit, builds sdist + wheel with the
+#      version pinned from the tag, publishes via PyPI trusted publishing,
+#      then creates the GitHub Release with generated notes.
+#
+# One-time setup (repository owner):
+#   - PyPI: project "cost-per-task", trusted publisher = GitHub, owner
+#     OptimNow, repository cost-per-task, workflow publish-pypi.yml,
+#     environment pypi. Register it as a "pending publisher" before the
+#     first release so the project name is claimed by this workflow.
+#   - GitHub: Settings -> Environments -> create "pypi" (optionally with a
+#     required reviewer, which turns every publish into an approval click).
+
+on:
+  push:
+    tags:
+      - 'v*'
+  workflow_dispatch:
+    inputs:
+      tag:
+        required: true
+        type: string
+        description: 'Existing tag to (re)publish, e.g. v0.4.0'
+
+# Read-only by default; id-token and contents:write are granted per job below,
+# so the build job (which executes the build backend) never holds a token
+# that can authenticate to PyPI.
+permissions:
+  contents: read
+
+jobs:
+  resolve-tag:
+    name: Resolve and validate the tag
+    runs-on: ubuntu-latest
+    outputs:
+      ref: ${{ steps.resolve.outputs.ref }}
+      version: ${{ steps.resolve.outputs.version }}
+    steps:
+      - name: Resolve tag
+        id: resolve
+        env:
+          INPUT_TAG: ${{ inputs.tag }}
+          FALLBACK_REF: ${{ github.ref }}
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -euo pipefail
+          if [[ -z "${INPUT_TAG}" ]]; then
+            tag="${FALLBACK_REF##*/}"
+          else
+            tag="${INPUT_TAG#V}"
+            tag="${tag#v}"
+            tag="v${tag}"
+            if [[ "$tag" != "${INPUT_TAG}" ]]; then
+              echo "::notice::normalised tag input '${INPUT_TAG}' to '${tag}'"
+            fi
+            # Works for private repositories too, unlike an anonymous ls-remote.
+            if ! gh api "repos/${GITHUB_REPOSITORY}/git/ref/tags/${tag}" >/dev/null 2>&1; then
+              echo "::error::tag ${tag} does not exist in ${GITHUB_REPOSITORY}; refs are case sensitive"
+              exit 1
+            fi
+          fi
+          version="${tag#v}"
+          if ! [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+([a-z0-9.]*)$ ]]; then
+            echo "::error::tag ${tag} is not a semantic version"
+            exit 1
+          fi
+          echo "ref=${tag}" >> "$GITHUB_OUTPUT"
+          echo "version=${version}" >> "$GITHUB_OUTPUT"
+          echo "Releasing cost-per-task ${version} from tag ${tag}"
+
+  test:
+    name: Test the tagged commit
+    needs: resolve-tag
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262  # v4.4.0
+        with:
+          ref: ${{ needs.resolve-tag.outputs.ref }}
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5.6.0
+        with:
+          python-version: '3.12'
+      - run: python -m pip install --upgrade pip
+      - run: pip install -e . pytest
+      - run: pytest -q
+
+  build:
+    name: Build sdist + wheel
+    needs: [resolve-tag, test]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262  # v4.4.0
+        with:
+          ref: ${{ needs.resolve-tag.outputs.ref }}
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5.6.0
+        with:
+          python-version: '3.12'
+
+      - name: Pin the package version to the tag
+        env:
+          VERSION: ${{ needs.resolve-tag.outputs.version }}
+        run: |
+          python - <<'PY'
+          import os, pathlib, re
+          version = os.environ["VERSION"]
+          for path, pattern, replacement in (
+              ("pyproject.toml", r'^version = ".*"', f'version = "{version}"'),
+              ("src/cost_per_task/__init__.py", r'^__version__ = ".*"', f'__version__ = "{version}"'),
+          ):
+              file = pathlib.Path(path)
+              text = file.read_text(encoding="utf-8")
+              new, count = re.subn(pattern, replacement, text, count=1, flags=re.M)
+              if count != 1:
+                  raise SystemExit(f"{path}: version line not found")
+              if new != text:
+                  print(f"::warning::{path} said a different version than the tag; pinned to {version}")
+              file.write_text(new, encoding="utf-8")
+          print(f"version pinned to {version}")
+          PY
+
+      - name: Build
+        run: |
+          python -m pip install --upgrade pip build
+          python -m build
+          ls -lh dist/
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
+        with:
+          name: pypi-dist
+          path: dist/
+          retention-days: 7
+
+  publish:
+    name: Publish to PyPI
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write   # PyPI trusted publishing (OIDC)
+    environment:
+      name: pypi
+      url: https://pypi.org/p/cost-per-task
+    steps:
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0
+        with:
+          name: pypi-dist
+          path: dist/
+      - name: Publish via trusted publishing
+        uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33  # v1.14.2
+        with:
+          packages-dir: dist/
+
+  github-release:
+    name: Create the GitHub Release
+    needs: [resolve-tag, publish]
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0
+        with:
+          name: pypi-dist
+          path: dist/
+      - uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65  # v2.6.2
+        with:
+          name: ${{ needs.resolve-tag.outputs.ref }}
+          tag_name: ${{ needs.resolve-tag.outputs.ref }}
+          files: dist/*
+          generate_release_notes: true
+          body: |
+            ## cost-per-task ${{ needs.resolve-tag.outputs.ref }}
+
+            Published to PyPI: https://pypi.org/project/cost-per-task/${{ needs.resolve-tag.outputs.version }}/
+
+            ```
+            pip install cost-per-task==${{ needs.resolve-tag.outputs.version }}
+            ```
+
+            See CHANGELOG.md for the curated notes; the list below is generated from merged pull requests.

cost_per_task-0.4.0/.gitignore

@@ -0,0 +1,14 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+dist/
+build/
+.venv/
+venv/
+.pytest_cache/
+cpt-log.jsonl
+cpt-labels.jsonl
+level1-log.jsonl
+level1-labels.jsonl
+level2-log.jsonl
+level2-labels.jsonl

cost_per_task-0.4.0/CHANGELOG.md

@@ -0,0 +1,71 @@
+# Changelog
+
+All notable changes to cost-per-task. The format follows
+[Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versions follow semantic
+versioning, with the minor digit bumped for any user-visible feature.
+
+## [Unreleased]
+
+- Release pipeline: tag `vX.Y.Z` on `main` to test, build, publish to PyPI via
+  trusted publishing and create the GitHub Release.
+
+## [0.4.0] - 2026-09-02
+
+### Added
+- OpenRouter and other OpenAI-compatible gateways as the `openai` upstream, with a
+  path prefix (`--openai-upstream https://openrouter.ai/api`).
+- OpenRouter usage accounting requested automatically; the cost the gateway
+  charged is stored as `reported_cost` and reconciled against the table price in
+  the report and the disclosure checklist.
+- Gateway model ids (`anthropic/claude-haiku-4.5`) match pricing tables without the
+  vendor prefix and with dots as hyphens.
+- `--prices` may be repeated to merge vendor tables; the merged `as_of` is the oldest.
+- The log's `provider` field names the gateway host when a call did not go to the
+  vendor directly.
+
+## [0.3.0] - 2026-09-02
+
+### Added
+- `cpt import langfuse` and `cpt import litellm`: convert usage exports (CSV, JSON,
+  JSONL) into cpt records with configurable task and attempt fields.
+- `cpt prices refresh`: diff a pricing table against the OptimNow AI Pricing Hub
+  catalogue; write only with `--write`; anomalous cache-read prices are flagged and
+  models the hub cannot price fully are skipped.
+- `cpt report --json` and `cpt compare --json`.
+- `cpt mcp`: MCP server with `cpt_report`, `cpt_compare` and `cpt_risk_denominator`,
+  via the optional `[mcp]` extra (the only optional dependency).
+- `prices/openai.json` with verified, dated rates for GPT-5.5, 5.4, 5.4-mini, 5.4-nano.
+
+## [0.2.0] - 2026-09-02
+
+### Added
+- OpenAI capture for the Chat Completions and Responses APIs, plain and streaming,
+  on the same proxy port as Anthropic (routed by path, then auth-header style).
+  `stream_options.include_usage` is injected on streaming chat requests.
+- `cpt label`: pass, fail and leak labels in a separate append-only
+  `cpt-labels.jsonl`, plus CSV import.
+- Statistics: Wilson interval, task-cluster bootstrap, P90, capped-retry success,
+  pass^k.
+- Report per model and task type with CPT_solved, cost per task attempted,
+  CPT_risk, and the disclosure checklist; `cpt compare` with break-even K*.
+- `--task-type` on `cpt run`.
+
+### Changed
+- `reasoning_per_mtok: null` now means "billed at the output rate" rather than
+  "not priced".
+
+## [0.1.0] - 2026-09-01
+
+### Added
+- Local capture proxy for the Anthropic Messages API, plain and streaming, logging
+  only usage metadata (never keys, headers, prompts or completions).
+- JSONL step schema aligned with the OpenTelemetry GenAI semantic conventions.
+- Dated pricing tables and per-attempt cost; `prices/anthropic.json` verified against
+  the Anthropic price list.
+- `cpt run`, `cpt serve`, `cpt report`.
+
+[Unreleased]: https://github.com/OptimNow/cost-per-task/compare/v0.4.0...HEAD
+[0.4.0]: https://github.com/OptimNow/cost-per-task/releases/tag/v0.4.0
+[0.3.0]: https://github.com/OptimNow/cost-per-task/releases/tag/v0.3.0
+[0.2.0]: https://github.com/OptimNow/cost-per-task/releases/tag/v0.2.0
+[0.1.0]: https://github.com/OptimNow/cost-per-task/releases/tag/v0.1.0

cost_per_task-0.4.0/CLAUDE.md

@@ -0,0 +1,50 @@
+# cost-per-task
+
+Vendor-neutral Python tool measuring the real cost per completed task of LLM agents. Implements the DoiT "Cost Per Task, Not Cost Per Token" measurement framework (13 August 2026): a local reverse proxy captures token usage per API call, calls are grouped into steps, attempts and tasks, priced from a dated table, and reported as cost per attempt, cost per solved task (CPT_solved = E[C_attempt] / p) and risk-adjusted cost (CPT_risk = CPT_solved + L x K).
+
+## Architecture
+
+- `src/cost_per_task/proxy.py`: local reverse proxy serving Anthropic and OpenAI on one port (routed by path, then auth-header style); the agent is pointed at it via ANTHROPIC_BASE_URL / OPENAI_BASE_URL. Upstreams may carry a path prefix (OpenRouter: `https://openrouter.ai/api`); any OpenAI-compatible gateway works as the `openai` upstream and the log's `provider` becomes the gateway host. Forwards requests unchanged except for injecting `stream_options.include_usage` (streaming OpenAI chat) and `usage.include` (OpenRouter); logs only usage metadata plus `reported_cost` when the gateway states what it charged.
+- `src/cost_per_task/providers/`: per-provider usage extraction (JSON and SSE): `anthropic.py`, `openai.py` (Chat Completions and Responses; subtracts cached and reasoning tokens out of the totals). Design allows Bedrock, Vertex, xAI.
+- `src/cost_per_task/schema.py`: StepRecord (OpenTelemetry GenAI aligned) + JSONL persistence.
+- `src/cost_per_task/pricing.py`: dated pricing tables, per-step cost; `reasoning_per_mtok: null` means output rate; `load_many` merges vendor tables (oldest `as_of` wins); `rates_for` strips gateway vendor prefixes and dot/hyphen differences.
+- `src/cost_per_task/labels.py`: pass/fail/leak labels in a separate append-only `cpt-labels.jsonl` (latest wins), CSV import.
+- `src/cost_per_task/stats.py`: Wilson interval, percentile, task-cluster bootstrap, p_N, pass^k. Standard library only.
+- `src/cost_per_task/metrics.py`: records to Attempt (primary model = largest cost share) to GroupSummary per model and task type; CPT_solved, CPT_risk, K*.
+- `src/cost_per_task/report.py`: text report, two-model comparison, disclosure checklist, JSON output.
+- `src/cost_per_task/analysis.py`: files-to-summaries loader shared by the CLI and the MCP server.
+- `src/cost_per_task/importers/`: Langfuse and LiteLLM export importers (`common.py` reads CSV/JSON/JSONL and groups rows into attempts). Built from documented schemas, not validated against live exports yet.
+- `src/cost_per_task/prices_hub.py`: `cpt prices refresh`; fetches `optimtoken.optimnow.io/api/llm-models` with urllib, maps ids per provider (Anthropic dots to hyphens), derives cache-write rates from documented rules, diffs by default, writes only with `--write`, flags anomalous cache-read ratios.
+- `src/cost_per_task/mcp_server.py`: MCP tools (`cpt_report`, `cpt_compare`, `cpt_risk_denominator`); the SDK is the optional `[mcp]` extra (mcp 2.x `MCPServer`, 1.x `FastMCP` fallback) and is imported lazily.
+- `src/cost_per_task/cli.py`: `cpt serve`, `run`, `label`, `report`, `compare`, `import`, `prices refresh`, `mcp`.
+- `prices/anthropic.json`, `prices/openai.json`: verified, dated rates; `prices/example.json` is a zero-value template. `tests/fixtures/hub_sample.json` is a recorded hub sample. Claude Fable 5.1's cache-read rate is a documented special 0.025x (0.25 USD/MTok); the refresh command flags it as suspicious by design, and Anthropic's table confirms it.
+- `docs/testing-guide.md` plus `examples/` (`ask.py` stdlib client, `level1.ps1`, `level2.ps1`, `level2/` tasks): the guided model comparison for non-developers.
+
+## Hard rules
+
+- **Never log secrets or content.** The proxy must never write API keys, headers, prompts or completions to the log. Only token counts, model, provider, ids, latency, tool names. Tests enforce this (`test_no_secrets_or_content_in_log`); keep them passing.
+- **Token counts come from the provider API response, never a local tokenizer.**
+- **No invented prices.** Pricing tables carry `as_of` dates and a source; an undated table is rejected at load. Do not add or update a price without a verifiable source.
+- **Zero runtime dependencies in the core.** Standard library only; pytest is the sole dev dependency. The only optional extra is `[mcp]` for `cpt mcp`. Any new dependency needs explicit justification and agreement.
+- **Hub prices never land unseen.** `cpt prices refresh` diffs by default; `--write` is a deliberate step after reading the diff and warnings.
+
+## Conventions
+
+- Python 3.11+, src/ layout, hatchling build, entry point `cpt`.
+- Tests with pytest (`python -m pytest -q`); proxy tests run against the fake upstream in `tests/conftest.py`, never the real API.
+- Documentation and user-facing text: British English, no em dashes, no emojis, facts before interpretation.
+- Semver: user-visible feature changes bump the minor digit (0.1.x to 0.2.0), not patch.
+- Streaming: Anthropic reports input usage in `message_start` and final output usage in `message_delta`; OpenAI Chat Completions only reports usage on a stream when `stream_options.include_usage` is set (the proxy injects it); the Responses API reports usage in `response.completed`. Anthropic does not report reasoning tokens separately (billed inside output_tokens), so `reasoning_tokens` is null for Anthropic rows; OpenAI reports reasoning as a subset of output and the adapter splits it out.
+- The usage log is never rewritten; outcomes go to the labels file.
+- Pricing tables hold one rate per token class; long-context tiers (Anthropic >200K, OpenAI >272K) and batch discounts are a documented limitation, not silently approximated.
+
+## Workflow and releases
+
+- Work on a branch and open a pull request; CI (`ci.yml`) must be green before merging to `main`. Do not push to `main` directly.
+- Release: bump `version` in `pyproject.toml` and `__version__` in `src/cost_per_task/__init__.py` (minor digit for features), move the `[Unreleased]` entries in `CHANGELOG.md` under the new version, merge, then `git tag vX.Y.Z` on `main` and push the tag. `publish-pypi.yml` tests the tagged commit, builds, publishes through PyPI trusted publishing (GitHub environment `pypi`, no stored token) and creates the GitHub Release. Re-publish an existing tag with `gh workflow run publish-pypi.yml -f tag=vX.Y.Z`.
+- One-time setup owned by the repository owner: the PyPI pending publisher for `cost-per-task` (owner OptimNow, repo cost-per-task, workflow `publish-pypi.yml`, environment `pypi`) and the GitHub environment `pypi`.
+
+## Roadmap
+
+Phases 1 to 3 shipped (0.3.0), OpenRouter and OpenAI-compatible gateways in 0.4.0: capture for both providers, labelling, statistics, CPT_solved / CPT_risk / K*, disclosure checklist, importers, Pricing Hub refresh, JSON output, MCP server, provider-reported cost reconciliation.
+Next: validate importers on real exports, further providers (Bedrock, Vertex, xAI), long-context price tiers, PyPI release and public repo (apply the OptimNow public-repo hardening standard first).

cost_per_task-0.4.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 OptimNow
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.