katalyst-forge 5.3.0__tar.gz → 5.4.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/CHANGELOG.md +43 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/PKG-INFO +1 -1
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/pyproject.toml +1 -1
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/metrics/core.py +56 -5
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/metrics/dashboard.py +120 -6
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/metrics/errors.py +20 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/metrics/generate_dashboard.py +72 -19
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/metrics/html_report.py +194 -28
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/metrics/providers/opencode.py +56 -22
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/metrics/run_health.py +18 -4
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/metrics/tiles.py +17 -5
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/test_metrics_billing_isolation.py +47 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/test_metrics_dashboard.py +126 -0
- katalyst_forge-5.4.1/tests/test_metrics_html_report.py +428 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/test_metrics_providers.py +245 -4
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/test_metrics_run_health.py +35 -0
- katalyst_forge-5.3.0/tests/test_metrics_html_report.py +0 -182
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/.gitignore +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/LICENSE +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/schemas/CHANGELOG.md +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/schemas/README.md +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/schemas/agents/agent.schema.json +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/schemas/agents/config.schema.json +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/schemas/agents/manifest.json +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/schemas/agents/skill.schema.json +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/schemas/environment.schema.json +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/schemas/hooks/hook-config.schema.json +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/schemas/katalyst-lock.schema.json +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/schemas/nodes/common.schema.json +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/schemas/nodes/design.schema.json +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/schemas/nodes/organizational.schema.json +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/schemas/nodes/portfolio.schema.json +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/schemas/nodes/technical.schema.json +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/schemas/taxonomy-node.schema.json +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/__init__.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/bundle.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/metrics/__init__.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/metrics/batch.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/metrics/billing.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/metrics/contracts.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/metrics/forge_stages.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/metrics/measurement.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/metrics/money.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/metrics/protocols.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/metrics/providers/__init__.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/metrics/providers/forge_model_router.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/metrics/providers/llm_gateway.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/metrics/redaction.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/metrics/registry.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/metrics/routing_policy.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/py.typed +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/relationships.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/creation/_base.yaml.j2 +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/creation/layer.yaml.j2 +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/creation/project.yaml.j2 +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/creation/stack.yaml.j2 +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/creation/system.yaml.j2 +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/creation/team.yaml.j2 +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/creation/team_member.yaml.j2 +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/app-docker/.dockerignore +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/app-docker/.env.example +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/app-docker/README.md +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/app-docker/base/Dockerfile +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/app-docker/docker-compose.yaml +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/app-docker/env-template/Dockerfile +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/app-docker/env-template/docker-compose.override.yaml +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/app-docker/env-template/environment.yaml +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/app-docker/layer.yaml +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/iac-terragrunt/README.md +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/iac-terragrunt/composite/terraform/main.tf +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/iac-terragrunt/composite/terraform/outputs.tf +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/iac-terragrunt/composite/terraform/variables.tf +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/iac-terragrunt/env-template/environment.yaml +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/iac-terragrunt/env-template/terragrunt.hcl +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/iac-terragrunt/global/iac/context.hcl +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/iac-terragrunt/global/iac/dev.yaml +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/iac-terragrunt/global/iac/prod.yaml +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/iac-terragrunt/global/iac/staging.yaml +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/iac-terragrunt/layer.yaml +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/iac-terragrunt/version.yaml +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/k8s-argocd/README.md +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/k8s-argocd/base/app/argocd-application.yaml +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/k8s-argocd/base/kustomization.yaml +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/k8s-argocd/base/namespace.yaml +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/k8s-argocd/environment-template/argocd-application.yaml +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/k8s-argocd/environment-template/kustomization.yaml +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/k8s-argocd/environment-template/patch-argocd-application.yaml +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/k8s-argocd/layer.yaml +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/k8s-kustomize/README.md +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/k8s-kustomize/base/deployment.yaml +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/k8s-kustomize/base/kustomization.yaml +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/k8s-kustomize/base/namespace.yaml +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/k8s-kustomize/base/service.yaml +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/k8s-kustomize/environment-template/kustomization.yaml +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/k8s-kustomize/environment-template/patch-deployment.yaml +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/k8s-kustomize/global/k8s/base/kustomization.yaml +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/k8s-kustomize/global/k8s/base/namespace.yaml +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/k8s-kustomize/global/k8s/kustomization.yaml +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/templates/scaffolds/k8s-kustomize/layer.yaml +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/wings/__init__.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/wings/_practice_refs.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/wings/agentic.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/wings/delivery.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/wings/design.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/wings/metrics.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/wings/org.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/wings/pipeline.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/wings/practices.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/wings/system.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/templates/CHANGELOG.md +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/conftest.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/fake_providers_opencode.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/fake_providers_router.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/forge-reconcile/fixtures/incident-ric1580/expected-modified.txt +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/forge-reconcile/fixtures/incident-ric1580/expected-new.txt +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/forge-reconcile/fixtures/incident-ric1580/modified-inplace.patch +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/forge-reconcile/fixtures/incident-ric1580/new/.global/taxonomy/delivery/work_items/ric-1580-forge-model-routing.yaml +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/forge-reconcile/fixtures/incident-ric1580/new/.global/taxonomy/design/capabilities/CAP-011-per-dispatch-model-tier-routing.md +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/forge-reconcile/fixtures/incident-ric1580/new/.global/taxonomy/design/capabilities/CAP-012-dispatch-parity-and-containment.md +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/forge-reconcile/fixtures/incident-ric1580/new/.global/taxonomy/design/capabilities/CAP-013-per-stage-model-token-telemetry.md +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/forge-reconcile/fixtures/incident-ric1580/new/.global/taxonomy/design/nfrs/dispatched-stage-executes-on-requested-tier.yaml +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/forge-reconcile/fixtures/incident-ric1580/new/.global/taxonomy/design/nfrs/routed-dispatch-preserves-task-parity.yaml +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/forge-reconcile/fixtures/incident-ric1580/new/.global/taxonomy/design/nfrs/tier-resolution-fails-before-any-child-session.yaml +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/forge-reconcile/fixtures/incident-ric1580/new/.global/taxonomy/design/nfrs/zero-committed-vendor-model-identity.yaml +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/forge-reconcile/fixtures/incident-ric1580/new/.global/taxonomy/design/user-needs/UN-024-operator-selects-model-tier-per-forge-run.md +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/forge-reconcile/fixtures/incident-ric1580/new/.global/taxonomy/design/user-needs/UN-025-model-tiers-from-env-vars-no-vendor-pins.md +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/forge-reconcile/fixtures/incident-ric1580/new/.global/taxonomy/design/user-needs/UN-026-routing-tier-recorded-on-work-item.md +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/forge-reconcile/fixtures/incident-ric1580/new/.global/taxonomy/design/user-needs/UN-027-per-stage-model-and-token-telemetry.md +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/forge-reconcile/fixtures/incident-ric1580/new/.global/taxonomy/design/user-needs/UN-028-dispatch-tool-enforces-agent-allowlist.md +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/forge-reconcile/fixtures/incident-ric1580/new/.global/taxonomy/design/user-needs/UN-029-silent-or-aborted-dispatch-yields-blocked.md +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/forge-reconcile/fixtures/incident-ric1580/new/.global/taxonomy/design/user-needs/UN-030-dispatched-write-stage-stays-in-worktree.md +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/forge-reconcile/fixtures/incident-ric1580/new/.global/taxonomy/design/user-needs/UN-031-single-stage-spike-preserves-eleven-stage-rollout.md +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/forge-reconcile/reconcile.test.mjs +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/test_agent_run.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/test_cost_observability_wing.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/test_delivery_doc_consistency.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/test_delivery_traceability.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/test_entry_point_discovery.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/test_forge.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/test_metrics_batch_hygiene.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/test_metrics_content_contract.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/test_metrics_contracts.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/test_metrics_dashboard_end_to_end.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/test_metrics_lazy_providers.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/test_metrics_money_and_redaction.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/test_metrics_resolve_compute.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/test_metrics_routing_policy.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/test_metrics_wing.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/test_practice_assignment.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/test_relationships.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/test_storage_contract.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/test_validate_templates_gates.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/test_wing_schemas.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/test_wings.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/test_writer_bundle_schema.py +0 -0
- {katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/tests/test_writer_kind_mapping.py +0 -0
|
@@ -16,6 +16,49 @@ PyPI package version. A breaking node-type/model change bumps `schema_version`.
|
|
|
16
16
|
|
|
17
17
|
---
|
|
18
18
|
|
|
19
|
+
## [5.4.1] -- 2026-08-09
|
|
20
|
+
|
|
21
|
+
### Fixed
|
|
22
|
+
|
|
23
|
+
- **The practice resolver reads the schema the taxonomy actually authors** (RIC-1532). It instructed agents to read three things that do not exist: `spec.practice.applicableTo`, a `skill_ref` on a practice, and an enforcement ladder of required/recommended/optional. Two TODO blocks admitted the schema was unresolved and asked callers not to depend on the paths, which left the practice-compliance gate documented against a contract no taxonomy can honour. Resolved by reading the shipped `wings/practices.py` and probing a real taxonomy rather than the docs example, which turns out not to match what is authored: `practice` is a real kind, no applicability field exists on it or on `practice_area` so context can only scope which areas are in play, `enforcement` lives on the `standard` node valued automated/review/advisory, and pillars live at `spec.pillars`. The graph cannot serve any of it, because `kata tax show --format json` reduces a `practice_area` spec to the generic envelope, so identity now comes from the probe and content from the source file the probe reports. The fail-loud probe contract from #41 is intact and gains one case: no resolved source is inconclusive, not a clean zero.
|
|
24
|
+
- **The auditor and reporter agree on what blocks** (RIC-1532). Both consumers of the resolver were corrected in the same change, because leaving them would have had the resolver grade every finding advisory while the reporter printed "Non-compliant (required)" and "Blocks: Yes" for that same finding. Blocking now originates only from a standard with `enforcement: automated` or from an NFR, and an inconclusive probe can no longer surface as a pass.
|
|
25
|
+
- **`kata-code-reviewer` no longer installs with unrestricted shell and dispatch** (RIC-1648). The agent shipped in `agents/` and did install globally, because `install-opencode.mjs` enumerates that directory rather than reading `opencode.json`, but it was absent from `opencode.json`'s agent map entirely. That absence was not cosmetic: `opencode.json` is the only place the per-agent permission blocks live in full, and the installer mirrors `permission` from it so an installed agent is no more privileged than its in-repo counterpart. With no entry to mirror, the installed reviewer carried only the `external_directory` rule from its own frontmatter, leaving `tools.bash` and `tools.task` true with no corresponding `permission` blocks, the one agent in the installed set with unbounded shell and unbounded sub-agent dispatch. It now carries an entry bounded to what its prompt does: `permission.task` names only `kata-forge-architect` and `kata-forge-security`, and `bash` is an allowlist over a `"*": "deny"` floor. NFR-023 recorded this agent as the live example of its fourth caller state, absent from the agent map, and the same NFR fails closed for unbounded callers until each carries an explicit entry, so a bounded entry satisfies the gate rather than widening it. Unbounded callers drop from three to two.
|
|
26
|
+
|
|
27
|
+
### Changed
|
|
28
|
+
|
|
29
|
+
- The fourth caller state in the NFR-023 differential gate is still tested, now built from a synthetic caller name instead of asserting that a real shipped agent is broken, which is what made that assertion fail the moment the agent was fixed.
|
|
30
|
+
- `kata-code-reviewer` is counted where it had been uncounted: `validate.sh`'s agent list, and a new `review` family in `openpackage.yml` rather than the `forge` family, since it is a standalone reviewer and not a Forge stage agent.
|
|
31
|
+
- Patch rather than minor: both changes correct a contract that was already published, and neither touches `src/` or `schemas/`, so the PyPI wheel and the JSON Schemas are unchanged from 5.4.0 and the bundle `schema_version` stays put. What moves is agent, skill and configuration content.
|
|
32
|
+
|
|
33
|
+
---
|
|
34
|
+
|
|
35
|
+
## [5.4.0] -- 2026-08-09
|
|
36
|
+
|
|
37
|
+
### Added
|
|
38
|
+
|
|
39
|
+
- **Per-dispatch model tier routing** (RIC-1580). A Forge dispatch can name the model tier it wants, and the orchestrator honours it through SDK child sessions rather than running every stage on the orchestrator model. Ships per-stage model and token telemetry alongside it, plus a containment and portability gate (`scripts/dispatch/check-dispatch.ts`) that reds on committed vendor model ids so a routing target cannot be hardcoded into the taxonomy.
|
|
40
|
+
- **`quantity_kind` on every cost metric definition** (RIC-1645). The audited catalogue now declares what each figure IS (`spend`, `saving`, `counterfactual`, `ratio`), not only which catalogue priced it. Read by both renderers so presentation cannot disagree with the document the figure was computed from.
|
|
41
|
+
|
|
42
|
+
### Fixed
|
|
43
|
+
|
|
44
|
+
- **The run report no longer presents savings as money that was billed** (RIC-1645). Every USD metric was captioned `Metered spend` and charted under "Metered spend per metric, in dollars actually billed", including three that the report itself described as savings: `cache-savings` ("Net spend avoided"), `routing-headroom` ("not yet captured this run") and `routing-savings-realized` ("Saving actually captured"). One figure put $14.17 of bars on an axis labelled dollars actually billed for a run that billed $1.66. `BillingMode` answers which catalogue priced a figure, not whether it was billed; for `agent-run-cost` and `cost-per-stage` the two coincide, which is why rendering the basis tag as the label read correctly and the conflation survived the whole RIC-1615..1629 series. Billed spend now has an axis of its own, savings and modelled figures have theirs, and neither is scaled against or totalled with the other.
|
|
45
|
+
- **The run report survives an unreadable message list instead of writing nothing** (RIC-1634). The cost path read each session's messages through an unguarded fetch while run health read the same endpoint through a degrading reader, so a persistent 500 on one session killed the whole report at the same moment run health was recording that session as degraded. Both halves now read through one path: the session degrades to its provider rollup, the run still reports, and the degraded-session panel names it. A transport failure is named (`ProviderTransportError` carries the endpoint) rather than surfacing as a traceback ending in a bare status code, and deliberately is not a `MetricsAdapterError`, so a host-level failure cannot be downgraded into a quietly missing source.
|
|
46
|
+
- **A rate no longer borrows an endpoint it never reached** (RIC-1635). A 99.998% cache hit rate rendered as `100.00%`, which reads as every token served from cache. A rate strictly inside its range keeps a comparator, and the denominator it is computed over is stated in the metric definition the report renders.
|
|
47
|
+
- **An absent measurement states itself** (RIC-1636). An unmeasured metric rendered as a numbered card with an empty body, which reads as a render bug, while `$0.00` would have read as measured and zero. It now carries the same explicit "not measured" marker the tiles use, and a billing-mode caption only ever appears under a money value, never under a ratio.
|
|
48
|
+
- **The report self-identifies** (RIC-1637). It carries a UTC generation timestamp, the installed generator version, and the pricing catalogue the figures were priced against, the last only once a price was actually looked up.
|
|
49
|
+
- **Rollup spend from a session that named no model is stated rather than left to vanish** (RIC-1638). It still gets no per-model row, because a fabricated model beside measured ones is worse than a gap, but the total is carried and both the tile and the detail note say so out loud.
|
|
50
|
+
- **Money-mixing guards hardened** (RIC-1639). The billing-mode tag used to fall away the moment a value left `MetricResult`, so a one-line sum over `items()` produced exactly the cross-mode total the type exists to prevent. Values now leave tagged and refuse addition, `combine()` refuses to merge a result with itself, and a threshold band judges one billing mode instead of any figure that happens to share its unit.
|
|
51
|
+
- **Message truncation is detected rather than silently shortening a run** (RIC-1631), and the RIC-1624 feature files are committed.
|
|
52
|
+
- **`opencode.json` points at agents that exist** (RIC-1643). Every agent prompt referenced `./.opencode/agents/<name>.md`, but nothing is tracked under `.opencode/` and all 40 agents live in `agents/`. OpenCode validates `{file:...}` references when it loads a config, so it refused to start from the repo root and the repository was unusable as an OpenCode working directory. `scripts/validate.sh` had been checking that each declared agent KEY had a matching file, never the prompt reference OpenCode actually resolves, so it stayed green throughout; it now resolves every `{file:...}` reference.
|
|
53
|
+
- **`kata-tax-validator-fixer` uses real kata commands** (RIC-1567). It invoked `kata tax json` and a `--path` flag, neither of which exists.
|
|
54
|
+
- **The practice resolver probe fails loud** (#41). A failed probe returned the same empty result as a genuine zero, so a gate whose probe never ran reported a clean pass.
|
|
55
|
+
|
|
56
|
+
### Changed
|
|
57
|
+
|
|
58
|
+
- Minor rather than patch: RIC-1580 is additive and RIC-1645 adds a metric-definition field with a backward-compatible default. No node-type or model contract broke, so the bundle `schema_version` is unchanged.
|
|
59
|
+
|
|
60
|
+
---
|
|
61
|
+
|
|
19
62
|
## [5.3.0] -- 2026-08-08
|
|
20
63
|
|
|
21
64
|
### Added
|
|
@@ -31,6 +31,44 @@ from katalyst_forge.metrics.protocols import (
|
|
|
31
31
|
)
|
|
32
32
|
|
|
33
33
|
|
|
34
|
+
class ModeTaggedMoney(Decimal):
|
|
35
|
+
"""A figure that remembers the billing mode it came from, and refuses addition.
|
|
36
|
+
|
|
37
|
+
MetricResult refuses to total across modes, but the tag used to disappear the
|
|
38
|
+
moment a value left it, so a plain `sum()` over `items()` produced exactly the
|
|
39
|
+
cross-mode total the class exists to prevent. Tagging the leaf keeps the
|
|
40
|
+
guarantee where the numbers actually travel.
|
|
41
|
+
|
|
42
|
+
Only addition is blocked. Comparison, division, quantizing and formatting all
|
|
43
|
+
still work, because addition is the single operation that silently merges two
|
|
44
|
+
quantities into a third one that describes neither.
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
__slots__ = ("billing_mode",)
|
|
48
|
+
|
|
49
|
+
def __new__(cls, value: Decimal, mode: BillingMode) -> ModeTaggedMoney:
|
|
50
|
+
instance = super().__new__(cls, value)
|
|
51
|
+
instance.billing_mode = mode
|
|
52
|
+
return instance
|
|
53
|
+
|
|
54
|
+
def _refuse(self, other: object) -> ModeTaggedMoney:
|
|
55
|
+
other_mode = getattr(other, "billing_mode", None)
|
|
56
|
+
detail = (
|
|
57
|
+
f"{self.billing_mode.value} + {other_mode.value}"
|
|
58
|
+
if isinstance(other_mode, BillingMode)
|
|
59
|
+
else f"a {self.billing_mode.value} figure with {type(other).__name__}"
|
|
60
|
+
)
|
|
61
|
+
raise CrossBillingModeError(
|
|
62
|
+
f"cannot add {detail}; read one mode at a time, or combine() within a mode"
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
def __add__(self, other: object) -> ModeTaggedMoney:
|
|
66
|
+
return self._refuse(other)
|
|
67
|
+
|
|
68
|
+
def __radd__(self, other: object) -> ModeTaggedMoney:
|
|
69
|
+
return self._refuse(other)
|
|
70
|
+
|
|
71
|
+
|
|
34
72
|
class MetricResult:
|
|
35
73
|
"""An immutable, mode-partitioned metric value. No cross-mode total exists."""
|
|
36
74
|
|
|
@@ -38,16 +76,21 @@ class MetricResult:
|
|
|
38
76
|
|
|
39
77
|
def __init__(self, metric_name: str, by_mode: Mapping[BillingMode, Decimal]) -> None:
|
|
40
78
|
self.metric_name = metric_name
|
|
41
|
-
|
|
79
|
+
# Stored untagged so internal within-mode arithmetic (combine) stays
|
|
80
|
+
# ordinary; the tag is applied on the way out, where values escape.
|
|
81
|
+
self._by_mode: Mapping[BillingMode, Decimal] = MappingProxyType(
|
|
82
|
+
{mode: Decimal(value) for mode, value in by_mode.items()}
|
|
83
|
+
)
|
|
42
84
|
|
|
43
85
|
def modes(self) -> tuple[BillingMode, ...]:
|
|
44
86
|
return tuple(self._by_mode.keys())
|
|
45
87
|
|
|
46
|
-
def for_mode(self, mode: BillingMode) ->
|
|
47
|
-
|
|
88
|
+
def for_mode(self, mode: BillingMode) -> ModeTaggedMoney | None:
|
|
89
|
+
value = self._by_mode.get(mode)
|
|
90
|
+
return None if value is None else ModeTaggedMoney(value, mode)
|
|
48
91
|
|
|
49
|
-
def items(self) -> tuple[tuple[BillingMode,
|
|
50
|
-
return tuple(self._by_mode.items())
|
|
92
|
+
def items(self) -> tuple[tuple[BillingMode, ModeTaggedMoney], ...]:
|
|
93
|
+
return tuple((mode, ModeTaggedMoney(value, mode)) for mode, value in self._by_mode.items())
|
|
51
94
|
|
|
52
95
|
def combine(self, other: MetricResult) -> MetricResult:
|
|
53
96
|
"""Merge two results of the same metric, within each billing mode only."""
|
|
@@ -55,6 +98,14 @@ class MetricResult:
|
|
|
55
98
|
raise CrossBillingModeError(
|
|
56
99
|
f"cannot combine different metrics {self.metric_name!r} and {other.metric_name!r}"
|
|
57
100
|
)
|
|
101
|
+
if self is other:
|
|
102
|
+
# Merging a result with itself doubles the run inside each mode, which
|
|
103
|
+
# is the same double count the cross-mode guard blocks, just wearing a
|
|
104
|
+
# different hat. Nothing legitimate combines a figure with itself.
|
|
105
|
+
raise CrossBillingModeError(
|
|
106
|
+
f"cannot combine metric {self.metric_name!r} with itself; "
|
|
107
|
+
"that would count the same spend twice"
|
|
108
|
+
)
|
|
58
109
|
merged: dict[BillingMode, Decimal] = dict(self._by_mode)
|
|
59
110
|
for mode, value in other._by_mode.items(): # noqa: SLF001 - same class
|
|
60
111
|
merged[mode] = merged.get(mode, ZERO_USD) + value
|
|
@@ -16,10 +16,12 @@ from collections.abc import Mapping, Sequence
|
|
|
16
16
|
from decimal import ROUND_HALF_UP, Decimal
|
|
17
17
|
from typing import Final, cast
|
|
18
18
|
|
|
19
|
-
from katalyst_forge.metrics.billing import BillingMode
|
|
19
|
+
from katalyst_forge.metrics.billing import BillingMode, parse_billing_mode
|
|
20
20
|
from katalyst_forge.metrics.core import MetricResult
|
|
21
21
|
|
|
22
22
|
CENTS: Final[Decimal] = Decimal("0.01")
|
|
23
|
+
ZERO_PERCENTAGE: Final[Decimal] = Decimal("0")
|
|
24
|
+
FULL_PERCENTAGE: Final[Decimal] = Decimal("100")
|
|
23
25
|
|
|
24
26
|
METERED_LABEL: Final[str] = "Metered spend"
|
|
25
27
|
NOTIONAL_LABEL: Final[str] = "Subscription notional (not money spent)"
|
|
@@ -29,8 +31,62 @@ _MODE_LABELS: Final[Mapping[BillingMode, str]] = {
|
|
|
29
31
|
BillingMode.SUBSCRIPTION_NOTIONAL: NOTIONAL_LABEL,
|
|
30
32
|
}
|
|
31
33
|
|
|
34
|
+
# What a figure IS, as distinct from the price basis it was computed on
|
|
35
|
+
# (RIC-1645). BillingMode answers "which catalogue priced this"; it does not
|
|
36
|
+
# answer "was this money billed". For a spend metric the two coincide, so the
|
|
37
|
+
# basis label reads correctly. For a saving it does not: cache-savings is spend
|
|
38
|
+
# avoided and routing-headroom was never incurred at all, yet both were captioned
|
|
39
|
+
# "Metered spend" and charted under "dollars actually billed".
|
|
40
|
+
QUANTITY_SPEND: Final[str] = "spend"
|
|
41
|
+
QUANTITY_SAVING: Final[str] = "saving"
|
|
42
|
+
QUANTITY_COUNTERFACTUAL: Final[str] = "counterfactual"
|
|
43
|
+
QUANTITY_RATIO: Final[str] = "ratio"
|
|
44
|
+
|
|
45
|
+
# An undeclared metric keeps the old caption, so a third-party bundle does not
|
|
46
|
+
# silently change meaning on upgrade. The shipped catalogue cannot rely on that:
|
|
47
|
+
# a test asserts every metric it ships declares a kind.
|
|
48
|
+
DEFAULT_QUANTITY_KIND: Final[str] = QUANTITY_SPEND
|
|
49
|
+
|
|
50
|
+
QUANTITIES_NOT_BILLED: Final[frozenset[str]] = frozenset({QUANTITY_SAVING, QUANTITY_COUNTERFACTUAL})
|
|
51
|
+
|
|
52
|
+
_QUANTITY_LABELS: Final[Mapping[tuple[str, BillingMode], str]] = {
|
|
53
|
+
(QUANTITY_SPEND, BillingMode.METERED): METERED_LABEL,
|
|
54
|
+
(QUANTITY_SPEND, BillingMode.SUBSCRIPTION_NOTIONAL): NOTIONAL_LABEL,
|
|
55
|
+
(QUANTITY_SAVING, BillingMode.METERED): "Metered basis, spend avoided (not billed)",
|
|
56
|
+
(QUANTITY_SAVING, BillingMode.SUBSCRIPTION_NOTIONAL): (
|
|
57
|
+
"Notional basis, allocation avoided (not money spent)"
|
|
58
|
+
),
|
|
59
|
+
(QUANTITY_COUNTERFACTUAL, BillingMode.METERED): (
|
|
60
|
+
"Metered basis, modelled and not captured (not billed)"
|
|
61
|
+
),
|
|
62
|
+
(QUANTITY_COUNTERFACTUAL, BillingMode.SUBSCRIPTION_NOTIONAL): (
|
|
63
|
+
"Notional basis, modelled and not captured (not money spent)"
|
|
64
|
+
),
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def is_billed(quantity_kind: str | None) -> bool:
|
|
69
|
+
"""Whether a figure of this kind is money that actually left the account."""
|
|
70
|
+
return (quantity_kind or DEFAULT_QUANTITY_KIND) not in QUANTITIES_NOT_BILLED
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def mode_label(mode: BillingMode, quantity_kind: str | None = None) -> str:
|
|
74
|
+
"""Label a figure by what it is, not only by the catalogue that priced it.
|
|
75
|
+
|
|
76
|
+
Falls back to the bare basis label for any mode/kind pair not enumerated, so
|
|
77
|
+
a new billing mode degrades to naming its basis rather than asserting spend.
|
|
78
|
+
"""
|
|
79
|
+
kind = quantity_kind or DEFAULT_QUANTITY_KIND
|
|
80
|
+
fallback = _MODE_LABELS.get(mode, str(mode))
|
|
81
|
+
return _QUANTITY_LABELS.get((kind, mode), fallback)
|
|
82
|
+
|
|
83
|
+
|
|
32
84
|
_OPERATORS: Final[Mapping[str, str]] = {"<": "<", "<=": "<=", ">": ">", ">=": ">="}
|
|
33
85
|
|
|
86
|
+
# A band with no declared mode is a spend threshold, because that is the only
|
|
87
|
+
# quantity a dollar ceiling can sensibly police.
|
|
88
|
+
_DEFAULT_BAND_MODE: Final[BillingMode] = BillingMode.METERED
|
|
89
|
+
|
|
34
90
|
|
|
35
91
|
UNIT_USD: Final[str] = "usd"
|
|
36
92
|
UNIT_PERCENTAGE: Final[str] = "percentage"
|
|
@@ -53,7 +109,20 @@ def format_usd(value: Decimal) -> str:
|
|
|
53
109
|
|
|
54
110
|
|
|
55
111
|
def format_percentage(value: Decimal) -> str:
|
|
56
|
-
|
|
112
|
+
"""Render a rate without letting rounding claim an endpoint it never reached.
|
|
113
|
+
|
|
114
|
+
A 99.998% cache hit rate quantizes to 100.00, which reads as "every billable
|
|
115
|
+
input token came from cache". The gap between nearly all and all is the whole
|
|
116
|
+
reason the rate is reported, so a value strictly inside the range keeps a
|
|
117
|
+
comparator instead of borrowing the endpoint. An exact 0 or 100 still prints
|
|
118
|
+
as itself.
|
|
119
|
+
"""
|
|
120
|
+
rounded = to_cents(value)
|
|
121
|
+
if rounded >= FULL_PERCENTAGE > value:
|
|
122
|
+
return f">{FULL_PERCENTAGE - CENTS}%"
|
|
123
|
+
if rounded <= ZERO_PERCENTAGE < value:
|
|
124
|
+
return f"<{CENTS}%"
|
|
125
|
+
return f"{rounded}%"
|
|
57
126
|
|
|
58
127
|
|
|
59
128
|
def format_value(value: Decimal, unit: str | None) -> str:
|
|
@@ -118,11 +187,27 @@ def _as_sequence(value: object) -> Sequence[object]:
|
|
|
118
187
|
return ()
|
|
119
188
|
|
|
120
189
|
|
|
190
|
+
def _entry_judges_mode(entry: Mapping[str, object], billing_mode: BillingMode | None) -> bool:
|
|
191
|
+
"""A band judges one billing mode, and by default that mode is real money.
|
|
192
|
+
|
|
193
|
+
A threshold written for dollars actually billed must not stamp a verdict on a
|
|
194
|
+
subscription-notional allocation, which is a different quantity that happens
|
|
195
|
+
to share a unit. A caller that does not say which mode it is judging is not
|
|
196
|
+
filtered, so text callers keep their existing behaviour.
|
|
197
|
+
"""
|
|
198
|
+
if billing_mode is None:
|
|
199
|
+
return True
|
|
200
|
+
declared = entry.get("billing_mode")
|
|
201
|
+
expected = parse_billing_mode(declared) if isinstance(declared, str) else _DEFAULT_BAND_MODE
|
|
202
|
+
return expected is billing_mode
|
|
203
|
+
|
|
204
|
+
|
|
121
205
|
def evaluate_band(
|
|
122
206
|
*,
|
|
123
207
|
metric_name: str,
|
|
124
208
|
value: Decimal,
|
|
125
209
|
threshold_profile_doc: Mapping[str, object],
|
|
210
|
+
billing_mode: BillingMode | None = None,
|
|
126
211
|
breach_bands: frozenset[str] = frozenset({"amber", "red"}),
|
|
127
212
|
) -> BandVerdict:
|
|
128
213
|
"""Find the band a value falls in and flag a breach (AC5).
|
|
@@ -137,6 +222,8 @@ def evaluate_band(
|
|
|
137
222
|
entry_mapping = _as_mapping(entry)
|
|
138
223
|
if entry_mapping.get("metric") != metric_name:
|
|
139
224
|
continue
|
|
225
|
+
if not _entry_judges_mode(entry_mapping, billing_mode):
|
|
226
|
+
continue
|
|
140
227
|
for band in _as_sequence(entry_mapping.get("bands")):
|
|
141
228
|
band_mapping = _as_mapping(band)
|
|
142
229
|
operator = band_mapping.get("operator")
|
|
@@ -160,7 +247,11 @@ def evaluate_band(
|
|
|
160
247
|
)
|
|
161
248
|
|
|
162
249
|
|
|
163
|
-
def render_savings_panel(
|
|
250
|
+
def render_savings_panel(
|
|
251
|
+
result: MetricResult,
|
|
252
|
+
unit: str | None = UNIT_USD,
|
|
253
|
+
quantity_kind: str | None = None,
|
|
254
|
+
) -> tuple[str, ...]:
|
|
164
255
|
"""Render one line per billing mode, never a combined figure (AC10).
|
|
165
256
|
|
|
166
257
|
Returns a line per mode rather than a total. There is deliberately no
|
|
@@ -169,7 +260,7 @@ def render_savings_panel(result: MetricResult, unit: str | None = UNIT_USD) -> t
|
|
|
169
260
|
"""
|
|
170
261
|
lines: list[str] = []
|
|
171
262
|
for mode, value in result.items():
|
|
172
|
-
label =
|
|
263
|
+
label = mode_label(mode, quantity_kind)
|
|
173
264
|
lines.append(f"{label}: {format_value(value, unit)}")
|
|
174
265
|
return tuple(lines)
|
|
175
266
|
|
|
@@ -180,24 +271,29 @@ def render_dashboard(
|
|
|
180
271
|
results: Sequence[MetricResult],
|
|
181
272
|
threshold_profile_doc: Mapping[str, object],
|
|
182
273
|
units: Mapping[str, str] | None = None,
|
|
274
|
+
quantity_kinds: Mapping[str, str] | None = None,
|
|
183
275
|
) -> str:
|
|
184
276
|
"""Render the whole dashboard as text, per metric and per billing mode.
|
|
185
277
|
|
|
186
278
|
`units` maps a metric name to its declared unit so a ratio is never
|
|
187
|
-
rendered with a dollar sign.
|
|
279
|
+
rendered with a dollar sign. `quantity_kinds` maps a metric name to what its
|
|
280
|
+
figure is, so a saving is not captioned as spend.
|
|
188
281
|
"""
|
|
189
282
|
unit_by_metric = units or {}
|
|
283
|
+
kind_by_metric = quantity_kinds or {}
|
|
190
284
|
lines: list[str] = [f"Run: {run_id}", ""]
|
|
191
285
|
|
|
192
286
|
for result in results:
|
|
193
287
|
unit = unit_by_metric.get(result.metric_name, UNIT_USD)
|
|
288
|
+
kind = kind_by_metric.get(result.metric_name)
|
|
194
289
|
lines.append(result.metric_name)
|
|
195
290
|
for mode, value in result.items():
|
|
196
|
-
label =
|
|
291
|
+
label = mode_label(mode, kind)
|
|
197
292
|
verdict = evaluate_band(
|
|
198
293
|
metric_name=result.metric_name,
|
|
199
294
|
value=value,
|
|
200
295
|
threshold_profile_doc=threshold_profile_doc,
|
|
296
|
+
billing_mode=mode,
|
|
201
297
|
)
|
|
202
298
|
suffix = ""
|
|
203
299
|
if verdict.band_name is not None:
|
|
@@ -219,3 +315,21 @@ def units_from_metric_definitions(
|
|
|
219
315
|
if isinstance(unit, str):
|
|
220
316
|
units[name] = unit
|
|
221
317
|
return units
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def quantity_kinds_from_metric_definitions(
|
|
321
|
+
metric_definitions: Mapping[str, Mapping[str, object]],
|
|
322
|
+
) -> dict[str, str]:
|
|
323
|
+
"""Read each metric's declared quantity kind from the audited catalogue.
|
|
324
|
+
|
|
325
|
+
Kept in the definitions rather than the renderer for the same reason units
|
|
326
|
+
are: whether a figure is spend or a saving is a property of the metric, and a
|
|
327
|
+
renderer that decided it locally could disagree with the document the figure
|
|
328
|
+
was computed from.
|
|
329
|
+
"""
|
|
330
|
+
kinds: dict[str, str] = {}
|
|
331
|
+
for name, doc in metric_definitions.items():
|
|
332
|
+
kind = _as_mapping(doc.get("spec")).get("quantity_kind")
|
|
333
|
+
if isinstance(kind, str):
|
|
334
|
+
kinds[name] = kind
|
|
335
|
+
return kinds
|
|
@@ -120,3 +120,23 @@ class ProviderPayloadError(MetricsAdapterError):
|
|
|
120
120
|
|
|
121
121
|
def __init__(self, *, field: str, value: object) -> None:
|
|
122
122
|
super().__init__(f"unparseable provider payload for {field!r}: {redact(value)}")
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
class ProviderTransportError(Exception):
|
|
126
|
+
"""Raised when a provider request fails before any payload exists.
|
|
127
|
+
|
|
128
|
+
Deliberately NOT a MetricsAdapterError. The per-source and per-metric
|
|
129
|
+
handlers skip a MetricsAdapterError and carry on, which is right for a
|
|
130
|
+
source that cannot be computed but wrong for a host that cannot be reached:
|
|
131
|
+
that leaves a hole of unknown size, so it has to stay a hard failure and
|
|
132
|
+
reach the entry point.
|
|
133
|
+
|
|
134
|
+
Carries the endpoint rather than a bare status line, because an operator
|
|
135
|
+
reading the failure needs to know which request died. Only the URL and the
|
|
136
|
+
exception type are used; no response body reaches the message.
|
|
137
|
+
"""
|
|
138
|
+
|
|
139
|
+
def __init__(self, *, url: str, detail: str) -> None:
|
|
140
|
+
self.url = url
|
|
141
|
+
self.detail = detail
|
|
142
|
+
super().__init__(f"request to {url} failed: {detail}")
|
{katalyst_forge-5.3.0 → katalyst_forge-5.4.1}/src/katalyst_forge/metrics/generate_dashboard.py
RENAMED
|
@@ -12,10 +12,12 @@ keeps its no-I/O guarantee. Credentials are read from the environment only.
|
|
|
12
12
|
from __future__ import annotations
|
|
13
13
|
|
|
14
14
|
import argparse
|
|
15
|
+
import importlib.metadata
|
|
15
16
|
import os
|
|
16
17
|
import sys
|
|
17
18
|
import time
|
|
18
19
|
from collections.abc import Callable, Iterator, Mapping, Sequence
|
|
20
|
+
from datetime import datetime, timezone
|
|
19
21
|
from decimal import Decimal
|
|
20
22
|
from pathlib import Path
|
|
21
23
|
from typing import Final, cast
|
|
@@ -24,8 +26,12 @@ import yaml
|
|
|
24
26
|
|
|
25
27
|
from katalyst_forge.metrics.batch import CanonicalSignalBatch
|
|
26
28
|
from katalyst_forge.metrics.core import MetricResult, compute, resolve
|
|
27
|
-
from katalyst_forge.metrics.dashboard import
|
|
28
|
-
|
|
29
|
+
from katalyst_forge.metrics.dashboard import (
|
|
30
|
+
quantity_kinds_from_metric_definitions,
|
|
31
|
+
render_dashboard,
|
|
32
|
+
units_from_metric_definitions,
|
|
33
|
+
)
|
|
34
|
+
from katalyst_forge.metrics.errors import MetricsAdapterError, ProviderTransportError
|
|
29
35
|
from katalyst_forge.metrics.html_report import render_html_report
|
|
30
36
|
from katalyst_forge.metrics.protocols import PriceCatalogue, RoutingPolicy
|
|
31
37
|
from katalyst_forge.metrics.registry import SignalProviderRegistry
|
|
@@ -246,6 +252,14 @@ def build_run_health(
|
|
|
246
252
|
return None
|
|
247
253
|
|
|
248
254
|
|
|
255
|
+
def _catalogue_source(signal_sources: Mapping[str, Document]) -> str | None:
|
|
256
|
+
"""Name the endpoint prices are read from, for the report's provenance line."""
|
|
257
|
+
source = signal_sources.get("opencode-session-cost", {})
|
|
258
|
+
query_config = _as_mapping(_as_mapping(source.get("spec")).get("query_config"))
|
|
259
|
+
endpoint = query_config.get("providers_endpoint")
|
|
260
|
+
return endpoint if isinstance(endpoint, str) else None
|
|
261
|
+
|
|
262
|
+
|
|
249
263
|
def _opencode_catalogue(
|
|
250
264
|
*, signal_sources: Mapping[str, Document], provider_env: Mapping[str, str]
|
|
251
265
|
) -> PriceCatalogue:
|
|
@@ -258,6 +272,14 @@ def _opencode_catalogue(
|
|
|
258
272
|
return provider.price_catalogue(query_config)
|
|
259
273
|
|
|
260
274
|
|
|
275
|
+
def _generator_identity() -> str:
|
|
276
|
+
"""Name the generator so a rendered report can be traced to a release."""
|
|
277
|
+
try:
|
|
278
|
+
return f"katalyst-forge {importlib.metadata.version('katalyst-forge')}"
|
|
279
|
+
except importlib.metadata.PackageNotFoundError:
|
|
280
|
+
return "katalyst-forge (working copy)"
|
|
281
|
+
|
|
282
|
+
|
|
261
283
|
class LazyPriceCatalogue:
|
|
262
284
|
"""Defers building the live catalogue until a price is actually needed.
|
|
263
285
|
|
|
@@ -271,9 +293,20 @@ class LazyPriceCatalogue:
|
|
|
271
293
|
needs a price never silently reads as free.
|
|
272
294
|
"""
|
|
273
295
|
|
|
274
|
-
def __init__(self, build: Callable[[], PriceCatalogue]) -> None:
|
|
296
|
+
def __init__(self, build: Callable[[], PriceCatalogue], *, source: str | None = None) -> None:
|
|
275
297
|
self._build = build
|
|
276
298
|
self._delegate: PriceCatalogue | None = None
|
|
299
|
+
self._source = source
|
|
300
|
+
|
|
301
|
+
@property
|
|
302
|
+
def priced_source(self) -> str | None:
|
|
303
|
+
"""The catalogue this run was actually priced against, or None if unused.
|
|
304
|
+
|
|
305
|
+
Reported only once a price has really been looked up. A run that needed
|
|
306
|
+
no price was not priced against anything, and claiming a source it never
|
|
307
|
+
read would be a provenance line that means nothing.
|
|
308
|
+
"""
|
|
309
|
+
return self._source if self._delegate is not None else None
|
|
277
310
|
|
|
278
311
|
def unit_price_usd(self, model_key: str, token_kind: str) -> Decimal | None:
|
|
279
312
|
if self._delegate is None:
|
|
@@ -307,34 +340,53 @@ def main(argv: Sequence[str] | None = None) -> int:
|
|
|
307
340
|
|
|
308
341
|
provider_env: Mapping[str, str] = dict(os.environ)
|
|
309
342
|
catalogue = LazyPriceCatalogue(
|
|
310
|
-
lambda: _opencode_catalogue(signal_sources=signal_sources, provider_env=provider_env)
|
|
343
|
+
lambda: _opencode_catalogue(signal_sources=signal_sources, provider_env=provider_env),
|
|
344
|
+
source=_catalogue_source(signal_sources),
|
|
311
345
|
)
|
|
312
346
|
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
347
|
+
try:
|
|
348
|
+
health = build_run_health(
|
|
349
|
+
run_id=run_id,
|
|
350
|
+
signal_sources=signal_sources,
|
|
351
|
+
provider_env=provider_env,
|
|
352
|
+
catalogue=catalogue,
|
|
353
|
+
now_ms=int(time.time() * 1000),
|
|
354
|
+
)
|
|
320
355
|
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
356
|
+
results = build_results(
|
|
357
|
+
run_id=run_id,
|
|
358
|
+
metric_definitions=metric_definitions,
|
|
359
|
+
signal_sources=signal_sources,
|
|
360
|
+
provider_env=provider_env,
|
|
361
|
+
catalogue=catalogue,
|
|
362
|
+
policy=build_policy(env=provider_env, catalogue=catalogue),
|
|
363
|
+
)
|
|
364
|
+
except ProviderTransportError as exc:
|
|
365
|
+
# Still a hard failure, and still nothing written. What changes is that
|
|
366
|
+
# the operator is told which endpoint died instead of reading a traceback
|
|
367
|
+
# whose last line is a status code with no subject.
|
|
368
|
+
print(
|
|
369
|
+
f"cannot report on run {run_id}: {exc}. "
|
|
370
|
+
"No report was written, because a request that fails outright leaves "
|
|
371
|
+
"a gap of unknown size rather than a measurable one.",
|
|
372
|
+
file=sys.stderr,
|
|
373
|
+
)
|
|
374
|
+
return 1
|
|
329
375
|
|
|
330
376
|
units = units_from_metric_definitions(metric_definitions)
|
|
377
|
+
quantity_kinds = quantity_kinds_from_metric_definitions(metric_definitions)
|
|
331
378
|
if args.format == "html":
|
|
332
379
|
rendered = render_html_report(
|
|
333
380
|
run_id=run_id,
|
|
334
381
|
results=results,
|
|
335
382
|
threshold_profile_doc=threshold_profile,
|
|
336
383
|
units=units,
|
|
384
|
+
generated=datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC"),
|
|
385
|
+
generator=_generator_identity(),
|
|
386
|
+
pricing_source=catalogue.priced_source,
|
|
337
387
|
health=health,
|
|
388
|
+
metric_docs=metric_definitions,
|
|
389
|
+
quantity_kinds=quantity_kinds,
|
|
338
390
|
)
|
|
339
391
|
else:
|
|
340
392
|
rendered = render_dashboard(
|
|
@@ -342,6 +394,7 @@ def main(argv: Sequence[str] | None = None) -> int:
|
|
|
342
394
|
results=results,
|
|
343
395
|
threshold_profile_doc=threshold_profile,
|
|
344
396
|
units=units,
|
|
397
|
+
quantity_kinds=quantity_kinds,
|
|
345
398
|
)
|
|
346
399
|
|
|
347
400
|
out_path = cast("str | None", args.out)
|