agent-lifecycle-kit 1.50.1__py3-none-any.whl

agent_lifecycle/metrics/__init__.py

@@ -0,0 +1,60 @@
+"""Lifecycle resource and process-cost metrics."""
+
+from agent_lifecycle.metrics.costs import (
+    COST_CATEGORIES,
+    DEFAULT_MODE_LIMITS,
+    require_lifecycle_cost_pass,
+    validate_lifecycle_cost_report,
+)
+from agent_lifecycle.metrics.cost_collection import build_lifecycle_cost_summary, generate_lifecycle_cost_report
+from agent_lifecycle.metrics.phase_resources import (
+    build_phase_resource_measurement,
+    require_phase_resource_measurement_pass,
+    validate_phase_resource_measurement,
+)
+from agent_lifecycle.metrics.outcome_index import (
+    build_quality_cost_signal_summary,
+    build_quality_cost_signals,
+    build_task_outcome_index,
+)
+from agent_lifecycle.metrics.recommendations import (
+    build_lifecycle_recommendation_summary,
+    recommend_lifecycle_mode,
+    recommend_from_quality_cost_signals,
+    require_lifecycle_recommendation_pass,
+    summarize_lifecycle_overhead,
+    validate_lifecycle_baselines,
+)
+from agent_lifecycle.metrics.regression_signals import summarize_regression_signals
+from agent_lifecycle.metrics.usage_export import (
+    build_usage_export,
+    require_usage_export_pass,
+    usage_export_totals,
+    validate_usage_export,
+)
+
+__all__ = [
+    "COST_CATEGORIES",
+    "DEFAULT_MODE_LIMITS",
+    "build_usage_export",
+    "build_phase_resource_measurement",
+    "build_lifecycle_cost_summary",
+    "build_lifecycle_recommendation_summary",
+    "build_quality_cost_signal_summary",
+    "build_quality_cost_signals",
+    "build_task_outcome_index",
+    "generate_lifecycle_cost_report",
+    "recommend_from_quality_cost_signals",
+    "recommend_lifecycle_mode",
+    "require_lifecycle_cost_pass",
+    "require_lifecycle_recommendation_pass",
+    "require_phase_resource_measurement_pass",
+    "require_usage_export_pass",
+    "summarize_lifecycle_overhead",
+    "summarize_regression_signals",
+    "usage_export_totals",
+    "validate_lifecycle_baselines",
+    "validate_lifecycle_cost_report",
+    "validate_phase_resource_measurement",
+    "validate_usage_export",
+]

agent_lifecycle/metrics/cost_collection.py

@@ -0,0 +1,315 @@
+"""Generate lifecycle cost reports from explicit local artifacts."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from agent_lifecycle.contracts import LifecycleError, canonical_bytes, canonical_digest, load_json_object, sha256_hex
+from agent_lifecycle.metrics.costs import (
+    COST_CATEGORIES,
+    COST_REPORT_SCHEMA,
+    COST_SUMMARY_SCHEMA,
+    cost_entry_totals,
+    cost_ratios,
+    summarize_usage_confidence,
+)
+
+
+def generate_lifecycle_cost_report(
+    *,
+    artifact_paths: list[Path],
+    mode: str = "standard",
+    root: Path | None = None,
+) -> dict[str, Any]:
+    """Generate a cost report from explicit JSON artifacts without host calls."""
+
+    if not artifact_paths:
+        raise LifecycleError("cost-report-artifacts-required", "at least one artifact path is required")
+    project_root = (root or Path.cwd()).resolve()
+    source_artifacts: list[dict[str, Any]] = []
+    entries: list[dict[str, Any]] = []
+    payloads: list[dict[str, Any]] = []
+    for index, raw_path in enumerate(artifact_paths):
+        payload, source = _load_source_artifact(raw_path, project_root)
+        source_artifacts.append(source)
+        entries.append(_entry_from_artifact(index, source, payload))
+        payloads.append(payload)
+
+    report = {
+        "schemaVersion": COST_REPORT_SCHEMA,
+        "mode": mode,
+        "generatedBy": "agent-lifecycle metrics cost-report",
+        "sourceArtifacts": source_artifacts,
+        "lineage": _lineage(source_artifacts, payloads),
+        "entries": entries,
+        "usageConfidence": summarize_usage_confidence(entries),
+        "productionPromotionClaimed": False,
+    }
+    report["compactSummary"] = build_lifecycle_cost_summary(report)
+    return report
+
+
+def build_lifecycle_cost_summary(report: dict[str, Any]) -> dict[str, Any]:
+    """Build a small-context summary from a lifecycle cost report."""
+
+    entries = report.get("entries") if isinstance(report.get("entries"), list) else []
+    totals = cost_entry_totals(entries)
+    ratios = cost_ratios(totals)
+    usage_confidence = summarize_usage_confidence(entries)
+    source_paths = _source_paths(report)
+    missing_usage = usage_confidence["missingEntries"]
+    next_action = "review missing usage entries" if missing_usage else "continue with the selected lifecycle policy"
+    return {
+        "schemaVersion": COST_SUMMARY_SCHEMA,
+        "mode": report.get("mode"),
+        "latestUserIntent": "Keep lifecycle overhead visible without reducing task quality.",
+        "activeDecisions": [
+            f"pipelineTokenShare={ratios['pipelineTokenShare']}",
+            f"pipelineStepShare={ratios['pipelineStepShare']}",
+            f"missingUsageEntries={missing_usage}",
+        ],
+        "openBlockers": [],
+        "acceptedEvidence": [
+            {
+                "id": "lifecycle-cost-report",
+                "status": "generated",
+                "sourceArtifactCount": len(source_paths),
+            }
+        ],
+        "changedFiles": source_paths[:20],
+        "nextRequiredAction": next_action,
+        "doNotDo": [
+            "Do not treat missing usage as zero.",
+            "Do not reduce required validation solely to improve cost ratios.",
+        ],
+        "categoryTotals": totals,
+        "ratios": ratios,
+        "usageConfidence": usage_confidence,
+        "usefulWorkTokens": totals["implementation"]["tokens"] + totals["productValidation"]["tokens"],
+        "processOverheadTokens": totals["pipelineCompliance"]["tokens"] + totals["coordination"]["tokens"],
+    }
+
+
+def _load_source_artifact(raw_path: Path, project_root: Path) -> tuple[dict[str, Any], dict[str, Any]]:
+    path = raw_path if raw_path.is_absolute() else project_root / raw_path
+    display_path = _display_path(path, project_root)
+    try:
+        data = path.read_bytes()
+    except OSError as exc:
+        raise LifecycleError(
+            "cost-report-artifact-unavailable",
+            "cost report artifact cannot be read",
+            {"path": display_path, "reason": type(exc).__name__},
+        ) from exc
+    payload = load_json_object(data, label=display_path)
+    source = {
+        "path": display_path,
+        "sha256": sha256_hex(data),
+        "bytes": len(data),
+        "schemaVersion": payload.get("schemaVersion"),
+        "payloadDigest": canonical_digest(payload),
+    }
+    return payload, source
+
+
+def _source_paths(report: dict[str, Any]) -> list[str]:
+    return [
+        item.get("path")
+        for item in report.get("sourceArtifacts", [])
+        if isinstance(item, dict) and isinstance(item.get("path"), str)
+    ]
+
+
+def _entry_from_artifact(index: int, source: dict[str, Any], payload: dict[str, Any]) -> dict[str, Any]:
+    usage = _usage_from_payload(payload)
+    entry = {
+        "id": f"artifact-{index + 1}",
+        "category": _classify_artifact(str(source.get("path")), payload),
+        "tokens": usage["tokens"],
+        "steps": usage["steps"],
+        "usageConfidence": usage["confidence"],
+        "sourcePath": source["path"],
+        "sourceSchemaVersion": source.get("schemaVersion"),
+        "sourceDigest": source["payloadDigest"],
+    }
+    for field in ("runId", "packageId", "taskId", "operationId", "planDigest", "sourceRevision"):
+        value = payload.get(field)
+        if isinstance(value, (str, int)):
+            entry[field] = value
+    return entry
+
+
+def _usage_from_payload(payload: dict[str, Any]) -> dict[str, Any]:
+    usage = payload.get("usage")
+    if isinstance(usage, dict):
+        billable = _int_value(usage.get("billableTokens"))
+        steps = _int_value(usage.get("toolCalls")) or 1
+        if billable is not None:
+            confidence = "ATTESTED" if _attested(payload.get("attestation")) else "ESTIMATED"
+            return {"tokens": billable, "steps": max(1, steps), "confidence": confidence}
+        return {"tokens": 0, "steps": 1, "confidence": "MISSING"}
+    for key in ("usageTotals", "counters"):
+        totals = payload.get(key)
+        if not isinstance(totals, dict):
+            continue
+        tokens = _first_int(totals, ("reportedTokens", "billableTokens", "tokens"))
+        if tokens is not None and tokens > 0:
+            steps = _first_int(totals, ("toolCalls", "validationRuns", "iterations", "steps")) or 1
+            return {"tokens": tokens, "steps": max(1, steps), "confidence": "ESTIMATED"}
+    if payload.get("schemaVersion") == "agent-lifecycle-model-usage-receipt.v1":
+        return {"tokens": 0, "steps": 1, "confidence": "MISSING"}
+    if _contains_pair(payload, "usageReceiptRequired", True):
+        return {"tokens": 0, "steps": 1, "confidence": "MISSING"}
+    return {"tokens": _estimate_tokens(payload), "steps": 1, "confidence": "ESTIMATED"}
+
+
+def _classify_artifact(path: str, payload: dict[str, Any]) -> str:
+    schema = str(payload.get("schemaVersion") or "")
+    if schema == "agent-lifecycle-model-usage-receipt.v1":
+        return _classify_usage_receipt(payload)
+    text = _classification_text(path, payload)
+    if schema in {"agent-workflow-state.v3", "agent-runner-state.v1"}:
+        return "coordination"
+    if schema == "agent-task-result.v2":
+        return "implementation"
+    if any(marker in text for marker in _PRODUCT_VALIDATION_MARKERS):
+        return "productValidation"
+    if any(marker in text for marker in _PIPELINE_MARKERS):
+        return "pipelineCompliance"
+    return "implementation"
+
+
+def _classify_usage_receipt(payload: dict[str, Any]) -> str:
+    text = _classification_text("", payload)
+    if any(marker in text for marker in _PRODUCT_VALIDATION_MARKERS):
+        return "productValidation"
+    if any(marker in text for marker in _PIPELINE_MARKERS):
+        return "pipelineCompliance"
+    return "implementation"
+
+
+def _classification_text(path: str, payload: dict[str, Any]) -> str:
+    return " ".join(
+        str(item)
+        for item in (
+            path,
+            payload.get("schemaVersion"),
+            payload.get("operationId"),
+            payload.get("modelClass"),
+            payload.get("stage"),
+            payload.get("role"),
+            payload.get("kind"),
+        )
+        if item is not None
+    ).lower()
+
+
+_PRODUCT_VALIDATION_MARKERS = (
+    "task-review",
+    "quality",
+    "test",
+    "security-review",
+    "performance-review",
+    "specialist-review",
+)
+
+_PIPELINE_MARKERS = (
+    "plan",
+    "workflow",
+    "runner",
+    "gate",
+    "neutrality",
+    "diagnostic",
+    "contract",
+    "release",
+    "cost",
+    "final-proof",
+    "handoff",
+    "followup",
+    "worktree",
+    "completion-check",
+)
+
+
+def _lineage(source_artifacts: list[dict[str, Any]], payloads: list[dict[str, Any]]) -> dict[str, list[Any]]:
+    values = {
+        "runIds": set(),
+        "packageIds": set(),
+        "taskIds": set(),
+        "planDigests": set(),
+        "sourceRevisions": set(),
+        "evidenceDigests": set(),
+        "sourceArtifactDigests": {item["sha256"] for item in source_artifacts},
+    }
+    for payload in payloads:
+        _collect_lineage_values(payload, values)
+    return {key: sorted(value) for key, value in values.items()}
+
+
+def _collect_lineage_values(value: Any, found: dict[str, set[Any]]) -> None:
+    if isinstance(value, dict):
+        for key, item in value.items():
+            if key == "runId" and isinstance(item, str):
+                found["runIds"].add(item)
+            elif key == "packageId" and isinstance(item, str):
+                found["packageIds"].add(item)
+            elif key == "taskId" and isinstance(item, str):
+                found["taskIds"].add(item)
+            elif key == "id" and isinstance(item, str) and item.startswith("WS-"):
+                found["taskIds"].add(item)
+            elif key == "planDigest" and isinstance(item, str):
+                found["planDigests"].add(item)
+            elif key == "sourceRevision" and isinstance(item, str):
+                found["sourceRevisions"].add(item)
+            elif _is_evidence_digest(key, item):
+                found["evidenceDigests"].add(item)
+            _collect_lineage_values(item, found)
+    elif isinstance(value, list):
+        for item in value:
+            _collect_lineage_values(item, found)
+
+
+def _is_evidence_digest(key: str, value: Any) -> bool:
+    keys = {"evidenceDigest", "resultDigest", "reviewDigest", "validationDigest", "finalProofDigest", "sha256"}
+    return key in keys and isinstance(value, str)
+
+
+def _contains_pair(value: Any, key: str, expected: Any) -> bool:
+    if isinstance(value, dict):
+        return any(
+            (item_key == key and item_value == expected) or _contains_pair(item_value, key, expected)
+            for item_key, item_value in value.items()
+        )
+    if isinstance(value, list):
+        return any(_contains_pair(item, key, expected) for item in value)
+    return False
+
+
+def _attested(value: Any) -> bool:
+    return isinstance(value, dict) and value.get("status") == "ATTESTED"
+
+
+def _first_int(payload: dict[str, Any], keys: tuple[str, ...]) -> int | None:
+    for key in keys:
+        value = _int_value(payload.get(key))
+        if value is not None:
+            return value
+    return None
+
+
+def _int_value(value: Any) -> int | None:
+    if isinstance(value, int) and not isinstance(value, bool) and value >= 0:
+        return value
+    return None
+
+
+def _estimate_tokens(value: Any) -> int:
+    return max(1, (len(canonical_bytes(value)) + 3) // 4)
+
+
+def _display_path(path: Path, root: Path) -> str:
+    try:
+        return path.resolve().relative_to(root).as_posix()
+    except ValueError:
+        return path.name

agent_lifecycle/metrics/costs.py

@@ -0,0 +1,203 @@
+"""Validate lifecycle cost reports for resource-disciplined runs."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from agent_lifecycle.contracts import LifecycleError, canonical_digest
+
+COST_REPORT_SCHEMA = "agent-lifecycle-cost-report.v1"
+COST_VALIDATION_SCHEMA = "agent-lifecycle-cost-validation.v1"
+COST_GENERATION_SCHEMA = "agent-lifecycle-cost-generation.v1"
+COST_SUMMARY_SCHEMA = "agent-lifecycle-cost-summary.v1"
+
+COST_CATEGORIES = ("implementation", "productValidation", "pipelineCompliance", "coordination")
+USAGE_CONFIDENCE_STATES = ("ATTESTED", "ESTIMATED", "MISSING")
+
+DEFAULT_MODE_LIMITS: dict[str, dict[str, float | int]] = {
+    "light": {"maxPipelineTokenShare": 0.20, "maxPipelineStepShare": 0.30, "maxPipelineTokens": 1500, "maxPipelineSteps": 3},
+    "standard": {"maxPipelineTokenShare": 0.30, "maxPipelineStepShare": 0.40, "maxPipelineTokens": 6000, "maxPipelineSteps": 6},
+    "strict": {"maxPipelineTokenShare": 0.45, "maxPipelineStepShare": 0.55, "maxPipelineTokens": 20000, "maxPipelineSteps": 12},
+    "release": {"maxPipelineTokenShare": 0.60, "maxPipelineStepShare": 0.70, "maxPipelineTokens": 50000, "maxPipelineSteps": 24},
+}
+
+
+def validate_lifecycle_cost_report(report: dict[str, Any]) -> dict[str, Any]:
+    """Validate a lifecycle cost report and return a compact receipt."""
+
+    blockers: list[dict[str, Any]] = []
+    if report.get("schemaVersion") != COST_REPORT_SCHEMA:
+        blockers.append({"code": "cost-report-schema", "message": "unsupported lifecycle cost report schemaVersion"})
+    if report.get("productionPromotionClaimed") is True:
+        blockers.append({"code": "cost-report-production-claim", "message": "cost reports must not claim production promotion"})
+    mode = report.get("mode")
+    if mode not in DEFAULT_MODE_LIMITS:
+        blockers.append({"code": "cost-report-mode", "mode": mode})
+        mode = None
+    entries = report.get("entries")
+    if not isinstance(entries, list) or not entries:
+        blockers.append({"code": "cost-report-entries", "message": "entries must be a non-empty array"})
+        entries = []
+    totals = _empty_totals()
+    for index, entry in enumerate(entries):
+        _add_entry(index, entry, totals, blockers)
+    totals["overall"] = {
+        "tokens": sum(totals[category]["tokens"] for category in COST_CATEGORIES),
+        "steps": sum(totals[category]["steps"] for category in COST_CATEGORIES),
+    }
+    usage_confidence = summarize_usage_confidence(entries)
+    limits = _limits(report.get("limits"), mode)
+    ratios = cost_ratios(totals)
+    if mode is not None:
+        _check_pipeline_limits(totals, ratios, limits, report.get("overLimitReason"), blockers)
+    body = {
+        "schemaVersion": COST_VALIDATION_SCHEMA,
+        "status": "PASS" if not blockers else "FAIL",
+        "mode": mode,
+        "totals": totals,
+        "ratios": ratios,
+        "usageConfidence": usage_confidence,
+        "limits": limits,
+        "blockers": blockers,
+        "reportDigest": canonical_digest(report),
+    }
+    return {**body, "validationDigest": canonical_digest(body)}
+
+
+def require_lifecycle_cost_pass(validation: dict[str, Any]) -> dict[str, Any]:
+    if validation.get("status") == "FAIL":
+        raise LifecycleError("lifecycle-cost-validation-failed", "lifecycle cost validation failed", {"validation": validation})
+    return validation
+
+
+def _empty_totals() -> dict[str, dict[str, int]]:
+    return {category: {"tokens": 0, "steps": 0} for category in COST_CATEGORIES}
+
+
+def cost_entry_totals(entries: list[Any]) -> dict[str, dict[str, int]]:
+    totals = _empty_totals()
+    for entry in entries:
+        if not isinstance(entry, dict):
+            continue
+        category = entry.get("category")
+        tokens = entry.get("tokens")
+        steps = entry.get("steps")
+        if category in COST_CATEGORIES and isinstance(tokens, int) and isinstance(steps, int):
+            totals[category]["tokens"] += max(0, tokens)
+            totals[category]["steps"] += max(0, steps)
+    totals["overall"] = {
+        "tokens": sum(totals[category]["tokens"] for category in COST_CATEGORIES),
+        "steps": sum(totals[category]["steps"] for category in COST_CATEGORIES),
+    }
+    return totals
+
+
+def _add_entry(index: int, entry: Any, totals: dict[str, dict[str, int]], blockers: list[dict[str, Any]]) -> None:
+    if not isinstance(entry, dict):
+        blockers.append({"code": "cost-entry-type", "index": index, "message": "entry must be an object"})
+        return
+    category = entry.get("category")
+    if category not in COST_CATEGORIES:
+        blockers.append({"code": "cost-entry-category", "index": index, "category": category})
+        return
+    tokens = _non_negative_int(entry.get("tokens"), field="tokens", index=index, blockers=blockers)
+    steps = _non_negative_int(entry.get("steps"), field="steps", index=index, blockers=blockers)
+    confidence = entry.get("usageConfidence")
+    if confidence is not None and confidence not in USAGE_CONFIDENCE_STATES:
+        blockers.append({"code": "cost-entry-usage-confidence", "index": index, "usageConfidence": confidence})
+    if tokens is None or steps is None:
+        return
+    if tokens == 0 and steps == 0:
+        blockers.append({"code": "cost-entry-empty", "index": index})
+        return
+    totals[category]["tokens"] += tokens
+    totals[category]["steps"] += steps
+
+
+def _non_negative_int(value: Any, *, field: str, index: int, blockers: list[dict[str, Any]]) -> int | None:
+    if not isinstance(value, int) or isinstance(value, bool) or value < 0:
+        blockers.append({"code": "cost-entry-numeric", "index": index, "field": field, "value": value})
+        return None
+    return value
+
+
+def summarize_usage_confidence(entries: list[Any]) -> dict[str, int]:
+    summary = {
+        "attestedEntries": 0,
+        "attestedTokens": 0,
+        "estimatedEntries": 0,
+        "estimatedTokens": 0,
+        "missingEntries": 0,
+        "missingTokens": 0,
+        "unspecifiedEntries": 0,
+        "unspecifiedTokens": 0,
+    }
+    for entry in entries:
+        if not isinstance(entry, dict):
+            continue
+        confidence = entry.get("usageConfidence")
+        tokens = entry.get("tokens") if isinstance(entry.get("tokens"), int) else 0
+        if confidence == "ATTESTED":
+            summary["attestedEntries"] += 1
+            summary["attestedTokens"] += max(0, tokens)
+        elif confidence == "ESTIMATED":
+            summary["estimatedEntries"] += 1
+            summary["estimatedTokens"] += max(0, tokens)
+        elif confidence == "MISSING":
+            summary["missingEntries"] += 1
+            summary["missingTokens"] += max(0, tokens)
+        else:
+            summary["unspecifiedEntries"] += 1
+            summary["unspecifiedTokens"] += max(0, tokens)
+    return summary
+
+
+def _limits(raw_limits: Any, mode: str | None) -> dict[str, float | int]:
+    if mode is None:
+        return {}
+    defaults = dict(DEFAULT_MODE_LIMITS[mode])
+    if not isinstance(raw_limits, dict):
+        return defaults
+    override = raw_limits.get(mode)
+    if not isinstance(override, dict):
+        return defaults
+    for key, default in defaults.items():
+        value = override.get(key)
+        if isinstance(default, float) and isinstance(value, (int, float)) and not isinstance(value, bool):
+            defaults[key] = float(value)
+        elif isinstance(default, int) and isinstance(value, int) and not isinstance(value, bool):
+            defaults[key] = value
+    return defaults
+
+
+def cost_ratios(totals: dict[str, dict[str, int]]) -> dict[str, float]:
+    overall_tokens = totals["overall"]["tokens"]
+    overall_steps = totals["overall"]["steps"]
+    pipeline_tokens = totals["pipelineCompliance"]["tokens"]
+    pipeline_steps = totals["pipelineCompliance"]["steps"]
+    return {
+        "pipelineTokenShare": round(pipeline_tokens / overall_tokens, 6) if overall_tokens else 0.0,
+        "pipelineStepShare": round(pipeline_steps / overall_steps, 6) if overall_steps else 0.0,
+    }
+
+
+def _check_pipeline_limits(
+    totals: dict[str, dict[str, int]],
+    ratios: dict[str, float],
+    limits: dict[str, float | int],
+    over_limit_reason: Any,
+    blockers: list[dict[str, Any]],
+) -> None:
+    over = []
+    if ratios["pipelineTokenShare"] > float(limits["maxPipelineTokenShare"]):
+        over.append("maxPipelineTokenShare")
+    if ratios["pipelineStepShare"] > float(limits["maxPipelineStepShare"]):
+        over.append("maxPipelineStepShare")
+    if totals["pipelineCompliance"]["tokens"] > int(limits["maxPipelineTokens"]):
+        over.append("maxPipelineTokens")
+    if totals["pipelineCompliance"]["steps"] > int(limits["maxPipelineSteps"]):
+        over.append("maxPipelineSteps")
+    if not over:
+        return
+    if not isinstance(over_limit_reason, str) or not over_limit_reason.strip():
+        blockers.append({"code": "pipeline-compliance-over-limit", "limits": sorted(set(over))})