algo-cli-runtime 0.14.0__py3-none-any.whl

algo_cli/evals/performance_regression.py

@@ -0,0 +1,136 @@
+"""CUSUM detection for sustained runtime performance changes (ALGO.md L3)."""
+
+from __future__ import annotations
+
+import math
+import statistics
+from dataclasses import asdict, dataclass
+from enum import Enum
+from typing import Iterable
+
+
+class RegressionState(str, Enum):
+    INSUFFICIENT_DATA = "insufficient_data"
+    STABLE = "stable"
+    IMPROVING = "improving"
+    REGRESSING = "regressing"
+
+
+@dataclass(frozen=True)
+class CUSUMResult:
+    state: RegressionState
+    sample_count: int
+    baseline: float | None
+    scale: float | None
+    positive_score: float
+    negative_score: float
+    change_index: int | None
+    reason: str
+
+    def to_dict(self) -> dict[str, object]:
+        payload = asdict(self)
+        payload["state"] = self.state.value
+        return payload
+
+
+def detect_cusum(
+    samples: Iterable[float],
+    *,
+    warmup: int = 5,
+    slack_scale: float = 0.5,
+    threshold_scale: float = 5.0,
+    min_consecutive: int = 2,
+) -> CUSUMResult:
+    """Detect a sustained upward/downward shift from a robust warmup baseline."""
+    values: list[float] = []
+    for sample in samples:
+        try:
+            value = float(sample)
+        except (TypeError, ValueError):
+            continue
+        if math.isfinite(value) and value >= 0.0:
+            values.append(value)
+    warmup = max(3, int(warmup))
+    min_consecutive = max(2, int(min_consecutive))
+    if len(values) < warmup + min_consecutive:
+        return CUSUMResult(
+            RegressionState.INSUFFICIENT_DATA,
+            len(values),
+            None,
+            None,
+            0.0,
+            0.0,
+            None,
+            f"need at least {warmup + min_consecutive} finite samples",
+        )
+
+    baseline_values = values[:warmup]
+    baseline = statistics.median(baseline_values)
+    deviations = [abs(value - baseline) for value in baseline_values]
+    robust_scale = statistics.median(deviations) * 1.4826
+    scale = max(robust_scale, abs(baseline) * 0.02, 1e-6)
+    slack = max(0.0, float(slack_scale)) * scale
+    threshold = max(1.0, float(threshold_scale)) * scale
+
+    positive = 0.0
+    negative = 0.0
+    positive_run = 0
+    negative_run = 0
+    positive_start: int | None = None
+    negative_start: int | None = None
+    for index, value in enumerate(values[warmup:], start=warmup):
+        delta = value - baseline
+        positive = max(0.0, positive + delta - slack)
+        negative = min(0.0, negative + delta + slack)
+
+        if delta > slack:
+            positive_run += 1
+            positive_start = index if positive_start is None else positive_start
+        else:
+            positive = 0.0
+            positive_run = 0
+            positive_start = None
+        if delta < -slack:
+            negative_run += 1
+            negative_start = index if negative_start is None else negative_start
+        else:
+            negative = 0.0
+            negative_run = 0
+            negative_start = None
+
+        if positive >= threshold and positive_run >= min_consecutive:
+            return CUSUMResult(
+                RegressionState.REGRESSING,
+                len(values),
+                baseline,
+                scale,
+                positive,
+                negative,
+                positive_start,
+                "sustained latency increase crossed the CUSUM decision threshold",
+            )
+        if abs(negative) >= threshold and negative_run >= min_consecutive:
+            return CUSUMResult(
+                RegressionState.IMPROVING,
+                len(values),
+                baseline,
+                scale,
+                positive,
+                negative,
+                negative_start,
+                "sustained latency decrease crossed the CUSUM decision threshold",
+            )
+
+    return CUSUMResult(
+        RegressionState.STABLE,
+        len(values),
+        baseline,
+        scale,
+        positive,
+        negative,
+        None,
+        "no sustained shift crossed the CUSUM decision threshold",
+    )
+
+
+__all__ = ["CUSUMResult", "RegressionState", "detect_cusum"]

algo_cli/evals/scorecard_grading.py

@@ -0,0 +1,308 @@
+"""Pure, fail-closed grading for the Algo CLI harness scorecard."""
+
+from __future__ import annotations
+
+import json
+import math
+from collections.abc import Iterable, Mapping
+from typing import Any
+
+
+SCHEMA_VERSION = 2
+SCORED_GATE_COUNT = 10
+MAX_SCORE = float(SCORED_GATE_COUNT)
+VALID_STATUSES = frozenset({"pass", "warn", "unavailable", "fail", "error"})
+STATUS_POINTS = {
+    "pass": 1.0,
+    "warn": 0.5,
+    "unavailable": 0.0,
+    "fail": 0.0,
+    "error": 0.0,
+}
+
+
+def _safe_text(value: Any) -> str:
+    if value is None:
+        return ""
+    try:
+        return str(value)
+    except Exception:
+        return f"<{type(value).__name__}>"
+
+
+def _json_safe(value: Any, *, _seen: set[int] | None = None, _depth: int = 0) -> Any:
+    """Return a JSON-compatible copy without trusting arbitrary evidence objects."""
+    if value is None or isinstance(value, (str, bool, int)):
+        return value
+    if isinstance(value, float):
+        return value if math.isfinite(value) else None
+    if _depth >= 8:
+        return _safe_text(value)
+
+    seen = _seen if _seen is not None else set()
+    identity = id(value)
+    if identity in seen:
+        return "<recursive>"
+
+    if isinstance(value, Mapping):
+        seen.add(identity)
+        try:
+            return {
+                _safe_text(key): _json_safe(item, _seen=seen, _depth=_depth + 1)
+                for key, item in value.items()
+            }
+        except Exception:
+            return _safe_text(value)
+        finally:
+            seen.discard(identity)
+    if isinstance(value, (list, tuple)):
+        seen.add(identity)
+        try:
+            return [
+                _json_safe(item, _seen=seen, _depth=_depth + 1)
+                for item in value
+            ]
+        except Exception:
+            return _safe_text(value)
+        finally:
+            seen.discard(identity)
+    if isinstance(value, (set, frozenset)):
+        seen.add(identity)
+        try:
+            items = [_json_safe(item, _seen=seen, _depth=_depth + 1) for item in value]
+            return sorted(items, key=_safe_text)
+        except Exception:
+            return _safe_text(value)
+        finally:
+            seen.discard(identity)
+    return _safe_text(value)
+
+
+def _evidence_text(value: Any) -> str:
+    safe = _json_safe(value)
+    if isinstance(safe, (dict, list)):
+        try:
+            return json.dumps(safe, ensure_ascii=False, sort_keys=True)
+        except (TypeError, ValueError):
+            return _safe_text(safe)
+    return _safe_text(safe)
+
+
+def _has_structured_evidence(value: Any) -> bool:
+    if value is None:
+        return False
+    if isinstance(value, (str, bytes, Mapping, list, tuple, set, frozenset)):
+        return bool(value)
+    return True
+
+
+def _materialize(values: Iterable[Any] | None, *, label: str) -> tuple[list[Any], str | None]:
+    if values is None:
+        return [], f"{label} must be an iterable, not null"
+    if isinstance(values, (str, bytes, Mapping)):
+        return [], f"{label} must be an iterable of mappings"
+    try:
+        return list(values), None
+    except Exception as exc:
+        return [], f"could not read {label}: {_safe_text(exc)}"
+
+
+def _normalize_check(raw: Any, position: int) -> tuple[dict[str, Any], list[str]]:
+    errors: list[str] = []
+    if not isinstance(raw, Mapping):
+        return (
+            {
+                "name": f"invalid-gate-{position + 1}",
+                "status": "error",
+                "critical": True,
+                "evidence": f"gate definition is {type(raw).__name__}, not a mapping",
+                "recommendation": "Provide a mapping with name, status, and evidence.",
+            },
+            [f"gate {position + 1} is not a mapping"],
+        )
+
+    name = _safe_text(raw.get("name")).strip()
+    if not name:
+        name = f"unnamed-gate-{position + 1}"
+        errors.append(f"gate {position + 1} is missing a name")
+
+    raw_status = raw.get("status")
+    status = _safe_text(raw_status).strip().lower()
+    if status not in VALID_STATUSES:
+        errors.append(f"gate {name!r} has invalid status {status or '<missing>'!r}")
+        status = "error"
+
+    raw_critical = raw.get("critical", False)
+    if isinstance(raw_critical, bool):
+        critical = raw_critical
+    else:
+        errors.append(f"gate {name!r} critical must be a boolean")
+        critical = True
+        status = "error"
+
+    evidence = _evidence_text(raw.get("evidence", "")).strip()
+    recommendation = _evidence_text(raw.get("recommendation", "")).strip()
+    metrics_present = "metrics" in raw
+    metrics = _json_safe(raw.get("metrics")) if metrics_present else None
+
+    if status in {"pass", "warn"} and not evidence and not _has_structured_evidence(metrics):
+        errors.append(f"gate {name!r} cannot be {status} without evidence")
+        status = "error"
+        critical = True
+        recommendation = recommendation or "Attach human-readable evidence or structured metrics."
+
+    if errors:
+        critical = True
+
+    normalized: dict[str, Any] = {
+        "name": name,
+        "status": status,
+        "critical": critical,
+        "evidence": evidence,
+        "recommendation": recommendation,
+    }
+    if metrics_present:
+        normalized["metrics"] = metrics
+    return normalized, errors
+
+
+def _normalize_capability(raw: Any, position: int) -> dict[str, Any]:
+    if not isinstance(raw, Mapping):
+        return {
+            "name": f"invalid-capability-{position + 1}",
+            "status": "error",
+            "evidence": f"capability definition is {type(raw).__name__}, not a mapping",
+            "recommendation": "Provide a mapping for this optional capability.",
+            "scored": False,
+            "points": 0.0,
+            "max_points": 0.0,
+        }
+
+    aliases = {
+        "ready": "pass",
+        "degraded": "warn",
+        "disabled": "unavailable",
+        "blocked": "fail",
+    }
+    raw_status = _safe_text(raw.get("status", "unavailable")).strip().lower()
+    status = aliases.get(raw_status, raw_status)
+    if status not in VALID_STATUSES:
+        status = "error"
+    normalized: dict[str, Any] = {
+        "name": _safe_text(raw.get("name")).strip() or f"capability-{position + 1}",
+        "status": status,
+        "evidence": _evidence_text(raw.get("evidence", "")).strip(),
+        "recommendation": _evidence_text(raw.get("recommendation", "")).strip(),
+        "scored": False,
+        "points": 0.0,
+        "max_points": 0.0,
+    }
+    if "metrics" in raw:
+        normalized["metrics"] = _json_safe(raw.get("metrics"))
+    return normalized
+
+
+def finalize_scorecard(
+    checks: Iterable[Mapping[str, Any]] | None,
+    capabilities: Iterable[Mapping[str, Any]] | None = (),
+) -> dict[str, Any]:
+    """Finalize exactly ten evidence-backed gates into a JSON-ready scorecard.
+
+    Optional capabilities are normalized for display but never affect points or
+    overall status. Invalid scored-gate schemas fail closed and are returned as
+    validation errors instead of raising.
+    """
+    raw_checks, collection_error = _materialize(checks, label="checks")
+    validation_errors = [collection_error] if collection_error else []
+    normalized_checks: list[dict[str, Any]] = []
+    for position, raw in enumerate(raw_checks):
+        normalized, errors = _normalize_check(raw, position)
+        normalized_checks.append(normalized)
+        validation_errors.extend(errors)
+
+    if len(normalized_checks) != SCORED_GATE_COUNT:
+        validation_errors.append(
+            f"expected exactly {SCORED_GATE_COUNT} scored gates, got {len(normalized_checks)}"
+        )
+        # With the wrong denominator there is no meaningful partial grade.
+        # Preserve the definitions for diagnosis but fail every supplied gate
+        # closed so the reported score remains the sum of normalized points.
+        for check in normalized_checks:
+            check["status"] = "error"
+            check["critical"] = True
+            if not check["recommendation"]:
+                check["recommendation"] = "Fix the scorecard gate schema before grading."
+
+    names: dict[str, list[int]] = {}
+    for position, check in enumerate(normalized_checks):
+        names.setdefault(check["name"].casefold(), []).append(position)
+    for positions in names.values():
+        if len(positions) < 2:
+            continue
+        duplicate_name = normalized_checks[positions[0]]["name"]
+        validation_errors.append(f"duplicate scored gate name: {duplicate_name!r}")
+        for position in positions:
+            normalized_checks[position]["status"] = "error"
+            normalized_checks[position]["critical"] = True
+            if not normalized_checks[position]["recommendation"]:
+                normalized_checks[position]["recommendation"] = "Use a unique scored gate name."
+
+    for check in normalized_checks:
+        check["points"] = STATUS_POINTS[check["status"]]
+        check["max_points"] = 1.0
+        check["scored"] = True
+
+    raw_capabilities, capability_error = _materialize(capabilities, label="capabilities")
+    normalized_capabilities = [
+        _normalize_capability(raw, position)
+        for position, raw in enumerate(raw_capabilities)
+    ]
+    if capability_error:
+        normalized_capabilities.append(
+            _normalize_capability(
+                {
+                    "name": "capability-schema",
+                    "status": "error",
+                    "evidence": capability_error,
+                },
+                len(normalized_capabilities),
+            )
+        )
+
+    score = float(sum(float(check["points"]) for check in normalized_checks))
+    critical_failure = any(
+        check["critical"] and check["status"] in {"fail", "error"}
+        for check in normalized_checks
+    )
+    all_pass = (
+        not validation_errors
+        and len(normalized_checks) == SCORED_GATE_COUNT
+        and all(check["status"] == "pass" for check in normalized_checks)
+    )
+    if all_pass and score == MAX_SCORE:
+        overall_status = "ready"
+    elif validation_errors or critical_failure:
+        overall_status = "blocked"
+    else:
+        overall_status = "degraded"
+
+    return {
+        "schema_version": SCHEMA_VERSION,
+        "score": score,
+        "max_score": MAX_SCORE,
+        "overall_status": overall_status,
+        "scored_gate_count": len(normalized_checks),
+        "checks": normalized_checks,
+        "capabilities": normalized_capabilities,
+        "validation_errors": validation_errors,
+    }
+
+
+__all__ = [
+    "MAX_SCORE",
+    "SCHEMA_VERSION",
+    "SCORED_GATE_COUNT",
+    "STATUS_POINTS",
+    "VALID_STATUSES",
+    "finalize_scorecard",
+]

algo_cli/evals/session_distribution.py

@@ -0,0 +1,84 @@
+"""Session distribution analysis (I11).
+
+Fable-5 traces are heavy-tailed: the top five sessions carried roughly a third
+of the corpus. This module makes that bias visible for harness stats and
+session-summary reporting.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class SessionDistribution:
+    session_count: int
+    total_rows: int
+    largest_session: int
+    median_session: float
+    top_sessions: tuple[tuple[str, int], ...]
+    heavy_tail: bool
+    effective_top_n: int
+    summary: str
+
+    def top_n_share(self, n: int) -> float:
+        if self.total_rows <= 0 or n <= 0:
+            return 0.0
+        return sum(count for _name, count in self.top_sessions[:n]) / self.total_rows
+
+    def to_dict(self) -> dict:
+        return {
+            "session_count": self.session_count,
+            "total_rows": self.total_rows,
+            "largest_session": self.largest_session,
+            "median_session": self.median_session,
+            "top_sessions": list(self.top_sessions),
+            "heavy_tail": self.heavy_tail,
+            "effective_top_n": self.effective_top_n,
+            "top_share": round(self.top_n_share(self.effective_top_n), 3),
+            "top5_share": round(self.top_n_share(5), 3),
+            "summary": self.summary,
+        }
+
+
+def _median(values: list[int]) -> float:
+    if not values:
+        return 0.0
+    ordered = sorted(values)
+    mid = len(ordered) // 2
+    if len(ordered) % 2:
+        return float(ordered[mid])
+    return (ordered[mid - 1] + ordered[mid]) / 2
+
+
+def summarize_session_distribution(session_counts: dict[str, int], *, heavy_tail_top_n: int = 5, heavy_tail_share: float = 0.33) -> SessionDistribution:
+    """Summarize per-session row counts and flag heavy-tail risk.
+
+    For large corpora this uses the Fable-5 top-5 concentration rule. For very
+    small corpora, ``top_n`` is adapted to roughly the largest half of sessions
+    so a four-session corpus can still reveal obvious concentration.
+    """
+    cleaned = {str(name): max(0, int(count)) for name, count in (session_counts or {}).items()}
+    sorted_counts = tuple(sorted(cleaned.items(), key=lambda item: item[1], reverse=True))
+    counts = [count for _name, count in sorted_counts]
+    total = sum(counts)
+    effective_top_n = min(max(1, heavy_tail_top_n), max(1, len(sorted_counts) // 2)) if sorted_counts else 0
+    top_share = 0.0 if total <= 0 or effective_top_n <= 0 else sum(count for _name, count in sorted_counts[:effective_top_n]) / total
+    heavy = len(sorted_counts) > 1 and top_share >= heavy_tail_share
+    summary = (
+        f"sessions={len(sorted_counts)}, rows={total}, largest={counts[0] if counts else 0}, "
+        f"median={_median(counts):.1f}, top{effective_top_n}_share={top_share:.2f}, "
+        f"heavy_tail={heavy}"
+    )
+    return SessionDistribution(
+        session_count=len(sorted_counts),
+        total_rows=total,
+        largest_session=counts[0] if counts else 0,
+        median_session=_median(counts),
+        top_sessions=sorted_counts,
+        heavy_tail=heavy,
+        effective_top_n=effective_top_n,
+        summary=summary,
+    )
+
+
+__all__ = ["SessionDistribution", "summarize_session_distribution"]