cortex-loop 0.1.0a1__py3-none-any.whl

cortex/requirements.py

@@ -0,0 +1,497 @@
+from __future__ import annotations
+
+import re
+import shlex
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Any, NamedTuple
+
+from .utils import _as_string_list, _normalize_repo_relative_path, _unique_list
+
+
+class RequirementAuditEvaluation(NamedTuple):
+    report: dict[str, Any] | None
+    details: dict[str, Any] | None
+    gap: bool
+    missing: bool
+    diagnostics: list[dict[str, Any]]
+    warnings: list[str]
+
+
+class TruthClaimsEvaluation(NamedTuple):
+    report: dict[str, Any] | None
+    gap: bool
+    diagnostics: list[dict[str, Any]]
+    warnings: list[str]
+
+
+def evaluate_requirement_audit_payload(
+    payload: Any,
+    *,
+    require_requirement_audit: bool,
+    require_evidence_for_passed_requirement: bool,
+    required_requirement_ids: list[str],
+    root: Path,
+    witness: Mapping[str, list[str]] | None = None,
+) -> RequirementAuditEvaluation:
+    if payload is None:
+        if require_requirement_audit:
+            return RequirementAuditEvaluation(
+                report=None,
+                details=None,
+                gap=False,
+                missing=True,
+                diagnostics=[_diagnostic([], ["requirement_audit with items covering required requirements"], "No requirement_audit was provided for a required requirement gate.", "comprehension_gap", "far")],
+                warnings=[
+                    "No requirement_audit provided in Stop payload. Include requirement_audit to prove "
+                    "prompt requirement coverage with evidence."
+                ],
+            )
+        return RequirementAuditEvaluation(
+            report=None,
+            details=None,
+            gap=False,
+            missing=False,
+            diagnostics=[],
+            warnings=[],
+        )
+
+    details = validate_requirement_audit(
+        payload,
+        require_evidence_for_passed_requirement=require_evidence_for_passed_requirement,
+        required_requirement_ids=required_requirement_ids,
+        root=root,
+        witness=witness,
+    )
+    warnings: list[str] = []
+    gap = not details["ok"]
+    if gap:
+        warnings.append("Requirement audit reported gaps: " + "; ".join(details.get("errors", [])))
+    warnings.extend(f"Requirement audit note: {note}" for note in details.get("warnings", []))
+    return RequirementAuditEvaluation(
+        report=minimal_requirement_audit_report(details),
+        details=details,
+        gap=gap,
+        missing=False,
+        diagnostics=[dict(item) for item in details.get("diagnostics", [])],
+        warnings=warnings,
+    )
+
+
+def evaluate_truth_claims_payload(
+    payload: Any,
+    *,
+    root: Path,
+    witness: Mapping[str, list[str]] | None = None,
+    observed_modified_files: list[str] | None = None,
+    modified_files_error: str | None = None,
+) -> TruthClaimsEvaluation:
+    if payload is None:
+        return TruthClaimsEvaluation(report=None, gap=False, diagnostics=[], warnings=[])
+
+    if not isinstance(payload, Mapping):
+        report = {
+            "ok": False,
+            "modified_files_claimed": [],
+            "modified_files_verified": [],
+            "modified_files_unverified": [],
+            "modified_files_uncheckable": [],
+            "tests_ran_claimed": [],
+            "tests_ran_verified": [],
+            "tests_ran_unverified": [],
+            "tests_ran_uncheckable": [],
+            "errors": ["Invalid truth_claims format; expected an object."],
+            "warnings": [],
+        }
+        return TruthClaimsEvaluation(
+            report=report,
+            gap=True,
+            diagnostics=[_diagnostic(["truth_claims payload with invalid shape"], ["truth_claims object"], "Truth claims used an invalid payload shape.", "comprehension_gap", "far")],
+            warnings=["Truth claims reported gaps: Invalid truth_claims format; expected an object."],
+        )
+
+    modified_claims = _normalize_modified_file_claims(payload.get("modified_files"), root=root.resolve())
+    tests_ran_claims = _unique_list(
+        [normalized for value in _as_string_list(payload.get("tests_ran")) if (normalized := _normalize_command(value))]
+    )
+    report_warnings: list[str] = []
+    errors: list[str] = []
+    diagnostics: list[dict[str, Any]] = []
+
+    modified_verified: list[str] = []
+    modified_unverified: list[str] = []
+    modified_uncheckable: list[str] = []
+    tests_ran_verified: list[str] = []
+    tests_ran_unverified: list[str] = []
+    tests_ran_uncheckable: list[str] = []
+
+    if not modified_claims and not tests_ran_claims:
+        report_warnings.append(
+            "truth_claims provided with no supported claims; expected modified_files and/or tests_ran."
+        )
+
+    if modified_claims:
+        changed_files = set(_as_string_list(observed_modified_files)) if observed_modified_files is not None else None
+        if changed_files is None:
+            modified_uncheckable = list(modified_claims)
+            error_reason = str(modified_files_error or "").strip()
+            report_warnings.append(
+                "truth_claims.modified_files are uncheckable: "
+                + (error_reason or "session-scoped modified-files evidence unavailable")
+            )
+        else:
+            for claimed_path in modified_claims:
+                if claimed_path in changed_files:
+                    modified_verified.append(claimed_path)
+                else:
+                    modified_unverified.append(claimed_path)
+            if modified_unverified:
+                errors.append(
+                    "truth_claims.modified_files not observed in repository changes: "
+                    + ", ".join(modified_unverified)
+                )
+                diagnostics.append(_diagnostic(modified_unverified, modified_claims, "Some claimed modified files were not observed in the session-scoped file delta.", "execution_gap", "moderate"))
+
+    if tests_ran_claims:
+        observed_commands = _unique_list(
+            [
+                normalized
+                for value in _as_string_list((witness or {}).get("commands"))
+                if (normalized := _normalize_command(value))
+            ]
+        )
+        if not observed_commands:
+            tests_ran_uncheckable = list(tests_ran_claims)
+            report_warnings.append("truth_claims.tests_ran are uncheckable: no observed command events in session")
+        else:
+            for claim in tests_ran_claims:
+                if any(_command_claim_matches(claim, observed) for observed in observed_commands):
+                    tests_ran_verified.append(claim)
+                else:
+                    tests_ran_unverified.append(claim)
+            if tests_ran_unverified:
+                errors.append(
+                    "truth_claims.tests_ran not witnessed in session events: " + ", ".join(tests_ran_unverified)
+                )
+                diagnostics.append(_diagnostic(tests_ran_verified, tests_ran_claims, "Some claimed test commands were not witnessed in session events.", "execution_gap", "moderate"))
+
+    report = {
+        "ok": len(errors) == 0,
+        "modified_files_claimed": modified_claims,
+        "modified_files_verified": modified_verified,
+        "modified_files_unverified": modified_unverified,
+        "modified_files_uncheckable": modified_uncheckable,
+        "tests_ran_claimed": tests_ran_claims,
+        "tests_ran_verified": tests_ran_verified,
+        "tests_ran_unverified": tests_ran_unverified,
+        "tests_ran_uncheckable": tests_ran_uncheckable,
+        "diagnostics": diagnostics,
+        "errors": errors,
+        "warnings": report_warnings,
+    }
+    warnings: list[str] = []
+    if errors:
+        warnings.append("Truth claims reported gaps: " + "; ".join(errors))
+    warnings.extend(f"Truth claims note: {warning}" for warning in report_warnings)
+    return TruthClaimsEvaluation(
+        report=report,
+        gap=bool(errors),
+        diagnostics=diagnostics,
+        warnings=warnings,
+    )
+
+
+def minimal_requirement_audit_report(report: Mapping[str, Any]) -> dict[str, Any]:
+    return {
+        "ok": bool(report.get("ok")),
+        "errors": [str(v) for v in _as_string_list(report.get("errors"))],
+        "missing_required_ids": [str(v) for v in _as_string_list(report.get("missing_required_ids"))],
+        "item_count": int(report.get("item_count") or 0),
+        "pass_count": int(report.get("pass_count") or 0),
+        "fail_count": int(report.get("fail_count") or 0),
+        "diagnostics": [dict(item) for item in report.get("diagnostics", []) if isinstance(item, Mapping)],
+    }
+
+
+def _diagnostic(
+    evidence_found: list[str],
+    evidence_expected: list[str],
+    gap_description: str,
+    gap_characterization: str,
+    distance_signal: str,
+) -> dict[str, Any]:
+    return {
+        "evidence_found": [str(item) for item in evidence_found if str(item).strip()],
+        "evidence_expected": [str(item) for item in evidence_expected if str(item).strip()],
+        "gap_description": gap_description,
+        "gap_characterization": gap_characterization,
+        "distance_signal": distance_signal,
+    }
+
+
+def evaluate_evidence_reference(
+    reference: str,
+    *,
+    root: Path,
+    witness: Mapping[str, list[str]] | None = None,
+) -> dict[str, str]:
+    observed_commands = [_normalize_command(v) for v in _as_string_list((witness or {}).get("commands"))]
+    observed_tools = {v.lower() for v in _as_string_list((witness or {}).get("tools"))}
+    return _evaluate_evidence_reference(
+        reference,
+        root=root,
+        observed_commands=observed_commands,
+        observed_tools=observed_tools,
+    )
+
+
+def validate_requirement_audit(
+    payload: Any,
+    *,
+    require_evidence_for_passed_requirement: bool,
+    required_requirement_ids: list[str],
+    root: Path,
+    witness: Mapping[str, list[str]] | None = None,
+) -> dict[str, Any]:
+    errors: list[str] = []
+    warnings: list[str] = []
+    diagnostics: list[dict[str, Any]] = []
+    pass_count = 0
+    fail_count = 0
+    unique_ids: set[str] = set()
+    missing_required_ids: list[str] = []
+    observed_commands = [_normalize_command(v) for v in _as_string_list((witness or {}).get("commands"))]
+    observed_tools = {v.lower() for v in _as_string_list((witness or {}).get("tools"))}
+
+    if not isinstance(payload, Mapping):
+        return {
+            "ok": False,
+            "item_count": 0,
+            "pass_count": 0,
+            "fail_count": 0,
+            "missing_required_ids": list(required_requirement_ids),
+            "diagnostics": [_diagnostic(["requirement_audit payload with invalid shape"], ["requirement_audit object with items"], "Requirement audit used an invalid payload shape.", "comprehension_gap", "far")],
+            "warnings": [],
+            "errors": ["Invalid requirement_audit format; expected an object."],
+        }
+
+    items = payload.get("items") if isinstance(payload.get("items"), list) else []
+    if not items:
+        errors.append("requirement_audit.items must be a non-empty list.")
+        diagnostics.append(_diagnostic([], ["requirement_audit.items with at least one requirement entry"], "Requirement audit did not provide any requirement items.", "comprehension_gap", "far"))
+
+    for idx, item in enumerate(items):
+        if not isinstance(item, Mapping):
+            errors.append(f"requirement_audit.items[{idx}] must be an object.")
+            diagnostics.append(_diagnostic([f"items[{idx}]={type(item).__name__}"], [f"items[{idx}] as requirement object"], "Requirement audit item used an invalid shape.", "comprehension_gap", "far"))
+            continue
+        item_id = str(item.get("id") or "").strip()
+        if not item_id:
+            errors.append(f"requirement_audit.items[{idx}] is missing a non-empty id.")
+            diagnostics.append(_diagnostic([f"items[{idx}] without id"], [f"items[{idx}].id"], "Requirement audit item is missing its requirement id.", "comprehension_gap", "far"))
+            continue
+        if item_id in unique_ids:
+            errors.append(f"requirement_audit.items[{idx}] has duplicate id '{item_id}'.")
+            diagnostics.append(_diagnostic([item_id], ["unique requirement ids"], f"Requirement '{item_id}' was reported more than once.", "comprehension_gap", "moderate"))
+        unique_ids.add(item_id)
+
+        status = str(item.get("status") or "").strip().lower()
+        if status == "pass":
+            pass_count += 1
+            evidence = _as_string_list(item.get("evidence"))
+            if require_evidence_for_passed_requirement and not evidence:
+                errors.append(f"requirement '{item_id}' is pass but has no evidence.")
+                diagnostics.append(_diagnostic([f"requirement '{item_id}' marked pass without evidence"], [f"verified evidence for requirement '{item_id}'"], f"Requirement '{item_id}' was marked pass without evidence.", "execution_gap", "close"))
+            for evidence_ref in evidence:
+                check = _evaluate_evidence_reference(
+                    evidence_ref,
+                    root=root,
+                    observed_commands=observed_commands,
+                    observed_tools=observed_tools,
+                )
+                if check["status"] == "unverified":
+                    errors.append(f"requirement '{item_id}' evidence is unverified: {check['reason']}")
+                    diagnostics.append(_diagnostic([evidence_ref], [f"verified evidence for requirement '{item_id}'"], f"Requirement '{item_id}' evidence could not be verified.", "execution_gap", "moderate"))
+                elif check["status"] == "uncheckable":
+                    warnings.append(f"requirement '{item_id}' evidence is uncheckable: {check['reason']}")
+        elif status == "fail":
+            fail_count += 1
+            if not str(item.get("gap") or "").strip():
+                errors.append(f"requirement '{item_id}' is fail but has no gap description.")
+                diagnostics.append(_diagnostic([f"requirement '{item_id}' marked fail"], [f"gap description for requirement '{item_id}'"], f"Requirement '{item_id}' failed without describing the gap.", "execution_gap", "close"))
+        else:
+            errors.append(f"requirement '{item_id}' has invalid status '{status}' (expected pass|fail).")
+            diagnostics.append(_diagnostic([f"requirement '{item_id}' status={status or 'missing'}"], ["status=pass|fail"], f"Requirement '{item_id}' used an invalid status value.", "comprehension_gap", "far"))
+
+    if required_requirement_ids:
+        missing_required_ids = [rid for rid in required_requirement_ids if rid not in unique_ids]
+        if missing_required_ids:
+            errors.append("requirement_audit missing required ids: " + ", ".join(missing_required_ids))
+            diagnostics.append(_diagnostic(sorted(unique_ids), required_requirement_ids, "Requirement audit omitted required requirement ids from the session contract.", "comprehension_gap", "moderate"))
+
+    expected_verdict = "pass" if (items and fail_count == 0 and not errors) else "fail"
+    completeness_verdict = payload.get("completeness_verdict")
+    if completeness_verdict is not None:
+        normalized_verdict = str(completeness_verdict).strip().lower()
+        if normalized_verdict not in {"pass", "fail"}:
+            errors.append("requirement_audit.completeness_verdict must be 'pass' or 'fail'.")
+            diagnostics.append(_diagnostic([f"completeness_verdict={normalized_verdict}"], ["completeness_verdict=pass|fail"], "Requirement audit used an invalid completeness verdict value.", "comprehension_gap", "far"))
+        elif normalized_verdict != expected_verdict:
+            errors.append(
+                f"requirement_audit.completeness_verdict={normalized_verdict} "
+                f"does not match computed verdict={expected_verdict}."
+            )
+            diagnostics.append(_diagnostic([f"completeness_verdict={normalized_verdict}"], [f"completeness_verdict={expected_verdict}"], "Requirement audit completeness verdict did not match the computed result.", "execution_gap", "close"))
+
+    return {
+        "ok": len(errors) == 0 and fail_count == 0,
+        "item_count": len(items),
+        "pass_count": pass_count,
+        "fail_count": fail_count,
+        "missing_required_ids": missing_required_ids,
+        "diagnostics": diagnostics,
+        "warnings": warnings,
+        "errors": errors,
+    }
+
+
+def _evaluate_evidence_reference(
+    reference: str,
+    *,
+    root: Path,
+    observed_commands: list[str],
+    observed_tools: set[str],
+) -> dict[str, str]:
+    kind, claim = _classify_evidence_reference(reference, root=root)
+    if kind == "path":
+        path = _evidence_reference_path(reference, root)
+        if path is not None and path.exists():
+            return {"kind": "path", "status": "verified", "reason": "path exists"}
+        return {
+            "kind": "path",
+            "status": "unverified",
+            "reason": f"path does not exist: {path}",
+        }
+
+    if kind == "tool":
+        if not observed_tools:
+            return {"kind": "tool", "status": "uncheckable", "reason": "no observed tool events in session"}
+        if claim in observed_tools:
+            return {"kind": "tool", "status": "verified", "reason": f"tool observed: {claim}"}
+        return {"kind": "tool", "status": "unverified", "reason": f"tool not observed: {claim}"}
+
+    if kind == "command":
+        if not observed_commands:
+            return {
+                "kind": "command",
+                "status": "uncheckable",
+                "reason": "no observed command events in session",
+            }
+        if claim and any(_command_claim_matches(claim, cmd) for cmd in observed_commands):
+            return {"kind": "command", "status": "verified", "reason": "command matched session witness"}
+        return {
+            "kind": "command",
+            "status": "unverified",
+            "reason": "command not witnessed in session events",
+        }
+
+    return {"kind": "note", "status": "uncheckable", "reason": "reference is non-verifiable note text"}
+
+
+def _classify_evidence_reference(reference: str, *, root: Path) -> tuple[str, str]:
+    text = str(reference).strip()
+    lower = text.lower()
+    if lower.startswith("tool:"):
+        return "tool", lower.split(":", 1)[1].strip()
+    if lower.startswith("cmd:"):
+        return "command", _normalize_command(text.split(":", 1)[1].strip())
+    if _looks_like_command(text):
+        return "command", _normalize_command(text)
+    if _evidence_reference_path(text, root) is not None:
+        return "path", text
+    return "note", ""
+
+
+def _evidence_reference_path(reference: str, root: Path) -> Path | None:
+    text = str(reference).strip()
+    if not text or text.startswith(("http://", "https://")):
+        return None
+
+    path_text = text.split("#", 1)[0].strip()
+    if " " in path_text:
+        first_token = path_text.split(None, 1)[0].strip()
+        if first_token and (
+            any(sep in first_token for sep in ("/", "\\"))
+            or first_token.startswith((".", "~"))
+        ):
+            path_text = first_token
+    path_text = re.sub(r":\d+(?::\d+|-\d+)?$", "", path_text).strip()
+    path_text = path_text.rstrip(".,;:")
+    if not path_text:
+        return None
+    if not any(sep in path_text for sep in ("/", "\\")) and not path_text.startswith((".", "~")):
+        return None
+
+    path = Path(path_text).expanduser()
+    if not path.is_absolute():
+        path = root / path
+    return path.resolve()
+
+
+def _looks_like_command(text: str) -> bool:
+    return bool(
+        re.search(
+            r"\b(pytest|npm|pnpm|yarn|npx|ruff|mypy|go test|cargo test|python\s+-m)\b",
+            text.lower(),
+        )
+    )
+
+
+def _normalize_command(text: str) -> str:
+    value = text.strip().strip("`").lower()
+    value = re.sub(r"[.,;:!?]+$", "", value).strip()
+    value = re.sub(r"\s+", " ", value)
+    value = re.sub(r"[:\-]\s*(ok|pass|passed|success|succeeded)$", "", value).strip()
+    return value
+
+
+def _command_claim_matches(claim: str, observed: str) -> bool:
+    claim_tokens = _command_tokens(claim)
+    observed_tokens = _command_tokens(observed)
+    if not claim_tokens or not observed_tokens:
+        return False
+    return claim_tokens == observed_tokens or (
+        len(claim_tokens) <= len(observed_tokens) and observed_tokens[: len(claim_tokens)] == claim_tokens
+    )
+
+
+def _command_tokens(command: str) -> tuple[str, ...]:
+    normalized = _normalize_command(command)
+    if not normalized:
+        return ()
+    try:
+        raw_tokens = shlex.split(normalized)
+    except ValueError:
+        raw_tokens = normalized.split()
+    tokens = [token.strip().lower() for token in raw_tokens if token.strip()]
+    if not tokens:
+        return ()
+
+    while True:
+        if len(tokens) >= 3 and tokens[0] in {"bash", "sh"} and tokens[1] == "-lc":
+            return _command_tokens(" ".join(tokens[2:]))
+        if len(tokens) >= 2 and tuple(tokens[:2]) in {("uv", "run"), ("poetry", "run"), ("pipenv", "run")}:
+            tokens = tokens[2:]
+            continue
+        if len(tokens) >= 3 and tokens[0] in {"python", "python3", "py"} and tokens[1:3] == ["-m", "pytest"]:
+            tokens = ["pytest", *tokens[3:]]
+            continue
+        break
+
+    return tuple(tokens)
+
+
+def _normalize_modified_file_claims(value: Any, *, root: Path) -> list[str]:
+    return _unique_list(
+        [normalized for raw in _as_string_list(value) if (normalized := _normalize_repo_relative_path(raw, root=root))]
+    )