failloud 0.2.0__py3-none-any.whl

failloud/__init__.py

@@ -0,0 +1,51 @@
+"""FailLoud — a verification harness for LLM agent outputs.
+
+Agents rarely fail loudly. They return ``status: done`` with an empty
+payload, claim an action no artifact backs, plan from state that went stale
+two steps ago, report counts their own itemized record contradicts, and
+ship deliverables with silent holes in them. FailLoud reads the output the
+way a skeptical reviewer would and produces a failure report instead of a
+shrug.
+
+Quick start::
+
+    import json
+    from failloud import check_output
+
+    result = check_output(json.load(open("agent_output.json")))
+    if not result.passed:
+        print(result.verdict)  # "FAIL"
+        for finding in result.errors:
+            print(finding.check_id, finding.path, finding.message)
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from .engine import run_checks, run_single
+from .models import CheckConfig, CheckOutcome, CheckResult, Finding, Severity
+from .receipts import Receipt, create_receipt, verify_chain, verify_receipt
+
+__version__ = "0.2.0"
+
+__all__ = [
+    "check_output",
+    "run_checks",
+    "run_single",
+    "CheckConfig",
+    "CheckOutcome",
+    "CheckResult",
+    "Finding",
+    "Severity",
+    "Receipt",
+    "create_receipt",
+    "verify_receipt",
+    "verify_chain",
+    "__version__",
+]
+
+
+def check_output(doc: Any, config: CheckConfig | None = None) -> CheckResult:
+    """Verify one agent output document. The main entry point."""
+    return run_checks(doc, config)

failloud/checks/__init__.py

@@ -0,0 +1,17 @@
+"""The check registry.
+
+Each module in this package detects one class of silent failure and exposes
+``CHECK_ID``, ``NAME``, ``DESCRIPTION`` and ``check(doc, config)``.
+"""
+
+from __future__ import annotations
+
+from types import ModuleType
+
+from . import completeness, counts, evidence, placeholder, stale
+
+MODULES: list[ModuleType] = [placeholder, evidence, stale, counts, completeness]
+
+BY_ID: dict[str, ModuleType] = {m.CHECK_ID: m for m in MODULES}
+
+__all__ = ["MODULES", "BY_ID"]

failloud/checks/completeness.py

@@ -0,0 +1,241 @@
+"""FL005 — missing_components.
+
+Catches deliverables that are structurally present but silently incomplete:
+a required field that exists on paper and is empty in fact, and — for
+segmented media — stretches where a component should be and nothing is.
+
+Production origin: a delivered video passed every existence check — right
+file, right length, right format — while one segment inside it was silent.
+Existence is not completeness. The check that exists because of that miss
+measures the inside of the deliverable, not just its outline.
+
+Three passes:
+
+1. Required paths (``--require output.url`` / config) and JSON Schema
+   ``required`` fields: missing *or* empty is an error.
+2. Heuristic emptiness: fields inside the payload that are present but
+   empty while their siblings carry content are warnings — often benign,
+   always worth a look.
+3. Segment integrity: zero-length segments and gaps between segments
+   longer than ``max_segment_gap_seconds`` are dead stretches, and are
+   errors.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from ..models import CheckConfig, Finding, Severity
+from ..util import is_empty, normalize_key, resolve_path
+
+CHECK_ID = "FL005"
+NAME = "missing_components"
+DESCRIPTION = (
+    "Required components missing or silently empty, and dead stretches "
+    "inside segmented deliverables."
+)
+
+#: Fields that are legitimately empty in healthy outputs.
+ALLOW_EMPTY_KEYS = {"error", "errors", "warnings", "cc", "bcc", "metadata", "parentid"}
+
+SEGMENT_LIST_KEYS = {"segments", "scenes", "tracks", "clips", "chapters", "shots"}
+START_KEYS = ("start", "start_sec", "start_time", "begin", "from")
+END_KEYS = ("end", "end_sec", "end_time", "to")
+DURATION_KEYS = ("duration", "duration_sec", "length", "duration_seconds")
+
+
+def _first_number(node: dict[str, Any], keys: tuple[str, ...]) -> float | None:
+    lowered = {normalize_key(k): v for k, v in node.items()}
+    for key in keys:
+        value = lowered.get(normalize_key(key))
+        if isinstance(value, (int, float)) and not isinstance(value, bool):
+            return float(value)
+    return None
+
+
+def _segment_span(segment: dict[str, Any]) -> tuple[float, float] | None:
+    start = _first_number(segment, START_KEYS)
+    end = _first_number(segment, END_KEYS)
+    duration = _first_number(segment, DURATION_KEYS)
+    if start is None and duration is None:
+        return None
+    start = start if start is not None else 0.0
+    if end is None and duration is not None:
+        end = start + duration
+    if end is None:
+        return None
+    return start, end
+
+
+def _check_segments(doc: Any, config: CheckConfig) -> list[Finding]:
+    findings: list[Finding] = []
+    from ..util import iter_nodes
+
+    for path, value in iter_nodes(doc):
+        key = path.rsplit(".", 1)[-1]
+        if normalize_key(key) not in SEGMENT_LIST_KEYS or not isinstance(value, list):
+            continue
+        spans: list[tuple[float, float, int]] = []
+        for index, segment in enumerate(value):
+            if not isinstance(segment, dict):
+                continue
+            span = _segment_span(segment)
+            if span is None:
+                continue
+            start, end = span
+            spans.append((start, end, index))
+            if end - start <= 0:
+                findings.append(
+                    Finding(
+                        check_id=CHECK_ID,
+                        code="zero_length_segment",
+                        severity=Severity.ERROR,
+                        path=f"{path}[{index}]",
+                        message=(
+                            f"Segment {index} has zero length "
+                            f"({start:g}s–{end:g}s) — a component that "
+                            "occupies a slot and contains nothing."
+                        ),
+                        detail={"start": start, "end": end},
+                    )
+                )
+        spans.sort()
+        for (prev_start, prev_end, _), (start, end, index) in zip(spans, spans[1:]):
+            gap = start - prev_end
+            if gap > config.max_segment_gap_seconds:
+                findings.append(
+                    Finding(
+                        check_id=CHECK_ID,
+                        code="dead_stretch",
+                        severity=Severity.ERROR,
+                        path=f"{path}[{index}]",
+                        message=(
+                            f"Dead stretch of {gap:.1f}s before segment "
+                            f"{index} (gap limit "
+                            f"{config.max_segment_gap_seconds:g}s) — the "
+                            "deliverable goes silent where content should be."
+                        ),
+                        detail={"gap_seconds": round(gap, 3), "before_segment": index},
+                    )
+                )
+    return findings
+
+
+def _check_schema(node: Any, schema: dict[str, Any], path: str) -> list[Finding]:
+    """Walk the supported JSON Schema subset: ``required`` + ``properties``
+    on objects, ``items`` on arrays."""
+    findings: list[Finding] = []
+    if not isinstance(schema, dict):
+        return findings
+    required = schema.get("required")
+    if isinstance(required, list) and isinstance(node, dict):
+        for key in required:
+            if key not in node:
+                findings.append(
+                    Finding(
+                        check_id=CHECK_ID,
+                        code="missing_component",
+                        severity=Severity.ERROR,
+                        path=f"{path}.{key}",
+                        message=f"Required component {key!r} is missing entirely.",
+                        detail={"required_by": "schema"},
+                    )
+                )
+            elif is_empty(node[key]):
+                findings.append(
+                    Finding(
+                        check_id=CHECK_ID,
+                        code="empty_component",
+                        severity=Severity.ERROR,
+                        path=f"{path}.{key}",
+                        message=(
+                            f"Required component {key!r} is present but "
+                            "empty — it satisfies the shape and delivers "
+                            "nothing."
+                        ),
+                        detail={"required_by": "schema"},
+                    )
+                )
+    properties = schema.get("properties")
+    if isinstance(properties, dict) and isinstance(node, dict):
+        for key, subschema in properties.items():
+            if key in node:
+                findings.extend(_check_schema(node[key], subschema, f"{path}.{key}"))
+    items = schema.get("items")
+    if isinstance(items, dict) and isinstance(node, list):
+        for index, item in enumerate(node):
+            findings.extend(_check_schema(item, items, f"{path}[{index}]"))
+    return findings
+
+
+def check(doc: Any, config: CheckConfig) -> list[Finding]:
+    findings: list[Finding] = []
+    if not isinstance(doc, dict):
+        return findings
+
+    # Pass 1a: explicitly required dotted paths.
+    for dotted in config.required_paths:
+        found, value = resolve_path(doc, dotted)
+        if not found:
+            findings.append(
+                Finding(
+                    check_id=CHECK_ID,
+                    code="missing_component",
+                    severity=Severity.ERROR,
+                    path=f"$.{dotted}",
+                    message=f"Required component {dotted!r} is missing entirely.",
+                    detail={"required_by": "required_paths"},
+                )
+            )
+        elif is_empty(value):
+            findings.append(
+                Finding(
+                    check_id=CHECK_ID,
+                    code="empty_component",
+                    severity=Severity.ERROR,
+                    path=f"$.{dotted}",
+                    message=(
+                        f"Required component {dotted!r} is present but empty "
+                        "— it satisfies the shape and delivers nothing."
+                    ),
+                    detail={"required_by": "required_paths"},
+                )
+            )
+
+    # Pass 1b: JSON Schema subset.
+    if config.schema:
+        findings.extend(_check_schema(doc, config.schema, "$"))
+
+    # Pass 2: heuristic emptiness inside the payload.
+    payload: Any = doc.get("output", doc)
+    if isinstance(payload, dict):
+        from ..util import iter_nodes
+
+        for path, value in iter_nodes(payload, "$.output" if "output" in doc else "$"):
+            if not isinstance(value, dict):
+                continue
+            populated = [k for k, v in value.items() if not is_empty(v)]
+            if not populated:
+                continue
+            for key, item in value.items():
+                if normalize_key(key) in ALLOW_EMPTY_KEYS:
+                    continue
+                if is_empty(item):
+                    findings.append(
+                        Finding(
+                            check_id=CHECK_ID,
+                            code="empty_field",
+                            severity=Severity.WARNING,
+                            path=f"{path}.{key}",
+                            message=(
+                                f"Field {key!r} is present but empty while "
+                                "its siblings carry content — verify it was "
+                                "meant to be blank."
+                            ),
+                            detail={},
+                        )
+                    )
+
+    # Pass 3: segment integrity.
+    findings.extend(_check_segments(doc, config))
+    return findings

failloud/checks/counts.py

@@ -0,0 +1,169 @@
+"""FL004 — count_mismatch.
+
+Catches reported numbers that disagree with the itemized record in the
+same document: the summary claims 12, the list contains 9. The claimed
+number is the one that travels — into dashboards, digests, and decisions —
+while the list is the only part that was ever true.
+
+Production origin: a report counted actions *attempted* while its audience
+read them as actions *completed*. Attempted and completed differed by the
+failures nobody itemized. The rule that came out of it: any count a reader
+will repeat must reconcile with a list a reader can count.
+
+What is verified (structured data only — prose numbers are out of scope):
+
+- A numeric claim (``counts.sent``, ``emails_sent_count``, ``total_sent``)
+  whose metric has an itemized list in the document must equal the list's
+  length.
+- A ``total`` inside a counts block must equal the sum of its siblings.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from ..models import CheckConfig, Finding, Severity
+from ..util import iter_nodes, key_tokens, normalize_key
+
+CHECK_ID = "FL004"
+NAME = "count_mismatch"
+DESCRIPTION = (
+    "A reported count that does not match the itemized record in the same "
+    "document (or a total that does not match its own parts)."
+)
+
+COUNT_CONTAINER_KEYS = {"counts", "stats", "summarycounts", "totals", "metrics"}
+COUNT_KEY_RES = (
+    re.compile(r"^(.+?)_count$"),
+    re.compile(r"^count_(.+)$"),
+    re.compile(r"^total_(.+)$"),
+    re.compile(r"^num_(.+)$"),
+)
+
+
+def _metric_of_count_key(key: str) -> str | None:
+    for pattern in COUNT_KEY_RES:
+        match = pattern.match(key.lower())
+        if match:
+            return match.group(1)
+    return None
+
+
+def _is_number(value: Any) -> bool:
+    return isinstance(value, (int, float)) and not isinstance(value, bool)
+
+
+def _names_match(claim_metric: str, array_metric: str) -> bool:
+    a, b = normalize_key(claim_metric), normalize_key(array_metric)
+    if a == b:
+        return True
+    if a.rstrip("s") == b.rstrip("s") and a.rstrip("s"):
+        return True
+    tokens_a, tokens_b = key_tokens(claim_metric), key_tokens(array_metric)
+    if tokens_a and tokens_b and (tokens_a <= tokens_b or tokens_b <= tokens_a):
+        return True
+    return False
+
+
+def check(doc: Any, config: CheckConfig) -> list[Finding]:
+    findings: list[Finding] = []
+    if not isinstance(doc, (dict, list)):
+        return findings
+
+    # Collect itemized lists: (metric name, path, length).
+    arrays: list[tuple[str, str, int]] = []
+    for path, value in iter_nodes(doc):
+        if isinstance(value, list) and path != "$":
+            metric = path.rsplit(".", 1)[-1]
+            if metric.startswith("$"):
+                continue
+            arrays.append((metric, path, len(value)))
+
+    # Collect numeric claims: (metric name, path, claimed value).
+    claims: list[tuple[str, str, float]] = []
+    seen: set[tuple[str, float]] = set()
+
+    def add_claim(metric: str, path: str, value: float) -> None:
+        dedupe_key = (normalize_key(metric), float(value))
+        if dedupe_key in seen:
+            return
+        seen.add(dedupe_key)
+        claims.append((metric, path, value))
+
+    for path, value in iter_nodes(doc):
+        if not isinstance(value, dict):
+            continue
+        parent_key = normalize_key(path.rsplit(".", 1)[-1]) if path != "$" else ""
+        for key, item in value.items():
+            if not _is_number(item):
+                continue
+            item_path = f"{path}.{key}"
+            if parent_key in COUNT_CONTAINER_KEYS:
+                add_claim(key, item_path, float(item))
+            else:
+                metric = _metric_of_count_key(key)
+                if metric is not None:
+                    add_claim(metric, item_path, float(item))
+
+    # A "total" must equal the sum of the siblings it totals.
+    for path, value in iter_nodes(doc):
+        if not isinstance(value, dict):
+            continue
+        parent_key = normalize_key(path.rsplit(".", 1)[-1]) if path != "$" else ""
+        if parent_key not in COUNT_CONTAINER_KEYS:
+            continue
+        numeric = {k: v for k, v in value.items() if _is_number(v)}
+        for key, total in numeric.items():
+            if normalize_key(key) in {"total", "grandtotal", "count"} and len(numeric) >= 3:
+                parts = sum(v for k, v in numeric.items() if k != key)
+                if float(total) != float(parts):
+                    findings.append(
+                        Finding(
+                            check_id=CHECK_ID,
+                            code="count_total_mismatch",
+                            severity=Severity.ERROR,
+                            path=f"{path}.{key}",
+                            message=(
+                                f"Reported total {total:g} does not equal "
+                                f"the sum of its parts ({parts:g})."
+                            ),
+                            detail={"claimed": total, "sum_of_parts": parts},
+                        )
+                    )
+
+    # Every claim with an itemized list must reconcile with it.
+    for metric, path, claimed in claims:
+        matching = [a for a in arrays if _names_match(metric, a[0])]
+        if not matching:
+            continue  # Nothing itemized: the claim is unverifiable, not wrong.
+        # Prefer the longest list among name matches — the fullest record.
+        array_metric, array_path, actual = max(matching, key=lambda a: a[2])
+        if float(actual) != claimed:
+            inflated = claimed > actual
+            findings.append(
+                Finding(
+                    check_id=CHECK_ID,
+                    code="count_mismatch",
+                    severity=Severity.ERROR,
+                    path=path,
+                    message=(
+                        f"Count for {metric!r} is reported as {claimed:g} "
+                        f"but the itemized record at {array_path} contains "
+                        f"{actual} item(s) — "
+                        + (
+                            "the report overstates what the record supports."
+                            if inflated
+                            else "the report understates its own record."
+                        )
+                    ),
+                    detail={
+                        "metric": metric,
+                        "claimed": claimed,
+                        "actual": actual,
+                        "inflated": inflated,
+                        "items_path": array_path,
+                    },
+                )
+            )
+    return findings

failloud/checks/evidence.py

@@ -0,0 +1,182 @@
+"""FL002 — unbacked_claim.
+
+Catches claims of completed real-world actions that carry no evidence: the
+agent says it sent, posted, published, updated, deployed — and nothing in
+the output (no message id, no URL, no receipt, no artifact) backs the claim.
+
+Production origin: a run reported a batch of messages as sent. One address
+had bounced; the report never knew, because "sent" was asserted by the same
+component that performed the send and nothing downstream demanded proof.
+The rule that came out of it: a claim without evidence is a rumor.
+
+Two passes:
+
+1. Structured claims — every entry in a ``claims``/``actions`` list that
+   asserts a completed action must carry at least one evidence field.
+2. Narrative claims — if a ``summary``/``message`` field asserts completed
+   actions in prose while the *entire document* contains no evidence at
+   all, the narrative is flagged once per field.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from ..models import CheckConfig, Finding, Severity
+from ..util import has_content, is_empty, iter_nodes, normalize_key
+from .placeholder import COMPLETION_VALUES
+
+CHECK_ID = "FL002"
+NAME = "unbacked_claim"
+DESCRIPTION = (
+    "A claimed action (sent, posted, published, updated…) with no evidence "
+    "— no id, URL, receipt, or artifact — anywhere to back it."
+)
+
+#: Keys whose non-empty value counts as evidence for a claim.
+EVIDENCE_KEYS = {
+    "evidence",
+    "proof",
+    "artifact",
+    "artifacts",
+    "url",
+    "urls",
+    "link",
+    "links",
+    "id",
+    "messageid",
+    "recordid",
+    "receipt",
+    "receiptid",
+    "confirmation",
+    "confirmationid",
+    "externalid",
+    "path",
+    "file",
+    "hash",
+    "sha",
+    "permalink",
+    "reference",
+    "ticket",
+    "postid",
+    "threadid",
+}
+
+#: Document keys that conventionally hold the list of claimed actions.
+CLAIM_LIST_KEYS = {"claims", "actions", "actionstaken", "steps", "completedactions"}
+
+#: Past-tense action verbs that assert something happened in the world.
+CLAIM_VERB_RE = re.compile(
+    r"\b(sent|posted|published|emailed|uploaded|deployed|updated|created|"
+    r"deleted|booked|charged|notified|delivered|filed|submitted|released|"
+    r"merged|announced)\b",
+    re.IGNORECASE,
+)
+
+URL_RE = re.compile(r"https?://[^\s)\]]+")
+
+#: Free-text fields where narrative claims usually live.
+TEXT_CLAIM_KEYS = {"summary", "message", "report", "description", "notes", "result"}
+
+
+def _value_has_evidence(value: Any) -> bool:
+    """True if *value* contains, at any depth, an evidence-named field
+    holding actual content. An ``evidence`` object whose fields are all
+    empty is a container, not evidence, and does not count."""
+    if isinstance(value, dict):
+        for key, item in value.items():
+            if normalize_key(key) in EVIDENCE_KEYS and has_content(item):
+                return True
+            if _value_has_evidence(item):
+                return True
+        return False
+    if isinstance(value, list):
+        return any(_value_has_evidence(item) for item in value)
+    return False
+
+
+def _claim_asserts_completion(claim: dict[str, Any]) -> str | None:
+    """Return the asserted action name if *claim* says something was done,
+    else None (planned, in-progress, or honestly failed claims pass)."""
+    status = claim.get("status") or claim.get("state")
+    if isinstance(status, str):
+        lowered = status.strip().lower()
+        if lowered in {"failed", "error", "skipped", "pending", "planned", "cancelled"}:
+            return None
+    for key in ("action", "type", "verb", "did", "name", "operation"):
+        value = claim.get(key)
+        if isinstance(value, str) and value.strip():
+            return value.strip()
+    if claim.get("done") is True or claim.get("completed") is True:
+        return "completed action"
+    if isinstance(status, str) and status.strip().lower() in COMPLETION_VALUES:
+        return "completed action"
+    return None
+
+
+def _document_has_evidence(doc: Any) -> bool:
+    for _, value in iter_nodes(doc):
+        if isinstance(value, dict):
+            for key, item in value.items():
+                if normalize_key(key) in EVIDENCE_KEYS and not is_empty(item):
+                    return True
+        if isinstance(value, str) and URL_RE.search(value):
+            return True
+    return False
+
+
+def check(doc: Any, config: CheckConfig) -> list[Finding]:
+    findings: list[Finding] = []
+    if not isinstance(doc, (dict, list)):
+        return findings
+
+    # Pass 1: structured claims.
+    for path, value in iter_nodes(doc):
+        key = path.rsplit(".", 1)[-1]
+        if normalize_key(key) not in CLAIM_LIST_KEYS or not isinstance(value, list):
+            continue
+        for index, claim in enumerate(value):
+            if not isinstance(claim, dict):
+                continue
+            action = _claim_asserts_completion(claim)
+            if action is None:
+                continue
+            if not _value_has_evidence(claim):
+                findings.append(
+                    Finding(
+                        check_id=CHECK_ID,
+                        code="unbacked_claim",
+                        severity=Severity.ERROR,
+                        path=f"{path}[{index}]",
+                        message=(
+                            f"Claimed action {action!r} has no evidence — no "
+                            "id, URL, receipt, or artifact in the claim "
+                            "backs that it happened."
+                        ),
+                        detail={"action": action},
+                    )
+                )
+
+    # Pass 2: narrative claims with zero evidence in the whole document.
+    if isinstance(doc, dict) and not _document_has_evidence(doc):
+        for key, value in doc.items():
+            if normalize_key(key) in {normalize_key(k) for k in TEXT_CLAIM_KEYS}:
+                if isinstance(value, str) and CLAIM_VERB_RE.search(value):
+                    verbs = sorted({m.lower() for m in CLAIM_VERB_RE.findall(value)})
+                    findings.append(
+                        Finding(
+                            check_id=CHECK_ID,
+                            code="unbacked_claim",
+                            severity=Severity.ERROR,
+                            path=f"$.{key}",
+                            message=(
+                                "Narrative claims completed actions "
+                                f"({', '.join(verbs)}) but the document "
+                                "contains no evidence — no id, URL, receipt, "
+                                "or artifact — anywhere."
+                            ),
+                            detail={"verbs": verbs},
+                        )
+                    )
+    return findings