agent-suite-conformance 1.0.0__py3-none-any.whl

agent_suite/conformance/__init__.py

@@ -0,0 +1,47 @@
+"""CLI contract v1 conformance kit (Plan 018 WI-2).
+
+One centrally versioned package, owned by agent-suite, consumed pinned by
+every component — never copied, so there is exactly one kit to drift from
+(the identifier-gate distribution model applied to conformance).
+
+``KIT_VERSION`` identifies the kit in recorded conformance results
+(``data/cli-conformance.json``); ``CLI_CONTRACT_VERSION`` is the contract
+revision the kit enforces (``docs/cli-contract.md``).
+"""
+
+from agent_suite.conformance.envelope import (
+    build_envelope,
+    emit_error,
+    validate_envelope,
+)
+from agent_suite.conformance.kit import (
+    BrokenPipeCase,
+    ErrorCase,
+    Framing,
+    SuccessCase,
+    UsageCase,
+    run_broken_pipe_case,
+    run_error_case,
+    run_success_case,
+    run_usage_case,
+)
+
+KIT_VERSION = "1.0.0"
+CLI_CONTRACT_VERSION = 1
+
+__all__ = [
+    "KIT_VERSION",
+    "CLI_CONTRACT_VERSION",
+    "BrokenPipeCase",
+    "ErrorCase",
+    "Framing",
+    "SuccessCase",
+    "UsageCase",
+    "build_envelope",
+    "emit_error",
+    "run_broken_pipe_case",
+    "run_error_case",
+    "run_success_case",
+    "run_usage_case",
+    "validate_envelope",
+]

agent_suite/conformance/envelope.py

@@ -0,0 +1,114 @@
+"""The common CLI error envelope (CLI contract v1 §3).
+
+``build_envelope`` constructs the shape, ``validate_envelope`` checks a
+parsed document against it (stdlib-only — the schema at
+``data/cli-error-envelope.schema.json`` is the normative statement; this
+validator mirrors it for consumers that must stay dependency-free), and
+``emit_error`` is the helper a conforming CLI calls on every error path.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import sys
+from typing import Any
+
+_CODE_RE = re.compile(r"^[A-Z][A-Z0-9_]*$")
+
+
+def build_envelope(
+    code: str,
+    message: str,
+    *,
+    detail: str | None = None,
+    retryable: bool = False,
+    partial: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Construct a contract-v1 error envelope."""
+    if not _CODE_RE.match(code):
+        raise ValueError(f"error code {code!r} is not SCREAMING_SNAKE")
+    return {
+        "ok": False,
+        "error": {
+            "code": code,
+            "message": message,
+            "detail": detail,
+            "retryable": retryable,
+            "partial": partial,
+        },
+    }
+
+
+def validate_envelope(document: Any) -> list[str]:
+    """Return contract violations for a parsed error document (empty = valid)."""
+    violations: list[str] = []
+    if not isinstance(document, dict):
+        return ["envelope is not a JSON object"]
+    if document.get("ok") is not False:
+        violations.append("envelope 'ok' is not false")
+    error = document.get("error")
+    if not isinstance(error, dict):
+        return violations + ["envelope 'error' is not an object"]
+    code = error.get("code")
+    if not isinstance(code, str) or not _CODE_RE.match(code):
+        violations.append(f"error 'code' {code!r} is not a SCREAMING_SNAKE string")
+    if not isinstance(error.get("message"), str):
+        violations.append("error 'message' is not a string")
+    if "detail" in error and not isinstance(error["detail"], (str, type(None))):
+        violations.append("error 'detail' is not a string or null")
+    if not isinstance(error.get("retryable"), bool):
+        violations.append("error 'retryable' is not a boolean")
+    partial = error.get("partial")
+    if partial is not None:
+        if not isinstance(partial, dict):
+            violations.append("error 'partial' is not an object or null")
+        else:
+            succeeded = partial.get("succeeded")
+            failed = partial.get("failed")
+            if not isinstance(succeeded, int) or succeeded < 0:
+                violations.append("partial 'succeeded' is not a non-negative integer")
+            if not isinstance(failed, int) or failed < 1:
+                violations.append("partial 'failed' is not a positive integer")
+    unknown = set(error) - {"code", "message", "detail", "retryable", "partial"}
+    if unknown:
+        violations.append(f"error object has unknown fields: {sorted(unknown)}")
+    unknown_top = set(document) - {"ok", "error"}
+    if unknown_top:
+        violations.append(f"envelope has unknown fields: {sorted(unknown_top)}")
+    return violations
+
+
+def emit_error(
+    code: str,
+    message: str,
+    *,
+    detail: str | None = None,
+    retryable: bool = False,
+    partial: dict[str, Any] | None = None,
+    json_mode: bool = False,
+) -> int:
+    """Report an error per the contract and return the exit code (1).
+
+    Under ``--json`` the envelope is the single stdout document; otherwise
+    the message (and detail) go to stderr. Either way the caller returns
+    the value returned here — no path prints an error and exits 0.
+    """
+    if json_mode:
+        print(
+            json.dumps(
+                build_envelope(
+                    code,
+                    message,
+                    detail=detail,
+                    retryable=retryable,
+                    partial=partial,
+                ),
+                indent=2,
+            )
+        )
+    else:
+        print(f"error: {message}", file=sys.stderr)
+        if detail:
+            print(f"  {detail}", file=sys.stderr)
+    return 1

agent_suite/conformance/kit.py

@@ -0,0 +1,215 @@
+"""Behavioral conformance checks for CLI contract v1 (Plan 018 WI-2).
+
+Each check runs a real subprocess against the component's CLI and returns
+a list of contract violations (empty = conformant). Components declare
+cases (fixtures) and parameterize pytest over them; the checks themselves
+live only here, centrally versioned.
+
+The checks prove the contract, not the implementation: stream discipline
+(§1), exit-code taxonomy (§2), the error envelope (§3), traceback and
+broken-pipe robustness (§4), and secret redaction (§3). Output honesty
+(§5) and manifest discovery (§6) are P1 — cases for them extend these
+dataclasses rather than forking the kit.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import sys
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Mapping
+
+from agent_suite.conformance.envelope import validate_envelope
+
+_TRACEBACK_MARKER = "Traceback (most recent call last)"
+
+# Value planted in per-case env vars to prove error paths never echo
+# secret material (contract §3 redaction). Cases list the env var *names*
+# whose values must never surface; the kit plants this sentinel there.
+SENTINEL_SECRET = "conformance-sentinel-3f9d2c"
+
+
+class Framing(Enum):
+    DOCUMENT = "document"
+    NDJSON = "ndjson"
+
+
+@dataclass(frozen=True)
+class SuccessCase:
+    """A verb invocation that must succeed with pure JSON stdout."""
+
+    name: str
+    argv: tuple[str, ...]
+    framing: Framing = Framing.DOCUMENT
+    env: Mapping[str, str] = field(default_factory=dict)
+    unset_env: tuple[str, ...] = ()
+
+
+@dataclass(frozen=True)
+class ErrorCase:
+    """A documented failure that must exit nonzero with the envelope."""
+
+    name: str
+    argv: tuple[str, ...]
+    expect_code: str | None = None
+    json_mode: bool = True
+    env: Mapping[str, str] = field(default_factory=dict)
+    unset_env: tuple[str, ...] = ()
+    secret_env_names: tuple[str, ...] = ()
+
+
+@dataclass(frozen=True)
+class UsageCase:
+    """A malformed invocation that must exit 2 (argparse usage error)."""
+
+    name: str
+    argv: tuple[str, ...]
+
+
+@dataclass(frozen=True)
+class BrokenPipeCase:
+    """A verb whose stdout is closed early; it must not traceback."""
+
+    name: str
+    argv: tuple[str, ...]
+    env: Mapping[str, str] = field(default_factory=dict)
+
+
+CASE_TIMEOUT: float = 60.0
+
+
+def _run(
+    argv: tuple[str, ...],
+    env: Mapping[str, str],
+    unset_env: tuple[str, ...] = (),
+    timeout: float = CASE_TIMEOUT,
+) -> subprocess.CompletedProcess[str]:
+    merged = {**os.environ, "PYTHONIOENCODING": "utf-8", **env}
+    for name in unset_env:
+        merged.pop(name, None)
+    try:
+        return subprocess.run(
+            list(argv),
+            capture_output=True,
+            text=True,
+            check=False,
+            env=merged,
+            timeout=timeout,
+        )
+    except subprocess.TimeoutExpired:
+        return subprocess.CompletedProcess(
+            args=list(argv),
+            returncode=124,
+            stdout="",
+            stderr=f"conformance: timed out after {timeout}s",
+        )
+
+
+def run_success_case(case: SuccessCase) -> list[str]:
+    """Contract §1 + §2: exit 0, stdout is pure JSON, no traceback."""
+    proc = _run(case.argv, case.env, case.unset_env)
+    violations: list[str] = []
+    if proc.returncode != 0:
+        violations.append(
+            f"exit {proc.returncode}, expected 0; stderr: {proc.stderr[-500:]!r}"
+        )
+    if _TRACEBACK_MARKER in proc.stderr:
+        violations.append("traceback on a documented success path")
+    stdout = proc.stdout.strip()
+    if not stdout:
+        violations.append("empty stdout on a JSON success path")
+        return violations
+    match case.framing:
+        case Framing.DOCUMENT:
+            try:
+                json.loads(stdout)
+            except json.JSONDecodeError as exc:
+                violations.append(
+                    f"stdout is not a single JSON document ({exc}); "
+                    f"first 200 bytes: {stdout[:200]!r}"
+                )
+        case Framing.NDJSON:
+            for lineno, line in enumerate(stdout.splitlines(), start=1):
+                if not line.strip():
+                    continue
+                try:
+                    json.loads(line)
+                except json.JSONDecodeError:
+                    violations.append(
+                        f"NDJSON line {lineno} is not JSON: {line[:200]!r}"
+                    )
+    return violations
+
+
+def run_error_case(case: ErrorCase) -> list[str]:
+    """Contract §2 + §3: nonzero exit, envelope on stdout, no leaks."""
+    env = dict(case.env)
+    for name in case.secret_env_names:
+        env[name] = SENTINEL_SECRET
+    proc = _run(case.argv, env, case.unset_env)
+    violations: list[str] = []
+    if proc.returncode == 0:
+        violations.append(
+            "exit 0 on a documented error path (the fail-open class); "
+            f"stdout: {proc.stdout[:200]!r} stderr: {proc.stderr[:200]!r}"
+        )
+    if proc.returncode == 2:
+        violations.append("exit 2 (usage) on an operational error path")
+    if _TRACEBACK_MARKER in proc.stderr or _TRACEBACK_MARKER in proc.stdout:
+        violations.append("traceback on a documented error path")
+    if case.secret_env_names and SENTINEL_SECRET in proc.stdout + proc.stderr:
+        violations.append("secret material leaked into error output")
+    if case.json_mode:
+        stdout = proc.stdout.strip()
+        if not stdout:
+            violations.append("no envelope on stdout for a --json error path")
+            return violations
+        try:
+            document = json.loads(stdout)
+        except json.JSONDecodeError:
+            violations.append(
+                f"--json error stdout is not JSON: {stdout[:200]!r}"
+            )
+            return violations
+        violations.extend(validate_envelope(document))
+        if case.expect_code is not None and not violations:
+            actual = document["error"]["code"]
+            if actual != case.expect_code:
+                violations.append(
+                    f"error code {actual!r}, expected {case.expect_code!r}"
+                )
+    return violations
+
+
+def run_usage_case(case: UsageCase) -> list[str]:
+    """Contract §2: malformed invocations exit 2, without traceback."""
+    proc = _run(case.argv, {})
+    violations: list[str] = []
+    if proc.returncode != 2:
+        violations.append(f"exit {proc.returncode}, expected 2 (usage error)")
+    if _TRACEBACK_MARKER in proc.stderr:
+        violations.append("traceback on a usage error")
+    return violations
+
+
+def run_broken_pipe_case(case: BrokenPipeCase) -> list[str]:
+    """Contract §4: closing stdout early must not produce a traceback."""
+    if sys.platform == "win32":
+        return []  # SIGPIPE semantics don't apply; nothing to prove here.
+    with subprocess.Popen(
+        list(case.argv),
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        env={**os.environ, **case.env},
+    ) as proc:
+        assert proc.stdout is not None
+        proc.stdout.read(1)
+        proc.stdout.close()
+        stderr = proc.stderr.read() if proc.stderr else b""
+        proc.wait(timeout=120)
+    if _TRACEBACK_MARKER.encode() in stderr:
+        return ["traceback on broken stdout pipe"]
+    return []

agent_suite_conformance-1.0.0.dist-info/METADATA

@@ -0,0 +1,53 @@
+Metadata-Version: 2.4
+Name: agent-suite-conformance
+Version: 1.0.0
+Summary: CLI contract v1 conformance kit for the agent suite (Plan 018 WI-2 / Plan 019 B1).
+License: MIT
+Keywords: agent-suite,cli-contract,conformance,testing
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+
+# agent-suite-conformance
+
+The CLI contract v1 conformance kit for the agent suite (Plan 018 WI-2). One
+centrally versioned, stdlib-only package that every suite
+component depends on as a normal pinned version — so there is exactly one kit,
+never per-repo copies to drift.
+
+## What it provides
+
+`agent_suite.conformance` — success/error/usage/broken-pipe case runners and the
+common error-envelope validator that hold each suite CLI to the contract in
+`docs/cli-contract.md`:
+
+- stdout under `--json` is exactly one JSON document (or documented NDJSON),
+  zero non-JSON bytes;
+- documented error paths exit nonzero with the common error envelope;
+- usage errors exit 2; broken pipe exits without a traceback;
+- error output carries no secret material.
+
+## Use
+
+```toml
+# in a component's dev/test dependencies
+"agent-suite-conformance==1.0.0"
+```
+
+```python
+from agent_suite.conformance import (
+    KIT_VERSION, CLI_CONTRACT_VERSION,
+    SuccessCase, ErrorCase, UsageCase,
+    run_success_case, run_error_case, run_usage_case,
+    validate_envelope,
+)
+```
+
+The kit discovers what to test from each component's CLI manifest
+(`<tool> contract --json`); see the agent-suite CLI contract for the manifest
+shape.
+
+## Provenance
+
+Built from the single source of truth at `src/agent_suite/conformance/` in the
+agent-suite repository. `version` here equals `agent_suite.conformance.KIT_VERSION`;
+a guard test fails CI if they diverge.

agent_suite_conformance-1.0.0.dist-info/RECORD

@@ -0,0 +1,6 @@
+agent_suite/conformance/__init__.py,sha256=tLw3hDbCRttlWMiaChqtCDjJ0jXwc6lp18r8zsd8S1o,1150
+agent_suite/conformance/envelope.py,sha256=efpArYbwx4TjjwZzg5gvlZd57ME_lv8XOntGyQYI9sY,4043
+agent_suite/conformance/kit.py,sha256=j-FlIHj4reinUQIPVAELAkSpUr9kENPA_UelrBnrIjI,7289
+agent_suite_conformance-1.0.0.dist-info/METADATA,sha256=RjpJmt88SbNZpmZx8ekXrmo_qObHHIYu1bG6qyfuk-Y,1724
+agent_suite_conformance-1.0.0.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
+agent_suite_conformance-1.0.0.dist-info/RECORD,,

agent_suite_conformance-1.0.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.31.0
+Root-Is-Purelib: true
+Tag: py3-none-any