spec2test 0.1.1__py3-none-any.whl

spec2test/adapters/api/codegen.py

@@ -0,0 +1,197 @@
+"""adapters/api/codegen.py — LLM writes pytest+httpx source for one APPROVED
+TestCase -> ast.parse + assertion-presence + import-whitelist validation ->
+retry <= MAX_CODEGEN_ATTEMPTS -> quarantine. See
+implementation/01-architecture.md 'Codegen strategy' and 'Codegen:
+code-execution trust boundary'.
+
+case_id / requirement_id / target_type are injected as module-level
+constants AFTER the model's output validates - never asked of the model,
+same pattern as core/generator.py never asking the model for
+id/requirement_id (02-components.md). runner.py recovers them by static
+ast parsing, never by importing/exec'ing generated code outside pytest's
+own subprocess.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from pydantic import BaseModel
+
+from spec2test.adapters.common.validate import categorize_error, validate_common
+from spec2test.core.config import settings
+from spec2test.core.exceptions import CodegenValidationError
+from spec2test.core.gate import require_approved
+from spec2test.core.llm import LLMHandle
+from spec2test.core.logging_config import get_logger
+from spec2test.core.metrics import record_codegen_attempt
+from spec2test.core.schema import TestCase
+
+log = get_logger("codegen.api")
+
+# Total attempts including the first draft - mirrors core/generator.py's
+# MAX_REPAIRS=2 (1 initial + 2 repairs = 3 total attempts), and matches the
+# M4 exit criterion "retry <=3, quarantine" read as a 3-attempt budget.
+MAX_CODEGEN_ATTEMPTS = 3
+
+# Hardening against the code-execution trust boundary (01-architecture.md
+# 'Codegen: code-execution trust boundary'): anything outside this set -
+# os, subprocess, socket, etc. - fails validation and is quarantined,
+# never executed.
+IMPORT_WHITELIST = {"pytest", "httpx"}
+
+GENERATED_DIR = Path("artifacts/generated")
+QUARANTINE_DIR = Path("artifacts/quarantine")
+
+SYSTEM_PROMPT = """You write a single self-contained pytest test function \
+that verifies one approved API test case, for a FastAPI service reachable \
+at {base_url}.
+
+Rules, all mandatory:
+- Start the file with `import httpx` on its own line. Add `import pytest` \
+too if you use it (e.g. pytest.raises). These are the ONLY imports allowed \
+- never `os`, `json`, `re`, or anything else. httpx.Response.json() gives \
+you parsed JSON without importing json. Every name you reference below \
+(httpx.Client, etc.) must come from one of these two import lines - do not \
+call httpx.anything without `import httpx` actually present in the file.
+- Define exactly one function, named `test_<snake_case_of_the_case_id>`, \
+taking no fixture arguments - create the client inside the function body: \
+`with httpx.Client(base_url="{base_url}") as client:`.
+- The function body must issue the HTTP calls described in the steps, in \
+order (is_setup steps first), matching the exact method and path in each \
+step's action, with data as the JSON request body, then assert the \
+expected result. Use real `assert` statements checking status codes and \
+response body fields - never a test with no assertions, never a trivially \
+always-true assertion like `assert True`.
+- Any identifier value you invent for a NEW entity this test creates (e.g. \
+a book's isbn, a member's email) must be derived at runtime from the \
+module-level constant `TESTFORGE_CASE_ID` (already defined above your \
+function in this file - reference it directly, do not redefine it), never \
+a fixed literal like "1234567890123" or "m@example.com". Two different \
+generated tests must never be able to collide on the same entity. Example \
+patterns: `isbn = str(abs(hash(TESTFORGE_CASE_ID)))[:13].ljust(13, "0")` \
+for a 13-digit isbn, `email = f"member+{{TESTFORGE_CASE_ID}}@example.com"` \
+for an email. Values the requirement itself pins down exactly (e.g. an \
+expected due_date offset, a loan limit) are NOT invented identifiers - \
+leave those as specified. If the test creates MORE THAN ONE new entity of \
+the SAME kind (e.g. two distinct new books), each one's derived value must \
+be distinguishable from the others' - append the step order or a short \
+suffix before hashing (e.g. `hash(TESTFORGE_CASE_ID + "-2")` for the \
+second one), never derive two entities of the same kind from the exact \
+same expression.
+- Output ONLY the Python source code. No markdown fences, no explanation, \
+no comments describing what you're doing.
+"""
+
+
+class GeneratedScript(BaseModel):
+    source: str
+
+
+def _build_prompt(tc: TestCase) -> list[tuple[str, str]]:
+    steps_desc = "\n".join(
+        f"  {s.order}. [{'setup' if s.is_setup else 'verify'}] {s.action}" + (f" data={s.data}" if s.data else "")
+        for s in tc.steps
+    )
+    user = (
+        f"case_id (for the function name, and available at runtime as the "
+        f"TESTFORGE_CASE_ID constant for deriving unique entity values - "
+        f"do not print it literally): {tc.id}\n"
+        f"title: {tc.title}\n"
+        f"requirement_excerpt: {tc.requirement_excerpt}\n"
+        f"preconditions: {tc.preconditions}\n"
+        f"steps:\n{steps_desc}\n"
+        f"expected_result: {tc.expected_result}\n"
+    )
+    return [("system", SYSTEM_PROMPT.format(base_url=settings.target_base_url)), ("user", user)]
+
+
+def _validate(source: str) -> list[str]:
+    """Every check here now lives in adapters/common/validate.py, shared
+    with the UI adapter. Extracted in M5 Phase 3 BEFORE the UI adapter was
+    written, specifically so it couldn't start from a hand-copied
+    pre-fix version of these checks - three of them are fixes for real
+    bugs that had already shipped once (module-scoped assertion check,
+    missing-import check, dynamic-import bypass). See that module's
+    docstring for the full history."""
+    _tree, errors = validate_common(source, IMPORT_WHITELIST)
+    return errors
+
+
+def _inject_metadata(source: str, tc: TestCase) -> str:
+    header = (
+        "# Generated by spec2test - DO NOT EDIT BY HAND\n"
+        f"TESTFORGE_CASE_ID = {tc.id!r}\n"
+        f"TESTFORGE_REQUIREMENT_ID = {tc.requirement_id!r}\n"
+        f"TESTFORGE_TARGET_TYPE = {tc.target_type.value!r}\n\n"
+    )
+    return header + source
+
+
+def generate_script(tc: TestCase, llm_handle: LLMHandle) -> Path:
+    """Hybrid loop: draft -> validate -> retry <= MAX_CODEGEN_ATTEMPTS ->
+    quarantine. Raises GateViolationError (via require_approved) if tc isn't
+    APPROVED (NFR-2) - this is the first line, before any LLM call. Raises
+    CodegenValidationError, and writes the last attempt + its errors to
+    artifacts/quarantine/, if nothing validates within the attempt budget -
+    the quarantined file is never executed by anything downstream."""
+    require_approved(tc)
+
+    base_messages = _build_prompt(tc)
+    last_errors: list[str] = []
+    last_source = ""
+
+    for attempt in range(1, MAX_CODEGEN_ATTEMPTS + 1):
+        messages = list(base_messages)
+        if last_errors:
+            errors_text = "\n".join(f"- {e}" for e in last_errors)
+            messages.append(
+                ("user", f"Your previous attempt had these problems - fix them and resend the FULL source:\n{errors_text}")
+            )
+
+        result = llm_handle.invoke_structured(GeneratedScript, messages)
+        # Inject before validating, not after: the prompt tells the model
+        # TESTFORGE_CASE_ID is "already defined above your function" (true
+        # only once this header is prepended), so validating the raw
+        # pre-injection source flags a correct script as referencing an
+        # unbound name. _inject_metadata is a pure, deterministic prepend
+        # of three fixed literals - injecting first can only add names the
+        # validator resolves, never hide a real unresolved-reference bug.
+        last_source = _inject_metadata(result.source, tc)
+        last_errors = _validate(last_source)
+
+        log.info(
+            "codegen_attempt",
+            extra={"event": "codegen_attempt", "case_id": tc.id, "attempt": attempt, "valid": not last_errors},
+        )
+
+        # Distinct event per implementation/06-observability.md's event
+        # taxonomy ("import_whitelist_violation ... always WARNING, case
+        # quarantined") - separate from the generic codegen_attempt/
+        # quarantined events so import-safety incidents are greppable/
+        # alertable on their own, not buried in ordinary validation noise.
+        import_errors = [
+            e for e in last_errors if e.startswith("disallowed import:") or e.startswith("disallowed dynamic import")
+        ]
+        if import_errors:
+            log.warning(
+                "import_whitelist_violation",
+                extra={"event": "import_whitelist_violation", "case_id": tc.id, "attempt": attempt, "violations": import_errors},
+            )
+
+        if not last_errors:
+            GENERATED_DIR.mkdir(parents=True, exist_ok=True)
+            path = GENERATED_DIR / f"{tc.id}.py"
+            path.write_text(last_source, encoding="utf-8")
+            log.info("codegen_success", extra={"event": "codegen_success", "case_id": tc.id, "path": str(path)})
+            record_codegen_attempt(succeeded=True)
+            return path
+
+    QUARANTINE_DIR.mkdir(parents=True, exist_ok=True)
+    (QUARANTINE_DIR / f"{tc.id}.py").write_text(last_source, encoding="utf-8")
+    (QUARANTINE_DIR / f"{tc.id}.errors.txt").write_text("\n".join(last_errors), encoding="utf-8")
+    log.error("codegen_quarantined", extra={"event": "quarantined", "case_id": tc.id, "errors": last_errors})
+    record_codegen_attempt(succeeded=False, quarantine_reason=categorize_error(last_errors[0]) if last_errors else "other")
+    raise CodegenValidationError(
+        f"TestCase {tc.id} quarantined after {MAX_CODEGEN_ATTEMPTS} attempts, never executed: {last_errors}"
+    )

spec2test/adapters/api/runner.py

@@ -0,0 +1,249 @@
+"""adapters/api/runner.py — executes generated pytest+httpx scripts and maps
+results to TestResult. Recovers case_id/requirement_id/target_type by
+STATIC ast parsing of each script's injected metadata constants (see
+codegen.py's _inject_metadata) - never by importing/exec'ing the file
+outside of pytest's own subprocess. See implementation/02-components.md
+'adapters/api/codegen.py + runner.py'.
+
+Found during M5 planning (B1/B3 in the deep bug sweep) - two gaps against
+implementation/05-test-data-and-state-strategy.md's LOCKED design, neither
+caught by the existing suite because the corpus was too small to trigger
+them:
+  - The doc requires "POST /_test/reset ... called once before each test
+    file" (file-level isolation). Nothing here ever called it - the only
+    reset in the whole codebase was cli.py's single pre-batch call. Two
+    book-creating scripts in one run collided on 409 DUPLICATE_ISBN, a
+    false failure attributed to the wrong requirement.
+  - settings.test_exec_timeout_s was defined and never passed to
+    subprocess.run() - no execution timeout existed at all. Harmless for
+    httpx (rarely hangs); a real risk once adapters/ui/runner.py reuses
+    this pattern for Playwright, where a hung selector wait can block
+    forever.
+"""
+
+from __future__ import annotations
+
+import ast
+import json
+import subprocess
+import sys
+import tempfile
+import time
+from pathlib import Path
+
+import httpx
+
+from spec2test.adapters.common.classify import classify_pytest_outcome
+from spec2test.adapters.common.process import run_with_tree_kill_timeout
+from spec2test.core.config import settings
+from spec2test.core.exceptions import TestForgeError
+from spec2test.core.logging_config import get_logger
+from spec2test.core.metrics import record_test_result
+from spec2test.core.schema import TargetType, TestResult
+
+log = get_logger("runner.api")
+
+_METADATA_NAMES = {"TESTFORGE_CASE_ID", "TESTFORGE_REQUIREMENT_ID", "TESTFORGE_TARGET_TYPE"}
+
+
+def _read_metadata(path: Path) -> dict[str, str]:
+    tree = ast.parse(path.read_text(encoding="utf-8"))
+    meta: dict[str, str] = {}
+    for node in tree.body:
+        if not isinstance(node, ast.Assign) or len(node.targets) != 1:
+            continue
+        target = node.targets[0]
+        if isinstance(target, ast.Name) and target.id in _METADATA_NAMES and isinstance(node.value, ast.Constant):
+            meta[target.id] = node.value.value
+    missing = _METADATA_NAMES - meta.keys()
+    if missing:
+        raise ValueError(f"{path} is missing spec2test metadata constants: {sorted(missing)}")
+    return meta
+
+
+def _reset_sandbox() -> None:
+    """B1 fix: file-level reset before each script, per the LOCKED design in
+    implementation/05-test-data-and-state-strategy.md ("called once before
+    each test file"). Previously never called here - cli.py's single
+    pre-batch reset was the only one in the codebase, so entity ids/state
+    from an earlier script leaked into every later one in the same run.
+
+    httpx errors are caught and re-raised as TestForgeError, matching the
+    precedent in cli.py's _wait_healthy - found during a deeper review pass
+    that an unwrapped httpx.ConnectError/HTTPStatusError here would escape
+    cli.py's TestForgeError-only exception handler as a raw traceback
+    instead of the one-line message the exception taxonomy promises. B1
+    calling this once PER SCRIPT (not once per batch) multiplies the
+    exposure to a transient failure here relative to before."""
+    try:
+        httpx.post(f"{settings.target_base_url}{settings.test_reset_path}", timeout=10).raise_for_status()
+    except httpx.HTTPError as exc:
+        raise TestForgeError(f"could not reset sandbox at {settings.target_base_url}: {exc}") from exc
+
+
+def _execute_once(path: Path, run_id: str, meta: dict[str, str]) -> tuple[TestResult, bool]:
+    """Runs the script exactly once via pytest. Returns (result, retryable) -
+    retryable is True only for a genuine reported test failure (pytest ran
+    the test and it failed), never for a timeout or a collection-error/crash
+    (report_path never written) - retrying either of those doesn't test
+    "is this flaky", it just wastes the timeout budget again on something
+    that isn't going to change. Split out of _run_one so the flaky-retry
+    wrapper (gap list item 6) can call this twice without duplicating the
+    execution/parsing logic."""
+    with tempfile.TemporaryDirectory() as tmp:
+        report_path = Path(tmp) / "report.json"
+        start = time.monotonic()
+        try:
+            proc = run_with_tree_kill_timeout(
+                [sys.executable, "-m", "pytest", str(path), "-q", "--json-report", f"--json-report-file={report_path}"],
+                timeout_s=settings.test_exec_timeout_s,
+            )
+        except subprocess.TimeoutExpired:
+            # B3 fix: settings.test_exec_timeout_s was defined but never
+            # passed to subprocess.run() - no execution timeout existed at
+            # all. Harmless for httpx; a real risk for Playwright (a hung
+            # selector wait blocks forever) - see
+            # adapters/common/process.py's run_with_tree_kill_timeout,
+            # which is what actually matters for that case: bare
+            # subprocess.run(timeout=...) only kills the immediate pytest
+            # child, not a grandchild browser process it launched, and that
+            # gap produced real orphaned Chromium processes in M5. Reported
+            # as a normal FAILED result, not a crash that would abort the
+            # rest of the batch - same "one bad case doesn't zero out the
+            # run" discipline as cli.py's per-case quarantine handling.
+            duration_ms = int(settings.test_exec_timeout_s * 1000)
+            result = TestResult(
+                test_case_id=meta["TESTFORGE_CASE_ID"],
+                requirement_id=meta["TESTFORGE_REQUIREMENT_ID"],
+                target_type=TargetType(meta["TESTFORGE_TARGET_TYPE"]),
+                status="FAILED",
+                duration_ms=duration_ms,
+                failure_message=f"test exceeded {settings.test_exec_timeout_s}s execution timeout",
+                script_path=str(path),
+                run_id=run_id,
+            )
+            log.warning(
+                "test_execution_timeout",
+                extra={
+                    "event": "test_execution_timeout",
+                    "case_id": result.test_case_id,
+                    "requirement_id": result.requirement_id,
+                    "timeout_s": settings.test_exec_timeout_s,
+                },
+            )
+            return result, False
+        duration_ms = int((time.monotonic() - start) * 1000)
+
+        failure_message: str | None = None
+        if report_path.exists():
+            report = json.loads(report_path.read_text(encoding="utf-8"))
+            tests = report.get("tests", [])
+            outcome = tests[0]["outcome"] if tests else "error"
+            longrepr = (tests[0].get("call") or {}).get("longrepr") if tests else None
+            status, is_assertion_failure, recognized = classify_pytest_outcome(outcome, longrepr)
+            if not recognized:
+                log.warning(
+                    "failure_classification_unrecognized_format",
+                    extra={"event": "failure_classification_unrecognized_format", "longrepr_tail": (longrepr or "")[-500:]},
+                )
+            if status != "PASSED":
+                failure_message = longrepr
+            # ERROR is never retryable - a TypeError/NameError/etc. is
+            # deterministic based on the code, not runtime flakiness;
+            # retrying a broken script wastes a full timeout budget for no
+            # chance of a different outcome. See R-23.
+            retryable = bool(tests) and is_assertion_failure
+        else:
+            # pytest itself didn't run (collection error, crash before the
+            # report plugin could write) - surface raw output rather than
+            # silently reporting a pass. ERROR, not FAILED: pytest never
+            # even ran the test body, so this can never be a genuine
+            # assertion outcome. Not retryable, same reasoning as above.
+            status = "ERROR"
+            failure_message = (proc.stdout + proc.stderr)[-4000:]
+            retryable = False
+
+    result = TestResult(
+        test_case_id=meta["TESTFORGE_CASE_ID"],
+        requirement_id=meta["TESTFORGE_REQUIREMENT_ID"],
+        target_type=TargetType(meta["TESTFORGE_TARGET_TYPE"]),
+        status=status,
+        duration_ms=duration_ms,
+        failure_message=failure_message,
+        script_path=str(path),
+        run_id=run_id,
+    )
+    # retryable is already False whenever status != "FAILED" (classify_pytest_outcome's
+    # own invariant: is_assertion_failure is only ever True alongside "FAILED").
+    return result, retryable
+
+
+def _run_one(path: Path, run_id: str) -> TestResult:
+    meta = _read_metadata(path)
+    _reset_sandbox()
+
+    result, retryable = _execute_once(path, run_id, meta)
+
+    # Gap list item 6: TestResult.status has always included "FLAKY" as a
+    # schema option, but nothing ever implemented the retry-and-reclassify
+    # step the original design intended - only a genuinely reported test
+    # failure (never a timeout, never a collection crash - see _execute_once)
+    # gets one retry. If the retry PASSES, the result is reclassified FLAKY,
+    # not PASSED - a test that fails once and passes once is not the same
+    # as a test that reliably passes, and silently reporting PASSED would
+    # hide real instability. If the retry ALSO fails, nothing changes: a
+    # mutation-caught failure is deterministic and will fail again on
+    # retry, staying correctly attributed as FAILED, not misclassified as
+    # flaky just because a retry happened.
+    if retryable:
+        first_failure_message = result.failure_message
+        log.info(
+            "test_retry_after_failure",
+            extra={"event": "test_retry_after_failure", "case_id": result.test_case_id, "requirement_id": result.requirement_id},
+        )
+        # Reset again before the retry - API scripts create their own
+        # entities (see module docstring's DUPLICATE_ISBN collision), so
+        # without this the retry runs against state the failed first
+        # attempt already mutated. That's not a re-run under the same
+        # conditions; a collision there would be misattributed as FLAKY or
+        # a second, differently-caused FAILED rather than reflecting the
+        # actual determinism of the test.
+        _reset_sandbox()
+        retry_result, _ = _execute_once(path, run_id, meta)
+        if retry_result.status == "PASSED":
+            result = retry_result.model_copy(
+                update={
+                    "status": "FLAKY",
+                    "failure_message": f"passed on retry; first attempt failed: {first_failure_message}",
+                }
+            )
+            log.warning(
+                "test_flaky",
+                extra={"event": "test_flaky", "case_id": result.test_case_id, "requirement_id": result.requirement_id},
+            )
+        else:
+            result = retry_result  # consistently failing, not flaky - report the retry's own detail verbatim
+
+    log.info(
+        "test_executed",
+        extra={
+            "event": "test_executed",
+            "case_id": result.test_case_id,
+            "requirement_id": result.requirement_id,
+            "status": result.status,
+            "duration_ms": result.duration_ms,
+        },
+    )
+    record_test_result(result.status)
+    return result
+
+
+def run(paths: list[Path], run_id: str) -> list[TestResult]:
+    """Runs each generated script in its own pytest subprocess - isolation,
+    so one script's collection error or crash can't take down the batch or
+    poison another script's process state. run_id is the caller's real CLI
+    session id (cli.py's own per-invocation run_id, threaded down through
+    core/registry.py) - not minted here anymore, so every case run within
+    one `report`/`mutate` invocation shares one true session id instead of
+    each adapter batch getting its own disconnected uuid."""
+    return [_run_one(path, run_id) for path in paths]

spec2test/adapters/common/classify.py

@@ -0,0 +1,69 @@
+"""Shared FAILED-vs-ERROR classification for both adapters' runners — see
+implementation/04-failure-analysis.md R-23.
+
+A pytest result that isn't "passed" can mean two very different things: the
+test's own assertion caught a real problem (AssertionError - a genuine
+signal), or the generated script itself is broken (TypeError, NameError, a
+bad Playwright locator call, etc. - not a real signal, would fail
+identically against a perfectly correct app). Collapsing both into the same
+"FAILED" string is exactly the class of bug this project has already found
+and fixed twice before in adapters/common/validate.py's docstring (a
+missing import, a bare __import__ call) - both times because a script that
+errors out LOOKS like a script that correctly caught a mutation.
+
+Extracted here, not duplicated per-adapter, for the same reason
+validate.py's checks are shared: a hand-copied version in one adapter would
+drift from a fix made in the other.
+"""
+
+from __future__ import annotations
+
+import re
+
+_FOOTER_PATTERN = re.compile(r":\s*([A-Za-z_][\w.]*)\s*$")
+
+
+def classify_pytest_outcome(outcome: str, longrepr: str | None) -> tuple[str, bool, bool]:
+    """Returns (status, is_assertion_failure, recognized_format).
+
+    status is "PASSED", "FAILED", or "ERROR" - never anything else.
+    is_assertion_failure is True only for a genuine AssertionError, which is
+    what callers should use to decide retry eligibility (retrying a broken
+    script is pointless - it's deterministic based on the code, not runtime
+    flakiness). recognized_format is False only in the fail-safe path below
+    - callers should log a warning when it's False, since it means pytest's
+    output didn't look like what this function expects.
+
+    Reads the last non-blank line of pytest's default --tb=long longrepr,
+    which always has the stable footer "{path}:{lineno}: {ExceptionClassName}"
+    for a call-phase failure (verified against real pytest output for a
+    bare assert, a message-carrying assert, AttributeError, and a custom
+    exception class). Deliberately does NOT parse the JSON report's
+    structured `call.crash.message` field - it omits the "AssertionError:"
+    prefix entirely for a bare `assert False` with no custom message, which
+    would misclassify the single most common generated-test shape as ERROR.
+
+    Fails safe to FAILED+retryable (today's pre-fix behavior) if the footer
+    doesn't match the expected shape - never invents a new false-ERROR for
+    an unanticipated pytest output format."""
+    if outcome == "passed":
+        return "PASSED", False, True
+
+    if not longrepr:
+        return "ERROR", False, True
+
+    last_line = ""
+    for line in reversed(longrepr.splitlines()):
+        if line.strip():
+            last_line = line.strip()
+            break
+
+    match = _FOOTER_PATTERN.search(last_line)
+    if not match:
+        return "FAILED", True, False  # unrecognized format - fail safe to today's behavior
+
+    exception_name = match.group(1).rsplit(".", 1)[-1]
+    if exception_name == "AssertionError":
+        return "FAILED", True, True
+
+    return "ERROR", False, True