ref-agents 1.0.0__py3-none-any.whl

ref_agents/tools/symbol_smoke_runner.py

@@ -0,0 +1,336 @@
+"""Symbol smoke runner for STORY-TQ-003 M3b.
+
+Reads a story's test plan, generates import-only smoke tests under
+``tests/generated/STORY-XXX/``, runs ``pytest`` against that directory, and
+emits ``symbol_drift.import_failed`` findings when a grounded symbol cannot
+be imported by its recorded module path.
+
+The generator handles only ``code_map`` and ``story_public_interface`` sources
+in v1. OpenAPI and ``.pyi`` sources are recorded but skipped here (deferred).
+"""
+
+from __future__ import annotations
+
+import json
+import shutil
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+# NOTE: ``shutil`` is imported for ``shutil.which`` only — never for ``rmtree``.
+# Generated test files are cleaned per-file with ``Path.unlink`` (see
+# ``_reset_generated_dir``) to stay inside the REF security_scan allowlist.
+import structlog
+
+logger = structlog.get_logger(__name__)
+
+
+SMOKE_RUN_TIMEOUT_SECONDS: int = 60
+SUPPORTED_SOURCES: frozenset[str] = frozenset({"code_map", "story_public_interface"})
+
+
+@dataclass(frozen=True)
+class SmokeFinding:
+    """Single ``symbol_drift.import_failed`` finding.
+
+    Attributes:
+        pattern_id: Constant ``symbol_drift.import_failed``.
+        severity: ``HIGH``.
+        expected_symbol: Symbol name from test plan.
+        expected_module_path: Module path from test plan.
+        error_message: Captured import/attribute error.
+        file_path: Path of the generated test file.
+        line: Line where the failing import occurs.
+        test_case_id: Originating test case identifier.
+    """
+
+    __test__ = False
+
+    pattern_id: str
+    severity: str
+    expected_symbol: str
+    expected_module_path: str
+    error_message: str
+    file_path: str
+    line: int
+    test_case_id: str
+
+
+def run_smoke_tests(
+    story_id: str,
+    project_root: Path,
+) -> list[SmokeFinding]:
+    """Generate + execute import-only smoke tests for one story.
+
+    Args:
+        story_id: Active story identifier.
+        project_root: Project root containing ``tests/plans/`` and source.
+
+    Returns:
+        List of findings. Empty when every grounded symbol imports cleanly OR
+        when the plan contains no grounded entries.
+    """
+    plan_path = project_root / "tests" / "plans" / f"{story_id}.json"
+    if not plan_path.exists():
+        logger.info(
+            "symbol_smoke_plan_missing",
+            story_id=story_id,
+            path=str(plan_path),
+        )
+        return []
+
+    try:
+        plan = json.loads(plan_path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as exc:
+        logger.warning("symbol_smoke_plan_unreadable", error=str(exc))
+        return []
+
+    grounded = _collect_grounded_entries(plan)
+    target_dir = project_root / "tests" / "generated" / story_id
+    _reset_generated_dir(target_dir)
+
+    if not grounded:
+        logger.info("symbol_smoke_no_grounded_entries", story_id=story_id)
+        return []
+
+    generated_files: dict[str, dict[str, Any]] = {}
+    for entry in grounded:
+        path = _write_smoke_test(target_dir, story_id, entry)
+        generated_files[path.name] = entry
+
+    pytest_output = _run_pytest_on_dir(target_dir, project_root)
+    findings = _parse_pytest_output(pytest_output, generated_files, target_dir)
+    logger.info(
+        "symbol_smoke_complete",
+        story_id=story_id,
+        generated=len(generated_files),
+        findings=len(findings),
+    )
+    return findings
+
+
+def _collect_grounded_entries(plan: dict[str, Any]) -> list[dict[str, Any]]:
+    test_cases = plan.get("test_cases", [])
+    grounded: list[dict[str, Any]] = []
+    if not isinstance(test_cases, list):
+        return grounded
+    for tc in test_cases:
+        if not isinstance(tc, dict):
+            continue
+        sut = tc.get("symbol_under_test")
+        if not isinstance(sut, dict):
+            continue
+        source = sut.get("source")
+        name = sut.get("name")
+        module_path = sut.get("module_path")
+        if source not in SUPPORTED_SOURCES:
+            continue
+        if not isinstance(name, str) or not name.strip():
+            continue
+        if not isinstance(module_path, str) or not module_path.strip():
+            # Cannot generate an import without a target module.
+            continue
+        grounded.append(
+            {
+                "test_case_id": str(tc.get("id", "UNKNOWN")),
+                "name": name,
+                "module_path": module_path,
+                "source": source,
+                "grounding": sut.get("grounding", ""),
+            }
+        )
+    return grounded
+
+
+def _reset_generated_dir(target_dir: Path) -> None:
+    """Reset target_dir bounded to its own python files (security: no rmtree).
+
+    Removes every ``test_*.py`` plus the ``__init__.py`` placeholder that the
+    runner itself produced. Foreign files are left in place — operator visibility.
+    """
+    target_dir.mkdir(parents=True, exist_ok=True)
+    for stale in target_dir.glob("test_*.py"):
+        try:
+            stale.unlink()
+        except OSError as exc:
+            logger.warning(
+                "symbol_smoke_unlink_failed",
+                path=str(stale),
+                error=str(exc),
+            )
+    init_file = target_dir / "__init__.py"
+    init_file.write_text("", encoding="utf-8")
+
+
+def _write_smoke_test(
+    target_dir: Path,
+    story_id: str,
+    entry: dict[str, Any],
+) -> Path:
+    test_case_id = entry["test_case_id"]
+    safe_id = test_case_id.replace("-", "_")
+    name = entry["name"]
+    module_path = entry["module_path"]
+    grounding = entry["grounding"]
+
+    # For class.method form, import the attribute root.
+    import_target = name.split(".", 1)[0]
+
+    body = (
+        f"# AUTO-GENERATED by REF symbol_smoke_runner (STORY-TQ-003 M3b).\n"
+        f"# Source story: {story_id}, test case {test_case_id}\n"
+        f"# Grounding: {grounding}\n"
+        f"# DO NOT EDIT — regenerated on every Tester run.\n"
+        f"\n"
+        f"from {module_path} import {import_target}  # noqa: F401\n"
+        f"\n"
+        f"def test_{safe_id}_symbol_present() -> None:\n"
+        f'    """Smoke test — confirms `{name}` is importable from `{module_path}`."""\n'
+        f"    assert {import_target} is not None\n"
+    )
+    path = target_dir / f"test_{safe_id}.py"
+    path.write_text(body, encoding="utf-8")
+    return path
+
+
+def _run_pytest_on_dir(
+    target_dir: Path,
+    project_root: Path,
+) -> str:
+    if not shutil.which("pytest") and not (project_root / ".venv").exists():
+        logger.warning("pytest_not_available")
+        return ""
+    # Resolve pytest via uv if available so we honour the project's venv.
+    pytest_cmd = (
+        ["uv", "run", "pytest", "--no-header", "-q", str(target_dir)]
+        if shutil.which("uv")
+        else ["pytest", "--no-header", "-q", str(target_dir)]
+    )
+    try:
+        result = subprocess.run(
+            pytest_cmd,
+            capture_output=True,
+            text=True,
+            timeout=SMOKE_RUN_TIMEOUT_SECONDS,
+            cwd=str(project_root),
+            check=False,
+        )
+    except (OSError, subprocess.TimeoutExpired) as exc:
+        logger.warning("symbol_smoke_pytest_failed", error=str(exc))
+        return ""
+    return (result.stdout or "") + "\n" + (result.stderr or "")
+
+
+def _parse_pytest_output(
+    output: str,
+    generated_files: dict[str, dict[str, Any]],
+    target_dir: Path,
+) -> list[SmokeFinding]:
+    """Best-effort extraction of import failures from pytest output.
+
+    pytest's collection failures surface as ``ERROR <path>`` and
+    ``ImportError: ...`` / ``ModuleNotFoundError: ...`` lines. We attribute
+    each failure back to its generating entry by matching the file basename.
+    """
+    findings: list[SmokeFinding] = []
+    if not output:
+        return findings
+
+    failure_files: set[str] = set()
+    for line in output.splitlines():
+        stripped = line.strip()
+        # Collection error format: "ERROR tests/generated/STORY-X/test_TC_001.py"
+        if stripped.startswith("ERROR ") and ".py" in stripped:
+            failure_files.add(Path(stripped.split()[1]).name)
+        # Test failure format: "FAILED tests/generated/STORY-X/test_TC_001.py::test_..."
+        elif stripped.startswith("FAILED ") and ".py" in stripped:
+            failure_files.add(Path(stripped.split()[1].split("::")[0]).name)
+
+    for filename in failure_files:
+        entry = generated_files.get(filename)
+        if entry is None:
+            continue
+        error_message = _extract_error_for_file(output, filename)
+        findings.append(
+            SmokeFinding(
+                pattern_id="symbol_drift.import_failed",
+                severity="HIGH",
+                expected_symbol=entry["name"],
+                expected_module_path=entry["module_path"],
+                error_message=error_message,
+                file_path=str(target_dir / filename),
+                line=_FROM_IMPORT_LINE,
+                test_case_id=entry["test_case_id"],
+            )
+        )
+    return findings
+
+
+_FROM_IMPORT_LINE: int = 6  # The `from X import Y` line inside generated files.
+
+
+def _extract_error_for_file(output: str, filename: str) -> str:
+    lines = output.splitlines()
+    capture = False
+    captured: list[str] = []
+    for line in lines:
+        if filename in line and (
+            line.lstrip().startswith("ERROR ")
+            or "ImportError" in line
+            or "ModuleNotFoundError" in line
+        ):
+            capture = True
+        if capture:
+            captured.append(line)
+            if "ImportError:" in line or "ModuleNotFoundError:" in line:
+                return "\n".join(captured[-5:])
+    return "\n".join(captured[-3:]) if captured else "Import failed (see pytest log)."
+
+
+def emit_findings_to_health_report(
+    findings: list[SmokeFinding],
+    project_root: Path,
+) -> None:
+    """Append ``symbol_drift.import_failed`` entries to ``health_report.json``.
+
+    Findings appear in the ``findings`` list with scanner ``"symbol_smoke"``
+    and severity ``"critical"`` so the existing auto-remediation bootstrap
+    picks them up (mirrors test_smell HIGH routing).
+
+    No-op when the report file is absent — caller should run a health scan
+    first if they want the routing.
+    """
+    if not findings:
+        return
+    report_path = (
+        project_root / "ref-reports" / "platform_engineer" / "health_report.json"
+    )
+    if not report_path.exists():
+        logger.info("symbol_smoke_skip_routing_no_report", path=str(report_path))
+        return
+    try:
+        data = json.loads(report_path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as exc:
+        logger.warning("symbol_smoke_route_failed", error=str(exc))
+        return
+    existing = data.get("findings", []) if isinstance(data, dict) else []
+    for finding in findings:
+        existing.append(
+            {
+                "scanner": "symbol_smoke",
+                "file": finding.file_path,
+                "line": finding.line,
+                "severity": "critical",
+                "rule": finding.pattern_id,
+                "message": (
+                    f"{finding.pattern_id}: expected `{finding.expected_symbol}` "
+                    f"from `{finding.expected_module_path}` — {finding.error_message[:120]}"
+                ),
+                "owner": "developer",
+            }
+        )
+    if isinstance(data, dict):
+        data["findings"] = existing
+        data["critical"] = int(data.get("critical", 0)) + len(findings)
+        report_path.write_text(json.dumps(data, indent=2), encoding="utf-8")

ref_agents/tools/test_plan_validator.py

@@ -0,0 +1,189 @@
+"""Test plan validator for STORY-TQ-003 — schema check for ``symbol_under_test``.
+
+Validates the optional ``symbol_under_test`` object on each test case in
+``tests/plans/STORY-XXX.json``. Pre-TQ-003 plans (no field) pass without
+findings. Malformed entries (missing required sub-fields, unknown ``source``
+values) produce per-entry validation errors.
+
+Cross-checking the resolver's output (FR-6 invented-name detection) is rule-text
+only in v1 — see ``docs/specs/SPEC-STORY-TQ-003.md`` Open Question 1.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from ref_agents.tools.symbol_resolver import VALID_SOURCES, is_valid_source
+
+
+@dataclass(frozen=True)
+class ValidationError:
+    """Single validator error referencing the offending test case.
+
+    Attributes:
+        test_case_id: Identifier of the offending test case.
+        field: Dotted path of the offending field (e.g. ``symbol_under_test.name``).
+        message: Human-readable description.
+    """
+
+    test_case_id: str
+    field: str
+    message: str
+
+
+def validate(plan: dict[str, Any]) -> list[ValidationError]:
+    """Return validation errors for ``plan``.
+
+    Empty list means the plan passes the v1 schema check. Pre-TQ-003 plans (no
+    ``symbol_under_test`` anywhere) return an empty list.
+
+    Args:
+        plan: Parsed test plan dict (loaded from
+            ``tests/plans/STORY-XXX.json``).
+
+    Returns:
+        Ordered list of validation errors. Empty when the plan is valid.
+    """
+    errors: list[ValidationError] = []
+    test_cases = plan.get("test_cases", [])
+    if not isinstance(test_cases, list):
+        errors.append(
+            ValidationError(
+                test_case_id="<root>",
+                field="test_cases",
+                message="`test_cases` must be a list.",
+            )
+        )
+        return errors
+
+    for tc in test_cases:
+        if not isinstance(tc, dict):
+            errors.append(
+                ValidationError(
+                    test_case_id="<unknown>",
+                    field="<test_case>",
+                    message="Each test case must be an object.",
+                )
+            )
+            continue
+        tc_id = str(tc.get("id", "<missing-id>"))
+        sut = tc.get("symbol_under_test")
+        if sut is None:
+            # Pre-TQ-003 / ungrounded path — FR-5, FR-15.
+            continue
+        errors.extend(_validate_symbol_under_test(tc_id, sut))
+    return errors
+
+
+def _validate_symbol_under_test(
+    tc_id: str,
+    sut: Any,
+) -> list[ValidationError]:
+    if not isinstance(sut, dict):
+        return [
+            ValidationError(
+                test_case_id=tc_id,
+                field="symbol_under_test",
+                message="`symbol_under_test` must be an object.",
+            )
+        ]
+
+    errors: list[ValidationError] = []
+
+    name = sut.get("name")
+    if not isinstance(name, str) or not name.strip():
+        errors.append(
+            ValidationError(
+                test_case_id=tc_id,
+                field="symbol_under_test.name",
+                message="`name` is required and must be a non-empty string.",
+            )
+        )
+
+    source = sut.get("source")
+    if not isinstance(source, str):
+        errors.append(
+            ValidationError(
+                test_case_id=tc_id,
+                field="symbol_under_test.source",
+                message=(
+                    "`source` is required and must be one of: "
+                    + ", ".join(VALID_SOURCES)
+                    + "."
+                ),
+            )
+        )
+    elif not is_valid_source(source):
+        errors.append(
+            ValidationError(
+                test_case_id=tc_id,
+                field="symbol_under_test.source",
+                message=(
+                    f"`source` value `{source}` is unknown. Use one of: "
+                    + ", ".join(VALID_SOURCES)
+                    + "."
+                ),
+            )
+        )
+
+    grounding = sut.get("grounding")
+    if not isinstance(grounding, str) or not grounding.strip():
+        errors.append(
+            ValidationError(
+                test_case_id=tc_id,
+                field="symbol_under_test.grounding",
+                message="`grounding` is required and must be a non-empty string.",
+            )
+        )
+
+    # `module_path` is optional; if present, must be a string.
+    module_path = sut.get("module_path")
+    if module_path is not None and not isinstance(module_path, str):
+        errors.append(
+            ValidationError(
+                test_case_id=tc_id,
+                field="symbol_under_test.module_path",
+                message="`module_path`, when present, must be a string.",
+            )
+        )
+
+    return errors
+
+
+def grounding_summary(plan: dict[str, Any]) -> dict[str, Any]:
+    """Compute the FR-14 grounding summary block.
+
+    Args:
+        plan: Parsed test plan dict.
+
+    Returns:
+        Dict with keys ``total_test_cases``, ``grounded``, ``ungrounded``,
+        ``by_source``.
+    """
+    test_cases = plan.get("test_cases", [])
+    total = 0
+    grounded = 0
+    by_source: dict[str, int] = {src: 0 for src in VALID_SOURCES if src != "none"}
+
+    for tc in test_cases:
+        if not isinstance(tc, dict):
+            continue
+        total += 1
+        sut = tc.get("symbol_under_test")
+        if not isinstance(sut, dict):
+            continue
+        source = sut.get("source")
+        if not isinstance(source, str) or not is_valid_source(source):
+            continue
+        if source == "none":
+            continue
+        grounded += 1
+        by_source[source] = by_source.get(source, 0) + 1
+
+    return {
+        "total_test_cases": total,
+        "grounded": grounded,
+        "ungrounded": total - grounded,
+        "by_source": by_source,
+    }