ref-agents 1.0.0__py3-none-any.whl

ref_agents/tools/browser/evidence_verifier.py

@@ -0,0 +1,302 @@
+"""Evidence verifier for browser test results.
+
+Verifies all test evidence (screenshots, logs, timestamps)
+to ensure test results are not hallucinated.
+"""
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from pathlib import Path
+
+import structlog
+
+from ref_agents.tools.browser.execution_logger import ExecutionLogger
+from ref_agents.tools.browser.screenshot_utils import (
+    get_screenshot_dir,
+    get_screenshot_mtime,
+    verify_screenshot_content,
+    verify_screenshot_exists,
+)
+
+logger = structlog.get_logger(__name__)
+
+
+@dataclass
+class VerificationResult:
+    """Result of evidence verification.
+
+    Attributes:
+        valid: Whether all evidence is valid.
+        issues: List of issues found.
+        screenshot_count: Number of screenshots verified.
+        log_entry_count: Number of log entries verified.
+        warnings: Non-critical issues.
+    """
+
+    valid: bool
+    issues: list[str] = field(default_factory=list)
+    screenshot_count: int = 0
+    log_entry_count: int = 0
+    warnings: list[str] = field(default_factory=list)
+
+    def to_dict(self) -> dict:
+        """Convert to dictionary."""
+        return {
+            "valid": self.valid,
+            "issues": self.issues,
+            "screenshot_count": self.screenshot_count,
+            "log_entry_count": self.log_entry_count,
+            "warnings": self.warnings,
+        }
+
+
+class EvidenceVerifier:
+    """Verify test execution evidence.
+
+    Ensures test results are backed by real evidence:
+    - Screenshots exist and are valid
+    - Execution log exists and is consistent
+    - Timestamps are sequential and realistic
+    """
+
+    def __init__(self, story_id: str, base_dir: Path | None = None):
+        """Initialize evidence verifier.
+
+        Args:
+            story_id: Story identifier.
+            base_dir: Base directory for evidence files.
+        """
+        self.story_id = story_id
+
+        if base_dir is None:
+            from ref_agents.session import SessionManager
+
+            session = SessionManager.get()
+            base_dir = session.project_root or Path.cwd()
+
+        self.base_dir = base_dir
+        self.execution_logger = ExecutionLogger(story_id, base_dir)
+        self.screenshot_dir = get_screenshot_dir(story_id, base_dir)
+
+    def verify_screenshot(self, path: Path) -> tuple[bool, str]:
+        """Verify a single screenshot file.
+
+        Args:
+            path: Path to screenshot file.
+
+        Returns:
+            Tuple of (valid, error_message).
+        """
+        # First check existence
+        valid, error = verify_screenshot_exists(path)
+        if not valid:
+            return False, error
+
+        # Then check content
+        valid, error = verify_screenshot_content(path)
+        if not valid:
+            return False, error
+
+        return True, ""
+
+    def verify_all_screenshots(self) -> tuple[bool, list[str], int]:
+        """Verify all screenshots in the screenshot directory.
+
+        Returns:
+            Tuple of (all_valid, issues, count).
+        """
+        issues = []
+        count = 0
+
+        if not self.screenshot_dir.exists():
+            return False, ["Screenshot directory does not exist"], 0
+
+        screenshots = list(self.screenshot_dir.glob("*.png"))
+
+        if not screenshots:
+            return False, ["No screenshots found"], 0
+
+        for screenshot in screenshots:
+            valid, error = self.verify_screenshot(screenshot)
+            if valid:
+                count += 1
+            else:
+                issues.append(error)
+
+        return len(issues) == 0, issues, count
+
+    def verify_log_integrity(self) -> tuple[bool, list[str]]:
+        """Verify execution log is valid.
+
+        Returns:
+            Tuple of (valid, issues).
+        """
+        return self.execution_logger.verify_log_integrity()
+
+    def verify_timestamp_consistency(self) -> tuple[bool, list[str]]:
+        """Verify timestamps are consistent.
+
+        Checks:
+        - Log timestamps are sequential
+        - Log timestamps are realistic (not too fast/slow)
+        - Screenshot mtimes are close to log timestamps
+
+        Returns:
+            Tuple of (valid, issues).
+        """
+        issues = []
+        entries = self.execution_logger.read_log()
+
+        if not entries:
+            return False, ["No log entries to verify"]
+
+        prev_timestamp = None
+
+        for entry in entries:
+            try:
+                timestamp = datetime.fromisoformat(entry.timestamp)
+
+                # Check sequential
+                if prev_timestamp:
+                    if timestamp < prev_timestamp:
+                        issues.append(
+                            f"Timestamp not sequential: {entry.test_id} step {entry.step_number}"
+                        )
+
+                    # Check realistic duration (between 50ms and 5 min)
+                    duration = (timestamp - prev_timestamp).total_seconds()
+                    if duration < 0.05:  # noqa: PLR2004  # TECH_DEBT: extract to named constant — G25
+                        issues.append(
+                            f"Suspiciously fast execution: {duration:.3f}s for {entry.test_id} step {entry.step_number}"
+                        )
+                    elif duration > 300:  # noqa: PLR2004  # TECH_DEBT: extract to named constant — G25
+                        issues.append(
+                            f"Suspiciously slow execution: {duration:.1f}s for {entry.test_id} step {entry.step_number}"
+                        )
+
+                # Check screenshot timestamp if available
+                if entry.screenshot_path:
+                    screenshot_path = Path(entry.screenshot_path)
+                    if screenshot_path.exists():
+                        try:
+                            mtime = get_screenshot_mtime(screenshot_path)
+                            screenshot_time = datetime.fromtimestamp(mtime)
+                            time_diff = abs(
+                                (timestamp - screenshot_time).total_seconds()
+                            )
+
+                            if (
+                                time_diff > 10  # noqa: PLR2004  # TECH_DEBT: extract to named constant — G25
+                            ):  # More than 10 seconds difference  # noqa: PLR2004  # TECH_DEBT: extract to named constant — G25
+                                issues.append(
+                                    f"Screenshot mtime mismatch: {time_diff:.1f}s diff for {entry.test_id} step {entry.step_number}"
+                                )
+                        except FileNotFoundError:
+                            pass  # Already caught by screenshot verification
+
+                prev_timestamp = timestamp
+
+            except ValueError as e:
+                issues.append(f"Invalid timestamp: {entry.timestamp} - {e}")
+
+        return len(issues) == 0, issues
+
+    def verify_log_screenshot_consistency(self) -> tuple[bool, list[str]]:
+        """Verify log entries have corresponding screenshots.
+
+        Returns:
+            Tuple of (valid, issues).
+        """
+        issues = []
+        entries = self.execution_logger.read_log()
+
+        for entry in entries:
+            if entry.screenshot_path:
+                screenshot = Path(entry.screenshot_path)
+                if not screenshot.exists():
+                    issues.append(
+                        f"Screenshot missing: {entry.screenshot_path} for {entry.test_id} step {entry.step_number}"
+                    )
+                elif screenshot.stat().st_size != entry.screenshot_size:
+                    issues.append(
+                        f"Screenshot size mismatch: {entry.test_id} step {entry.step_number}"
+                    )
+
+        return len(issues) == 0, issues
+
+    def cross_validate(self) -> VerificationResult:
+        """Cross-validate all evidence sources.
+
+        Performs comprehensive verification:
+        1. Verify all screenshots exist and are valid
+        2. Verify execution log integrity
+        3. Verify timestamp consistency
+        4. Verify log-screenshot consistency
+
+        Returns:
+            VerificationResult with all findings.
+        """
+        all_issues = []
+        warnings = []
+
+        # 1. Verify screenshots
+        screenshots_valid, screenshot_issues, screenshot_count = (
+            self.verify_all_screenshots()
+        )
+        if not screenshots_valid:
+            all_issues.extend(screenshot_issues)
+
+        # 2. Verify log integrity
+        log_valid, log_issues = self.verify_log_integrity()
+        if not log_valid:
+            all_issues.extend(log_issues)
+
+        log_entry_count = self.execution_logger.get_entry_count()
+
+        # 3. Verify timestamp consistency
+        timestamp_valid, timestamp_issues = self.verify_timestamp_consistency()
+        if not timestamp_valid:
+            # Timestamp issues are warnings, not critical
+            warnings.extend(timestamp_issues)
+
+        # 4. Verify log-screenshot consistency
+        consistency_valid, consistency_issues = self.verify_log_screenshot_consistency()
+        if not consistency_valid:
+            all_issues.extend(consistency_issues)
+
+        valid = len(all_issues) == 0
+
+        result = VerificationResult(
+            valid=valid,
+            issues=all_issues,
+            screenshot_count=screenshot_count,
+            log_entry_count=log_entry_count,
+            warnings=warnings,
+        )
+
+        logger.info(
+            "evidence_verification_complete",
+            valid=valid,
+            screenshot_count=screenshot_count,
+            log_entry_count=log_entry_count,
+            issue_count=len(all_issues),
+            warning_count=len(warnings),
+        )
+
+        return result
+
+
+def verify_test_evidence(
+    story_id: str, base_dir: Path | None = None
+) -> VerificationResult:
+    """Verify all test evidence for a story.
+
+    Args:
+        story_id: Story identifier.
+        base_dir: Base directory for evidence files.
+
+    Returns:
+        VerificationResult with all findings.
+    """
+    verifier = EvidenceVerifier(story_id, base_dir)
+    return verifier.cross_validate()

ref_agents/tools/browser/execution_logger.py

@@ -0,0 +1,249 @@
+"""Execution logger for browser test audit trail.
+
+Logs all MCP tool calls with timestamps, arguments, responses,
+and screenshot paths for complete audit trail.
+"""
+
+import json
+from dataclasses import asdict, dataclass, field
+from datetime import datetime
+from pathlib import Path
+
+import structlog
+
+logger = structlog.get_logger(__name__)
+
+
+@dataclass
+class ExecutionLogEntry:
+    """Single execution log entry.
+
+    Attributes:
+        timestamp: ISO8601 timestamp of execution.
+        test_id: Test case identifier.
+        step_number: Step number within test case.
+        tool_name: Name of MCP tool called.
+        tool_args: Arguments passed to tool.
+        response: Response from tool.
+        screenshot_path: Path to captured screenshot.
+        screenshot_size: Size of screenshot file in bytes.
+        duration_ms: Execution duration in milliseconds.
+        status: Step status (PASS/FAIL).
+        error: Error message if failed.
+    """
+
+    timestamp: str
+    test_id: str
+    step_number: int
+    tool_name: str
+    tool_args: dict = field(default_factory=dict)
+    response: dict = field(default_factory=dict)
+    screenshot_path: str | None = None
+    screenshot_size: int = 0
+    duration_ms: int = 0
+    status: str = "PASS"
+    error: str | None = None
+
+    def to_json(self) -> str:
+        """Convert to JSON string."""
+        return json.dumps(asdict(self), ensure_ascii=False)
+
+    @classmethod
+    def from_json(cls, json_str: str) -> "ExecutionLogEntry":
+        """Create from JSON string."""
+        data = json.loads(json_str)
+        return cls(**data)
+
+
+def get_log_path(story_id: str, base_dir: Path | None = None) -> Path:
+    """Get execution log path for a story.
+
+    Args:
+        story_id: Story identifier.
+        base_dir: Base directory. Defaults to project root.
+
+    Returns:
+        Path to log file.
+    """
+    if base_dir is None:
+        from ref_agents.session import SessionManager
+
+        session = SessionManager.get()
+        base_dir = session.project_root or Path.cwd()
+
+    return base_dir / "ref-reports" / "qa_lead" / f"test_execution_log_{story_id}.jsonl"
+
+
+class ExecutionLogger:
+    """Log all test execution for audit trail.
+
+    Writes JSONL format with one entry per line.
+    """
+
+    def __init__(self, story_id: str, base_dir: Path | None = None):
+        """Initialize execution logger.
+
+        Args:
+            story_id: Story identifier.
+            base_dir: Base directory for log files.
+        """
+        self.story_id = story_id
+        self.log_file = get_log_path(story_id, base_dir)
+        self._ensure_log_dir()
+
+    def _ensure_log_dir(self) -> None:
+        """Ensure log directory exists."""
+        self.log_file.parent.mkdir(parents=True, exist_ok=True)
+
+    def log_execution(self, entry: ExecutionLogEntry) -> None:
+        """Append execution entry to log file.
+
+        Args:
+            entry: Execution log entry to append.
+        """
+        with open(self.log_file, "a", encoding="utf-8") as f:
+            f.write(entry.to_json() + "\n")
+
+        logger.info(
+            "execution_logged",
+            test_id=entry.test_id,
+            step=entry.step_number,
+            tool=entry.tool_name,
+            status=entry.status,
+        )
+
+    def log_step(  # noqa: PLR0913  # TECH_DEBT: refactor to TypedDict — STORY-046
+        self,
+        test_id: str,
+        step_number: int,
+        tool_name: str,
+        tool_args: dict,
+        response: dict,
+        screenshot_path: Path | None = None,
+        duration_ms: int = 0,
+        status: str = "PASS",
+        error: str | None = None,
+    ) -> ExecutionLogEntry:
+        """Log a test step execution.
+
+        Args:
+            test_id: Test case identifier.
+            step_number: Step number.
+            tool_name: MCP tool name.
+            tool_args: Tool arguments.
+            response: Tool response.
+            screenshot_path: Path to screenshot.
+            duration_ms: Execution duration.
+            status: Step status.
+            error: Error message if failed.
+
+        Returns:
+            The logged entry.
+        """
+        screenshot_size = 0
+        screenshot_str = None
+
+        if screenshot_path and screenshot_path.exists():
+            screenshot_size = screenshot_path.stat().st_size
+            screenshot_str = str(screenshot_path)
+
+        entry = ExecutionLogEntry(
+            timestamp=datetime.now().isoformat(),
+            test_id=test_id,
+            step_number=step_number,
+            tool_name=tool_name,
+            tool_args=tool_args,
+            response=response,
+            screenshot_path=screenshot_str,
+            screenshot_size=screenshot_size,
+            duration_ms=duration_ms,
+            status=status,
+            error=error,
+        )
+
+        self.log_execution(entry)
+        return entry
+
+    def read_log(self) -> list[ExecutionLogEntry]:
+        """Read all entries from log file.
+
+        Returns:
+            List of log entries.
+        """
+        if not self.log_file.exists():
+            return []
+
+        entries = []
+        with open(self.log_file, "r", encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if line:
+                    try:
+                        entries.append(ExecutionLogEntry.from_json(line))
+                    except json.JSONDecodeError as e:
+                        logger.warning("invalid_log_entry", line=line, error=str(e))
+
+        return entries
+
+    def clear_log(self) -> None:
+        """Clear the log file."""
+        if self.log_file.exists():
+            self.log_file.unlink()
+        self._ensure_log_dir()
+
+    def verify_log_integrity(self) -> tuple[bool, list[str]]:
+        """Verify log file integrity.
+
+        Checks:
+        - Log file exists
+        - All entries are valid JSON
+        - Timestamps are sequential
+        - No gaps in step numbers per test
+
+        Returns:
+            Tuple of (valid, list of issues).
+        """
+        issues = []
+
+        if not self.log_file.exists():
+            return False, ["Log file does not exist"]
+
+        entries = self.read_log()
+
+        if not entries:
+            return False, ["Log file is empty"]
+
+        # Check timestamps are sequential
+        prev_timestamp = None
+        for entry in entries:
+            try:
+                timestamp = datetime.fromisoformat(entry.timestamp)
+                if prev_timestamp and timestamp < prev_timestamp:
+                    issues.append(
+                        f"Timestamp not sequential: {entry.timestamp} < {prev_timestamp.isoformat()}"
+                    )
+                prev_timestamp = timestamp
+            except ValueError:
+                issues.append(f"Invalid timestamp format: {entry.timestamp}")
+
+        # Check step numbers per test
+        test_steps: dict[str, list[int]] = {}
+        for entry in entries:
+            if entry.test_id not in test_steps:
+                test_steps[entry.test_id] = []
+            test_steps[entry.test_id].append(entry.step_number)
+
+        for test_id, steps in test_steps.items():
+            expected = list(range(1, len(steps) + 1))
+            if sorted(steps) != expected:
+                issues.append(f"Step numbers not sequential for {test_id}: {steps}")
+
+        return len(issues) == 0, issues
+
+    def get_entry_count(self) -> int:
+        """Get number of entries in log.
+
+        Returns:
+            Number of log entries.
+        """
+        return len(self.read_log())