patchi 0.7.5__py3-none-any.whl

patchi/core/testing/security_test_agent.py

@@ -0,0 +1,253 @@
+"""
+Security Test Agent — generates security-focused tests per route/endpoint.
+
+For each route found by RouteMapScanner, generates:
+- Authentication bypass tests
+- SQL injection payload tests
+- XSS payload injection tests
+- CSRF token validation tests
+- Rate limiting tests
+- Header validation tests
+- CORS origin tests
+
+Outputs pytest files in tests/security/.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+from patchi.core.agents.base import (
+    AgentDomain,
+    AgentGroup,
+    AgentInput,
+    AgentResult,
+    BaseAgent,
+    Finding,
+    Severity,
+    register,
+)
+
+# ── Test templates ────────────────────────────────────────────────────────────
+
+_AUTH_BYPASS_TEMPLATE = '''\
+def test_{func}_requires_auth(client):
+    """{route} should reject unauthenticated requests."""
+    resp = client.{method}("{route}")
+    assert resp.status_code in (401, 403), \
+        f"Expected 401/403, got {{resp.status_code}}"
+'''
+
+_SQLI_TEMPLATE = '''\
+def test_{func}_sqli_protection(client):
+    """{route} should reject SQL injection payloads."""
+    payloads = ["' OR '1'='1", "1; DROP TABLE--", "' UNION SELECT NULL--"]
+    for payload in payloads:
+        resp = client.{method}("{route}", json={{"input": payload}})
+        assert resp.status_code != 500, \
+            f"Server error on SQLi payload: {{payload}}"
+'''
+
+_XSS_TEMPLATE = '''\
+def test_{func}_xss_protection(client):
+    """{route} should sanitize or escape XSS payloads."""
+    payloads = ["<script>alert(1)</script>", "<img onerror=alert(1)>"]
+    for payload in payloads:
+        resp = client.{method}("{route}", json={{"input": payload}})
+        if resp.status_code == 200:
+            assert payload not in resp.text, \
+                f"XSS payload reflected in response: {{payload}}"
+'''
+
+_CSRF_TEMPLATE = '''\
+def test_{func}_csrf_protection(client):
+    """{route} should validate CSRF tokens for state-changing operations."""
+    resp = client.{method}("{route}", headers={{"X-Requested-With": ""}})
+    assert resp.status_code in (400, 403), \
+        f"Expected CSRF rejection, got {{resp.status_code}}"
+'''
+
+_HEADER_TEMPLATE = '''\
+def test_{func}_security_headers(client):
+    """{route} should include security headers."""
+    resp = client.get("{route}")
+    headers = {h.lower() for h in resp.headers}
+    # At minimum, these should be present on sensitive endpoints
+    recommended = ["x-content-type-options", "x-frame-options"]
+    for h in recommended:
+        # Only warn, don't fail — not all routes need all headers
+        pass
+'''
+
+_RATE_LIMIT_TEMPLATE = '''\
+def test_{func}_rate_limiting(client):
+    """{route} should enforce rate limiting on repeated requests."""
+    responses = [client.{method}("{route}") for _ in range(50)]
+    status_codes = [r.status_code for r in responses]
+    # At least some requests should be throttled (429) or the endpoint
+    # should handle rapid requests gracefully (no 500s)
+    assert 500 not in status_codes, "Server crashed under rapid requests"
+'''
+
+_CORS_TEMPLATE = '''\
+def test_{func}_cors_restrictions(client):
+    """{route} should not allow arbitrary origins."""
+    resp = client.options("{route}", headers={{{{ \
+        "Origin": "https://evil.com", \
+        "Access-Control-Request-Method": "{method_upper}", \
+    }}}})
+    acao = resp.headers.get("access-control-allow-origin", "")
+    assert acao != "*", \
+        f"CORS allows all origins on {route}"
+'''
+
+
+# ── Agent ─────────────────────────────────────────────────────────────────────
+
+
+@register
+class SecurityTestAgent(BaseAgent):
+    """Generates security-focused pytest tests for detected routes."""
+
+    name = "SecurityTestAgent"
+    group = AgentGroup.TEST
+    domain = AgentDomain.TESTING
+    timeout = 120
+
+    def _run(self, inp: AgentInput, result: AgentResult) -> None:
+        routes = inp.brain.get("routes", [])
+        if not routes:
+            result.add_finding(
+                Finding(
+                    agent=self.name,
+                    type="info",
+                    severity=Severity.INFO,
+                    file="",
+                    message="No routes detected — security tests require a web app with routes",
+                )
+            )
+            return
+
+        test_dir = inp.root / "tests" / "security"
+        test_files: list[Path] = []
+
+        framework = self._detect_framework(inp.brain)
+        for route_info in routes:
+            route = (
+                route_info.get("path", "")
+                if isinstance(route_info, dict)
+                else getattr(route_info, "path", "")
+            )
+            method = (
+                route_info.get("method", "get")
+                if isinstance(route_info, dict)
+                else getattr(route_info, "method", "get")
+            ).lower()
+            func_name = self._route_to_func(route, method)
+
+            if not func_name:
+                continue
+
+            tests = self._generate_tests(route, method, func_name)
+            if tests:
+                test_file = test_dir / f"test_security_{func_name}.py"
+                test_file.parent.mkdir(parents=True, exist_ok=True)
+                header = self._build_header(framework, route, method)
+                test_file.write_text(header + tests, encoding="utf-8")
+                test_files.append(test_file)
+                result.files_scanned += 1
+
+        if test_files:
+            result.data["generated_files"] = [str(f.relative_to(inp.root)) for f in test_files]
+            result.data["test_count"] = len(test_files)
+
+    def _route_to_func(self, route: str, method: str) -> str:
+        """Convert /api/users/{id} -> api_users_id."""
+        clean = re.sub(r"[^a-zA-Z0-9/]", "", route)
+        parts = [p for p in clean.split("/") if p]
+        name = "_".join(parts[:4])  # limit depth
+        return f"{method}_{name}" if name else ""
+
+    def _detect_framework(self, brain: dict) -> str:
+        fw = brain.get("framework", "")
+        fw_raw = brain.get("frameworks", fw)
+        if isinstance(fw_raw, list):
+            fw = fw_raw[0] if fw_raw else ""
+        else:
+            fw = fw_raw or ""
+        fw_lower = fw.lower()
+        if "fastapi" in fw_lower or "starlette" in fw_lower:
+            return "fastapi"
+        if "flask" in fw_lower:
+            return "flask"
+        if "django" in fw_lower:
+            return "django"
+        if "express" in fw_lower or "node" in fw_lower:
+            return "express"
+        return "generic"
+
+    def _build_header(self, framework: str, route: str, method: str) -> str:
+        fixtures = {
+            "fastapi": (
+                "@pytest.fixture\ndef client():\n"
+                "    from fastapi.testclient import TestClient\n"
+                "    from app import app\n"
+                "    return TestClient(app)\n"
+            ),
+            "flask": (
+                "@pytest.fixture\ndef client():\n"
+                "    from flask import Flask\n"
+                "    app = Flask(__name__)\n"
+                "    return app.test_client()\n"
+            ),
+            "django": (
+                "@pytest.fixture\ndef client():\n"
+                "    from django.test import Client\n"
+                "    return Client()\n"
+            ),
+        }
+        fixture = fixtures.get(
+            framework,
+            (
+                "@pytest.fixture\ndef client():\n"
+                '    """Override with your app\'s test client."""\n'
+                "    import pytest\n"
+                '    raise NotImplementedError("Define a client fixture for your framework")\n'
+            ),
+        )
+        return (
+            '"""Auto-generated security tests by Patchi SecurityTestAgent."""\n'
+            "import pytest\n\n\n"
+            f"{fixture}\n\n"
+        )
+
+    def _generate_tests(self, route: str, method: str, func: str) -> str:
+        """Generate test code for a single route."""
+        tests = []
+        templates = [
+            ("auth", _AUTH_BYPASS_TEMPLATE),
+            ("sqli", _SQLI_TEMPLATE),
+            ("xss", _XSS_TEMPLATE),
+        ]
+        # Only add CSRF for state-changing methods
+        if method in ("post", "put", "patch", "delete"):
+            templates.append(("csrf", _CSRF_TEMPLATE))
+
+        # CORS for all
+        templates.append(("cors", _CORS_TEMPLATE))
+
+        # Rate limit for all
+        templates.append(("rate_limit", _RATE_LIMIT_TEMPLATE))
+
+        for _name, tpl in templates:
+            tests.append(
+                tpl.format(
+                    func=func,
+                    route=route,
+                    method=method,
+                    method_upper=method.upper(),
+                )
+            )
+
+        return "\n".join(tests)

patchi/core/testing/stress_test_agent.py

@@ -0,0 +1,203 @@
+"""
+StressTestAgent — Load testing via StressOrchestrator.
+
+Thin wrapper around StressOrchestrator that runs as a registered BaseAgent.
+Supports load, spike, soak, and breakpoint testing scenarios.
+
+Uses aiohttp for async HTTP requests (no external binary needed).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from pathlib import Path
+
+from ..agents.base import (
+    AgentDomain,
+    AgentGroup,
+    AgentInput,
+    AgentResult,
+    AgentStatus,
+    BaseAgent,
+    Severity,
+    make_finding,
+    register,
+)
+from ..testing.live_v2.stress_orchestrator import (
+    StressConfig,
+    StressOrchestrator,
+    StressTestReport,
+    run_stress_test,
+)
+
+_log = logging.getLogger("patchi.testing.stress_test_agent")
+
+
+@register
+class StressTestAgent(BaseAgent):
+    """Load/stress testing agent using StressOrchestrator."""
+
+    group = AgentGroup.TEST
+    domain = AgentDomain.TESTING
+    name = "StressTestAgent"
+    description = "Stress testing: load, spike, soak, breakpoint scenarios via async HTTP"
+    timeout = 600  # 10 minutes max for stress tests
+
+    def _run(self, inp: AgentInput, result: AgentResult) -> None:
+        target_url = self._detect_target(inp)
+        if not target_url:
+            self.skip(result, "No running web application detected. Start the web server first.")
+            return
+
+        _log.info("StressTestAgent: Testing %s with StressOrchestrator", target_url)
+
+        # Get scenario from config (default: load)
+        scenario = inp.config.get("stress_scenario", "load")
+        users = inp.config.get("stress_users", 10)
+        duration = inp.config.get("stress_duration", 60)
+
+        # Build endpoints from route map
+        route_map = inp.brain.get("route_map", {})
+        endpoints = []
+        for k, v in route_map.items():
+            if isinstance(v, dict):
+                path = v.get("path", k.split(" ", 1)[-1])
+                endpoints.append({"method": "GET", "path": path})
+        if not endpoints:
+            endpoints = [{"method": "GET", "path": "/"}]
+
+        # Run stress test
+        try:
+            report = asyncio.run(self._run_stress(target_url, scenario, users, duration, endpoints))
+            self._process_report(report, result)
+        except Exception as e:
+            _log.error("StressTestAgent failed: %s", e)
+            result.add_error(f"Stress test failed: {e}")
+            result.status = AgentStatus.FAILED
+
+    async def _run_stress(
+        self,
+        target_url: str,
+        scenario: str,
+        users: int,
+        duration: int,
+        endpoints: list[dict],
+    ) -> StressTestReport:
+        """Run the stress test via StressOrchestrator."""
+        config = StressConfig(
+            base_url=target_url,
+            scenario=scenario,
+            users=users,
+            duration_seconds=duration,
+            endpoints=endpoints,
+        )
+        orchestrator = StressOrchestrator(config)
+        return await orchestrator.run()
+
+    def _process_report(self, report: StressTestReport, result: AgentResult) -> None:
+        """Process StressTestReport into AgentResult findings."""
+        result.data["report"] = report.to_dict()
+        result.data["scenario"] = report.config.scenario
+        result.data["users"] = report.config.users
+        result.data["duration_seconds"] = report.config.duration_seconds
+
+        # Add latency findings
+        latency = report.latency
+        if latency:
+            p95 = latency.get("p95", 0)
+            p99 = latency.get("p99", 0)
+
+            if p95 > 2000:
+                result.add_finding(
+                    make_finding(
+                        agent=self.name,
+                        finding_type="slow_p95",
+                        severity=Severity.MEDIUM,
+                        file="stress_test",
+                        message=f"Slow p95 latency under load: {p95:.0f}ms",
+                    )
+                )
+            if p99 > 5000:
+                result.add_finding(
+                    make_finding(
+                        agent=self.name,
+                        finding_type="slow_p99",
+                        severity=Severity.HIGH,
+                        file="stress_test",
+                        message=f"Slow p99 latency under load: {p99:.0f}ms",
+                    )
+                )
+
+        # Add error rate findings
+        if report.total_requests > 0:
+            error_rate = report.failed_requests / report.total_requests
+            if error_rate > 0.05:  # 5%
+                result.add_finding(
+                    make_finding(
+                        agent=self.name,
+                        finding_type="high_error_rate",
+                        severity=Severity.HIGH,
+                        file="stress_test",
+                        message=f"High load-test error rate: {error_rate:.1%}",
+                    )
+                )
+
+        # Add breakpoint findings
+        if report.breakpoint_found:
+            result.add_finding(
+                make_finding(
+                    agent=self.name,
+                    finding_type="breakpoint_found",
+                    severity=Severity.HIGH,
+                    file="stress_test",
+                    message=f"Breakpoint found at {report.breakpoint_users} users",
+                )
+            )
+
+        # Add soak stability findings
+        if not report.soak_stability:
+            result.add_finding(
+                make_finding(
+                    agent=self.name,
+                    finding_type="soak_instability",
+                    severity=Severity.MEDIUM,
+                    file="stress_test",
+                    message="System instability detected during soak test",
+                )
+            )
+
+        result.status = AgentStatus.DONE
+        result.files_scanned = 1
+
+    def _detect_target(self, inp: AgentInput) -> str | None:
+        """Detect the target URL for testing."""
+        import socket
+
+        # Check for explicit base_url in config
+        if inp.config.get("base_url"):
+            return inp.config["base_url"]
+
+        # Try common ports
+        ports = [1612, 8000, 3000, 8080, 80]
+
+        for port in ports:
+            try:
+                with socket.create_connection(("127.0.0.1", port), timeout=1):
+                    return f"http://127.0.0.1:{port}"
+            except (ConnectionRefusedError, OSError):
+                continue
+
+        # Check config for web port
+        try:
+            from patchi.core import config as cfg
+
+            config = cfg.load(inp.root)
+            web_config = config.get("web", {})
+            host = web_config.get("host", "127.0.0.1")
+            port = web_config.get("port", 1612)
+            return f"http://{host}:{port}"
+        except Exception as _exc:
+            _log.warning("_detect_target failed: %s", _exc)
+
+        return None

patchi/core/testing/test_agents.py

@@ -0,0 +1,50 @@
+"""
+Test agent aggregation module for Patchi.
+
+Exports all test agent classes and shared test helper types.
+"""
+
+from __future__ import annotations
+
+from ..agents.base import register
+from .app_discovery_agent import AppDiscoveryAgent  # noqa: F401
+from .api_contract_agent import APIContractAgent
+from .browser_test_agent import BrowserTestAgent
+from .e2e_flow_agent import E2EFlowAgent
+from .flake_detector_agent import FlakeDetectorAgent  # noqa: F401
+from .regression_agent import RegressionAgent
+from .security_test_agent import SecurityTestAgent
+from .stress_test_agent import StressTestAgent
+from .ui_accessibility_agent import UIAccessibilityAgent
+from .ui_button_agent import UIButtonAgent
+from .ui_layout_agent import UILayoutAgent
+from .unit_test_agent import TestCase, TestSuite, UnitTestAgent, _run
+from .visual_regression_agent import VisualRegressionAgent
+
+# Live v2 runner registers its agent on import (additive — never breaks v1)
+try:
+    from .live_v2.runner import LiveTestRunnerV2Agent  # noqa: F401
+except Exception as _e:  # pragma: no cover — playwright-less envs still work
+    import logging as _logging
+
+    _logging.getLogger("patchi.testing").warning("LiveTestRunnerV2Agent unavailable: %s", _e)
+
+__all__ = [
+    "AppDiscoveryAgent",
+    "APIContractAgent",
+    "BrowserTestAgent",
+    "E2EFlowAgent",
+    "FlakeDetectorAgent",
+    "RegressionAgent",
+    "SecurityTestAgent",
+    "StressTestAgent",
+    "TestCase",
+    "TestSuite",
+    "UIAccessibilityAgent",
+    "UIButtonAgent",
+    "UILayoutAgent",
+    "UnitTestAgent",
+    "VisualRegressionAgent",
+    "_run",
+    "register",
+]

patchi/core/testing/test_config.py

@@ -0,0 +1,194 @@
+"""
+Test configuration for Patchi.
+
+Reads/writes .patchi/tests.json — project-level test settings.
+
+Config shape:
+{
+  "base_url": "http://localhost:3000",
+  "viewport_widths": [375, 768, 1024, 1440],
+  "timeout_per_agent": 120,
+  "parallel": false,
+  "skip_agents": [],
+  "only_agents": [],
+  "ai_generate": true,
+  "screenshots_dir": ".patchi/screenshots",
+  "baselines_dir": ".patchi/baselines",
+  "custom_flows": [
+    {"name": "login", "url": "/login", "steps": ["fill email", "fill password", "click submit"]}
+  ],
+  "exclude_paths": ["node_modules", ".venv", "dist"],
+  "report_format": "rich"
+}
+"""
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+from typing import Any
+
+from patchi.core.brain.languages import DEFAULT_IGNORE_DIRS
+
+from ..constants import PATCHI_DIR
+
+TESTS_CONFIG = f"{PATCHI_DIR}/tests.json"
+
+DEFAULT_TEST_CONFIG = {
+    "base_url": None,
+    "viewport_widths": [375, 768, 1024, 1440],
+    "timeout_per_agent": 120,
+    "parallel": False,
+    "skip_agents": [],
+    "only_agents": [],
+    "ai_generate": True,
+    "screenshots_dir": f"{PATCHI_DIR}/screenshots",
+    "baselines_dir": f"{PATCHI_DIR}/baselines",
+    "custom_flows": [],
+    "exclude_paths": sorted(DEFAULT_IGNORE_DIRS),
+    "report_format": "rich",
+}
+
+
+_log = logging.getLogger("patchi.testing.test_config")
+
+
+def _config_path(root: Path) -> Path:
+    return root / TESTS_CONFIG
+
+
+def load(root: Path | None = None) -> dict:
+    """Load test config, merging with defaults."""
+    from ..config import require_project_root
+
+    root = root or require_project_root()
+    path = _config_path(root)
+    if not path.exists():
+        return dict(DEFAULT_TEST_CONFIG)
+    try:
+        raw = json.loads(path.read_text(encoding="utf-8"))
+        return {**DEFAULT_TEST_CONFIG, **raw}
+    except Exception as e:
+        _log.warning("load failed: %s", e)
+        return dict(DEFAULT_TEST_CONFIG)
+
+
+def save(config: dict, root: Path | None = None) -> None:
+    """Write test config to disk."""
+    from ..config import require_project_root
+
+    root = root or require_project_root()
+    path = _config_path(root)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(config, indent=2), encoding="utf-8")
+
+
+def get(key: str, root: Path | None = None) -> Any:
+    """Get a single test config value."""
+    return load(root).get(key)
+
+
+def set_value(key: str, value: Any, root: Path | None = None) -> None:
+    """Update a single test config key."""
+    config = load(root)
+    config[key] = value
+    save(config, root)
+
+
+def add_custom_flow(name: str, url: str, steps: list[str], root: Path | None = None) -> None:
+    """Add a custom test flow."""
+    config = load(root)
+    flows = config.get("custom_flows", [])
+    flows.append({"name": name, "url": url, "steps": steps})
+    config["custom_flows"] = flows
+    save(config, root)
+
+
+def remove_custom_flow(name: str, root: Path | None = None) -> bool:
+    """Remove a custom test flow by name."""
+    config = load(root)
+    flows = config.get("custom_flows", [])
+    before = len(flows)
+    config["custom_flows"] = [f for f in flows if f.get("name") != name]
+    save(config, root)
+    return len(config["custom_flows"]) < before
+
+
+def get_test_history(root: Path | None = None) -> list[dict]:
+    """Load test run history from memory."""
+    from .. import memory as mem
+    from ..config import require_project_root
+
+    root = root or require_project_root()
+    brain = mem.get_brain(root)
+    return brain.get("test_history", [])
+
+
+def append_test_history(entry: dict, root: Path | None = None) -> None:
+    """Append a test run entry to history."""
+    from .. import memory as mem
+    from ..config import require_project_root
+
+    root = root or require_project_root()
+    brain = mem.get_brain(root)
+    history = brain.get("test_history", [])
+    history.append(entry)
+    # Keep last 50 runs
+    if len(history) > 50:
+        history = history[-50:]
+    brain["test_history"] = history
+    mem.save_brain(brain, root)
+
+
+def get_coverage(root: Path | None = None) -> dict:
+    """Get cumulative test coverage data."""
+    from .. import memory as mem
+    from ..config import require_project_root
+
+    root = root or require_project_root()
+    brain = mem.get_brain(root)
+    return brain.get(
+        "test_coverage",
+        {
+            "files_tested": [],
+            "agents_run": {},
+            "total_runs": 0,
+            "last_run": None,
+        },
+    )
+
+
+def update_coverage(run_result: dict, root: Path | None = None) -> None:
+    """Update coverage data after a test run."""
+    from .. import memory as mem
+    from ..config import require_project_root
+
+    root = root or require_project_root()
+    brain = mem.get_brain(root)
+    coverage = brain.get(
+        "test_coverage",
+        {
+            "files_tested": [],
+            "agents_run": {},
+            "total_runs": 0,
+            "last_run": None,
+        },
+    )
+
+    coverage["total_runs"] = coverage.get("total_runs", 0) + 1
+    coverage["last_run"] = run_result.get("timestamp")
+
+    # Track which agents have been run
+    for agent_name in run_result.get("agents", []):
+        counts = coverage.get("agents_run", {})
+        counts[agent_name] = counts.get(agent_name, 0) + 1
+        coverage["agents_run"] = counts
+
+    # Track files tested
+    tested_files = set(coverage.get("files_tested", []))
+    for f in run_result.get("files_tested", []):
+        tested_files.add(f)
+    coverage["files_tested"] = sorted(tested_files)
+
+    brain["test_coverage"] = coverage
+    mem.save_brain(brain, root)