shadowcat 2.0.0__py3-none-any.whl

backend/crawler/spider.py

@@ -0,0 +1,550 @@
+"""Async concurrent BFS web spider for pre-scan attack surface discovery.
+
+Runs before the LLM agent to give it a complete map of endpoints, forms, and
+interesting parameters so the agent spends its budget testing, not exploring.
+
+Improvements over the initial version:
+  Phase 1  — Authenticated crawling: accepts session cookies so the spider sees
+             authenticated pages the anonymous spider misses.
+  Phase 3  — JavaScript API discovery: after the HTML crawl, fetches linked .js
+             files and scans them for fetch/axios/route API path patterns.
+  Phase 6  — Concurrent worker pool: N workers (default 10) share a global
+             token-bucket rate limiter, giving ~10× throughput vs sequential
+             without changing the per-second request budget.
+
+Design constraints (unchanged from v1):
+  - Zero new runtime deps (stdlib html.parser + httpx already present).
+  - Stays within authorized scope; skips static assets by extension.
+  - Parses robots.txt; honours Disallow unless respect_robots=False.
+  - Prioritises dynamic endpoints in the summary.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import logging
+import os.path
+import re
+import time
+import urllib.parse
+from dataclasses import dataclass, field
+from html.parser import HTMLParser
+from typing import Any
+
+import httpx
+
+log = logging.getLogger(__name__)
+
+_SKIP_EXTENSIONS = frozenset(
+    {
+        ".css",
+        ".js",
+        ".ts",
+        ".mjs",
+        ".jsx",
+        ".tsx",
+        ".png",
+        ".jpg",
+        ".jpeg",
+        ".gif",
+        ".svg",
+        ".webp",
+        ".ico",
+        ".avif",
+        ".woff",
+        ".woff2",
+        ".ttf",
+        ".eot",
+        ".otf",
+        ".pdf",
+        ".zip",
+        ".tar",
+        ".gz",
+        ".rar",
+        ".7z",
+        ".dmg",
+        ".exe",
+        ".mp4",
+        ".mp3",
+        ".avi",
+        ".mov",
+        ".webm",
+        ".map",
+    }
+)
+
+_ID_SEGMENT = re.compile(
+    r"(?:^|\/)(\d{1,12}|[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:\/|$)",
+    re.IGNORECASE,
+)
+
+
+# ---------------------------------------------------------------------------
+# HTML parser helpers
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class FormInfo:
+    page_url: str
+    action: str
+    method: str
+    fields: list[str] = field(default_factory=list)
+
+    def summary(self) -> str:
+        fields_str = ", ".join(self.fields[:8]) or "(none)"
+        return f"{self.method} {self.action} [fields: {fields_str}]"
+
+
+class _PageParser(HTMLParser):
+    def __init__(self, base_url: str) -> None:
+        super().__init__()
+        self._base = base_url
+        self.links: list[str] = []
+        self.forms: list[FormInfo] = []
+        self._form: FormInfo | None = None
+
+    def _abs(self, href: str) -> str:
+        return urllib.parse.urljoin(self._base, href.strip())
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        a = dict(attrs)
+        tag = tag.lower()
+        href = a.get("href")
+        if tag in ("a", "area") and href:
+            self.links.append(self._abs(href))
+        elif tag == "form":
+            action = self._abs(a.get("action") or self._base)
+            self._form = FormInfo(
+                page_url=self._base,
+                action=action,
+                method=(a.get("method") or "get").upper(),
+            )
+        elif tag in ("input", "textarea", "select", "button") and self._form is not None:
+            name = a.get("name") or a.get("id")
+            if name and name not in ("submit", "reset", "button"):
+                self._form.fields.append(name)
+
+    def handle_endtag(self, tag: str) -> None:
+        if tag.lower() == "form" and self._form is not None:
+            self.forms.append(self._form)
+            self._form = None
+
+
+# ---------------------------------------------------------------------------
+# Attack surface output
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class AttackSurface:
+    """Structured output from a Spider crawl."""
+
+    base_url: str
+    pages_visited: int = 0
+    dynamic_endpoints: list[str] = field(default_factory=list)
+    forms: list[FormInfo] = field(default_factory=list)
+    technologies: list[str] = field(default_factory=list)
+    robots_disallowed: list[str] = field(default_factory=list)
+    js_api_endpoints: list[str] = field(default_factory=list)  # Phase 3
+    subdomains: list[str] = field(default_factory=list)  # Phase 4
+
+    def summary(self, max_endpoints: int = 30, max_forms: int = 20) -> str:
+        lines: list[str] = [
+            f"PRE-SCAN ATTACK SURFACE ({self.base_url})",
+            f"Pages crawled: {self.pages_visited}  |  "
+            f"Dynamic endpoints: {len(self.dynamic_endpoints)}  |  "
+            f"Forms: {len(self.forms)}  |  "
+            f"JS API paths: {len(self.js_api_endpoints)}",
+        ]
+        if self.technologies:
+            lines.append(f"Detected stack: {', '.join(self.technologies)}")
+        if self.subdomains:
+            lines.append(f"Live subdomains: {', '.join(self.subdomains[:15])}")
+
+        if self.forms:
+            lines.append("\nFORMS (test for SQLi, XSS, CSRF):")
+            for f in self.forms[:max_forms]:
+                lines.append(f"  {f.summary()}")
+            if len(self.forms) > max_forms:
+                lines.append(f"  … and {len(self.forms) - max_forms} more")
+
+        if self.dynamic_endpoints:
+            lines.append("\nDYNAMIC ENDPOINTS (test for IDOR, SQLi, traversal):")
+            for ep in self.dynamic_endpoints[:max_endpoints]:
+                lines.append(f"  {ep}")
+            if len(self.dynamic_endpoints) > max_endpoints:
+                lines.append(f"  … and {len(self.dynamic_endpoints) - max_endpoints} more")
+
+        if self.js_api_endpoints:
+            lines.append("\nJS-DISCOVERED API ENDPOINTS (not visible in HTML):")
+            for ep in self.js_api_endpoints[:20]:
+                lines.append(f"  {ep}")
+            if len(self.js_api_endpoints) > 20:
+                lines.append(f"  … and {len(self.js_api_endpoints) - 20} more")
+
+        if self.robots_disallowed:
+            interesting = [
+                p for p in self.robots_disallowed if "admin" in p.lower() or "api" in p.lower()
+            ]
+            if interesting:
+                lines.append(
+                    f"\nINTERESTING robots.txt Disallow paths: {', '.join(interesting[:10])}"
+                )
+
+        lines.append(
+            "\nFocus your testing budget on the forms and dynamic endpoints above. "
+            "Test each form field for SQLi and XSS. Test path parameters for IDOR and traversal."
+        )
+        return "\n".join(lines)
+
+
+# ---------------------------------------------------------------------------
+# Spider
+# ---------------------------------------------------------------------------
+
+
+class Spider:
+    """Concurrent BFS web crawler.
+
+    Parameters
+    ----------
+    allowed_hosts:
+        Additional hostnames / netlocs the crawler may follow links to.
+        The base URL's host is always allowed.
+    respect_robots:
+        Parse ``/robots.txt`` and skip disallowed paths when True.
+    timeout:
+        Per-request timeout in seconds.
+    workers:
+        Number of concurrent fetcher coroutines (Phase 6).
+    analyze_js:
+        Run JS API discovery after the HTML crawl (Phase 3).
+    """
+
+    def __init__(
+        self,
+        *,
+        allowed_hosts: set[str] | None = None,
+        respect_robots: bool = True,
+        timeout: float = 15.0,
+        workers: int = 10,
+        analyze_js: bool = True,
+        render_js: bool = False,
+    ) -> None:
+        self._allowed_hosts = allowed_hosts or set()
+        self._respect_robots = respect_robots
+        self._timeout = timeout
+        self._workers = workers
+        self._analyze_js = analyze_js
+        # Phase 7: render the root in a real headless browser so JS-rendered SPAs
+        # (which return an almost-empty shell over plain httpx) are seen properly.
+        self._render_js = render_js
+
+    async def crawl(
+        self,
+        start_url: str,
+        *,
+        max_pages: int = 150,
+        max_depth: int = 3,
+        rate_limit_ms: int = 300,
+        emitter: Any = None,
+        session_cookies: dict[str, str] | None = None,  # Phase 1: auth crawl
+    ) -> AttackSurface:
+        """Run a concurrent BFS crawl from *start_url*."""
+        parsed = urllib.parse.urlparse(start_url)
+        base_netloc = parsed.netloc
+        allowed = self._allowed_hosts | {base_netloc}
+        surface = AttackSurface(base_url=start_url)
+
+        # O(1) dedup mirrors of surface's ordered lists. FormInfo is not
+        # hashable, so forms are keyed by a value tuple (page_url included to
+        # preserve the original dataclass-__eq__ dedup semantics).
+        seen_endpoints: set[str] = set()
+        seen_tech: set[str] = set()
+        seen_forms: set[tuple[str, str, str, tuple[str, ...]]] = set()
+
+        disallowed: set[str] = set()
+        visited: set[str] = set()
+        visited_lock = asyncio.Lock()
+
+        # Global token-bucket rate limiter shared across all workers.
+        last_req_time = 0.0
+        rate_lock = asyncio.Lock()
+
+        # Phase 3: capture root-page HTML for JS analysis.
+        root_html: list[str] = []
+
+        async def _rate_wait() -> None:
+            nonlocal last_req_time
+            if rate_limit_ms <= 0:
+                return
+            async with rate_lock:
+                now = time.monotonic()
+                gap = rate_limit_ms / 1000.0
+                to_sleep = last_req_time + gap - now
+                if to_sleep > 0:
+                    await asyncio.sleep(to_sleep)
+                last_req_time = time.monotonic()
+
+        work_queue: asyncio.Queue[tuple[str, int]] = asyncio.Queue()
+
+        async def _process(url: str, depth: int, client: httpx.AsyncClient) -> None:
+            norm = _normalise(url)
+            if not norm:
+                return
+            async with visited_lock:
+                if norm in visited:
+                    return
+                if not _in_scope(norm, allowed):
+                    return
+                if _skip_by_extension(norm):
+                    return
+                if self._respect_robots and _is_disallowed(norm, disallowed):
+                    return
+                if surface.pages_visited >= max_pages:
+                    return
+                visited.add(norm)
+
+            await _rate_wait()
+
+            try:
+                resp = await client.get(norm)
+            except Exception as exc:
+                log.debug("Spider fetch error %s: %s", norm, exc)
+                return
+
+            surface.pages_visited += 1
+
+            for header in ("server", "x-powered-by", "x-aspnet-version"):
+                val = resp.headers.get(header)
+                if val and val not in seen_tech:
+                    seen_tech.add(val)
+                    surface.technologies.append(val)
+
+            if _is_dynamic(norm) and norm not in seen_endpoints:
+                seen_endpoints.add(norm)
+                surface.dynamic_endpoints.append(norm)
+
+            ct = resp.headers.get("content-type", "")
+            if "html" not in ct:
+                return
+
+            html_text = resp.text
+            # Capture root page HTML for JS analysis (Phase 3).
+            if not root_html and norm == _normalise(start_url):
+                root_html.append(html_text)
+
+            page_parser = _PageParser(norm)
+            with contextlib.suppress(Exception):
+                page_parser.feed(html_text)
+
+            for form in page_parser.forms:
+                if not form.action:
+                    continue
+                form_key = (form.page_url, form.action, form.method, tuple(form.fields))
+                if form_key not in seen_forms:
+                    seen_forms.add(form_key)
+                    surface.forms.append(form)
+                    if form.action not in seen_endpoints:
+                        seen_endpoints.add(form.action)
+                        surface.dynamic_endpoints.append(form.action)
+
+            if depth < max_depth:
+                for link in page_parser.links:
+                    lnorm = _normalise(link)
+                    if lnorm and _in_scope(lnorm, allowed):
+                        async with visited_lock:
+                            if lnorm not in visited:
+                                work_queue.put_nowait((lnorm, depth + 1))
+
+        async def _worker(client: httpx.AsyncClient) -> None:
+            while True:
+                url, depth = await work_queue.get()
+                try:
+                    await _process(url, depth, client)
+                except Exception as exc:
+                    log.debug("Spider worker error on %s: %s", url, exc)
+                finally:
+                    work_queue.task_done()
+
+        log.info(
+            "Spider starting: %s (max_pages=%d depth=%d workers=%d)",
+            start_url,
+            max_pages,
+            max_depth,
+            self._workers,
+        )
+
+        client_cookies = dict(session_cookies or {})
+
+        # Block redirects that would leave the authorized scope: an in-scope page
+        # 3xx-ing to an internal/metadata host would otherwise be fetched here,
+        # outside the agent's scope boundary (SSRF). Same-host redirects are
+        # unaffected; an out-of-scope hop raises and the worker skips that page.
+        async def _redirect_scope_guard(response: httpx.Response) -> None:
+            if response.is_redirect:
+                loc = response.headers.get("location")
+                if loc and not _in_scope(str(response.url.join(loc)), allowed):
+                    raise RuntimeError(f"redirect out of scope: {loc}")
+
+        async with httpx.AsyncClient(
+            verify=False,
+            timeout=self._timeout,
+            follow_redirects=True,
+            cookies=client_cookies,
+            headers={"User-Agent": "Mozilla/5.0 (compatible; ShadowCat-Spider/2.0)"},
+            event_hooks={"response": [_redirect_scope_guard]},
+        ) as client:
+            # Fetch robots.txt first (best-effort).
+            if self._respect_robots:
+                robots_url = f"{parsed.scheme}://{base_netloc}/robots.txt"
+                try:
+                    resp = await client.get(robots_url)
+                    if resp.status_code == 200:
+                        disallowed, extra_paths = _parse_robots(
+                            resp.text, parsed.scheme, base_netloc
+                        )
+                        surface.robots_disallowed = list(disallowed)
+                        for path in extra_paths:
+                            work_queue.put_nowait((path, 1))
+                except Exception:
+                    pass
+
+            # Seed the queue.
+            work_queue.put_nowait((start_url, 0))
+
+            # Start N workers.
+            worker_tasks = [asyncio.create_task(_worker(client)) for _ in range(self._workers)]
+
+            # Wait until all queued items are processed.
+            await work_queue.join()
+
+            # Stop workers (they are blocked on queue.get()).
+            for task in worker_tasks:
+                task.cancel()
+            await asyncio.gather(*worker_tasks, return_exceptions=True)
+
+            # Phase 3: JS API analysis on the root page.
+            if self._analyze_js and root_html:
+                try:
+                    from backend.crawler.js_analyzer import JsApiExtractor
+
+                    extractor = JsApiExtractor()
+                    js_paths = await extractor.extract(root_html[0], start_url, client)
+                    surface.js_api_endpoints = js_paths
+                    # Promote JS-discovered paths to dynamic_endpoints.
+                    for path in js_paths:
+                        abs_url = urllib.parse.urljoin(start_url, path)
+                        norm = _normalise(abs_url)
+                        if norm and norm not in seen_endpoints:
+                            seen_endpoints.add(norm)
+                            surface.dynamic_endpoints.append(norm)
+                except Exception as exc:
+                    log.debug("JS analysis failed: %s", exc)
+
+            # Phase 7: headless render of the root for JS-rendered SPAs. The root
+            # IS the whole app for a SPA, so one render captures its real DOM and
+            # the API endpoints it calls. Optional + best-effort: skipped silently
+            # if Playwright (or its browser) isn't installed.
+            if self._render_js:
+                from backend.crawler import headless
+
+                if headless.available():
+                    try:
+                        rendered = await headless.fetch_rendered(
+                            start_url,
+                            timeout_ms=int(self._timeout * 1000) + 15000,
+                            interact=True,  # also click buttons to trigger click-only APIs
+                        )
+                        if surface.pages_visited == 0:
+                            surface.pages_visited = 1
+                        # Forms/links the JS produced that plain httpx never saw.
+                        rp = _PageParser(start_url)
+                        with contextlib.suppress(Exception):
+                            rp.feed(rendered.html)
+                        for form in rp.forms:
+                            key = (form.page_url, form.action, form.method, tuple(form.fields))
+                            if form.action and key not in seen_forms:
+                                seen_forms.add(key)
+                                surface.forms.append(form)
+                        # The real SPA surface: every API the page called at load.
+                        for call in rendered.api_calls:
+                            url_part = call.split(" ", 1)[-1]
+                            norm = _normalise(url_part)
+                            if norm and norm not in seen_endpoints:
+                                seen_endpoints.add(norm)
+                                surface.dynamic_endpoints.append(norm)
+                                surface.js_api_endpoints.append(call)
+                        log.info("Headless render: %d API calls captured", len(rendered.api_calls))
+                    except Exception as exc:
+                        log.debug("Headless render failed: %s", exc)
+                else:
+                    log.info("render_js requested but Playwright not available — skipping")
+
+        log.info(
+            "Spider done: %d pages, %d dynamic endpoints, %d forms, %d JS paths",
+            surface.pages_visited,
+            len(surface.dynamic_endpoints),
+            len(surface.forms),
+            len(surface.js_api_endpoints),
+        )
+        return surface
+
+
+# ---------------------------------------------------------------------------
+# Helpers (unchanged from v1)
+# ---------------------------------------------------------------------------
+
+
+def _normalise(url: str) -> str:
+    try:
+        p = urllib.parse.urlparse(url)
+        clean = urllib.parse.urlunparse((p.scheme, p.netloc, p.path, p.params, p.query, ""))
+        return clean if clean.startswith("http") else ""
+    except Exception:
+        return ""
+
+
+def _in_scope(url: str, allowed: set[str]) -> bool:
+    try:
+        return urllib.parse.urlparse(url).netloc in allowed
+    except Exception:
+        return False
+
+
+def _skip_by_extension(url: str) -> bool:
+    path = urllib.parse.urlparse(url).path.lower()
+    _, ext = os.path.splitext(path)
+    return ext in _SKIP_EXTENSIONS
+
+
+def _is_dynamic(url: str) -> bool:
+    p = urllib.parse.urlparse(url)
+    if p.query:
+        return True
+    return bool(_ID_SEGMENT.search(p.path))
+
+
+def _parse_robots(content: str, scheme: str, netloc: str) -> tuple[set[str], list[str]]:
+    disallowed: set[str] = set()
+    notable: list[str] = []
+    for line in content.splitlines():
+        line = line.split("#")[0].strip()
+        if line.lower().startswith("disallow:"):
+            path = line[len("disallow:") :].strip()
+            if path and path != "/":
+                disallowed.add(path)
+                lower = path.lower()
+                if any(
+                    k in lower
+                    for k in ("admin", "api", "manage", "secret", "backup", "config", "staff")
+                ):
+                    notable.append(f"{scheme}://{netloc}{path}")
+    return disallowed, notable
+
+
+def _is_disallowed(url: str, disallowed: set[str]) -> bool:
+    path = urllib.parse.urlparse(url).path
+    return any(path.startswith(d) for d in disallowed)