nat-engine 1__py3-none-any.whl

mannf/core/agents/web_crawler_agent.py

@@ -0,0 +1,510 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""WebCrawlerAgent – autonomously discovers and maps a target web application.
+
+Extends :class:`PlannerAgent` (which itself extends :class:`BDIAgent`) and
+**composes** a Playwright browser lifecycle following the same lazy-init pattern
+used by :class:`BrowserExecutorAgent` (rather than inheriting from it, since the
+crawler is semantically a planner).
+
+The agent performs a bounded BFS crawl over the target site and produces a
+:class:`~mannf.core.browser.discovery_model.DiscoveryModel` — a structured JSON
+artefact consumed by downstream Phase 2 components (Scenario Config Generator).
+
+Key responsibilities
+--------------------
+* **BFS crawl loop** — visits pages up to ``max_pages`` / ``max_depth`` /
+  ``crawl_timeout_s`` budget limits.
+* **Form detection** — extracts fields with ``required``, ``label``, ``pattern``,
+  and ``submit_selector`` via enhanced JS.
+* **Interactive element discovery** — buttons, links, dropdowns, tabs.
+* **SPA support** — detects client-side route changes after clicks.
+* **Belief integration** — updates :class:`~mannf.core.agents.belief_state.BeliefState`
+  based on page load stability.
+* **Progress / result publishing** — emits ``CRAWL_PROGRESS`` messages during
+  crawling and a single ``CRAWL_RESULT`` when done.
+
+Import note: Playwright is imported **lazily** inside ``start_browser()`` so
+this module can be imported and tested without Playwright installed.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+from collections import deque
+from datetime import datetime, timezone
+from typing import Any
+from urllib.parse import urljoin, urlparse, urlunparse
+
+from mannf.core.agents.planner_agent import PlannerAgent
+from mannf.core.browser.discovery_model import (
+    DiscoveredField,
+    DiscoveredForm,
+    DiscoveredPage,
+    DiscoveryModel,
+    InteractiveElement,
+    PageEdge,
+)
+from mannf.core.browser.dom_snapshot import _ENHANCED_FORM_EXTRACTOR_JS, build_selector
+from mannf.core.messaging.bus import MessageBus
+from mannf.core.messaging.messages import Message, MessageType
+
+logger = logging.getLogger(__name__)
+
+# JavaScript to extract all outgoing same-page links.
+_LINK_EXTRACTOR_JS = """
+() => {
+    const hrefs = [];
+    document.querySelectorAll('a[href]').forEach(a => {
+        const h = a.getAttribute('href');
+        if (h && !h.startsWith('javascript:') && !h.startsWith('mailto:')
+                && !h.startsWith('tel:')) {
+            hrefs.push(h);
+        }
+    });
+    return hrefs;
+}
+"""
+
+# JavaScript to extract non-form interactive elements.
+_INTERACTIVE_EXTRACTOR_JS = """
+() => {
+    function buildSelector(el) {
+        if (el.id) return '#' + el.id;
+        const testid = el.getAttribute('data-testid');
+        if (testid) return '[data-testid="' + testid + '"]';
+        const name = el.getAttribute('name');
+        if (name) return el.tagName.toLowerCase() + '[name="' + name + '"]';
+        const classes = Array.from(el.classList);
+        if (classes.length) return el.tagName.toLowerCase() + '.' + classes.join('.');
+        return el.tagName.toLowerCase();
+    }
+
+    const elements = [];
+    const selectors = [
+        { sel: 'button:not([type="submit"]):not([type="reset"])', type: 'button' },
+        { sel: 'a[href]', type: 'link' },
+        { sel: 'select', type: 'dropdown' },
+        { sel: '[role="tab"]', type: 'tab' },
+    ];
+    selectors.forEach(({ sel, type }) => {
+        document.querySelectorAll(sel).forEach(el => {
+            const text = (el.innerText || el.textContent || '').trim().slice(0, 200);
+            elements.push({
+                selector: buildSelector(el),
+                text,
+                type,
+                tag: el.tagName.toLowerCase(),
+            });
+        });
+    });
+    return elements;
+}
+"""
+
+
+def _normalize_url(url: str) -> str:
+    """Strip fragment and normalize trailing slash for deduplication."""
+    parsed = urlparse(url)
+    # Remove fragment
+    normalized = parsed._replace(fragment="")
+    # Normalize path: ensure non-empty paths end consistently (no trailing slash
+    # for paths that are not just "/")
+    path = normalized.path
+    if path != "/" and path.endswith("/"):
+        path = path.rstrip("/")
+    normalized = normalized._replace(path=path)
+    return urlunparse(normalized)
+
+
+def _same_origin(url: str, base_url: str) -> bool:
+    """Return True when *url* shares the scheme + netloc of *base_url*."""
+    p = urlparse(url)
+    b = urlparse(base_url)
+    return p.scheme == b.scheme and p.netloc == b.netloc
+
+
+def _url_to_path(url: str, base_url: str) -> str:
+    """Return the path (relative to origin) for display in the discovery model."""
+    parsed = urlparse(url)
+    path = parsed.path or "/"
+    if parsed.query:
+        path = f"{path}?{parsed.query}"
+    return path
+
+
+class WebCrawlerAgent(PlannerAgent):
+    """Autonomously crawls a web application and produces a :class:`DiscoveryModel`.
+
+    Parameters
+    ----------
+    agent_id:
+        Unique identifier for this agent.
+    bus:
+        Shared message bus.
+    service_names:
+        Services tracked by the BDI belief state.
+    base_url:
+        Root URL of the application to crawl.
+    browser_type:
+        Playwright browser family: ``"chromium"``, ``"firefox"``, or ``"webkit"``.
+    headless:
+        Run the browser in headless mode.
+    viewport_width:
+        Browser viewport width in pixels.
+    viewport_height:
+        Browser viewport height in pixels.
+    max_pages:
+        Hard cap on the number of pages to visit.
+    max_depth:
+        Maximum BFS depth to follow links.
+    crawl_timeout_s:
+        Total crawl time budget in seconds (default 3 minutes).
+    page_timeout_ms:
+        Per-page navigation timeout in milliseconds.
+    """
+
+    def __init__(
+        self,
+        agent_id: str,
+        bus: MessageBus,
+        service_names: list[str],
+        *,
+        base_url: str,
+        browser_type: str = "chromium",
+        headless: bool = True,
+        viewport_width: int = 1280,
+        viewport_height: int = 720,
+        max_pages: int = 50,
+        max_depth: int = 5,
+        crawl_timeout_s: float = 180.0,
+        page_timeout_ms: int = 10_000,
+    ) -> None:
+        super().__init__(agent_id, bus, service_names)
+        self._base_url = base_url.rstrip("/")
+        self._browser_type = browser_type
+        self._headless = headless
+        self._viewport_width = viewport_width
+        self._viewport_height = viewport_height
+        self._max_pages = max_pages
+        self._max_depth = max_depth
+        self._crawl_timeout_s = crawl_timeout_s
+        self._page_timeout_ms = page_timeout_ms
+
+        # Composed Playwright state (lazy-initialised by start_browser)
+        self._playwright: Any = None
+        self._browser: Any = None
+        self._context: Any = None
+        self._page: Any = None
+
+        # Crawl state
+        self._crawl_started = False
+        self._discovery_model: DiscoveryModel | None = None
+
+    # ------------------------------------------------------------------
+    # Subscription
+    # ------------------------------------------------------------------
+
+    @property
+    def subscribed_types(self) -> set[MessageType]:
+        return super().subscribed_types | {
+            MessageType.CRAWL_PROGRESS,
+            MessageType.CRAWL_RESULT,
+        }
+
+    # ------------------------------------------------------------------
+    # Idle hook — drives the crawl
+    # ------------------------------------------------------------------
+
+    async def _on_idle(self) -> None:
+        if self._crawl_started:
+            return
+        self._crawl_started = True
+        try:
+            await self._run_crawl()
+        except Exception:  # noqa: BLE001
+            logger.exception(
+                "WebCrawlerAgent %s: unhandled error during crawl", self.agent_id
+            )
+
+    # ------------------------------------------------------------------
+    # Browser lifecycle (composed, not inherited)
+    # ------------------------------------------------------------------
+
+    async def start_browser(self) -> None:
+        """Launch Playwright and open a browser context + page (idempotent)."""
+        if self._playwright is not None:
+            return
+
+        from playwright.async_api import async_playwright  # lazy import
+
+        self._playwright = await async_playwright().start()
+        launcher = getattr(self._playwright, self._browser_type)
+        self._browser = await launcher.launch(headless=self._headless)
+        self._context = await self._browser.new_context(
+            viewport={"width": self._viewport_width, "height": self._viewport_height}
+        )
+        self._page = await self._context.new_page()
+        logger.info(
+            "WebCrawlerAgent %s: browser started (%s, headless=%s)",
+            self.agent_id,
+            self._browser_type,
+            self._headless,
+        )
+
+    async def stop_browser(self) -> None:
+        """Close the browser and Playwright instance (idempotent)."""
+        if self._playwright is None:
+            return
+
+        try:
+            if self._context is not None:
+                await self._context.close()
+            if self._browser is not None:
+                await self._browser.close()
+            await self._playwright.stop()
+        except Exception:  # noqa: BLE001
+            logger.exception(
+                "WebCrawlerAgent %s: error during stop_browser", self.agent_id
+            )
+        finally:
+            self._page = None
+            self._context = None
+            self._browser = None
+            self._playwright = None
+            logger.info("WebCrawlerAgent %s: browser stopped", self.agent_id)
+
+    # ------------------------------------------------------------------
+    # Core crawl loop
+    # ------------------------------------------------------------------
+
+    async def _run_crawl(self) -> None:
+        """Run the full BFS crawl and publish the discovery model."""
+        start_time = time.monotonic()
+        start_norm = _normalize_url(self._base_url)
+
+        model = DiscoveryModel(base_url=self._base_url)
+        visited: set[str] = set()
+        # Queue entries: (normalized_url, depth)
+        queue: deque[tuple[str, int]] = deque()
+        queue.append((start_norm, 0))
+
+        await self.start_browser()
+        try:
+            while queue:
+                if len(visited) >= self._max_pages:
+                    logger.info(
+                        "WebCrawlerAgent %s: max_pages=%d reached, stopping",
+                        self.agent_id, self._max_pages,
+                    )
+                    break
+                elapsed = time.monotonic() - start_time
+                if elapsed >= self._crawl_timeout_s:
+                    logger.info(
+                        "WebCrawlerAgent %s: crawl_timeout=%.1fs reached, stopping",
+                        self.agent_id, self._crawl_timeout_s,
+                    )
+                    break
+
+                url, depth = queue.popleft()
+                if url in visited:
+                    continue
+                if depth > self._max_depth:
+                    continue
+
+                visited.add(url)
+                page_data = await self._visit_page(url, depth, model, visited, queue)
+                if page_data is not None:
+                    model.pages.append(page_data)
+                    # Register page as a "service" in the belief state (dynamic)
+                    page_path = _url_to_path(url, self._base_url)
+                    if page_path not in self.beliefs.fault_likelihood:
+                        self.beliefs.update(page_path, 0.5)
+
+                    await self._publish_progress(model)
+        finally:
+            await self.stop_browser()
+
+        elapsed = time.monotonic() - start_time
+        model.crawl_duration_s = round(elapsed, 3)
+        model.crawl_timestamp = datetime.now(timezone.utc).isoformat()
+        self._discovery_model = model
+
+        await self._publish_result(model)
+        logger.info(
+            "WebCrawlerAgent %s: crawl complete — %d pages, %d edges in %.1fs",
+            self.agent_id, len(model.pages), len(model.edges), elapsed,
+        )
+
+    async def _visit_page(
+        self,
+        url: str,
+        depth: int,
+        model: DiscoveryModel,
+        visited: set[str],
+        queue: deque[tuple[str, int]],
+    ) -> DiscoveredPage | None:
+        """Navigate to *url*, extract page data, and enqueue new links.
+
+        Returns the :class:`DiscoveredPage` on success, or ``None`` on error.
+        """
+        assert self._page is not None, "Browser page is not initialised"
+
+        try:
+            await self._page.goto(
+                url,
+                wait_until="networkidle",
+                timeout=self._page_timeout_ms,
+            )
+        except Exception:  # noqa: BLE001
+            try:
+                await self._page.goto(
+                    url,
+                    wait_until="load",
+                    timeout=self._page_timeout_ms,
+                )
+            except Exception:  # noqa: BLE001
+                logger.warning(
+                    "WebCrawlerAgent %s: failed to load %r, skipping", self.agent_id, url
+                )
+                # Update belief — page failed to load → higher fault likelihood
+                page_path = _url_to_path(url, self._base_url)
+                if page_path in self.beliefs.fault_likelihood:
+                    self.beliefs.update(page_path, 1.0)
+                return None
+
+        # Wait for any SPA hydration
+        try:
+            await self._page.wait_for_load_state(
+                "networkidle", timeout=self._page_timeout_ms
+            )
+        except Exception:  # noqa: BLE001
+            pass  # best-effort; continue with whatever the DOM has
+
+        current_url = _normalize_url(self._page.url)
+        title = await self._page.title()
+
+        # Extract links
+        raw_links: list[str] = await self._page.evaluate(_LINK_EXTRACTOR_JS)
+        links: list[str] = []
+        for href in raw_links:
+            absolute = urljoin(current_url, href)
+            norm = _normalize_url(absolute)
+            if _same_origin(norm, self._base_url):
+                links.append(_url_to_path(norm, self._base_url))
+                if norm not in visited and depth + 1 <= self._max_depth:
+                    queue.append((norm, depth + 1))
+                    # Record graph edge
+                    parsed = urlparse(href)
+                    link_sel = (
+                        f'a[href="{href}"]'
+                        if not parsed.fragment
+                        else f'a[href="{href}"]'
+                    )
+                    model.edges.append(
+                        PageEdge(
+                            from_url=_url_to_path(current_url, self._base_url),
+                            to_url=_url_to_path(norm, self._base_url),
+                            action="click",
+                            selector=link_sel,
+                        )
+                    )
+
+        # Extract enhanced forms
+        raw_forms: list[dict[str, Any]] = await self._page.evaluate(
+            _ENHANCED_FORM_EXTRACTOR_JS
+        )
+        forms = _parse_forms(raw_forms)
+
+        # Extract interactive elements
+        raw_interactive: list[dict[str, Any]] = await self._page.evaluate(
+            _INTERACTIVE_EXTRACTOR_JS
+        )
+        interactive_elements = [
+            InteractiveElement(
+                selector=r.get("selector", ""),
+                text=r.get("text", ""),
+                type=r.get("type", "button"),
+                tag=r.get("tag", ""),
+            )
+            for r in raw_interactive
+        ]
+
+        # Belief update — page loaded successfully → lower fault likelihood
+        page_path = _url_to_path(current_url, self._base_url)
+        if page_path in self.beliefs.fault_likelihood:
+            self.beliefs.update(page_path, 0.0)
+
+        logger.debug(
+            "WebCrawlerAgent %s: visited %r depth=%d links=%d forms=%d",
+            self.agent_id, page_path, depth, len(links), len(forms),
+        )
+
+        return DiscoveredPage(
+            url=page_path,
+            title=title,
+            forms=forms,
+            links=links,
+            interactive_elements=interactive_elements,
+        )
+
+    # ------------------------------------------------------------------
+    # Message publishing
+    # ------------------------------------------------------------------
+
+    async def _publish_progress(self, model: DiscoveryModel) -> None:
+        """Publish a CRAWL_PROGRESS message with current crawl statistics."""
+        payload = {
+            "pages_discovered": len(model.pages),
+            "forms_found": sum(len(p.forms) for p in model.pages),
+            "edges_found": len(model.edges),
+        }
+        await self._publish(
+            Message(MessageType.CRAWL_PROGRESS, self.agent_id, payload)
+        )
+
+    async def _publish_result(self, model: DiscoveryModel) -> None:
+        """Publish a CRAWL_RESULT message carrying the full discovery model."""
+        await self._publish(
+            Message(MessageType.CRAWL_RESULT, self.agent_id, model.to_dict())
+        )
+
+
+# ---------------------------------------------------------------------------
+# Parsing helpers
+# ---------------------------------------------------------------------------
+
+def _parse_forms(raw_forms: list[dict[str, Any]]) -> list[DiscoveredForm]:
+    """Convert raw JS-extracted form dicts into :class:`DiscoveredForm` objects."""
+    forms: list[DiscoveredForm] = []
+    for rf in raw_forms:
+        raw_fields = rf.get("fields", [])
+        fields: list[DiscoveredField] = []
+        for f in raw_fields:
+            # Determine the input type robustly
+            field_type = f.get("type", "") or f.get("tag", "text") or "text"
+            fields.append(
+                DiscoveredField(
+                    selector=f.get("selector", ""),
+                    type=field_type,
+                    required=bool(f.get("required", False)),
+                    label=f.get("label", ""),
+                    name=f.get("name", ""),
+                    placeholder=f.get("placeholder", ""),
+                    pattern=f.get("pattern", ""),
+                )
+            )
+        forms.append(
+            DiscoveredForm(
+                action=rf.get("action", ""),
+                method=(rf.get("method", "GET") or "GET").upper(),
+                fields=fields,
+                submit_selector=rf.get("submit_selector", ""),
+            )
+        )
+    return forms