agent-ready-audit 0.1.0__tar.gz

agent_ready_audit-0.1.0/.gitignore

@@ -0,0 +1,12 @@
+.venv/
+.pytest_cache/
+.ruff_cache/
+.mypy_cache/
+.coverage
+__pycache__/
+*.py[cod]
+storage/
+.env
+dist/
+build/
+*.egg-info/

agent_ready_audit-0.1.0/PKG-INFO

@@ -0,0 +1,16 @@
+Metadata-Version: 2.5
+Name: agent-ready-audit
+Version: 0.1.0
+Summary: Deterministic audits of website readiness for agents.
+Requires-Python: >=3.12
+Requires-Dist: beautifulsoup4<5,>=4.13
+Requires-Dist: defusedxml<1,>=0.7
+Requires-Dist: httpx<1,>=0.28
+Requires-Dist: pydantic<3,>=2.12
+Requires-Dist: pyyaml<7,>=6
+Requires-Dist: typer<1,>=0.16
+Description-Content-Type: text/markdown
+
+# Agent-Ready Audit
+
+`agent-ready-audit https://example.com` runs a bounded, robots-aware audit of one public website root.

agent_ready_audit-0.1.0/README.md

@@ -0,0 +1,3 @@
+# Agent-Ready Audit
+
+`agent-ready-audit https://example.com` runs a bounded, robots-aware audit of one public website root.

agent_ready_audit-0.1.0/pyproject.toml

@@ -0,0 +1,17 @@
+[build-system]
+requires=["hatchling"]
+build-backend="hatchling.build"
+[project]
+name="agent-ready-audit"
+version="0.1.0"
+description="Deterministic audits of website readiness for agents."
+readme="README.md"
+requires-python=">=3.12"
+dependencies=["beautifulsoup4>=4.13,<5","defusedxml>=0.7,<1","httpx>=0.28,<1","pydantic>=2.12,<3","pyyaml>=6,<7","typer>=0.16,<1"]
+[project.scripts]
+agent-ready-audit="agent_ready_audit.cli:app"
+[tool.hatch.build.targets.wheel]
+packages=["src/agent_ready_audit"]
+[tool.hatch.build.targets.wheel.force-include]
+"src/agent_ready_audit/data/checks.yaml"="agent_ready_audit/data/checks.yaml"
+"src/agent_ready_audit/data/report.schema.json"="agent_ready_audit/data/report.schema.json"

agent_ready_audit-0.1.0/src/agent_ready_audit/__init__.py

@@ -0,0 +1,14 @@
+from agent_ready_audit._version import __version__
+from agent_ready_audit.analyzer import audit
+from agent_ready_audit.models import AuditConfig, AuditFinding, AuditReport, AuditRequest, CategoryScore, PageFetch
+
+__all__ = [
+    "AuditConfig",
+    "AuditFinding",
+    "AuditReport",
+    "AuditRequest",
+    "CategoryScore",
+    "PageFetch",
+    "audit",
+    "__version__",
+]

agent_ready_audit-0.1.0/src/agent_ready_audit/__main__.py

@@ -0,0 +1,3 @@
+from agent_ready_audit.cli import app
+
+app()

agent_ready_audit-0.1.0/src/agent_ready_audit/_version.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

agent_ready_audit-0.1.0/src/agent_ready_audit/analyzer.py

@@ -0,0 +1,589 @@
+# ruff: noqa: E501, E701, E702
+from __future__ import annotations
+
+import asyncio
+import heapq
+import json
+import re
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from importlib.resources import files
+from urllib.parse import urljoin, urlsplit
+
+import yaml
+from bs4 import BeautifulSoup
+from defusedxml import ElementTree
+
+from agent_ready_audit._version import __version__
+from agent_ready_audit.errors import (
+    BudgetExceededError,
+    FetchError,
+    FetchTimeoutError,
+    RobotsDisallowedError,
+    TargetSecurityError,
+)
+from agent_ready_audit.http import FetchBudget, FetchedResource, SafeHttpClient, origin, safe_url
+from agent_ready_audit.models import (
+    AuditFinding,
+    AuditLimits,
+    AuditReport,
+    AuditRequest,
+    CategoryScore,
+    Evidence,
+    FetchSummary,
+    FindingStatus,
+    PageFetch,
+    ScoreSummary,
+    Severity,
+    TargetSummary,
+)
+from agent_ready_audit.robots import AI_AGENTS, Policy, RobotsManager
+
+PROBES = (
+    "/sitemap.xml",
+    "/llms.txt",
+    "/openapi.json",
+    "/swagger.json",
+    "/.well-known/security.txt",
+    "/metrics",
+    "/health",
+    "/healthz",
+    "/readyz",
+    "/debug",
+)
+EXPOSURE = {"/metrics", "/health", "/healthz", "/readyz", "/debug"}
+
+
+@dataclass
+class Facts:
+    url: str
+    links: list[str] = field(default_factory=list)
+    text: str = ""
+    canonical: str | None = None
+    canonical_count: int = 0
+    jsonld: int = 0
+    bad_jsonld: int = 0
+    types: set[str] = field(default_factory=set)
+    social: int = 0
+    code: int = 0
+    sitemap: list[str] = field(default_factory=list)
+    sitemap_bad: bool = False
+    openapi: str | None = None
+    paths: int = 0
+    auth: bool = False
+    mcp: set[str] = field(default_factory=set)
+
+    @property
+    def json_ld_types(self) -> set[str]:
+        return self.types
+
+    @property
+    def code_blocks(self) -> int:
+        return self.code
+
+    @property
+    def sitemap_urls(self) -> list[str]:
+        return self.sitemap
+
+    @property
+    def openapi_valid(self) -> bool:
+        return bool(self.openapi and self.openapi.startswith(("3.0.", "3.1.")) and self.paths)
+
+    @property
+    def openapi_paths(self) -> int:
+        return self.paths
+
+    @property
+    def openapi_auth_documented(self) -> bool:
+        return self.auth
+
+    @property
+    def mcp_signals(self) -> set[str]:
+        values: set[str] = set()
+        if "transport" in self.mcp:
+            values.add("protocolReference")
+        if "endpoint" in self.mcp:
+            values.add("endpointOrLink")
+        if "config" in self.mcp:
+            values.add("clientConfig")
+        return values
+
+
+def _links(values: list[str], base: str, limit: int) -> list[str]:
+    out = []
+    for raw in values:
+        try:
+            p = urlsplit(urljoin(base, raw))
+            if p.scheme not in {"http", "https"} or not p.hostname or p.username or p.password:
+                continue
+            value = safe_url(p.geturl())
+        except ValueError:
+            continue
+        if value not in out:
+            out.append(value)
+        if len(out) >= limit:
+            break
+    return out
+
+
+def parse(fetched: FetchedResource, request: AuditRequest) -> Facts:
+    page = fetched.public
+    url = page.final_url or page.requested_url
+    path = urlsplit(url).path.lower()
+    kind = page.content_type or ""
+    facts = Facts(url)
+    if page.status_code != 200:
+        return facts
+    if kind in {"text/html", "application/xhtml+xml"}:
+        soup = BeautifulSoup(fetched.body, "html.parser")
+        facts.text = re.sub(r"\s+", " ", soup.get_text(" ", strip=True)).lower()[:200000]
+        facts.code = len(soup.find_all(["code", "pre"]))
+        facts.links = _links(
+            [str(x.get("href")) for x in soup.find_all(["a", "link"], href=True)],
+            url,
+            request.config.max_links_per_document,
+        )
+        canonical = []
+        for node in soup.find_all("link", href=True):
+            if "canonical" in {str(x).lower() for x in node.get("rel") or []}:
+                canonical += _links([str(node["href"])], url, 1)
+        facts.canonical_count = len(canonical)
+        facts.canonical = canonical[0] if canonical else None
+        for node in soup.find_all("meta"):
+            name = str(node.get("property") or node.get("name") or "").lower()
+            facts.social += name.startswith("og:") or name.startswith("twitter:")
+        for node in soup.find_all("script"):
+            if str(node.get("type") or "").lower() != "application/ld+json":
+                continue
+            facts.jsonld += 1
+            try:
+                value = json.loads(node.string or node.get_text())
+                items = value if isinstance(value, list) else [value]
+                for item in items[:1000]:
+                    if isinstance(item, dict) and isinstance(item.get("@type"), str):
+                        facts.types.add(item["@type"])
+            except (ValueError, TypeError):
+                facts.bad_jsonld += 1
+        combined = facts.text + " " + " ".join(facts.links)
+        if "model context protocol" in combined or "streamable http" in combined:
+            facts.mcp.add("transport")
+        if re.search(r"(?:^|[/_-])mcp(?:$|[/_.-])", combined):
+            facts.mcp.add("endpoint")
+        if "mcpservers" in combined or "client config" in combined:
+            facts.mcp.add("config")
+    elif "sitemap" in path or "xml" in kind:
+        try:
+            root = ElementTree.fromstring(fetched.body)
+            locations = [x.text.strip() for x in root.iter() if x.tag.rsplit("}", 1)[-1].lower() == "loc" and x.text][
+                : request.config.max_sitemap_urls
+            ]
+            facts.sitemap = _links(locations, url, request.config.max_sitemap_urls)
+            facts.sitemap_bad = root.tag.rsplit("}", 1)[-1].lower() not in {"urlset", "sitemapindex"}
+        except Exception:
+            facts.sitemap_bad = True
+    elif path.endswith(("openapi.json", "swagger.json", "openapi.yaml", "openapi.yml")):
+        try:
+            text = fetched.body.decode()
+            value = json.loads(text) if text.lstrip().startswith("{") else yaml.safe_load(text)
+            if isinstance(value, dict):
+                version = value.get("openapi")
+                paths = value.get("paths")
+                facts.openapi = version if isinstance(version, str) else None
+                facts.paths = len(paths) if isinstance(paths, dict) else 0
+                components = value.get("components")
+                facts.auth = bool(value.get("security")) or (
+                    isinstance(components, dict) and bool(components.get("securitySchemes"))
+                )
+        except (ValueError, UnicodeError, yaml.YAMLError):
+            pass
+    elif kind.startswith("text/"):
+        facts.text = fetched.body.decode("utf-8", errors="replace").lower()[:200000]
+    return facts
+
+
+def _finding(
+    identifier: str,
+    category: str,
+    status: FindingStatus,
+    description: str,
+    *,
+    evidence: list[Evidence] | None = None,
+    penalty: float = 0,
+) -> AuditFinding:
+    titles = {
+        "HTTP_REACHABLE": "Website root is reachable",
+        "HTTP_HTTPS": "Final site root uses HTTPS",
+        "ROBOTS_VALID": "Robots policy is available and parseable",
+        "ROBOTS_AI_ACCESS": "AI crawler access is documented",
+        "CANONICAL": "Canonical identity agrees with the final URL",
+        "SITEMAP": "A usable sitemap is discoverable",
+        "MACHINE": "Machine-facing surfaces are linked",
+        "OPENAPI": "A usable OpenAPI document is discoverable",
+        "MCP": "An actionable agent integration is discoverable",
+        "METADATA": "Structured metadata identifies the site",
+        "DOCS": "Documentation is discoverable and useful",
+        "PRICING": "Pricing or custom pricing is discoverable",
+        "EXPOSURE_METRICS": "Public operational surfaces avoid sensitive telemetry",
+    }
+    severity = {
+        FindingStatus.FAIL: Severity.HIGH,
+        FindingStatus.WARN: Severity.MEDIUM,
+        FindingStatus.NOT_TESTED: Severity.LOW,
+    }.get(status, Severity.INFO)
+    return AuditFinding(
+        id=identifier,
+        category=category,
+        status=status,
+        severity=severity,
+        title=titles[identifier],
+        description=description,
+        evidence=evidence or [],
+        remediation="Review the affected public surface and make it deliberately agent-readable."
+        if status in {FindingStatus.FAIL, FindingStatus.WARN}
+        else None,
+        security_penalty=penalty,
+    )
+
+
+def checks(
+    root: PageFetch | None,
+    facts: dict[str, Facts],
+    policies: dict[str, Policy],
+    resources: dict[str, PageFetch],
+    final: str | None,
+) -> list[AuditFinding]:
+    rootfacts = facts.get(root.final_url or "") if root else None
+    reachable = bool(root and root.status_code and 200 <= root.status_code < 300)
+    policy = policies.get(origin(final)) if final else None
+    result = [
+        _finding(
+            "HTTP_REACHABLE",
+            "http",
+            FindingStatus.PASS if reachable else FindingStatus.FAIL,
+            "The root returned 2xx." if reachable else "The root did not return 2xx.",
+        ),
+        _finding(
+            "HTTP_HTTPS",
+            "http",
+            FindingStatus.PASS if final and urlsplit(final).scheme == "https" else FindingStatus.FAIL,
+            "The final root uses HTTPS."
+            if final and urlsplit(final).scheme == "https"
+            else "The final root is not HTTPS.",
+        ),
+    ]
+    robot_status = (
+        FindingStatus.PASS
+        if policy and policy.status != "unavailable" and policy.malformed == 0
+        else FindingStatus.NOT_TESTED
+    )
+    result.append(
+        _finding(
+            "ROBOTS_VALID",
+            "crawlability",
+            robot_status,
+            "robots.txt was interpreted."
+            if robot_status == FindingStatus.PASS
+            else "robots.txt was unavailable or malformed.",
+        )
+    )
+    allowed = sum(policy.can_fetch(x, final) for x in AI_AGENTS) if policy and final else 0
+    result.append(
+        _finding(
+            "ROBOTS_AI_ACCESS",
+            "crawlability",
+            FindingStatus.PASS
+            if allowed == len(AI_AGENTS)
+            else FindingStatus.WARN
+            if allowed
+            else FindingStatus.NOT_TESTED,
+            f"{allowed} of {len(AI_AGENTS)} tracked AI identities may fetch the root.",
+        )
+    )
+    canonical = bool(
+        rootfacts
+        and rootfacts.canonical
+        and root
+        and rootfacts.canonical.rstrip("/") == str(root.final_url).rstrip("/")
+        and rootfacts.canonical_count == 1
+    )
+    result.append(
+        _finding(
+            "CANONICAL",
+            "canonicalIdentity",
+            FindingStatus.PASS if canonical else FindingStatus.FAIL,
+            "Canonical identity is consistent."
+            if canonical
+            else "Canonical identity is missing, duplicated, or conflicting.",
+        )
+    )
+    maps = [x for x in facts.values() if x.sitemap and not x.sitemap_bad]
+    result.append(
+        _finding(
+            "SITEMAP",
+            "machineDiscovery",
+            FindingStatus.PASS if maps else FindingStatus.WARN,
+            "A usable sitemap was found." if maps else "No usable sitemap was found.",
+        )
+    )
+    links = rootfacts.links if rootfacts else []
+    machine = [x for x in links if any(t in x.lower() for t in ("docs", "api", "developer", "mcp", "agent", "pricing"))]
+    result.append(
+        _finding(
+            "MACHINE",
+            "machineDiscovery",
+            FindingStatus.PASS if machine else FindingStatus.WARN,
+            f"{len(machine)} machine-facing root links were found.",
+        )
+    )
+    specs = [x for x in facts.values() if x.openapi and x.openapi.startswith(("3.0.", "3.1.")) and x.paths]
+    result.append(
+        _finding(
+            "OPENAPI",
+            "apiDiscovery",
+            FindingStatus.PASS if specs else FindingStatus.FAIL,
+            "A usable OpenAPI document was found." if specs else "No usable OpenAPI document was found.",
+        )
+    )
+    mcp = set().union(*(x.mcp for x in facts.values())) if facts else set()
+    result.append(
+        _finding(
+            "MCP",
+            "agentDiscovery",
+            FindingStatus.PASS
+            if {"transport", "endpoint"} <= mcp
+            else FindingStatus.WARN
+            if mcp
+            else FindingStatus.FAIL,
+            "MCP discovery is actionable."
+            if {"transport", "endpoint"} <= mcp
+            else "MCP discovery is absent or incomplete.",
+        )
+    )
+    relevant = {"Organization", "Product", "SoftwareApplication", "WebSite", "Service"}
+    meta = bool(rootfacts and rootfacts.jsonld and not rootfacts.bad_jsonld and rootfacts.types & relevant)
+    result.append(
+        _finding(
+            "METADATA",
+            "structuredMetadata",
+            FindingStatus.PASS if meta else FindingStatus.WARN,
+            "Relevant valid JSON-LD was found." if meta else "Relevant valid JSON-LD was not found.",
+        )
+    )
+    docs = [x for x in facts.values() if any(t in x.url.lower() for t in ("docs", "documentation", "developer"))]
+    useful = bool(
+        docs
+        and any(x.code and any(t in x.text for t in ("authentication", "error", "rate limit", "status")) for x in docs)
+    )
+    result.append(
+        _finding(
+            "DOCS",
+            "docsUsability",
+            FindingStatus.PASS if useful else FindingStatus.WARN,
+            "Useful documentation was found." if useful else "Documentation or operational examples were incomplete.",
+        )
+    )
+    pricing = [x for x in facts.values() if "pricing" in x.url.lower()]
+    clear = bool(
+        pricing
+        and any(any(t in x.text for t in ("$", "custom pricing", "contact sales", "free", "plan")) for x in pricing)
+    )
+    result.append(
+        _finding(
+            "PRICING",
+            "pricingDiscovery",
+            FindingStatus.PASS if clear else FindingStatus.WARN,
+            "Pricing is discoverable." if clear else "Pricing was absent or unclear.",
+        )
+    )
+    exposed = [
+        x
+        for x in facts.values()
+        if urlsplit(x.url).path in EXPOSURE
+        and any(t in x.text for t in ("# help", "# type", "traceback", "hostname", "database", "build sha", "backend"))
+    ]
+    penalty = min(20, 10 if any(urlsplit(x.url).path in {"/metrics", "/debug"} for x in exposed) else 5 * len(exposed))
+    result.append(
+        _finding(
+            "EXPOSURE_METRICS",
+            "securityExposure",
+            FindingStatus.FAIL if exposed else FindingStatus.PASS,
+            "Sensitive operational metadata was public."
+            if exposed
+            else "No sensitive operational metadata was observed.",
+            evidence=[Evidence(kind="publicOperationalSurface", url=x.url) for x in exposed],
+            penalty=penalty,
+        )
+    )
+    return result
+
+
+def score(findings: list[AuditFinding], weights: dict[str, float]) -> tuple[list[AuditFinding], ScoreSummary]:
+    categories = []
+    earned_total = 0.0
+    evaluated = 0
+    total = 0
+    rendered = []
+    factors = {FindingStatus.PASS: 1, FindingStatus.WARN: 0.5, FindingStatus.FAIL: 0, FindingStatus.NOT_TESTED: 0}
+    for category, weight in weights.items():
+        items = [x for x in findings if x.category == category and x.status != FindingStatus.INFO]
+        each = weight / len(items) if items else 0
+        earned = sum(each * factors[x.status] for x in items)
+        earned_total += earned
+        total += len(items)
+        evaluated += sum(x.status != FindingStatus.NOT_TESTED for x in items)
+        categories.append(
+            CategoryScore(
+                category=category,
+                weight=weight,
+                score=round(earned / weight * 100, 1),
+                earned_points=round(earned, 2),
+                possible_points=weight,
+            )
+        )
+        for item in items:
+            rendered.append(item.model_copy(update={"score_delta": round(each * (factors[item.status] - 1), 4)}))
+    rendered.extend(x for x in findings if x.category not in weights)
+    penalty = min(20, sum(x.security_penalty for x in findings))
+    return rendered, ScoreSummary(
+        overall=max(0, min(100, round(earned_total - penalty))),
+        completeness=round(evaluated / total, 4) if total else 1,
+        security_penalty=penalty,
+        categories=categories,
+    )
+
+
+async def audit(request: AuditRequest, *, client: SafeHttpClient | None = None) -> AuditReport:
+    raw = yaml.safe_load(files("agent_ready_audit.data").joinpath("checks.yaml").read_text())
+    weights = {str(k): float(v) for k, v in raw["categories"].items()}
+    version = str(raw["version"])
+    active = client or SafeHttpClient(request.config, version=__version__)
+    budget = FetchBudget(request.config.max_pages, request.config.max_total_bytes)
+    resources: dict[str, PageFetch] = {}
+    documents: dict[str, Facts] = {}
+    limitations: list[str] = []
+    disallowed = 0
+    timedout = 0
+
+    def record(page: PageFetch) -> None:
+        key = safe_url(page.requested_url)
+        old = resources.get(key)
+        if old is None or page.method == "GET" or old.status_code is None:
+            resources[key] = page
+
+    robots = RobotsManager(active, budget, record)
+    root = None
+    final = None
+    status = "failed"
+    try:
+        async with asyncio.timeout(request.config.overall_timeout_seconds):
+            try:
+                rootraw = await active.fetch(request.url, budget=budget, authorize=robots.authorize)
+                root = rootraw.public
+                record(root)
+                rootfacts = parse(rootraw, request)
+                documents[rootfacts.url] = rootfacts
+                final = root.final_url or request.url
+            except RobotsDisallowedError as exc:
+                disallowed += 1
+                record(PageFetch(requested_url=request.url, error_code=exc.code))
+                limitations.append("robots.txt disallowed the requested root")
+                status = "restricted"
+                raise StopAsyncIteration from None
+            except (FetchError, FetchTimeoutError) as exc:
+                timedout += isinstance(exc, FetchTimeoutError)
+                record(PageFetch(requested_url=request.url, error_code=exc.code))
+                limitations.append("the requested root was unreachable")
+                status = "unreachable"
+                raise StopAsyncIteration from None
+            site = origin(final)
+            queue: list[tuple[int, str]] = []
+            queued: set[str] = set()
+
+            def enqueue(url: str) -> None:
+                value = safe_url(url)
+                if value not in queued and value not in resources and origin(value) == site:
+                    queued.add(value)
+                    heapq.heappush(queue, (0 if any(x in value for x in ("openapi", "swagger", "mcp")) else 1, value))
+
+            for p in robots.policies.values():
+                for item in p.sitemaps:
+                    enqueue(item)
+            if request.config.standard_probes:
+                for path in PROBES:
+                    enqueue(site + path)
+            for item in rootfacts.links:
+                enqueue(item)
+            while queue:
+                _, url = heapq.heappop(queue)
+
+                async def authorize(value: str) -> None:
+                    if origin(value) != site:
+                        raise TargetSecurityError("cross-origin redirect is not allowed")
+                    await robots.authorize(value)
+
+                try:
+                    if urlsplit(url).path in EXPOSURE:
+                        head = await active.fetch(url, budget=budget, method="HEAD", authorize=authorize)
+                        record(head.public)
+                        if not head.public.status_code or not 200 <= head.public.status_code < 300:
+                            continue
+                    fetched = await active.fetch(
+                        url,
+                        budget=budget,
+                        max_bytes=256 * 1024 if urlsplit(url).path in EXPOSURE else None,
+                        authorize=authorize,
+                    )
+                    record(fetched.public)
+                    facts = parse(fetched, request)
+                    documents[facts.url] = facts
+                    for item in facts.links + facts.sitemap:
+                        enqueue(item)
+                except RobotsDisallowedError as exc:
+                    disallowed += 1
+                    record(PageFetch(requested_url=url, error_code=exc.code))
+                    limitations.append("robots.txt prevented planned checks")
+                except BudgetExceededError:
+                    limitations.append("resource or byte budget was exhausted")
+                    break
+                except (FetchError, TargetSecurityError) as exc:
+                    record(PageFetch(requested_url=url, error_code=exc.code))
+                    limitations.append("a discovered resource could not be fetched safely")
+            status = "success" if root.status_code and 200 <= root.status_code < 300 else "unreachable"
+            if limitations and status == "success":
+                status = "partial"
+    except StopAsyncIteration:
+        pass
+    except TimeoutError:
+        timedout += 1
+        limitations.append("the overall audit deadline was reached")
+        status = "partial" if resources else "unreachable"
+    findings, summary = score(checks(root, documents, robots.policies, resources, final), weights)
+    pages = sorted(resources.values(), key=lambda x: (x.requested_url, x.method))
+    return AuditReport(
+        generated_at=datetime.now(UTC),
+        analyzer_version=__version__,
+        rules_version=version,
+        status=status,
+        target=TargetSummary(requested_root=request.url, final_root=final),
+        limits=AuditLimits(
+            max_pages=request.config.max_pages,
+            request_timeout_seconds=request.config.request_timeout_seconds,
+            overall_timeout_seconds=request.config.overall_timeout_seconds,
+            max_response_bytes=request.config.max_response_bytes,
+            max_total_bytes=request.config.max_total_bytes,
+            standard_probes=request.config.standard_probes,
+        ),
+        fetch_summary=FetchSummary(
+            resources_attempted=len(budget.urls),
+            resources_fetched=sum(x.status_code is not None for x in pages),
+            http_requests=active.request_count,
+            http_retries=active.retry_count,
+            bytes_processed=budget.bytes_processed,
+            robots_disallowed=disallowed,
+            timed_out=timedout,
+        ),
+        score=summary,
+        resources=pages,
+        findings=findings[: request.config.max_findings],
+        limitations=list(dict.fromkeys(limitations)),
+    )