webarchiver 1.0.0__py3-none-any.whl

webarchiver/__init__.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+
+import importlib
+import inspect
+import warnings
+
+warnings.filterwarnings("ignore", message=".*urllib3.*or chardet.*")
+
+__all__: list[str] = []
+
+CORE_MODULES = [
+    "webarchiver.api",
+    "webarchiver.error_authority",
+]
+
+OPTIONAL_MODULES = {
+    "webarchiver.mcp_server": "mcp",
+}
+
+
+def _import_module_safely(module_name: str):
+    """Try to import a module and return it, or None if not available."""
+    try:
+        return importlib.import_module(module_name)
+    except ImportError:
+        return None
+
+
+def _expose_members(module):
+    """Expose public classes and functions from a module into globals and __all__."""
+    for name, obj in inspect.getmembers(module):
+        if (inspect.isclass(obj) or inspect.isfunction(obj)) and not name.startswith(
+            "_"
+        ):
+            globals()[name] = obj
+            __all__.append(name)
+
+
+for module_name in CORE_MODULES:
+    try:
+        module = importlib.import_module(module_name)
+        _expose_members(module)
+    except ImportError:
+        pass
+
+for module_name, extra_name in OPTIONAL_MODULES.items():
+    module = _import_module_safely(module_name)
+    if module is not None:
+        _expose_members(module)
+        globals()[f"_{extra_name.upper()}_AVAILABLE"] = True
+    else:
+        globals()[f"_{extra_name.upper()}_AVAILABLE"] = False
+
+__all__.extend(["_MCP_AVAILABLE"])

webarchiver/__main__.py

@@ -0,0 +1,4 @@
+from .mcp_server import mcp_server
+
+if __name__ == "__main__":
+    mcp_server()

webarchiver/_version.py

@@ -0,0 +1,3 @@
+"""Single source of the package version (read by bump2version)."""
+
+__version__ = "1.0.0"

webarchiver/api/__init__.py

@@ -0,0 +1,10 @@
+from .api_client_system import ApiClientSystem, ExtractResult, LinksResult
+from .policy import FetchPolicy, PolicyRefused
+
+__all__ = [
+    "ApiClientSystem",
+    "ExtractResult",
+    "FetchPolicy",
+    "LinksResult",
+    "PolicyRefused",
+]

webarchiver/api/api_client_system.py

@@ -0,0 +1,166 @@
+"""The webarchiver operations: one public method per MCP tool.
+
+CONN-WEBARCHIVER-WEB-SOURCE-R001. ``register_tool_surface`` turns each public
+method of :class:`ApiClientSystem` into one typed tool named
+``webarchiver_<method>``: ``fetch``, ``extract``, ``render``, ``screenshot``
+and ``links``. All five only read the web, so each is declared ``GET`` and
+served read-only. Every operation honours robots.txt and the per-domain rate
+limit; a refusal is returned as a typed :class:`PolicyRefused`.
+"""
+
+from __future__ import annotations
+
+import datetime as _dt
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict
+
+from .browser import Browser, RenderedPage, Screenshot
+from .extraction import ExtractedPage, PageLink, extract_links, extract_page
+from .fetcher import FetchedResponse, Fetcher
+from .policy import FetchPolicy, PolicyRefused, origin_of
+
+__all__ = ["ApiClientSystem", "ExtractResult", "LinksResult"]
+
+#: Rendered HTML returned by ``render`` is capped to keep tool results bounded.
+_MAX_RENDERED_HTML = 2 * 1024 * 1024
+
+
+def _read_only(method: Any) -> Any:
+    method.__http_method__ = "GET"
+    return method
+
+
+class ExtractResult(BaseModel):
+    """Extracted main content plus the provenance of how it was fetched."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    backend: Literal["webarchiver"] = "webarchiver"
+    fetch_mode: Literal["http", "render"]
+    url: str
+    final_url: str
+    status: int
+    content_type: str
+    byte_count: int
+    content_digest: str
+    retrieved_at: str
+    robots: Literal["allowed", "not_checked"]
+    page: ExtractedPage
+
+
+class LinksResult(BaseModel):
+    """The outbound links of one page."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    url: str
+    final_url: str
+    links: tuple[PageLink, ...]
+
+
+class ApiClientSystem:
+    """Policy-governed web fetch, extraction, rendering and screenshots."""
+
+    def __init__(
+        self,
+        policy: FetchPolicy | None = None,
+        *,
+        allow_loopback: bool = False,
+        allowed_private_hosts: tuple[str, ...] = (),
+    ) -> None:
+        self.policy = policy or FetchPolicy()
+        self.fetcher = Fetcher(
+            self.policy,
+            allow_loopback=allow_loopback,
+            allowed_private_hosts=allowed_private_hosts,
+        )
+        self.browser = Browser(allow_loopback=allow_loopback)
+
+    async def _admit(self, url: str) -> None:
+        """Robots and rate-limit admission for browser-driven operations."""
+        if self.policy.obey_robots and not await self.fetcher.robots.allowed(
+            url, self.policy.user_agent
+        ):
+            raise PolicyRefused("robots_disallowed", url)
+        await self.fetcher.limiter.acquire(origin_of(url))
+
+    @_read_only
+    async def fetch(self, url: str) -> FetchedResponse:
+        """GET one URL under robots.txt, rate-limit, size and content-type policy."""
+        return await self.fetcher.fetch(url)
+
+    @_read_only
+    async def extract(self, url: str, render: bool = False) -> ExtractResult:
+        """Fetch one URL and extract its main content as text and markdown.
+
+        Returns title, author, published date, canonical URL and outbound
+        links. ``render=True`` runs the page's scripts in headless Chromium
+        first, for pages that build their content client-side.
+        """
+        if render:
+            rendered = await self.render(url)
+            page = extract_page(rendered.html, rendered.final_url)
+            return ExtractResult(
+                fetch_mode="render",
+                url=url,
+                final_url=rendered.final_url,
+                status=rendered.status,
+                content_type="text/html",
+                byte_count=len(rendered.html.encode("utf-8")),
+                content_digest=page.content_sha256,
+                retrieved_at=_now(),
+                robots=self._robots_decision(),
+                page=page.model_copy(update={"url": url}),
+            )
+        fetched = await self.fetcher.fetch(url)
+        page = extract_page(fetched.text, fetched.final_url)
+        return ExtractResult(
+            fetch_mode="http",
+            url=url,
+            final_url=fetched.final_url,
+            status=fetched.status,
+            content_type=fetched.content_type,
+            byte_count=fetched.bytes,
+            content_digest=page.content_sha256,
+            retrieved_at=fetched.fetched_at,
+            robots=self._robots_decision(),
+            page=page.model_copy(update={"url": url}),
+        )
+
+    def _robots_decision(self) -> Literal["allowed", "not_checked"]:
+        return "allowed" if self.policy.obey_robots else "not_checked"
+
+    @_read_only
+    async def render(self, url: str) -> RenderedPage:
+        """Render one URL in ad-blocked headless Chromium and return its DOM."""
+        await self._admit(url)
+        rendered = await self.browser.render(url, user_agent=self.policy.user_agent)
+        if len(rendered.html) > _MAX_RENDERED_HTML:
+            raise PolicyRefused("size_exceeded", url, "rendered DOM")
+        return rendered
+
+    @_read_only
+    async def screenshot(self, url: str, full_page: bool = True) -> Screenshot:
+        """Take a PNG screenshot of one URL, the full page by default."""
+        await self._admit(url)
+        return await self.browser.screenshot(
+            url, user_agent=self.policy.user_agent, full_page=full_page
+        )
+
+    @_read_only
+    async def links(self, url: str, same_site_only: bool = False) -> LinksResult:
+        """List the outbound http(s) links of one URL."""
+        fetched = await self.fetcher.fetch(url)
+        found = extract_links(fetched.text, fetched.final_url)
+        if same_site_only:
+            found = tuple(link for link in found if link.same_site)
+        return LinksResult(url=url, final_url=fetched.final_url, links=found)
+
+    async def _close(self) -> None:
+        """Release the shared browser."""
+        await self.browser.close()
+
+
+def _now() -> str:
+    return _dt.datetime.now(_dt.UTC).isoformat()

webarchiver/api/browser.py

@@ -0,0 +1,247 @@
+"""Headless, ad-blocked page rendering and full-page screenshots.
+
+CONN-WEBARCHIVER-WEB-SOURCE-R001. Chromium runs through Playwright. Every
+request the page makes, sub-resources included, is routed through
+:meth:`Browser._route`: requests to a known ad or tracker host, or to an
+address outside the public egress boundary, are aborted. Before a page loads,
+fixed and sticky overlays are removed and scrolling is re-enabled, so a full
+page screenshot shows the content rather than a cookie banner (the behaviour
+the original Selenium archiver had).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+import socket
+from dataclasses import dataclass, field
+from functools import lru_cache
+from typing import Any
+from urllib.parse import urlsplit
+
+from agent_connector_sdk.http.egress import egress_ip_is_blocked
+from pydantic import BaseModel, ConfigDict
+
+from .policy import PolicyRefused
+
+__all__ = ["AD_HOST_SUFFIXES", "Browser", "RenderedPage", "Screenshot"]
+
+#: Ad and tracker hosts aborted at the network layer (suffix match).
+AD_HOST_SUFFIXES = (
+    "doubleclick.net",
+    "googlesyndication.com",
+    "googleadservices.com",
+    "google-analytics.com",
+    "googletagmanager.com",
+    "googletagservices.com",
+    "adservice.google.com",
+    "amazon-adsystem.com",
+    "adnxs.com",
+    "adsrvr.org",
+    "criteo.com",
+    "criteo.net",
+    "taboola.com",
+    "outbrain.com",
+    "scorecardresearch.com",
+    "quantserve.com",
+    "hotjar.com",
+    "facebook.net",
+    "connect.facebook.net",
+    "ads-twitter.com",
+    "rubiconproject.com",
+    "pubmatic.com",
+    "openx.net",
+    "moatads.com",
+    "chartbeat.com",
+    "segment.io",
+    "mixpanel.com",
+)
+
+#: Removes fixed/sticky overlays and re-enables scrolling.
+_DECLUTTER_JS = """
+() => {
+  for (const el of Array.from(document.querySelectorAll('body *'))) {
+    const pos = getComputedStyle(el).position;
+    if (pos === 'fixed' || pos === 'sticky' || pos === '-webkit-sticky') {
+      el.remove();
+    }
+  }
+  for (const el of [document.documentElement, document.body]) {
+    if (el) { el.style.overflow = 'visible'; el.style.position = 'relative'; }
+  }
+}
+"""
+
+#: Scrolls through the page so lazy content loads, then back to the top.
+_SCROLL_JS = """
+async (maxHeight) => {
+  const step = Math.max(window.innerHeight, 200);
+  const height = Math.min(document.body.scrollHeight, maxHeight);
+  for (let y = 0; y < height; y += step) {
+    window.scrollTo(0, y);
+    await new Promise(r => setTimeout(r, 50));
+  }
+  window.scrollTo(0, 0);
+}
+"""
+
+
+class RenderedPage(BaseModel):
+    """The DOM of a page after scripts ran."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    url: str
+    final_url: str
+    status: int
+    title: str
+    html: str
+    blocked_requests: int
+
+
+class Screenshot(BaseModel):
+    """A PNG screenshot, base64-encoded."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    url: str
+    final_url: str
+    full_page: bool
+    width: int
+    png_base64: str
+    bytes: int
+
+
+def is_ad_host(host: str) -> bool:
+    host = host.lower().rstrip(".")
+    return any(host == s or host.endswith(f".{s}") for s in AD_HOST_SUFFIXES)
+
+
+@lru_cache(maxsize=1024)
+def _resolve(host: str) -> tuple[str, ...]:
+    try:
+        infos = socket.getaddrinfo(host, None, proto=socket.IPPROTO_TCP)
+    except OSError:
+        return ()
+    return tuple(sorted({str(info[4][0]) for info in infos}))
+
+
+@dataclass
+class Browser:
+    """One lazily launched Chromium shared by render and screenshot."""
+
+    allow_loopback: bool = False
+    navigation_timeout_ms: int = 30_000
+    max_scroll_height: int = 20_000
+    viewport_width: int = 1366
+    viewport_height: int = 900
+    _playwright: Any = field(default=None, init=False)
+    _browser: Any = field(default=None, init=False)
+    _lock: asyncio.Lock = field(default_factory=asyncio.Lock, init=False)
+
+    def _egress_blocked(self, host: str) -> bool:
+        addresses = _resolve(host)
+        if not addresses:
+            return True
+        return any(
+            egress_ip_is_blocked(ip, allow_loopback=self.allow_loopback)
+            for ip in addresses
+        )
+
+    async def _ensure(self) -> Any:
+        async with self._lock:
+            if self._browser is None:
+                from playwright.async_api import async_playwright
+
+                self._playwright = await async_playwright().start()
+                try:
+                    self._browser = await asyncio.wait_for(
+                        self._playwright.chromium.launch(
+                            headless=True, args=["--disable-dev-shm-usage"]
+                        ),
+                        self.navigation_timeout_ms / 1000,
+                    )
+                except Exception as exc:  # noqa: BLE001 - launch failure is typed
+                    await self._playwright.stop()
+                    self._playwright = None
+                    raise PolicyRefused(
+                        "render_failed", "chromium", type(exc).__name__
+                    ) from exc
+        return self._browser
+
+    async def close(self) -> None:
+        if self._browser is not None:
+            await self._browser.close()
+            self._browser = None
+        if self._playwright is not None:
+            await self._playwright.stop()
+            self._playwright = None
+
+    async def _open(self, url: str, user_agent: str) -> tuple[Any, Any, int, list[int]]:
+        host = urlsplit(url).hostname or ""
+        if self._egress_blocked(host):
+            raise PolicyRefused("egress_denied", url)
+        browser = await self._ensure()
+        context = await browser.new_context(
+            user_agent=user_agent,
+            viewport={"width": self.viewport_width, "height": self.viewport_height},
+            java_script_enabled=True,
+            accept_downloads=False,
+        )
+        blocked = [0]
+
+        async def _route(route: Any) -> None:
+            req_host = urlsplit(route.request.url).hostname or ""
+            if req_host and (is_ad_host(req_host) or self._egress_blocked(req_host)):
+                blocked[0] += 1
+                await route.abort()
+            else:
+                await route.continue_()
+
+        await context.route("**/*", _route)
+        page = await context.new_page()
+        try:
+            response = await page.goto(
+                url, wait_until="networkidle", timeout=self.navigation_timeout_ms
+            )
+        except Exception as exc:  # noqa: BLE001 - Playwright raises its own types
+            await context.close()
+            raise PolicyRefused("upstream_failed", url, type(exc).__name__) from exc
+        status = response.status if response is not None else 0
+        if status >= 400:
+            await context.close()
+            raise PolicyRefused("upstream_failed", url, str(status))
+        await page.evaluate(_SCROLL_JS, self.max_scroll_height)
+        await page.evaluate(_DECLUTTER_JS)
+        return context, page, status, blocked
+
+    async def render(self, url: str, *, user_agent: str) -> RenderedPage:
+        context, page, status, blocked = await self._open(url, user_agent)
+        try:
+            return RenderedPage(
+                url=url,
+                final_url=page.url,
+                status=status,
+                title=await page.title(),
+                html=await page.content(),
+                blocked_requests=blocked[0],
+            )
+        finally:
+            await context.close()
+
+    async def screenshot(
+        self, url: str, *, user_agent: str, full_page: bool = True
+    ) -> Screenshot:
+        context, page, _, _ = await self._open(url, user_agent)
+        try:
+            png = await page.screenshot(full_page=full_page, type="png")
+            return Screenshot(
+                url=url,
+                final_url=page.url,
+                full_page=full_page,
+                width=self.viewport_width,
+                png_base64=base64.b64encode(png).decode("ascii"),
+                bytes=len(png),
+            )
+        finally:
+            await context.close()

webarchiver/api/extraction.py

@@ -0,0 +1,130 @@
+"""Main-content extraction from fetched HTML (CONN-WEBARCHIVER-WEB-SOURCE-R001).
+
+Readability-style boilerplate removal is trafilatura's; this module only maps
+its output onto one typed :class:`ExtractedPage` and collects outbound links
+with lxml. A page with no extractable main content is refused rather than
+returned empty, so a caller never ingests a fabricated blank document.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from urllib.parse import urldefrag, urljoin, urlsplit
+
+import lxml.html
+import trafilatura
+from pydantic import BaseModel, ConfigDict
+
+from .policy import PolicyRefused
+
+__all__ = ["ExtractedPage", "PageLink", "extract_links", "extract_page"]
+
+_MAX_LINKS = 2000
+
+
+class PageLink(BaseModel):
+    """One outbound hyperlink, resolved to an absolute URL."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    url: str
+    text: str = ""
+    same_site: bool = False
+
+
+class ExtractedPage(BaseModel):
+    """The main content of one page plus its document metadata."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    url: str
+    canonical_url: str
+    title: str = ""
+    author: str = ""
+    published: str = ""
+    site_name: str = ""
+    language: str = ""
+    description: str = ""
+    text: str
+    markdown: str
+    content_sha256: str
+    links: tuple[PageLink, ...] = ()
+
+
+def _canonical(html_doc: lxml.html.HtmlElement, base: str) -> str:
+    for href in html_doc.xpath('//link[@rel="canonical"]/@href'):
+        if str(href).strip():
+            return urldefrag(urljoin(base, str(href).strip()))[0]
+    return urldefrag(base)[0]
+
+
+def extract_links(html: str, base_url: str) -> tuple[PageLink, ...]:
+    """Every distinct http(s) ``<a href>`` in ``html``, absolute and de-fragmented."""
+    if not html.strip():
+        return ()
+    doc = lxml.html.fromstring(html)
+    site = (urlsplit(base_url).hostname or "").lower()
+    seen: dict[str, PageLink] = {}
+    for anchor in doc.iter("a"):
+        href = (anchor.get("href") or "").strip()
+        if not href:
+            continue
+        url = urldefrag(urljoin(base_url, href))[0]
+        parts = urlsplit(url)
+        if parts.scheme not in {"http", "https"} or url in seen:
+            continue
+        seen[url] = PageLink(
+            url=url,
+            text=" ".join(anchor.text_content().split())[:300],
+            same_site=(parts.hostname or "").lower() == site,
+        )
+        if len(seen) >= _MAX_LINKS:
+            break
+    return tuple(seen.values())
+
+
+def extract_page(html: str, url: str) -> ExtractedPage:
+    """Extract ``html``'s main content as text and markdown with metadata."""
+    if not html.strip():
+        raise PolicyRefused("extract_empty", url, "empty body")
+    doc = lxml.html.fromstring(html)
+    canonical = _canonical(doc, url)
+    raw = trafilatura.extract(
+        html,
+        url=url,
+        output_format="json",
+        with_metadata=True,
+        include_comments=False,
+        include_tables=True,
+        favor_precision=True,
+    )
+    meta = json.loads(raw) if raw else {}
+    text = str(meta.get("text") or "").strip()
+    if not text:
+        raise PolicyRefused("extract_empty", url, "no main content")
+    markdown = (
+        trafilatura.extract(
+            html,
+            url=url,
+            output_format="markdown",
+            include_comments=False,
+            include_tables=True,
+            favor_precision=True,
+        )
+        or text
+    )
+    return ExtractedPage(
+        url=url,
+        canonical_url=canonical,
+        title=str(meta.get("title") or ""),
+        author=str(meta.get("author") or ""),
+        published=str(meta.get("date") or ""),
+        site_name=str(meta.get("sitename") or ""),
+        language=str(meta.get("language") or ""),
+        description=str(meta.get("description") or ""),
+        text=text,
+        markdown=markdown.strip(),
+        content_sha256=hashlib.sha256(text.encode("utf-8")).hexdigest(),
+        links=extract_links(html, url),
+    )