webrouter 0.11.0__py3-none-any.whl

web_router/__init__.py

@@ -0,0 +1,293 @@
+"""web-router: one interface over many web search, extraction, answer, research and browser APIs.
+
+Point the router at a provider and call it; swapping backends is a one-word
+change rather than a rewrite.
+
+    >>> from web_router import WebRouter
+    >>> router = WebRouter("nimble")
+    >>> response = router.search("what is retrieval augmented generation", max_results=5)
+    >>> pages = router.extract(response.urls[:3], format="markdown")
+
+:meth:`~web_router.WebRouter.run_template` runs a backend's Extract Template --
+a saved, site-specific extraction that turns one page into a record -- with
+:meth:`~web_router.WebRouter.templates` and :meth:`~web_router.WebRouter.template`
+to find one and read the inputs it takes:
+
+    >>> record = router.run_template("books_pdp", {"url": "https://books.example/..."}).data
+
+:meth:`~web_router.WebRouter.answer` asks one question and returns the
+backend's written answer with the sources it rests on, in a single call:
+
+    >>> reply = router.answer("who won the 2022 world cup final", provider="linkup")
+
+:meth:`~web_router.WebRouter.research` runs an agentic backend
+over several turns and answers in minutes rather than milliseconds:
+
+    >>> answer = router.research("how did nvidia's datacenter revenue mix shift in 2025")
+
+:meth:`~web_router.WebRouter.browse` is the same shape with a hosted browser
+agent doing the work -- Browser Use, Skyvern or Browserbase:
+
+    >>> response = router.browse("find the top story", start_url="https://news.ycombinator.com")
+
+And :meth:`~web_router.WebRouter.open_browser` hands back a browser session --
+a cloud vendor's, or a Chromium on this machine through ``local`` -- to drive
+over CDP or through the handle's own page actions, released when the block
+holding it ends:
+
+    >>> with router.open_browser() as browser:
+    ...     browser.goto("https://example.com")
+    ...     tree = browser.snapshot().tree
+"""
+
+from web_router._version import __version__
+from web_router.client import AsyncWebRouter, WebRouter
+from web_router.client_scope import AsyncBudgetScope, BudgetScope
+from web_router.constants import DEFAULT_PROVIDER
+from web_router.exceptions import (
+    BenchmarkTimeoutError,
+    BrowserActionError,
+    BrowserActionTimeoutError,
+    BrowseTimeoutError,
+    ConfigurationError,
+    DeadlineExceededError,
+    ElementNotFoundError,
+    MissingParameterError,
+    NavigationError,
+    NoAnswerProviderError,
+    PageClosedError,
+    ProviderError,
+    ProviderNotFoundError,
+    ProviderNotInstalledError,
+    ResearchTimeoutError,
+    RouterError,
+    StaleElementError,
+    TemplateNotFoundError,
+    UnsupportedFormatError,
+    UnsupportedParameterError,
+    WebRouterError,
+)
+from web_router.exceptions_budget import BudgetExceededError
+from web_router.providers.answer import AnswerProvider
+from web_router.providers.base import (
+    AsyncBrowserHandle,
+    BrowseJob,
+    BrowseProvider,
+    BrowserHandle,
+    BrowserProvider,
+    ExtractProvider,
+    ResearchJob,
+    ResearchProvider,
+    SearchProvider,
+    TemplateProvider,
+)
+from web_router.registry import (
+    available_browse_providers,
+    available_browser_providers,
+    available_extract_providers,
+    available_providers,
+    available_research_providers,
+    available_template_providers,
+    configured_browse_providers,
+    configured_browser_providers,
+    configured_extract_providers,
+    configured_providers,
+    configured_research_providers,
+    configured_template_providers,
+    register_browse_provider,
+    register_browser_provider,
+    register_extract_provider,
+    register_provider,
+    register_research_provider,
+    register_template_provider,
+    unregister_browse_provider,
+    unregister_browser_provider,
+    unregister_extract_provider,
+    unregister_provider,
+    unregister_research_provider,
+    unregister_template_provider,
+)
+from web_router.registry_answer import (
+    available_answer_providers,
+    configured_answer_providers,
+    register_answer_provider,
+    unregister_answer_provider,
+)
+from web_router.routing import Budget, Candidate, Plan, RouteDecision, RouteRequest, RouterLike, ToolName
+from web_router.routing.pool_status import BudgetStatus
+from web_router.types import (
+    BROWSE_PARAMETERS,
+    BROWSER_PARAMETERS,
+    EXTRACT_PARAMETERS,
+    NORMALIZED_PARAMETERS,
+    RESEARCH_PARAMETERS,
+    TEMPLATE_PARAMETERS,
+    BrowseOutcome,
+    BrowseProgress,
+    BrowseRequest,
+    BrowseResponse,
+    BrowserRequest,
+    BrowserSession,
+    BrowserState,
+    BrowseStep,
+    Citation,
+    ExtractedDocument,
+    ExtractFailure,
+    ExtractFormat,
+    ExtractRequest,
+    ExtractResponse,
+    Provider,
+    ResearchEffort,
+    ResearchOutcome,
+    ResearchProgress,
+    ResearchRequest,
+    ResearchResponse,
+    ResearchState,
+    SearchDepth,
+    SearchRequest,
+    SearchResponse,
+    SearchResult,
+    StealthLevel,
+    Template,
+    TemplatePage,
+    TemplateRequest,
+    TemplateResponse,
+    TemplateSample,
+    Viewport,
+)
+from web_router.types_answer import ANSWER_PARAMETERS, AnswerRequest, AnswerResponse, Recency
+from web_router.types_benchmark import BenchmarkReport, BenchmarkRun, ExtractBenchmarkReport, ExtractBenchmarkRun
+from web_router.types_decision import DecisionAttempt, DecisionHop, DecisionRecord
+from web_router.types_page import PageFormat, PageLoadState, PageSnapshot, PageState, Screenshot
+
+__all__ = [
+    "ANSWER_PARAMETERS",
+    "BROWSER_PARAMETERS",
+    "BROWSE_PARAMETERS",
+    "DEFAULT_PROVIDER",
+    "EXTRACT_PARAMETERS",
+    "NORMALIZED_PARAMETERS",
+    "RESEARCH_PARAMETERS",
+    "TEMPLATE_PARAMETERS",
+    "AnswerProvider",
+    "AnswerRequest",
+    "AnswerResponse",
+    "AsyncBrowserHandle",
+    "AsyncBudgetScope",
+    "AsyncWebRouter",
+    "BenchmarkReport",
+    "BenchmarkRun",
+    "BenchmarkTimeoutError",
+    "BrowseJob",
+    "BrowseOutcome",
+    "BrowseProgress",
+    "BrowseProvider",
+    "BrowseRequest",
+    "BrowseResponse",
+    "BrowseStep",
+    "BrowseTimeoutError",
+    "BrowserActionError",
+    "BrowserActionTimeoutError",
+    "BrowserHandle",
+    "BrowserProvider",
+    "BrowserRequest",
+    "BrowserSession",
+    "BrowserState",
+    "Budget",
+    "BudgetExceededError",
+    "BudgetScope",
+    "BudgetStatus",
+    "Candidate",
+    "Citation",
+    "ConfigurationError",
+    "DeadlineExceededError",
+    "DecisionAttempt",
+    "DecisionHop",
+    "DecisionRecord",
+    "ElementNotFoundError",
+    "ExtractBenchmarkReport",
+    "ExtractBenchmarkRun",
+    "ExtractFailure",
+    "ExtractFormat",
+    "ExtractProvider",
+    "ExtractRequest",
+    "ExtractResponse",
+    "ExtractedDocument",
+    "MissingParameterError",
+    "NavigationError",
+    "NoAnswerProviderError",
+    "PageClosedError",
+    "PageFormat",
+    "PageLoadState",
+    "PageSnapshot",
+    "PageState",
+    "Plan",
+    "Provider",
+    "ProviderError",
+    "ProviderNotFoundError",
+    "ProviderNotInstalledError",
+    "Recency",
+    "ResearchEffort",
+    "ResearchJob",
+    "ResearchOutcome",
+    "ResearchProgress",
+    "ResearchProvider",
+    "ResearchRequest",
+    "ResearchResponse",
+    "ResearchState",
+    "ResearchTimeoutError",
+    "RouteDecision",
+    "RouteRequest",
+    "RouterError",
+    "RouterLike",
+    "Screenshot",
+    "SearchDepth",
+    "SearchProvider",
+    "SearchRequest",
+    "SearchResponse",
+    "SearchResult",
+    "StaleElementError",
+    "StealthLevel",
+    "Template",
+    "TemplateNotFoundError",
+    "TemplatePage",
+    "TemplateProvider",
+    "TemplateRequest",
+    "TemplateResponse",
+    "TemplateSample",
+    "ToolName",
+    "UnsupportedFormatError",
+    "UnsupportedParameterError",
+    "Viewport",
+    "WebRouter",
+    "WebRouterError",
+    "__version__",
+    "available_answer_providers",
+    "available_browse_providers",
+    "available_browser_providers",
+    "available_extract_providers",
+    "available_providers",
+    "available_research_providers",
+    "available_template_providers",
+    "configured_answer_providers",
+    "configured_browse_providers",
+    "configured_browser_providers",
+    "configured_extract_providers",
+    "configured_providers",
+    "configured_research_providers",
+    "configured_template_providers",
+    "register_answer_provider",
+    "register_browse_provider",
+    "register_browser_provider",
+    "register_extract_provider",
+    "register_provider",
+    "register_research_provider",
+    "register_template_provider",
+    "unregister_answer_provider",
+    "unregister_browse_provider",
+    "unregister_browser_provider",
+    "unregister_extract_provider",
+    "unregister_provider",
+    "unregister_research_provider",
+    "unregister_template_provider",
+]

web_router/_benchmark.py

@@ -0,0 +1,205 @@
+"""The fan-out behind :meth:`~web_router.WebRouter.benchmark` and :meth:`~web_router.WebRouter.benchmark_extract`.
+
+A comparison is generic in the method it fans out: the provider it calls,
+the response that call returns, and the row the report keeps per provider.
+What the two methods share is here -- the members resolved before any row
+starts, the threads or the tasks, the deadline and the rows that outlive
+it -- and each router method passes in its own call and row builder. Private
+to the routers: nothing here is a public name.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextvars
+import threading
+import time
+from collections.abc import Awaitable, Callable, Sequence
+from concurrent.futures import Future, ThreadPoolExecutor, wait
+from typing import Any, TypeVar
+
+from web_router.constants import MSG_BENCHMARK_ROW_FAILED
+from web_router.exceptions import BenchmarkTimeoutError, ProviderError, WebRouterError
+from web_router.providers.base import ExtractProvider, SearchProvider
+from web_router.types import ExtractResponse, Provider, SearchResponse, provider_name
+from web_router.types_benchmark import BenchmarkRun, ExtractBenchmarkRun
+
+_MemberT = TypeVar("_MemberT", bound="SearchProvider | ExtractProvider")
+_AnswerT = TypeVar("_AnswerT", bound="SearchResponse | ExtractResponse")
+_RunT = TypeVar("_RunT", bound="BenchmarkRun | ExtractBenchmarkRun")
+
+
+class Stragglers:
+    """Rows the sync fan-out gave up on whose calls still run.
+
+    The router's ``close()`` joins them, so it never shuts a provider under a
+    live call; each is dropped the moment it finishes.
+    """
+
+    def __init__(self) -> None:
+        self._lock = threading.Lock()
+        self._futures: set[Future[Any]] = set()
+
+    def track(self, future: Future[Any]) -> None:
+        with self._lock:
+            self._futures.add(future)
+        future.add_done_callback(self._forget)
+
+    def _forget(self, future: Future[Any]) -> None:
+        with self._lock:
+            self._futures.discard(future)
+
+    def join(self) -> None:
+        """Block until every call still tracked has finished -- bounded by each provider's own HTTP timeout."""
+        with self._lock:
+            pending = tuple(self._futures)
+        if pending:
+            wait(pending)
+
+
+def members_of(
+    providers: Sequence[str | Provider | _MemberT] | None,
+    *,
+    default: Callable[[], list[str]],
+    build: Callable[[str | Provider | _MemberT], _MemberT],
+    none_named: str,
+) -> list[tuple[str, _MemberT | WebRouterError]]:
+    """The rows to run, as ``(row name, the provider or what stopped it)``, each name once, in the order asked.
+
+    Resolved on the caller's thread and before any row starts, for two
+    reasons: a backend whose extra is not installed or whose key is missing
+    is that row's failure rather than the comparison's, and a provider built
+    inside a worker after the router was closed would be cached by a router
+    that will never close it. ``build`` is the method's own getter, so the
+    provider is the one a plain call would use.
+    """
+    members: list[tuple[str, _MemberT | WebRouterError]] = []
+    seen: set[str] = set()
+    for item in default() if providers is None else providers:
+        name = provider_name(item) if isinstance(item, (str, Provider)) else item.name
+        if name in seen:
+            continue
+        seen.add(name)
+        try:
+            members.append((name, build(item)))
+        except Exception as exc:
+            members.append((name, _failure(name, exc)))
+    if not members:
+        raise ValueError(none_named)
+    return members
+
+
+def _failure(name: str, exc: Exception) -> WebRouterError:
+    """What a call raised, as its row's error: a router error as it is, anything else wrapped.
+
+    The public methods wrap every backend failure, so only a router error
+    should reach here; when something else does -- a provider bug, a caller's
+    own provider -- the report still comes back with this row saying what raised.
+    """
+    if isinstance(exc, WebRouterError):
+        return exc
+    wrapped = ProviderError(name, MSG_BENCHMARK_ROW_FAILED.format(error=exc), original=exc)
+    wrapped.__cause__ = exc  # Returned, not raised, so ``raise ... from`` cannot set it; the traceback stays reachable.
+    return wrapped
+
+
+def fan_out(
+    members: list[tuple[str, _MemberT | WebRouterError]],
+    call: Callable[[_MemberT], _AnswerT],
+    row_of: Callable[[str, float | None, _AnswerT | WebRouterError], _RunT],
+    timeout: float | None,
+    *,
+    stragglers: Stragglers,
+) -> list[_RunT]:
+    """Every member's row at once, each timed around its own ``call``, the whole bounded by ``timeout``.
+
+    ``row_of`` builds the method's own row from a name, when its call
+    started, and what it got or raised -- ``BenchmarkRun.of`` or
+    ``ExtractBenchmarkRun.of``. A row still unanswered at the
+    deadline is reported as timed out and its call left to finish on its
+    thread, tracked in ``stragglers`` until it does.
+    """
+    started = time.perf_counter()
+    # A member that could not be built is its row already, with nothing to time.
+    runs = {name: row_of(name, None, member) for name, member in members if isinstance(member, WebRouterError)}
+    futures = _start(members, call, row_of)
+    wait(futures.values(), timeout=timeout)
+    runs.update(_collect(futures, row_of, started, timeout, stragglers))
+    return [runs[name] for name, _ in members]
+
+
+def _start(
+    members: list[tuple[str, _MemberT | WebRouterError]],
+    call: Callable[[_MemberT], _AnswerT],
+    row_of: Callable[[str, float | None, _AnswerT | WebRouterError], _RunT],
+) -> dict[str, Future[_RunT]]:
+    """Each buildable member's row started on a thread of its own, by name."""
+
+    def row(name: str, provider: _MemberT) -> _RunT:
+        own = time.perf_counter()
+        try:
+            outcome: _AnswerT | WebRouterError = call(provider)
+        except Exception as exc:
+            outcome = _failure(name, exc)
+        return row_of(name, own, outcome)
+
+    live = [(name, member) for name, member in members if not isinstance(member, WebRouterError)]
+    # Not a `with` block: leaving one joins every thread, which would hold
+    # the report until the slowest backend answered and make the timeout
+    # a fiction. A row given up on finishes on its own and is discarded;
+    # its provider was built before it started, so the router closes it.
+    pool = ThreadPoolExecutor(max_workers=max(1, len(live)))
+    try:
+        # Each in a copy of the caller's context, so a deadline set around
+        # the call reaches the member on its worker thread.
+        return {name: pool.submit(contextvars.copy_context().run, row, name, member) for name, member in live}
+    finally:
+        pool.shutdown(wait=False)
+
+
+def _collect(
+    futures: dict[str, Future[_RunT]],
+    row_of: Callable[[str, float | None, Any], _RunT],
+    started: float,
+    timeout: float | None,
+    stragglers: Stragglers,
+) -> dict[str, _RunT]:
+    """Each started row as it stands once the deadline has passed: its result, or a timeout."""
+    runs: dict[str, _RunT] = {}
+    for name, future in futures.items():
+        if future.done():
+            runs[name] = future.result()
+            continue
+        # A row given up on reports how long it was waited for. Its call
+        # keeps running, on a provider this router will close, so close()
+        # waits for it -- tracked here, dropped the moment it finishes.
+        runs[name] = row_of(name, started, BenchmarkTimeoutError(name, timeout or 0.0))
+        stragglers.track(future)
+    return runs
+
+
+async def afan_out(
+    members: list[tuple[str, _MemberT | WebRouterError]],
+    call: Callable[[_MemberT], Awaitable[_AnswerT]],
+    row_of: Callable[[str, float | None, _AnswerT | WebRouterError], _RunT],
+    timeout: float | None,
+) -> list[_RunT]:
+    """The async twin of :func:`fan_out`: the rows gathered on the loop under one deadline, late ones cancelled."""
+    deadline = None if timeout is None else time.perf_counter() + timeout
+
+    async def row(name: str, member: _MemberT | WebRouterError) -> _RunT:
+        if isinstance(member, WebRouterError):
+            return row_of(name, None, member)
+        own = time.perf_counter()
+        # One deadline for the whole comparison: a row that starts late
+        # gets what is left of it, not a fresh timeout of its own.
+        remaining = None if deadline is None else max(0.0, deadline - own)
+        try:
+            outcome: _AnswerT | WebRouterError = await asyncio.wait_for(call(member), remaining)
+        except asyncio.TimeoutError:
+            outcome = BenchmarkTimeoutError(name, timeout or 0.0)
+        except Exception as exc:
+            outcome = _failure(name, exc)
+        return row_of(name, own, outcome)
+
+    return list(await asyncio.gather(*(row(name, member) for name, member in members)))

web_router/_chains.py

@@ -0,0 +1,35 @@
+"""The default ``auto`` orders that route-eval ranked, read from ``data/chains.json``.
+
+Search's keyed members and answer's members are ranked from the frozen benchmark
+results by ``make chains`` (``benchmarks/providers/chains.py``),
+which writes the file; never edit it by hand. A new backend joins by hand at the
+end of its method's ``appended`` (search) or ``preferred`` (answer) list, where
+the next ``make chains`` keeps it until frozen results rank it. Experimental: the
+file's shape may change before 1.0.
+
+Imports nothing from ``web_router``: :mod:`web_router.constants` and
+:mod:`web_router.constants_answer` read it while they load.
+"""
+
+from __future__ import annotations
+
+import json
+from functools import lru_cache
+from importlib import resources
+from typing import Any
+
+CHAINS_DATA_DIR = "data"
+CHAINS_DATA_FILE = "chains.json"
+
+
+@lru_cache(maxsize=1)
+def chains() -> dict[str, Any]:
+    """The whole generated file."""
+    text = (resources.files("web_router") / CHAINS_DATA_DIR / CHAINS_DATA_FILE).read_text(encoding="utf-8")
+    data: dict[str, Any] = json.loads(text)
+    return data
+
+
+def chain(method: str, part: str) -> tuple[str, ...]:
+    """One list of a method's generated order: ``preferred``, or search's ``appended``."""
+    return tuple(chains()[method][part])

web_router/_hosts.py

@@ -0,0 +1,105 @@
+"""What a URL's host is: loopback, a cloud metadata endpoint, or a private network -- read as a browser reads it.
+
+Shared by the MCP browser tools' navigation guard and the routers'
+``url_policy``. Every check here reads the host as written: a literal address
+in any spelling a browser accepts (decimal, hex, octal, short IPv4, and IPv4
+mapped into IPv6) is judged as the address it is, and nothing resolves DNS.
+"""
+
+from __future__ import annotations
+
+import ipaddress
+import re
+import socket
+
+from web_router.constants_urls import (
+    IPV4_COMPATIBLE_PREFIX,
+    LEGACY_IPV4_PATTERN,
+    LOOPBACK_NAMES,
+    METADATA_HOSTS,
+    NAT64_PREFIX,
+)
+
+_LEGACY_IPV4 = re.compile(LEGACY_IPV4_PATTERN)
+_BARE_HEX = re.compile(r"(?:^|(?<=\.))0x(?=\.|$)")
+
+
+def host_address(host: str) -> ipaddress.IPv4Address | ipaddress.IPv6Address | None:
+    """``host`` as the address it spells, an IPv4-mapped IPv6 address as its IPv4; ``None`` for a name."""
+    address: ipaddress.IPv4Address | ipaddress.IPv6Address
+    try:
+        address = ipaddress.ip_address(host)
+    except ValueError:
+        legacy = legacy_ipv4(host)
+        if legacy is None:
+            return None
+        address = legacy
+    if isinstance(address, ipaddress.IPv6Address):
+        return _embedded_ipv4(address) or address
+    return address
+
+
+def _embedded_ipv4(address: ipaddress.IPv6Address) -> ipaddress.IPv4Address | None:
+    """The IPv4 address an IPv6 one carries and reaches: IPv4-mapped, NAT64 (``64:ff9b::/96``) or the deprecated
+    IPv4-compatible form (``::a.b.c.d``, but not ``::`` or ``::1``, which are IPv6's own).
+
+    A 6to4 address (``2002::/16``) gives its IPv4 only when that is no public host's: older Pythons call every
+    6to4 address global, newer ones none, so the IPv4 decides a non-public one and the rest stay as before.
+    """
+    if address.ipv4_mapped is not None:
+        return address.ipv4_mapped
+    sixtofour = address.sixtofour
+    if sixtofour is not None and (not sixtofour.is_global or sixtofour.is_multicast):
+        return sixtofour
+    low = ipaddress.IPv4Address(int(address) & 0xFFFFFFFF)
+    if address in NAT64_PREFIX:
+        return low
+    if address in IPV4_COMPATIBLE_PREFIX and int(address) > 1:
+        return low
+    return None
+
+
+def numeric_host(host: str) -> bool:
+    """Whether ``host`` has an IPv4 address's shape: up to four dot-separated parts, each digits or ``0x`` hex.
+
+    Whether or not ``inet_aton`` reads it: ``127.0.0.09`` is no octal, but macOS's getaddrinfo reads it as decimal.
+    """
+    return _LEGACY_IPV4.fullmatch(host) is not None
+
+
+def legacy_ipv4(host: str) -> ipaddress.IPv4Address | None:
+    """``host`` read as a browser reads an IPv4 address: ``2130706433``, ``0x7f000001``, ``127.1`` or ``0177.0.0.1``.
+
+    ``ipaddress`` accepts only dotted quads, but a browser canonicalizes all of
+    these to one before it connects, so each is checked as the address it is.
+    """
+    if not numeric_host(host):
+        return None
+    try:
+        # A browser reads a bare ``0x`` part as 0; inet_aton may refuse it or not, by platform.
+        return ipaddress.IPv4Address(socket.inet_aton(_BARE_HEX.sub("0x0", host)))
+    except (OSError, ValueError):
+        return None
+
+
+def is_loopback(host: str) -> bool:
+    """``localhost``, ``0.0.0.0``, or any loopback or unspecified address."""
+    address = host_address(host)
+    return host in LOOPBACK_NAMES or (address is not None and (address.is_loopback or address.is_unspecified))
+
+
+def is_metadata(host: str) -> bool:
+    """A cloud metadata endpoint by name, or any link-local address."""
+    address = host_address(host)
+    # All of link-local, not only the one address: the metadata service answers on 169.254/16 on several clouds.
+    return host in METADATA_HOSTS or (address is not None and address.is_link_local)
+
+
+def is_private(host: str) -> bool:
+    """A literal address no public host has: private (RFC 1918, ULA), shared (CGNAT), reserved, or multicast.
+
+    Read as :func:`ipaddress.ip_address` reads ``is_global``, so loopback and
+    link-local are in it too. A name is never private here: nothing resolves it.
+    """
+    address = host_address(host)
+    return address is not None and (not address.is_global or address.is_multicast)