webless 0.1.0__py3-none-any.whl

webless/_http.py

@@ -0,0 +1,281 @@
+"""The shared HTTP layer behind `search` and `fetch`.
+
+Every request this package makes goes through `fetch_url`, which exists to
+make two promises the rest of the package depends on.
+
+The first is that no request needs a key. Every search engine reached from
+here is a public endpoint, so the package works on a fresh install with
+nothing configured.
+
+The second is that a request is hard to block and never blocks. Hosts that
+refuse a request do it on fingerprint reputation, so a retry after a 403 or a
+429 presents a different browser user agent rather than repeating the one that
+was just refused; that alone clears most transient blocks. And every call is
+async with a hard timeout, so a slow host delays its own result and nothing
+else -- a search queries its engines concurrently and keeps whatever comes
+back in time.
+
+Requests are also guarded against pointing back inward. A URL that resolves to
+a private, loopback, or link-local address is refused before it is sent, on
+the original URL and again on every redirect hop, so a caller cannot be
+steered into the machine's own network.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import ipaddress
+import random
+import socket
+from dataclasses import dataclass, field
+from urllib.parse import urlsplit, urlunsplit
+
+import httpx
+
+USER_AGENTS = (
+    (
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
+        "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
+    ),
+    (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+        "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
+    ),
+    "Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0",
+    (
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 "
+        "(KHTML, like Gecko) Version/17.4 Safari/605.1.15"
+    ),
+)
+
+DEFAULT_HEADERS = {
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.9",
+}
+
+BLOCKED_STATUSES = frozenset({401, 403, 429})
+RETRYABLE_STATUSES = frozenset({408, 425, 429, 500, 502, 503, 504})
+
+DEFAULT_TIMEOUT_S = 15.0
+MAX_REDIRECTS = 5
+MAX_ATTEMPTS = 3
+MAX_BODY_BYTES = 5 * 1024 * 1024
+
+
+class WebError(Exception):
+    """A request that failed in a way the caller should report, not retry."""
+
+    def __init__(self, message: str, *, status: int | None = None) -> None:
+        super().__init__(message)
+        self.status = status
+
+
+def next_user_agent(previous: str | None = None) -> str:
+    """Pick a user agent, guaranteed to differ from `previous` when given.
+
+    A cold request draws at random so concurrent engines do not all present
+    the same fingerprint; a retry walks to the next agent in the pool so the
+    host sees a different browser than the one it just refused.
+    """
+    if previous is None:
+        return random.choice(USER_AGENTS)
+    try:
+        index = USER_AGENTS.index(previous)
+    except ValueError:
+        return USER_AGENTS[0]
+    return USER_AGENTS[(index + 1) % len(USER_AGENTS)]
+
+
+def normalize_url(url: str) -> str:
+    """Return `url` with a scheme, lowercased host, and no fragment."""
+    candidate = url.strip()
+    if candidate.startswith("//"):
+        candidate = f"https:{candidate}"
+    if not candidate.lower().startswith(("http://", "https://")):
+        candidate = f"https://{candidate}"
+    parts = urlsplit(candidate)
+    host = parts.hostname or ""
+    netloc = host.lower()
+    if parts.port:
+        netloc = f"{netloc}:{parts.port}"
+    if parts.username:
+        credentials = parts.username
+        if parts.password:
+            credentials = f"{credentials}:{parts.password}"
+        netloc = f"{credentials}@{netloc}"
+    return urlunsplit((parts.scheme.lower(), netloc, parts.path or "/", parts.query, ""))
+
+
+def dedup_key(url: str) -> str:
+    """A comparison key that treats trivial URL variants as one page."""
+    parts = urlsplit(normalize_url(url))
+    host = (parts.hostname or "").removeprefix("www.")
+    path = parts.path.rstrip("/") or "/"
+    return f"{host}{path}?{parts.query}" if parts.query else f"{host}{path}"
+
+
+def host_of(url: str) -> str:
+    """The hostname of `url`, or an empty string when it has none."""
+    try:
+        return urlsplit(normalize_url(url)).hostname or ""
+    except ValueError:
+        return ""
+
+
+def _is_public_address(address: str) -> bool:
+    """True when `address` is a routable public IP."""
+    try:
+        ip = ipaddress.ip_address(address)
+    except ValueError:
+        return False
+    return not (
+        ip.is_private
+        or ip.is_loopback
+        or ip.is_link_local
+        or ip.is_reserved
+        or ip.is_multicast
+        or ip.is_unspecified
+    )
+
+
+async def _guard_host(url: str) -> None:
+    """Refuse a URL that resolves into private or loopback address space."""
+    parts = urlsplit(url)
+    if parts.scheme not in {"http", "https"}:
+        raise WebError(f"Unsupported URL scheme: {parts.scheme or 'none'}.")
+    host = parts.hostname
+    if not host:
+        raise WebError(f"{url} has no host.")
+    try:
+        infos = await asyncio.get_running_loop().getaddrinfo(
+            host, parts.port or (443 if parts.scheme == "https" else 80),
+            proto=socket.IPPROTO_TCP,
+        )
+    except OSError as exc:
+        raise WebError(f"Cannot resolve {host}: {exc}") from exc
+    addresses = {info[4][0] for info in infos}
+    if not any(_is_public_address(a) for a in addresses):
+        raise WebError(
+            f"{host} resolves to a private or loopback address; refusing to fetch it."
+        )
+
+
+@dataclass(slots=True)
+class Response:
+    """A completed fetch."""
+
+    url: str
+    final_url: str
+    status: int
+    text: str
+    content_type: str
+    headers: dict[str, str] = field(default_factory=dict)
+
+    @property
+    def is_html(self) -> bool:
+        """True when the body looks like markup rather than data."""
+        return "html" in self.content_type or "xml" in self.content_type
+
+    @property
+    def is_json(self) -> bool:
+        """True when the server labelled the body as JSON."""
+        return "json" in self.content_type
+
+
+async def fetch_url(
+    url: str,
+    *,
+    headers: dict[str, str] | None = None,
+    timeout: float = DEFAULT_TIMEOUT_S,
+    attempts: int = MAX_ATTEMPTS,
+    abort: asyncio.Event | None = None,
+) -> Response:
+    """Fetch `url` and return its body, rotating fingerprint on a block.
+
+    Redirects are followed manually so that each hop can be re-guarded against
+    private address space. A retryable status or transport error backs off and
+    tries again with a fresh user agent, up to `attempts` times; anything else
+    is raised as a `WebError` carrying the status.
+    """
+    target = normalize_url(url)
+    user_agent: str | None = None
+    last_error: WebError | None = None
+
+    for attempt in range(attempts):
+        if abort is not None and abort.is_set():
+            raise WebError("Aborted before the request completed.")
+        if attempt:
+            await asyncio.sleep(min(0.5 * 2 ** (attempt - 1), 4.0) + random.random() * 0.3)
+        user_agent = next_user_agent(user_agent)
+        request_headers = {
+            **DEFAULT_HEADERS,
+            "User-Agent": user_agent,
+            **(headers or {}),
+        }
+        try:
+            return await _fetch_once(target, request_headers, timeout)
+        except WebError as exc:
+            last_error = exc
+            if exc.status is None or exc.status not in (
+                RETRYABLE_STATUSES | BLOCKED_STATUSES
+            ):
+                raise
+        except (httpx.TransportError, httpx.HTTPError) as exc:
+            last_error = WebError(f"Request to {target} failed: {exc}")
+
+    raise last_error or WebError(f"Request to {target} failed.")
+
+
+async def _fetch_once(
+    url: str, headers: dict[str, str], timeout: float
+) -> Response:
+    """Perform one request, following and re-guarding redirects by hand."""
+    current = url
+    async with httpx.AsyncClient(
+        follow_redirects=False,
+        timeout=httpx.Timeout(timeout),
+        http2=False,
+    ) as client:
+        for _ in range(MAX_REDIRECTS + 1):
+            await _guard_host(current)
+            try:
+                response = await client.get(current, headers=headers)
+            except httpx.TimeoutException as exc:
+                raise WebError(f"{current} timed out after {timeout:g}s.") from exc
+
+            if response.is_redirect:
+                location = response.headers.get("location", "")
+                if not location:
+                    raise WebError(f"{current} redirected without a target.")
+                current = normalize_url(str(response.url.join(location)))
+                continue
+
+            if response.status_code >= 400:
+                raise WebError(
+                    f"{current} returned HTTP {response.status_code}.",
+                    status=response.status_code,
+                )
+
+            body = response.content[:MAX_BODY_BYTES]
+            return Response(
+                url=url,
+                final_url=str(response.url),
+                status=response.status_code,
+                text=body.decode(response.encoding or "utf-8", errors="replace"),
+                content_type=response.headers.get("content-type", "").lower(),
+                headers=dict(response.headers),
+            )
+
+    raise WebError(f"{url} exceeded {MAX_REDIRECTS} redirects.")
+
+
+__all__ = [
+    "USER_AGENTS",
+    "Response",
+    "WebError",
+    "fetch_url",
+    "next_user_agent",
+    "normalize_url",
+    "dedup_key",
+    "host_of",
+]

webless/_search.py

@@ -0,0 +1,90 @@
+"""The public search API.
+
+One call, `search`, queries several keyless engines at once and returns a
+single fused ranking. The engines are public endpoints, so nothing has to be
+configured before the first call works, and none of them is treated as
+required: an engine that is blocked, rate limited, or slow costs recall and is
+named in `SearchResult.failed`, while the rest of the ranking stands.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from ._engines import GENERAL_ENGINES, WIDE_ENGINES, Engine, SearchHit, search_web
+
+DEFAULT_LIMIT = 10
+DEFAULT_TIMEOUT_S = 12.0
+
+
+@dataclass(slots=True)
+class SearchResult:
+    """A fused ranking, plus what each engine did to produce it.
+
+    `failed` is part of the result rather than an exception because a partial
+    answer is still an answer: knowing that Mojeek was blocked while
+    DuckDuckGo answered lets a caller decide whether to trust the ranking,
+    without having to lose it.
+    """
+
+    query: str
+    hits: list[SearchHit]
+    succeeded: list[str] = field(default_factory=list)
+    failed: dict[str, str] = field(default_factory=dict)
+
+    def __len__(self) -> int:
+        return len(self.hits)
+
+    def __iter__(self):
+        return iter(self.hits)
+
+    def __getitem__(self, index):
+        return self.hits[index]
+
+    @property
+    def ok(self) -> bool:
+        """Whether at least one engine answered."""
+        return bool(self.succeeded)
+
+
+async def search(
+    query: str,
+    *,
+    limit: int = DEFAULT_LIMIT,
+    wide: bool = False,
+    engines: tuple[type[Engine], ...] | None = None,
+    timeout: float = DEFAULT_TIMEOUT_S,
+    allowed_domains: list[str] | None = None,
+    blocked_domains: list[str] | None = None,
+) -> SearchResult:
+    """Search the web without an API key.
+
+    `wide` adds the long-tail engines -- Marginalia's small-web index and
+    Hacker News -- to the general set, which helps on niche or discussion
+    oriented queries and adds latency on ordinary ones. Pass `engines`
+    directly to choose the set yourself.
+
+    `allowed_domains` and `blocked_domains` filter on host, and an entry
+    covers its subdomains, so ``python.org`` keeps ``docs.python.org``.
+    """
+    if not query.strip():
+        raise ValueError("query is empty")
+
+    chosen = engines if engines is not None else (WIDE_ENGINES if wide else GENERAL_ENGINES)
+    report = await search_web(
+        query,
+        limit=limit,
+        engines=chosen,
+        timeout=timeout,
+        allowed_domains=allowed_domains,
+        blocked_domains=blocked_domains,
+    )
+    return SearchResult(
+        query=query,
+        hits=report.hits,
+        succeeded=report.succeeded,
+        failed=report.failed,
+    )
+
+
+__all__ = ["search", "SearchResult", "SearchHit"]

webless/_sync.py

@@ -0,0 +1,45 @@
+"""Blocking wrappers over the async API.
+
+The core is async because a search is several concurrent requests and a slow
+engine must not hold up the rest. Plenty of callers -- a script, a notebook
+cell, a Django view -- have no loop to await in, so each entry point gets a
+``*_sync`` twin that runs one for the duration of the call.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+
+from ._fetch import Page, fetch
+from ._search import SearchResult, search
+
+
+def _run(coro: Any) -> Any:
+    """Run `coro` to completion, refusing to do it from inside a loop.
+
+    `asyncio.run` inside a running loop raises a message about the loop rather
+    than about the call the caller made, so the check is done here where the
+    right advice -- await the async twin -- can be given.
+    """
+    try:
+        asyncio.get_running_loop()
+    except RuntimeError:
+        return asyncio.run(coro)
+    coro.close()
+    raise RuntimeError(
+        "a sync call cannot run inside an event loop; await the async version instead"
+    )
+
+
+def search_sync(query: str, **kwargs: Any) -> SearchResult:
+    """Blocking `search`. Takes the same keyword arguments."""
+    return _run(search(query, **kwargs))
+
+
+def fetch_sync(url: str, **kwargs: Any) -> Page:
+    """Blocking `fetch`. Takes the same keyword arguments."""
+    return _run(fetch(url, **kwargs))
+
+
+__all__ = ["search_sync", "fetch_sync"]

webless/cli.py

@@ -0,0 +1,121 @@
+"""The `webless` command.
+
+A thin shell over `search` and `fetch`, so the package is usable from a
+terminal or a pipe without writing a script. Output is human readable by
+default and JSON with ``--json``, which makes it composable with `jq`.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import sys
+from dataclasses import asdict
+
+from . import __version__
+from ._http import WebError
+from ._fetch import fetch
+from ._search import DEFAULT_LIMIT, search
+
+
+def build_parser() -> argparse.ArgumentParser:
+    """Assemble the argument parser for both subcommands."""
+    parser = argparse.ArgumentParser(
+        prog="webless",
+        description="Search the web and fetch pages, with no API keys.",
+    )
+    parser.add_argument("--version", action="version", version=f"webless {__version__}")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    find = sub.add_parser("search", help="search the web")
+    find.add_argument("query", nargs="+", help="what to search for")
+    find.add_argument("-n", "--limit", type=int, default=DEFAULT_LIMIT)
+    find.add_argument("-w", "--wide", action="store_true", help="add the long-tail engines")
+    find.add_argument("--allow", action="append", metavar="DOMAIN", default=None)
+    find.add_argument("--block", action="append", metavar="DOMAIN", default=None)
+    find.add_argument("--timeout", type=float, default=12.0)
+    find.add_argument("--json", action="store_true")
+
+    get = sub.add_parser("fetch", help="fetch a page as Markdown")
+    get.add_argument("url")
+    get.add_argument("-f", "--format", choices=("markdown", "text", "raw"), default="markdown")
+    get.add_argument("--full-page", action="store_true", help="keep navigation and footers")
+    get.add_argument("-c", "--max-chars", type=int, default=None)
+    get.add_argument("--offset", type=int, default=0)
+    get.add_argument("--timeout", type=float, default=None)
+    get.add_argument("--json", action="store_true")
+    return parser
+
+
+async def run_search(args: argparse.Namespace) -> int:
+    """Print a fused ranking, and report any engine that did not answer."""
+    result = await search(
+        " ".join(args.query),
+        limit=args.limit,
+        wide=args.wide,
+        timeout=args.timeout,
+        allowed_domains=args.allow,
+        blocked_domains=args.block,
+    )
+
+    if args.json:
+        print(json.dumps(asdict(result), indent=2))
+        return 0 if result.ok else 1
+
+    for index, hit in enumerate(result.hits, start=1):
+        print(f"{index}. {hit.title}")
+        print(f"   {hit.url}")
+        if hit.snippet:
+            print(f"   {hit.snippet}")
+        print(f"   [{', '.join(hit.engines)}]")
+        print()
+
+    if not result.hits:
+        print("no results", file=sys.stderr)
+    for name, reason in result.failed.items():
+        print(f"{name}: {reason}", file=sys.stderr)
+    return 0 if result.ok else 1
+
+
+async def run_fetch(args: argparse.Namespace) -> int:
+    """Print one page's content, or its full record with ``--json``."""
+    page = await fetch(
+        args.url,
+        format=args.format,
+        full_page=args.full_page,
+        max_chars=args.max_chars,
+        offset=args.offset,
+        timeout=args.timeout,
+    )
+
+    if args.json:
+        print(json.dumps(asdict(page), indent=2))
+        return 0
+
+    if page.redirected:
+        print(f"(redirected to {page.final_url})", file=sys.stderr)
+    print(page.content)
+    if page.truncated:
+        print(
+            f"[truncated; continue with --offset {args.offset + len(page.content)}]",
+            file=sys.stderr,
+        )
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Entry point. Network failures print a message rather than a traceback."""
+    args = build_parser().parse_args(argv)
+    runner = run_search if args.command == "search" else run_fetch
+    try:
+        return asyncio.run(runner(args))
+    except (WebError, ValueError) as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 1
+    except KeyboardInterrupt:
+        return 130
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())