wxpath 0.4.0__py3-none-any.whl

wxpath/hooks/registry.py

@@ -0,0 +1,145 @@
+"""
+Pluggable hook system for wxpath.
+
+Write once:
+
+    from wxpath import hooks
+
+    @hooks.register
+    class OnlyEnglish:
+        def post_parse(self, ctx, elem):
+            lang = elem.xpath('string(/html/@lang)').lower()[:2]
+            return elem if lang in ("en", "") else None
+
+... and wxpath.engine will call it automatically.
+"""
+
+from __future__ import annotations
+
+import functools
+from collections.abc import Generator
+from dataclasses import dataclass, field
+from typing import Any, Iterable, List, Optional, Protocol
+
+from lxml import html
+
+from wxpath.util.logging import get_logger
+
+log = get_logger(__name__)
+
+
+# --------------------------------------------------------------------------- #
+# Dataclass describing the crawl context for a single URL
+# --------------------------------------------------------------------------- #
+@dataclass
+class FetchContext:
+    url: str
+    backlink: Optional[str]
+    depth: int
+    segments: list        # remaining op/value pairs
+    user_data: dict = field(default_factory=dict)
+
+
+# --------------------------------------------------------------------------- #
+# Hook protocol - every method is optional
+# --------------------------------------------------------------------------- #
+class Hook(Protocol):
+    # Return False to abort fetching this URL
+    # def pre_fetch(self, ctx: FetchContext) -> bool: ...
+
+    # May return modified HTML bytes or None to drop this branch entirely
+    def post_fetch(self, ctx: FetchContext, html_bytes: bytes) -> bytes | None: ...
+
+    # May return modified element or None to drop this branch entirely
+    def post_parse(
+        self, ctx: FetchContext, elem: html.HtmlElement
+    ) -> html.HtmlElement | None: ...
+
+    # Called for every candidate link; return False to prevent enqueueing it
+    # def pre_queue(self, ctx: FetchContext, url: str) -> bool: ...
+
+    # Called for every extracted value; may transform or drop it
+    def post_extract(self, value: Any) -> Any: ...
+
+
+# --------------------------------------------------------------------------- #
+# Global registry helpers
+# --------------------------------------------------------------------------- #
+_global_hooks: dict[str, Hook] = dict()
+
+
+def register(hook: Hook | type) -> Hook:
+    """Decorator/helper to add a Hook to the global list.
+
+    Args:
+        hook: A Hook class or instance to register.
+
+    Returns:
+        The registered hook (instantiated if a class was provided).
+
+    Example:
+        >>> @register
+        ... class DebugHook:
+        ...     def post_fetch(self, ctx, html_bytes):
+        ...         print("Fetched", ctx.url)
+        ...         return html_bytes
+    """
+
+    hook_name = getattr(hook, '__name__', hook.__class__.__name__)
+    if hook_name in _global_hooks:
+        return hook
+
+    instance = hook() if isinstance(hook, type) else hook
+    _global_hooks[hook_name] = instance
+    return hook
+
+
+def get_hooks() -> List[Hook]:
+    """Return the list of globally-registered hooks (read-only)."""
+    return list(_global_hooks.values())
+
+
+def iter_post_extract_hooks() -> Iterable[Hook]:
+    yield from (h for h in _global_hooks.values() if hasattr(h, "post_extract"))
+
+
+def pipe_post_extract(gen_func):
+    """Wrap a generator function to pipe yielded values through post_extract hooks.
+
+    Args:
+        gen_func: A generator function to wrap.
+
+    Returns:
+        A wrapped generator that filters values through registered hooks.
+    """
+    @functools.wraps(gen_func)
+    def wrapper(*args, **kwargs) -> Generator:
+        for item in gen_func(*args, **kwargs):
+            for hook in iter_post_extract_hooks():
+                item = hook.post_extract(item)
+                if item is None:       # hook decided to drop it
+                    break
+            if item is not None:
+                yield item
+    return wrapper
+
+
+def pipe_post_extract_async(async_gen_func):
+    """Wrap an async generator function to pipe yielded values through hooks.
+
+    Args:
+        async_gen_func: An async generator function to wrap.
+
+    Returns:
+        A wrapped async generator that filters values through registered hooks.
+    """
+    @functools.wraps(async_gen_func)
+    async def wrapper(*args, **kwargs):
+        async for item in async_gen_func(*args, **kwargs):
+            for hook in iter_post_extract_hooks():
+                item = hook.post_extract(item)
+                if item is None:
+                    break
+            if item is not None:
+                yield item
+    return wrapper

wxpath/http/client/__init__.py

@@ -0,0 +1,9 @@
+from wxpath.http.client.crawler import Crawler
+from wxpath.http.client.request import Request
+from wxpath.http.client.response import Response
+
+__all__ = [
+    "Crawler",
+    "Request",
+    "Response"
+]

wxpath/http/client/cache.py

@@ -0,0 +1,43 @@
+try:
+    from aiohttp_client_cache import SQLiteBackend
+except ImportError:
+    CachedSession = None
+
+from wxpath.settings import SETTINGS
+from wxpath.util.logging import get_logger
+
+log = get_logger(__name__)
+
+CACHE_SETTINGS = SETTINGS.http.client.cache
+
+def get_cache_backend():
+    log.info("cache backend", extra={"backend": CACHE_SETTINGS.backend})
+    if CACHE_SETTINGS.backend == "redis":
+        from aiohttp_client_cache.backends.redis import RedisBackend
+        return RedisBackend(
+            expire_after=CACHE_SETTINGS.expire_after,
+            urls_expire_after=CACHE_SETTINGS.urls_expire_after or None,
+            allowed_methods=CACHE_SETTINGS.allowed_methods,
+            allowed_codes=CACHE_SETTINGS.allowed_codes,
+            include_headers=CACHE_SETTINGS.include_headers,
+            ignored_parameters=CACHE_SETTINGS.ignored_parameters,
+            **CACHE_SETTINGS.redis
+            # cache_name=CACHE_SETTINGS.redis.cache_name,
+            # host=CACHE_SETTINGS.redis.host,
+            # port=CACHE_SETTINGS.redis.port,
+            # db=CACHE_SETTINGS.redis.db,
+            # cache_control=CACHE_SETTINGS.cache_control,
+        )
+    elif CACHE_SETTINGS.backend == "sqlite":
+        return SQLiteBackend(
+            cache_name=CACHE_SETTINGS.sqlite.cache_name,
+            expire_after=CACHE_SETTINGS.expire_after,
+            urls_expire_after=CACHE_SETTINGS.urls_expire_after or None,
+            allowed_methods=CACHE_SETTINGS.allowed_methods,
+            allowed_codes=CACHE_SETTINGS.allowed_codes,
+            include_headers=CACHE_SETTINGS.include_headers,
+            ignored_parameters=CACHE_SETTINGS.ignored_parameters,
+            # cache_control=CACHE_SETTINGS.cache_control,
+        )
+    else:
+        raise ValueError(f"Unknown cache backend: {CACHE_SETTINGS.backend}")

wxpath/http/client/crawler.py

@@ -0,0 +1,315 @@
+import aiohttp
+
+try:
+    from aiohttp_client_cache import CachedSession, SQLiteBackend
+except ImportError:
+    CachedSession = None
+
+import asyncio
+import time
+import urllib.parse
+from collections import defaultdict
+from socket import gaierror
+from typing import AsyncIterator
+
+from wxpath.http.client.cache import get_cache_backend
+from wxpath.http.client.request import Request
+from wxpath.http.client.response import Response
+from wxpath.http.policy.retry import RetryPolicy
+from wxpath.http.policy.robots import RobotsTxtPolicy
+from wxpath.http.policy.throttler import AbstractThrottler, AutoThrottler
+from wxpath.http.stats import CrawlerStats, build_trace_config
+from wxpath.settings import SETTINGS
+from wxpath.util.logging import get_logger
+
+log = get_logger(__name__)
+
+CACHE_SETTINGS = SETTINGS.http.client.cache
+CRAWLER_SETTINGS = SETTINGS.http.client.crawler
+
+def get_async_session(
+        headers: dict | None = None,
+        timeout: aiohttp.ClientTimeout | None = None,
+        connector: aiohttp.TCPConnector | None = None,
+        trace_config: aiohttp.TraceConfig | None = None
+) -> aiohttp.ClientSession:
+    """
+    Create and return a new aiohttp session. If aiohttp-client-cache is available
+    and enabled, return a new CachedSession bound to the configured SQLite backend.
+    The caller is responsible for closing the session.
+    """
+
+    if timeout is None:
+        timeout = aiohttp.ClientTimeout(total=CRAWLER_SETTINGS.timeout)
+
+    if CACHE_SETTINGS.enabled and CachedSession and SQLiteBackend:
+        log.info("using aiohttp-client-cache")
+        return CachedSession(
+            cache=get_cache_backend(),
+            headers=headers,
+            timeout=timeout,
+            connector=connector,
+            trace_configs=[trace_config] if trace_config is not None else None
+        )
+
+    return aiohttp.ClientSession(
+        headers=headers, 
+        timeout=timeout, 
+        connector=connector, 
+        trace_configs=[trace_config] if trace_config is not None else None
+    )
+
+
+class Crawler:
+    """Concurrent HTTP crawler that manages throttling, retries, and robots."""
+
+    def __init__(
+        self,
+        concurrency: int = None,
+        per_host: int = None,
+        timeout: int = None,
+        *,
+        headers: dict | None = None,
+        proxies: dict | None = None,
+        retry_policy: RetryPolicy | None = None,
+        throttler: AbstractThrottler | None = None,
+        auto_throttle_target_concurrency: float = None,
+        auto_throttle_start_delay: float = None,
+        auto_throttle_max_delay: float = None,
+        respect_robots: bool = True,
+    ):
+        cfg = CRAWLER_SETTINGS
+
+        self.concurrency = concurrency if concurrency is not None else cfg.concurrency
+        self.per_host = per_host if per_host is not None else cfg.per_host
+
+        timeout = timeout if timeout is not None else cfg.timeout
+        self._timeout = aiohttp.ClientTimeout(total=timeout)
+
+        self._headers = cfg.headers | (headers or {}) # merge headers
+        
+        _proxies = proxies if proxies is not None else cfg.proxies
+        self._proxies = _proxies if (isinstance(_proxies, defaultdict) or _proxies) else {}
+        
+        self.retry_policy = retry_policy or RetryPolicy()
+
+        # auto-throttle defaults
+        auto_throttle_target_concurrency = auto_throttle_target_concurrency \
+            if auto_throttle_target_concurrency is not None \
+            else cfg.auto_throttle_target_concurrency
+        
+        auto_throttle_start_delay = auto_throttle_start_delay \
+            if auto_throttle_start_delay is not None \
+            else cfg.auto_throttle_start_delay
+
+        auto_throttle_max_delay = auto_throttle_max_delay \
+            if auto_throttle_max_delay is not None \
+            else cfg.auto_throttle_max_delay
+
+        self.throttler = throttler or AutoThrottler(
+            target_concurrency=auto_throttle_target_concurrency or self.concurrency/4.0,
+            start_delay=auto_throttle_start_delay,
+            max_delay=auto_throttle_max_delay,
+        )
+
+        self._sem_global = asyncio.Semaphore(self.concurrency)
+        self._sem_host = defaultdict(lambda: asyncio.Semaphore(self.per_host))
+
+        self._pending: asyncio.Queue[Request] = asyncio.Queue()
+        self._results: asyncio.Queue[Response] = asyncio.Queue()
+
+        self._session: aiohttp.ClientSession | None = None
+        self._workers: list[asyncio.Task] = []
+        self._closed = False
+        self._stats = CrawlerStats()
+
+        self.respect_robots = respect_robots if respect_robots is not None else cfg.respect_robots
+        self._robots_policy: RobotsTxtPolicy | None = None
+
+        # WARN: If SQLiteBackend caching is enabled and min(concurrency, per_host) > 1,
+        #       write-contention is likely to occur.
+        if (CACHE_SETTINGS.enabled 
+            and CACHE_SETTINGS.backend == "sqlite"
+            and min(self.concurrency, self.per_host) > 1
+            ):
+            log.warning(
+                "SQLiteBackend caching is enabled and min(concurrency, per_host) > 1. "
+                "Write-contention is likely to occur. Consider using RedisBackend."
+            )
+
+    def build_session(self) -> aiohttp.ClientSession:
+        """Construct an `aiohttp.ClientSession` with tracing and pooling."""
+        trace_config = build_trace_config(self._stats)
+        # Need to build the connector as late as possible as it requires the loop
+        connector = aiohttp.TCPConnector(limit=self.concurrency*2, ttl_dns_cache=300)
+        return get_async_session(
+            headers=self._headers,
+            timeout=self._timeout,
+            connector=connector,
+            trace_config=trace_config
+        )
+
+    async def __aenter__(self) -> "Crawler":
+        """Initialize HTTP session and start background workers."""
+        if self._session is None:
+            # self._session = aiohttp.ClientSession(timeout=self._timeout)
+            self._session = self.build_session()
+
+        # Note: Set robots policy after session is created
+        if self.respect_robots:
+            self._robots_policy = RobotsTxtPolicy(self._session)
+
+        self._workers = [
+            asyncio.create_task(self._worker(), name=f"crawler-worker-{i}")
+            for i in range(self.concurrency)
+        ]
+        return self
+
+    async def __aexit__(self, *_) -> None:
+        """Tear down workers and close the HTTP session."""
+        self._closed = True
+        for w in self._workers:
+            w.cancel()
+
+        await asyncio.gather(*self._workers, return_exceptions=True)
+
+        if self._session:
+            await self._session.close()
+
+    def submit(self, req: Request) -> None:
+        """Queue a request for fetching or raise if crawler already closed."""
+        if self._closed:
+            raise RuntimeError("crawler is closed")
+        self._pending.put_nowait(req)
+
+    def __aiter__(self) -> AsyncIterator[Response]:
+        return self._result_iter()
+
+    async def _result_iter(self) -> AsyncIterator[Response]:
+        """Async iterator yielding responses as workers produce them."""
+        # while not self._closed:
+        while not (self._closed and self._results.empty()):
+            resp = await self._results.get()
+            self._results.task_done()
+            yield resp
+
+    def _proxy_for(self, url: str) -> str | None:
+        host = urllib.parse.urlsplit(url).hostname
+        try:
+            # bracket notation first, for defaultdicts
+            value = self._proxies[host]
+        except KeyError:
+            value = self._proxies.get(host)
+        
+        if not value:
+            log.debug("proxy", extra={"host": host, "value": value})
+        return value
+
+    async def _worker(self) -> None:
+        """Worker loop that fetches pending requests and enqueues results."""
+        while True:
+            req = await self._pending.get()
+            try:
+                resp = await self._fetch_one(req)
+                if resp is not None:
+                    await self._results.put(resp)
+
+            except asyncio.CancelledError:
+                # Must propagate cancellation
+                log.debug("cancelled error", extra={"url": req.url})
+                raise
+
+            except gaierror:
+                # Ignore DNS errors
+                log.warning("DNS error", extra={"url": req.url})
+                pass
+
+            except Exception as exc:
+                log.warning("exception", extra={"url": req.url})
+                # Last-resort safety: never drop a request silently
+                await self._results.put(Response(req, 0, b"", error=exc))
+            finally:
+                self._pending.task_done()
+
+    async def _fetch_one(self, req: Request) -> Response | None:
+        """Fetch a single request, handling robots, throttling, and retries."""
+        host = req.hostname
+
+        if self._robots_policy:
+            can_fetch = await self._robots_policy.can_fetch(
+                req.url, self._headers.get("User-Agent")
+            )
+            if not can_fetch:
+                log.debug("disallowed by robots.txt", extra={"url": req.url})
+                return Response(req, 403, b"", error=RuntimeError("Disallowed by robots.txt"))
+
+        # TODO: Move this filter to hooks
+        if req.url.lower().endswith((".pdf", ".zip", ".exe")):
+            req.max_retries = 0
+
+        async with self._sem_global, self._sem_host[host]:
+            t0 = asyncio.get_running_loop().time()
+            await self.throttler.wait(host)
+            dt = asyncio.get_running_loop().time() - t0
+
+            self._stats.throttle_waits += 1
+            self._stats.throttle_wait_time += dt
+            self._stats.throttle_waits_by_host[host] += 1
+
+            start = time.monotonic()
+            try:
+                log.info("fetching", extra={"url": req.url})
+                async with self._session.get(
+                    req.url,
+                    headers=self._headers | req.headers,
+                    proxy=self._proxy_for(req.url),
+                    timeout=req.timeout or self._timeout,
+                ) as resp:
+                    from_cache = getattr(resp, "from_cache", False)
+                    if from_cache:
+                        # NOTE: This is a bit of a hack, but it works. aiohttp-client-cache does not
+                        #  interface with TraceConfigs on cache hit, so we have to do it here.
+                        self._stats.requests_cache_hit += 1
+                        log.info("[CACHE HIT]", extra={"req.url": req.url, "resp.url": resp.url})
+                    else:
+                        log.info("[CACHE MISS]", extra={"req.url": req.url, "resp.url": resp.url})
+
+                    body = await resp.read()
+
+                    latency = time.monotonic() - start
+                    self.throttler.record_latency(host, latency)
+
+                    if self.retry_policy.should_retry(req, response=resp):
+                        await self._retry(req)
+                        return None
+
+                    return Response(req, resp.status, body, dict(resp.headers))
+            except asyncio.CancelledError:
+                # Normal during shutdown / timeout propagation
+                log.debug("cancelled error", extra={"url": req.url})
+                raise
+            except Exception as exc:
+                latency = time.monotonic() - start
+                self.throttler.record_latency(host, latency)
+
+                if self.retry_policy.should_retry(req, exception=exc):
+                    await self._retry(req)
+                    return None
+                
+                log.error("request failed", extra={"url": req.url}, exc_info=exc)
+                return Response(req, 0, b"", error=exc)
+
+    async def _retry(self, req: Request) -> None:
+        """Reschedule a request according to the retry policy."""
+        req.retries += 1
+        delay = self.retry_policy.get_delay(req)
+
+        log.warning(
+            "retrying",
+            extra={"url": req.url, "retry": req.retries, "delay": delay},
+        )
+
+        if delay:
+            await asyncio.sleep(delay)
+
+        self.submit(req)

wxpath/http/client/request.py

@@ -0,0 +1,38 @@
+import time
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class Request:
+    """HTTP request envelope used by the crawler."""
+    url: str
+    method: str = "GET"
+    headers: dict[str, str] = field(default_factory=dict)
+    timeout: float = 15.0
+
+    retries: int = 0
+    max_retries: int | None = None
+    dont_retry: bool = False
+
+    meta: dict[str, Any] = field(default_factory=dict)
+
+    created_at: float = field(default_factory=time.monotonic)
+
+    def copy_for_retry(self) -> "Request":
+        """Create a copy incrementing the retry counter for scheduling."""
+        return Request(
+            url=self.url,
+            method=self.method,
+            headers=self.headers,
+            timeout=self.timeout,
+            retries=self.retries + 1,
+            max_retries=self.max_retries,
+            dont_retry=self.dont_retry,
+            meta=self.meta,
+        )
+    
+    @property
+    def hostname(self) -> str:
+        from urllib.parse import urlsplit
+        return urlsplit(self.url).hostname or ""

wxpath/http/client/response.py

@@ -0,0 +1,14 @@
+# wxpath/http/response.py
+from dataclasses import dataclass, field
+from typing import Optional
+
+from wxpath.http.client.request import Request
+
+
+@dataclass
+class Response:
+    request: Request
+    status: int
+    body: bytes
+    headers: dict[str, str] | None = None
+    error: Optional[Exception] = field(default=None, kw_only=True)

wxpath/http/policy/backoff.py

@@ -0,0 +1,16 @@
+import random
+
+
+def exponential_backoff(
+    attempt: int,
+    base: float = 0.5,
+    cap: float = 30.0,
+    jitter: bool = True,
+) -> float:
+    """
+    Exponential backoff with optional jitter.
+    """
+    delay = min(cap, base * (2 ** attempt))
+    if jitter:
+        delay *= random.uniform(0.7, 1.3)
+    return delay

wxpath/http/policy/retry.py

@@ -0,0 +1,35 @@
+from wxpath.http.policy.backoff import exponential_backoff
+from wxpath.util.logging import get_logger
+
+log = get_logger(__name__)
+
+
+class RetryPolicy:
+    def __init__(
+        self,
+        max_retries: int = 3,
+        retry_statuses: set[int] = None,
+    ):
+        self.max_retries = max_retries
+        self.retry_statuses = retry_statuses or {500, 502, 503, 504}
+
+    def should_retry(self, request, response=None, exception=None) -> bool:
+        if request.dont_retry:
+            return False
+
+        if request.max_retries is not None and request.retries >= request.max_retries:
+            return False
+        
+        if request.retries >= self.max_retries:
+            return False
+
+        if response is not None and response.status in self.retry_statuses:
+            return True
+        
+        if exception is not None:
+            return True
+
+        return False
+
+    def get_delay(self, request) -> float:
+        return exponential_backoff(request.retries)