wxpath 0.1.1__py3-none-any.whl → 0.3.0__py3-none-any.whl

wxpath/hooks/registry.py

@@ -0,0 +1,145 @@
+"""
+Pluggable hook system for wxpath.
+
+Write once:
+
+    from wxpath import hooks
+
+    @hooks.register
+    class OnlyEnglish:
+        def post_parse(self, ctx, elem):
+            lang = elem.xpath('string(/html/@lang)').lower()[:2]
+            return elem if lang in ("en", "") else None
+
+... and wxpath.engine will call it automatically.
+"""
+
+from __future__ import annotations
+
+import functools
+from collections.abc import Generator
+from dataclasses import dataclass, field
+from typing import Any, Iterable, List, Optional, Protocol
+
+from lxml import html
+
+from wxpath.util.logging import get_logger
+
+log = get_logger(__name__)
+
+
+# --------------------------------------------------------------------------- #
+# Dataclass describing the crawl context for a single URL
+# --------------------------------------------------------------------------- #
+@dataclass
+class FetchContext:
+    url: str
+    backlink: Optional[str]
+    depth: int
+    segments: list        # remaining op/value pairs
+    user_data: dict = field(default_factory=dict)
+
+
+# --------------------------------------------------------------------------- #
+# Hook protocol - every method is optional
+# --------------------------------------------------------------------------- #
+class Hook(Protocol):
+    # Return False to abort fetching this URL
+    # def pre_fetch(self, ctx: FetchContext) -> bool: ...
+
+    # May return modified HTML bytes or None to drop this branch entirely
+    def post_fetch(self, ctx: FetchContext, html_bytes: bytes) -> bytes | None: ...
+
+    # May return modified element or None to drop this branch entirely
+    def post_parse(
+        self, ctx: FetchContext, elem: html.HtmlElement
+    ) -> html.HtmlElement | None: ...
+
+    # Called for every candidate link; return False to prevent enqueueing it
+    # def pre_queue(self, ctx: FetchContext, url: str) -> bool: ...
+
+    # Called for every extracted value; may transform or drop it
+    def post_extract(self, value: Any) -> Any: ...
+
+
+# --------------------------------------------------------------------------- #
+# Global registry helpers
+# --------------------------------------------------------------------------- #
+_global_hooks: dict[str, Hook] = dict()
+
+
+def register(hook: Hook | type) -> Hook:
+    """Decorator/helper to add a Hook to the global list.
+
+    Args:
+        hook: A Hook class or instance to register.
+
+    Returns:
+        The registered hook (instantiated if a class was provided).
+
+    Example:
+        >>> @register
+        ... class DebugHook:
+        ...     def post_fetch(self, ctx, html_bytes):
+        ...         print("Fetched", ctx.url)
+        ...         return html_bytes
+    """
+
+    hook_name = getattr(hook, '__name__', hook.__class__.__name__)
+    if hook_name in _global_hooks:
+        return hook
+
+    instance = hook() if isinstance(hook, type) else hook
+    _global_hooks[hook_name] = instance
+    return hook
+
+
+def get_hooks() -> List[Hook]:
+    """Return the list of globally-registered hooks (read-only)."""
+    return list(_global_hooks.values())
+
+
+def iter_post_extract_hooks() -> Iterable[Hook]:
+    yield from (h for h in _global_hooks.values() if hasattr(h, "post_extract"))
+
+
+def pipe_post_extract(gen_func):
+    """Wrap a generator function to pipe yielded values through post_extract hooks.
+
+    Args:
+        gen_func: A generator function to wrap.
+
+    Returns:
+        A wrapped generator that filters values through registered hooks.
+    """
+    @functools.wraps(gen_func)
+    def wrapper(*args, **kwargs) -> Generator:
+        for item in gen_func(*args, **kwargs):
+            for hook in iter_post_extract_hooks():
+                item = hook.post_extract(item)
+                if item is None:       # hook decided to drop it
+                    break
+            if item is not None:
+                yield item
+    return wrapper
+
+
+def pipe_post_extract_async(async_gen_func):
+    """Wrap an async generator function to pipe yielded values through hooks.
+
+    Args:
+        async_gen_func: An async generator function to wrap.
+
+    Returns:
+        A wrapped async generator that filters values through registered hooks.
+    """
+    @functools.wraps(async_gen_func)
+    async def wrapper(*args, **kwargs):
+        async for item in async_gen_func(*args, **kwargs):
+            for hook in iter_post_extract_hooks():
+                item = hook.post_extract(item)
+                if item is None:
+                    break
+            if item is not None:
+                yield item
+    return wrapper

wxpath/http/client/__init__.py

@@ -0,0 +1,9 @@
+from wxpath.http.client.crawler import Crawler
+from wxpath.http.client.request import Request
+from wxpath.http.client.response import Response
+
+__all__ = [
+    "Crawler",
+    "Request",
+    "Response"
+]

wxpath/http/client/crawler.py

@@ -0,0 +1,231 @@
+import asyncio
+import time
+import urllib.parse
+from collections import defaultdict
+from socket import gaierror
+from typing import AsyncIterator
+
+import aiohttp
+
+from wxpath.http.client.request import Request
+from wxpath.http.client.response import Response
+from wxpath.http.policy.retry import RetryPolicy
+from wxpath.http.policy.robots import RobotsTxtPolicy
+from wxpath.http.policy.throttler import AbstractThrottler, AutoThrottler
+from wxpath.http.stats import CrawlerStats, build_trace_config
+from wxpath.util.logging import get_logger
+
+log = get_logger(__name__)
+
+HEADERS = {"User-Agent": ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)" 
+                   "AppleWebKit/537.36 (KHTML, like Gecko) "
+                   "Chrome/142.0.0.0 Safari/537.36")}
+
+
+class Crawler:
+    """Concurrent HTTP crawler that manages throttling, retries, and robots."""
+
+    def __init__(
+        self,
+        concurrency: int = 16,
+        per_host: int = 8,
+        timeout: int = 15,
+        *,
+        headers: dict | None = None,
+        proxies: dict | None = None,
+        retry_policy: RetryPolicy | None = None,
+        throttler: AbstractThrottler | None = None,
+        auto_throttle_target_concurrency: float = None,
+        auto_throttle_start_delay: float = 0.25,
+        auto_throttle_max_delay: float = 10.0,
+        respect_robots: bool = True,
+    ):
+        self.concurrency = concurrency
+        self._timeout = aiohttp.ClientTimeout(total=timeout)
+        self._headers   = HEADERS | (headers or {}) # merge headers
+        self._proxies = proxies if (isinstance(proxies, defaultdict) or proxies) else {}
+        self.respect_robots = respect_robots
+
+        self.retry_policy = retry_policy or RetryPolicy()
+        self.throttler = throttler or AutoThrottler(
+            target_concurrency=auto_throttle_target_concurrency or concurrency/4.0,
+            start_delay=auto_throttle_start_delay,
+            max_delay=auto_throttle_max_delay,
+        )
+        self._sem_global = asyncio.Semaphore(concurrency)
+        self._sem_host = defaultdict(lambda: asyncio.Semaphore(per_host))
+
+        self._pending: asyncio.Queue[Request] = asyncio.Queue()
+        self._results: asyncio.Queue[Response] = asyncio.Queue()
+
+        self._session: aiohttp.ClientSession | None = None
+        self._workers: list[asyncio.Task] = []
+        self._closed = False
+        self._stats = CrawlerStats()
+        self._robots_policy: RobotsTxtPolicy | None = None
+
+    def build_session(self) -> aiohttp.ClientSession:
+        """Construct an `aiohttp.ClientSession` with tracing and pooling."""
+        trace_config = build_trace_config(self._stats)
+        # Need to build the connector as late as possible as it requires the loop
+        connector = aiohttp.TCPConnector(limit=self.concurrency*2, ttl_dns_cache=300)
+        return aiohttp.ClientSession(
+            headers=self._headers, 
+            timeout=self._timeout, 
+            connector=connector, 
+            trace_configs=[trace_config]
+        )
+
+    async def __aenter__(self) -> "Crawler":
+        """Initialize HTTP session and start background workers."""
+        if self._session is None:
+            # self._session = aiohttp.ClientSession(timeout=self._timeout)
+            self._session = self.build_session()
+
+        if self.respect_robots:
+            self._robots_policy = RobotsTxtPolicy(self._session)
+
+        self._workers = [
+            asyncio.create_task(self._worker(), name=f"crawler-worker-{i}")
+            for i in range(self.concurrency)
+        ]
+        return self
+
+    async def __aexit__(self, *_) -> None:
+        """Tear down workers and close the HTTP session."""
+        self._closed = True
+        for w in self._workers:
+            w.cancel()
+
+        await asyncio.gather(*self._workers, return_exceptions=True)
+
+        if self._session:
+            await self._session.close()
+
+    def submit(self, req: Request) -> None:
+        """Queue a request for fetching or raise if crawler already closed."""
+        if self._closed:
+            raise RuntimeError("crawler is closed")
+        self._pending.put_nowait(req)
+
+    def __aiter__(self) -> AsyncIterator[Response]:
+        return self._result_iter()
+
+    async def _result_iter(self) -> AsyncIterator[Response]:
+        """Async iterator yielding responses as workers produce them."""
+        # while not self._closed:
+        while not (self._closed and self._results.empty()):
+            resp = await self._results.get()
+            self._results.task_done()
+            yield resp
+
+    def _proxy_for(self, url: str) -> str | None:
+        host = urllib.parse.urlsplit(url).hostname
+        try:
+            # bracket notation first, for defaultdicts
+            value = self._proxies[host]
+        except KeyError:
+            value = self._proxies.get(host)
+        
+        if not value:
+            log.debug("proxy", extra={"host": host, "value": value})
+        return value
+
+    async def _worker(self) -> None:
+        """Worker loop that fetches pending requests and enqueues results."""
+        while True:
+            req = await self._pending.get()
+            try:
+                resp = await self._fetch_one(req)
+                if resp is not None:
+                    await self._results.put(resp)
+
+            except asyncio.CancelledError:
+                # Must propagate cancellation
+                log.debug("cancelled error", extra={"url": req.url})
+                raise
+
+            except gaierror:
+                # Ignore DNS errors
+                log.warning("DNS error", extra={"url": req.url})
+                pass
+
+            except Exception as exc:
+                log.warning("exception", extra={"url": req.url})
+                # Last-resort safety: never drop a request silently
+                await self._results.put(Response(req, 0, b"", error=exc))
+            finally:
+                self._pending.task_done()
+
+    async def _fetch_one(self, req: Request) -> Response | None:
+        """Fetch a single request, handling robots, throttling, and retries."""
+        host = req.hostname
+
+        if self._robots_policy:
+            can_fetch = await self._robots_policy.can_fetch(
+                req.url, self._headers.get("User-Agent")
+            )
+            if not can_fetch:
+                log.debug("disallowed by robots.txt", extra={"url": req.url})
+                return Response(req, 403, b"", error=RuntimeError("Disallowed by robots.txt"))
+
+        # TODO: Move this filter to hooks
+        if req.url.lower().endswith((".pdf", ".zip", ".exe")):
+            req.max_retries = 0
+
+        async with self._sem_global, self._sem_host[host]:
+            t0 = asyncio.get_running_loop().time()
+            await self.throttler.wait(host)
+            dt = asyncio.get_running_loop().time() - t0
+
+            self._stats.throttle_waits += 1
+            self._stats.throttle_wait_time += dt
+            self._stats.throttle_waits_by_host[host] += 1
+
+            start = time.monotonic()
+            try:
+                async with self._session.get(
+                    req.url,
+                    headers=self._headers | req.headers,
+                    proxy=self._proxy_for(req.url),
+                    timeout=req.timeout or self._timeout,
+                ) as resp:
+                    body = await resp.read()
+
+                    latency = time.monotonic() - start
+                    self.throttler.record_latency(host, latency)
+
+                    if self.retry_policy.should_retry(req, response=resp):
+                        await self._retry(req)
+                        return None
+
+                    return Response(req, resp.status, body, dict(resp.headers))
+            except asyncio.CancelledError:
+                # Normal during shutdown / timeout propagation
+                log.debug("cancelled error", extra={"url": req.url})
+                raise
+            except Exception as exc:
+                latency = time.monotonic() - start
+                self.throttler.record_latency(host, latency)
+
+                if self.retry_policy.should_retry(req, exception=exc):
+                    await self._retry(req)
+                    return None
+                
+                log.error("request failed", extra={"url": req.url}, exc_info=exc)
+                return Response(req, 0, b"", error=exc)
+
+    async def _retry(self, req: Request) -> None:
+        """Reschedule a request according to the retry policy."""
+        req.retries += 1
+        delay = self.retry_policy.get_delay(req)
+
+        log.warning(
+            "retrying",
+            extra={"url": req.url, "retry": req.retries, "delay": delay},
+        )
+
+        if delay:
+            await asyncio.sleep(delay)
+
+        self.submit(req)

wxpath/http/client/request.py

@@ -0,0 +1,38 @@
+import time
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class Request:
+    """HTTP request envelope used by the crawler."""
+    url: str
+    method: str = "GET"
+    headers: dict[str, str] = field(default_factory=dict)
+    timeout: float = 15.0
+
+    retries: int = 0
+    max_retries: int | None = None
+    dont_retry: bool = False
+
+    meta: dict[str, Any] = field(default_factory=dict)
+
+    created_at: float = field(default_factory=time.monotonic)
+
+    def copy_for_retry(self) -> "Request":
+        """Create a copy incrementing the retry counter for scheduling."""
+        return Request(
+            url=self.url,
+            method=self.method,
+            headers=self.headers,
+            timeout=self.timeout,
+            retries=self.retries + 1,
+            max_retries=self.max_retries,
+            dont_retry=self.dont_retry,
+            meta=self.meta,
+        )
+    
+    @property
+    def hostname(self) -> str:
+        from urllib.parse import urlsplit
+        return urlsplit(self.url).hostname or ""

wxpath/http/client/response.py

@@ -0,0 +1,14 @@
+# wxpath/http/response.py
+from dataclasses import dataclass, field
+from typing import Optional
+
+from wxpath.http.client.request import Request
+
+
+@dataclass
+class Response:
+    request: Request
+    status: int
+    body: bytes
+    headers: dict[str, str] | None = None
+    error: Optional[Exception] = field(default=None, kw_only=True)

wxpath/http/policy/backoff.py

@@ -0,0 +1,16 @@
+import random
+
+
+def exponential_backoff(
+    attempt: int,
+    base: float = 0.5,
+    cap: float = 30.0,
+    jitter: bool = True,
+) -> float:
+    """
+    Exponential backoff with optional jitter.
+    """
+    delay = min(cap, base * (2 ** attempt))
+    if jitter:
+        delay *= random.uniform(0.7, 1.3)
+    return delay

wxpath/http/policy/retry.py

@@ -0,0 +1,35 @@
+from wxpath.http.policy.backoff import exponential_backoff
+from wxpath.util.logging import get_logger
+
+log = get_logger(__name__)
+
+
+class RetryPolicy:
+    def __init__(
+        self,
+        max_retries: int = 3,
+        retry_statuses: set[int] = None,
+    ):
+        self.max_retries = max_retries
+        self.retry_statuses = retry_statuses or {500, 502, 503, 504}
+
+    def should_retry(self, request, response=None, exception=None) -> bool:
+        if request.dont_retry:
+            return False
+
+        if request.max_retries is not None and request.retries >= request.max_retries:
+            return False
+        
+        if request.retries >= self.max_retries:
+            return False
+
+        if response is not None and response.status in self.retry_statuses:
+            return True
+        
+        if exception is not None:
+            return True
+
+        return False
+
+    def get_delay(self, request) -> float:
+        return exponential_backoff(request.retries)

wxpath/http/policy/robots.py

@@ -0,0 +1,82 @@
+import asyncio
+import urllib.parse
+import urllib.robotparser
+
+import aiohttp
+
+from wxpath.util.logging import get_logger
+
+log = get_logger(__name__)
+
+
+class RobotsTxtPolicy:
+    """Caches and evaluates robots.txt rules for crawler requests."""
+
+    def __init__(self, 
+                 session: aiohttp.ClientSession, 
+                 default_parser: type['RobotsParserBase'] | None = None):
+        self._session = session
+        self._parsers: dict[str, "RobotsParserBase"] = {}
+        self._lock = asyncio.Lock()
+        self._default_parser = default_parser or UrllibRobotParser
+
+    async def can_fetch(self, url: str, user_agent: str | None) -> bool:
+        """Return whether the crawler is allowed to fetch `url`."""
+        host = urllib.parse.urlsplit(url).hostname
+        if not host:
+            return False
+
+        # Due to multiple aiohttp workers running concurrently, we need to lock
+        async with self._lock:
+            if host not in self._parsers:
+                self._parsers[host] = await self._fetch_robots_txt(host)
+
+        return self._parsers[host].can_fetch(url, user_agent)
+
+    async def _fetch_robots_txt(self, host: str) -> "RobotsParserBase":
+        """Retrieve and parse the robots.txt for `host`, failing open on errors."""
+        url = f"http://{host}/robots.txt"
+        try:
+            async with self._session.get(url) as response:
+                if response.status == 200:
+                    text = await response.text()
+                    # Pass the text as-is to the parser, let it handle the format
+                    if self._default_parser == UrllibRobotParser:
+                        return self._default_parser(text.splitlines())
+                    else:
+                        return self._default_parser(text)
+                else:
+                    # Empty robots.txt - allow all
+                    if self._default_parser == UrllibRobotParser:
+                        return self._default_parser([])
+                    else:
+                        return self._default_parser("")
+        except Exception:
+            # If robots.txt is unavailable, allow all requests (fail open)
+            log.debug(f"Failed to fetch robots.txt from {host}, allowing all requests")
+            if self._default_parser == UrllibRobotParser:
+                return self._default_parser([])
+            else:
+                return self._default_parser("")
+
+
+class RobotsParserBase:
+    """Base type for robots.txt parsers used by the policy."""
+
+
+class UrllibRobotParser(RobotsParserBase):
+    """Adapter around `urllib.robotparser.RobotFileParser`."""
+
+    def __init__(self, text):
+        self._parser = urllib.robotparser.RobotFileParser()
+        # urllib.robotparser.RobotFileParser.parse() expects a list of lines
+        if isinstance(text, str):
+            lines = text.splitlines() if text else []
+        else:
+            lines = text if text else []
+        self._parser.parse(lines)
+
+    def can_fetch(self, url, user_agent):
+        """Return whether the URL is allowed for the given user agent."""
+        return self._parser.can_fetch(user_agent, url)
+

wxpath/http/policy/throttler.py

@@ -0,0 +1,114 @@
+import asyncio
+from abc import ABC, abstractmethod
+from collections import defaultdict
+
+from wxpath.util.logging import get_logger
+
+log = get_logger(__name__)
+
+
+# Abstract Base Class
+class AbstractThrottler(ABC):
+    @abstractmethod
+    async def wait(self, host: str):
+        pass
+
+    @abstractmethod
+    def record_latency(self, host: str, latency: float):
+        pass
+
+
+class AutoThrottler(AbstractThrottler):
+    """
+    Scrapy-inspired auto-throttle, simplified:
+    - increases delay when latency increases
+    - decreases delay when responses are fast
+
+    Explanation: 
+    - target_concurrency is the desired number of concurrent requests
+    - start_delay is the initial delay
+    - max_delay is the maximum delay
+    - smoothing is the exponential smoothing factor
+    """
+
+    def __init__(
+        self,
+        start_delay: float = 0.25,
+        max_delay: float = 10.0,
+        target_concurrency: float = 1.0,
+        smoothing: float = 0.7,
+    ):
+        self.start_delay = start_delay
+        self.max_delay = max_delay
+        self.target_concurrency = target_concurrency
+        self.smoothing = smoothing
+
+        self._delay = defaultdict(lambda: start_delay)
+        self._latency = defaultdict(lambda: None)
+
+    def record_latency(self, host: str, latency: float):
+        prev = self._latency[host]
+        if prev is None:
+            self._latency[host] = latency
+        else:
+            self._latency[host] = (
+                # exponential smoothing
+                self.smoothing * prev + (1 - self.smoothing) * latency
+            )
+
+        self._recalculate_delay(host)
+
+    def _recalculate_delay(self, host: str):
+        latency = self._latency[host]
+        if not latency:
+            return
+
+        target_delay = latency / self.target_concurrency
+        delay = min(self.max_delay, max(0.0, target_delay))
+        self._delay[host] = delay
+
+        log.debug(
+            "auto-throttle",
+            extra={"host": host, "latency": latency, "delay": delay},
+        )
+
+    async def wait(self, host: str):
+        delay = self._delay[host]
+        if delay > 0:
+            await asyncio.sleep(delay)
+
+
+class ImpoliteThrottle(AbstractThrottler):
+    """
+    Zero delay throttler
+    """
+
+    async def wait(self, host: str):
+        pass
+
+    def record_latency(self, host: str, latency: float):
+        pass
+
+
+ZeroWaitThrottler = ImpoliteThrottle
+
+
+class SimpleThrottler(AbstractThrottler):
+    """
+    Fixed delay throttler. Optionally provide per-host delays via `per_host_delays`.
+    """
+    def __init__(self, delay: float, per_host_delays: dict[str, float] = None):
+        self.delay = delay
+        self._delays = per_host_delays or defaultdict(lambda: delay)
+
+    async def wait(self, host: str):
+        if host in self._delays:
+            await asyncio.sleep(self._delays[host])
+        else:
+            await asyncio.sleep(self.delay)
+    
+    def record_latency(self, host: str, latency: float):
+        pass
+
+
+FixedDelayThrottler = SimpleThrottler