wxpath 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl

wxpath/hooks/builtin.py

@@ -0,0 +1,113 @@
+import atexit
+import json
+import os
+import queue
+import threading
+import time
+
+from elementpath.serialization import XPathMap, XPathNode
+
+from wxpath.util.logging import get_logger
+
+log = get_logger(__name__)
+
+
+class SerializeXPathMapAndNodeHook:
+    """
+    Serialize XPathMap and XPathNode objects to plain Python types.
+    This is enabled by default (once this module is imported).
+    """
+    def post_extract(self, value):
+        if isinstance(value, (list, tuple, set)):
+            return type(value)(self.post_extract(v) for v in value)
+        if isinstance(value, XPathMap):
+            return {k: self.post_extract(v) for k, v in value.items()}
+        if isinstance(value, XPathNode):
+            return self.post_extract(value.obj)
+        return value
+
+
+class JSONLWriter:
+    """
+    Efficient writer that mirrors items to an NDJSON file.
+    - Non-blocking: post_extract enqueues and returns immediately.
+    - Background thread flushes to disk.
+    - Skips non-JSONable values (e.g., raw HtmlElement) by default.
+      Customize _jsonable() to change behavior.
+    """
+    def __init__(self, path=None):
+        self.path = path or os.getenv("WXPATH_OUT", "extractions.ndjson")
+        self._q: "queue.Queue[str]" = queue.Queue(maxsize=10000)
+        self._written = 0
+        self._dropped = 0
+        self._stop = False
+        self._t = threading.Thread(target=self._writer, name="wxpath-ndjson-writer", daemon=True)
+        self._t.start()
+        atexit.register(self._shutdown)
+
+    # ---- hook API ----
+    def post_extract(self, value):
+        js = self._jsonable(value)
+        if js is not None:
+            line = json.dumps(js, ensure_ascii=False, separators=(",", ":"))
+            try:
+                self._q.put_nowait(line)
+            except queue.Full:
+                self._dropped += 1
+                if self._dropped in (1, 100, 1000) or self._dropped % 10000 == 0:
+                    log.warning("NDJSON queue full; dropping items",
+                                extra={"dropped": self._dropped, "written": self._written})
+        return value  # always pass-through
+
+    # ---- internals ----
+    def _writer(self):
+        # Open lazily to avoid creating files when nothing is produced.
+        f = None
+        try:
+            last_flush = time.time()
+            while not self._stop or not self._q.empty():
+                try:
+                    line = self._q.get(timeout=0.5)
+                except queue.Empty:
+                    line = None
+                if line is not None:
+                    if f is None:
+                        f = open(self.path, "a", buffering=1, encoding="utf-8")  # line-buffered
+                    f.write(line)
+                    f.write("\n")
+                    self._written += 1
+                # periodic flush guard for OS buffers even with line buffering
+                if f and (time.time() - last_flush) > 1.0:
+                    f.flush()
+                    last_flush = time.time()
+        finally:
+            if f:
+                f.flush()
+                f.close()
+            if self._dropped:
+                log.warning("NDJSON writer finished with drops",
+                            extra={"dropped": self._dropped, "written": self._written})
+
+    def _shutdown(self):
+        self._stop = True
+        if self._t.is_alive():
+            self._t.join(timeout=2)
+
+    def _jsonable(self, v):
+        # Keep it conservative: only write JSON-friendly shapes by default.
+        # You can relax this if you want to serialize HtmlElement metadata, etc.
+        if v is None or isinstance(v, (bool, int, float, str, list, dict)):
+            return v
+        # Handle common wxpath types gently:
+        # - WxStr: stringify
+        if v.__class__.__name__ == "WxStr":
+            return str(v)
+        # - lxml HtmlElement: record minimal metadata instead of the whole DOM
+        base_url = getattr(v, "base_url", None)
+        tag = getattr(v, "tag", None)
+        if base_url or tag:
+            return {"_element": tag, "url": base_url}
+        return None  # skip unknowns
+    
+    
+NDJSONWriter = JSONLWriter

wxpath/hooks/registry.py

@@ -0,0 +1,133 @@
+"""
+Pluggable hook system for wxpath.
+
+Write once:
+
+    from wxpath import hooks
+
+    @hooks.register
+    class OnlyEnglish:
+        def post_parse(self, ctx, elem):
+            lang = elem.xpath('string(/html/@lang)').lower()[:2]
+            return elem if lang in ("en", "") else None
+
+... and wxpath.engine will call it automatically.
+"""
+
+from __future__ import annotations
+
+import functools
+from collections import OrderedDict
+from collections.abc import Generator
+from dataclasses import dataclass, field
+from typing import Any, Iterable, List, Optional, Protocol
+
+from lxml import html
+
+from wxpath.util.logging import get_logger
+
+log = get_logger(__name__)
+
+
+# --------------------------------------------------------------------------- #
+# Dataclass describing the crawl context for a single URL
+# --------------------------------------------------------------------------- #
+@dataclass
+class FetchContext:
+    url: str
+    backlink: Optional[str]
+    depth: int
+    segments: list        # remaining op/value pairs
+    user_data: dict = field(default_factory=dict)
+
+
+# --------------------------------------------------------------------------- #
+# Hook protocol - every method is optional
+# --------------------------------------------------------------------------- #
+class Hook(Protocol):
+    # Return False to abort fetching this URL
+    # def pre_fetch(self, ctx: FetchContext) -> bool: ...
+
+    # May return modified HTML bytes or None to drop this branch entirely
+    def post_fetch(self, ctx: FetchContext, html_bytes: bytes) -> bytes | None: ...
+
+    # May return modified element or None to drop this branch entirely
+    def post_parse(
+        self, ctx: FetchContext, elem: html.HtmlElement
+    ) -> html.HtmlElement | None: ...
+
+    # Called for every candidate link; return False to prevent enqueueing it
+    # def pre_queue(self, ctx: FetchContext, url: str) -> bool: ...
+
+    # Called for every extracted value; may transform or drop it
+    def post_extract(self, value: Any) -> Any: ...
+
+
+# --------------------------------------------------------------------------- #
+# Global registry helpers
+# --------------------------------------------------------------------------- #
+_global_hooks: OrderedDict[str, Hook] = OrderedDict()
+
+
+def register(hook: Hook | type) -> Hook:
+    """
+    Decorator / helper to add a Hook to the global list.
+
+    Example
+    -------
+    >>> @register
+    ... class DebugHook:
+    ...     def post_fetch(self, ctx, html_bytes):
+    ...         print("Fetched", ctx.url)
+    ...         return html_bytes
+    """
+
+    hook_name = getattr(hook, '__name__', hook.__class__.__name__)
+    if hook_name in _global_hooks:
+        return hook
+
+    instance = hook() if isinstance(hook, type) else hook
+    _global_hooks[hook_name] = instance
+    return hook
+
+
+def get_hooks() -> List[Hook]:
+    """Return the list of globally-registered hooks (read-only)."""
+    return list(_global_hooks.values())
+
+
+def iter_post_extract_hooks() -> Iterable[Hook]:
+    yield from (h for h in _global_hooks.values() if hasattr(h, "post_extract"))
+
+
+def pipe_post_extract(gen_func):
+    """
+    Decorator: wrap a *generator function* so every yielded value
+    is piped through the registered post_extract hooks.
+    """
+    @functools.wraps(gen_func)
+    def wrapper(*args, **kwargs) -> Generator:
+        for item in gen_func(*args, **kwargs):
+            for hook in iter_post_extract_hooks():
+                item = hook.post_extract(item)
+                if item is None:       # hook decided to drop it
+                    break
+            if item is not None:
+                yield item
+    return wrapper
+
+
+def pipe_post_extract_async(async_gen_func):
+    """
+    Async variant - wraps an *async* generator function.
+    """
+    @functools.wraps(async_gen_func)
+    async def wrapper(*args, **kwargs):
+        async for item in async_gen_func(*args, **kwargs):
+            for hook in iter_post_extract_hooks():
+                item = hook.post_extract(item)
+                if item is None:
+                    break
+            if item is not None:
+                yield item
+    return wrapper

wxpath/http/client/__init__.py

@@ -0,0 +1,9 @@
+from wxpath.http.client.crawler import Crawler
+from wxpath.http.client.request import Request
+from wxpath.http.client.response import Response
+
+__all__ = [
+    "Crawler",
+    "Request",
+    "Response"
+]

wxpath/http/client/crawler.py

@@ -0,0 +1,196 @@
+import asyncio
+import time
+import urllib.parse
+from collections import defaultdict
+from socket import gaierror
+from typing import AsyncIterator
+
+import aiohttp
+
+from wxpath.http.client.request import Request
+from wxpath.http.client.response import Response
+from wxpath.http.policy.retry import RetryPolicy
+from wxpath.http.policy.throttler import AbstractThrottler, AutoThrottler
+from wxpath.http.stats import CrawlerStats, build_trace_config
+from wxpath.util.logging import get_logger
+
+log = get_logger(__name__)
+
+HEADERS = {"User-Agent": ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)" 
+                   "AppleWebKit/537.36 (KHTML, like Gecko) "
+                   "Chrome/142.0.0.0 Safari/537.36")}
+
+
+class Crawler:
+    def __init__(
+        self,
+        concurrency: int = 16,
+        per_host: int = 8,
+        timeout: int = 15,
+        *,
+        headers: dict | None = None,
+        proxies: dict | None = None,
+        retry_policy: RetryPolicy | None = None,
+        throttler: AbstractThrottler | None = None,
+        auto_throttle_target_concurrency: float = None,
+        auto_throttle_start_delay: float = 0.25,
+        auto_throttle_max_delay: float = 10.0,
+    ):
+        self.concurrency = concurrency
+        self._timeout = aiohttp.ClientTimeout(total=timeout)
+        self._headers   = HEADERS | (headers or {}) # merge headers
+        self._proxies = proxies or {}
+
+        self.retry_policy = retry_policy or RetryPolicy()
+        self.throttler = throttler or AutoThrottler(
+            target_concurrency=auto_throttle_target_concurrency or concurrency/4.0,
+            start_delay=auto_throttle_start_delay,
+            max_delay=auto_throttle_max_delay,
+        )
+        self._sem_global = asyncio.Semaphore(concurrency)
+        self._sem_host = defaultdict(lambda: asyncio.Semaphore(per_host))
+
+        self._pending: asyncio.Queue[Request] = asyncio.Queue()
+        self._results: asyncio.Queue[Response] = asyncio.Queue()
+
+        self._session: aiohttp.ClientSession | None = None
+        self._workers: list[asyncio.Task] = []
+        self._closed = False
+        self._stats = CrawlerStats()
+
+    def build_session(self):
+        trace_config = build_trace_config(self._stats)
+        # Need to build the connector as late as possible as it requires the loop
+        connector = aiohttp.TCPConnector(limit=self.concurrency*2, ttl_dns_cache=300)
+        return aiohttp.ClientSession(
+            headers=self._headers, 
+            timeout=self._timeout, 
+            connector=connector, 
+            trace_configs=[trace_config]
+        )
+
+    async def __aenter__(self):
+        if self._session is None:
+            # self._session = aiohttp.ClientSession(timeout=self._timeout)
+            self._session = self.build_session()
+
+        self._workers = [
+            asyncio.create_task(self._worker(), name=f"crawler-worker-{i}")
+            for i in range(self.concurrency)
+        ]
+        return self
+
+    async def __aexit__(self, *_):
+        self._closed = True
+        for w in self._workers:
+            w.cancel()
+        await asyncio.gather(*self._workers, return_exceptions=True)
+        if self._session:
+            await self._session.close()
+
+    def submit(self, req: Request):
+        if self._closed:
+            raise RuntimeError("crawler is closed")
+        self._pending.put_nowait(req)
+
+    def __aiter__(self) -> AsyncIterator[Response]:
+        return self._result_iter()
+
+    async def _result_iter(self):
+        # while not self._closed:
+        while not (self._closed and self._results.empty()):
+            resp = await self._results.get()
+            self._results.task_done()
+            yield resp
+
+    def _proxy_for(self, url: str):
+        host = urllib.parse.urlsplit(url).hostname
+        return self._proxies.get(host)
+
+    async def _worker(self):
+        while True:
+            req = await self._pending.get()
+            try:
+                resp = await self._fetch_one(req)
+                if resp is not None:
+                    await self._results.put(resp)
+
+            except asyncio.CancelledError:
+                # Must propagate cancellation
+                log.debug("cancelled error", extra={"url": req.url})
+                raise
+
+            except gaierror:
+                # Ignore DNS errors
+                log.warning("DNS error", extra={"url": req.url})
+                pass
+
+            except Exception as exc:
+                log.warning("exception", extra={"url": req.url})
+                # Last-resort safety: never drop a request silently
+                await self._results.put(Response(req, 0, b"", error=exc))
+            finally:
+                self._pending.task_done()
+
+    async def _fetch_one(self, req: Request) -> Response | None:
+        host = req.hostname
+
+        # TODO: Move this filter to hooks
+        if req.url.lower().endswith((".pdf", ".zip", ".exe")):
+            req.max_retries = 0
+
+        async with self._sem_global, self._sem_host[host]:
+            t0 = asyncio.get_running_loop().time()
+            await self.throttler.wait(host)
+            dt = asyncio.get_running_loop().time() - t0
+
+            self._stats.throttle_waits += 1
+            self._stats.throttle_wait_time += dt
+            self._stats.throttle_waits_by_host[host] += 1
+
+            start = time.monotonic()
+            try:
+                async with self._session.get(
+                    req.url,
+                    headers=self._headers | req.headers,
+                    proxy=self._proxy_for(req.url),
+                    timeout=req.timeout or self._timeout,
+                ) as resp:
+                    body = await resp.read()
+
+                    latency = time.monotonic() - start
+                    self.throttler.record_latency(host, latency)
+
+                    if self.retry_policy.should_retry(req, response=resp):
+                        await self._retry(req)
+                        return None
+
+                    return Response(req, resp.status, body, dict(resp.headers))
+            except asyncio.CancelledError:
+                # Normal during shutdown / timeout propagation
+                log.debug("cancelled error", extra={"url": req.url})
+                raise
+            except Exception as exc:
+                latency = time.monotonic() - start
+                self.throttler.record_latency(host, latency)
+
+                if self.retry_policy.should_retry(req, exception=exc):
+                    await self._retry(req)
+                    return None
+                
+                log.error("request failed", extra={"url": req.url}, exc_info=exc)
+                return Response(req, 0, b"", error=exc)
+
+    async def _retry(self, req: Request):
+        req.retries += 1
+        delay = self.retry_policy.get_delay(req)
+
+        log.warning(
+            "retrying",
+            extra={"url": req.url, "retry": req.retries, "delay": delay},
+        )
+
+        if delay:
+            await asyncio.sleep(delay)
+
+        self.submit(req)

wxpath/http/client/request.py

@@ -0,0 +1,35 @@
+import time
+from dataclasses import dataclass, field
+from typing import Any, Dict
+
+
+@dataclass
+class Request:
+    url: str
+    method: str = "GET"
+    headers: Dict[str, str] = field(default_factory=dict)
+    timeout: float = 15.0
+
+    retries: int = 0
+    max_retries: int | None = None
+    dont_retry: bool = False
+
+    meta: Dict[str, Any] = field(default_factory=dict)
+
+    created_at: float = field(default_factory=time.monotonic)
+
+    def copy_for_retry(self) -> "Request":
+        return Request(
+            url=self.url,
+            method=self.method,
+            headers=self.headers,
+            timeout=self.timeout,
+            retries=self.retries + 1,
+            max_retries=self.max_retries,
+            meta=self.meta,
+        )
+    
+    @property
+    def hostname(self) -> str:
+        from urllib.parse import urlsplit
+        return urlsplit(self.url).hostname or ""

wxpath/http/client/response.py

@@ -0,0 +1,14 @@
+# wxpath/http/response.py
+from dataclasses import dataclass, field
+from typing import Optional
+
+from wxpath.http.client.request import Request
+
+
+@dataclass
+class Response:
+    request: Request
+    status: int
+    body: bytes
+    headers: dict | None = None
+    error: Optional[Exception] = field(default=None, kw_only=True)

wxpath/http/policy/backoff.py

@@ -0,0 +1,16 @@
+import random
+
+
+def exponential_backoff(
+    attempt: int,
+    base: float = 0.5,
+    cap: float = 30.0,
+    jitter: bool = True,
+) -> float:
+    """
+    Exponential backoff with optional jitter.
+    """
+    delay = min(cap, base * (2 ** attempt))
+    if jitter:
+        delay *= random.uniform(0.7, 1.3)
+    return delay

wxpath/http/policy/retry.py

@@ -0,0 +1,35 @@
+from wxpath.http.policy.backoff import exponential_backoff
+from wxpath.util.logging import get_logger
+
+log = get_logger(__name__)
+
+
+class RetryPolicy:
+    def __init__(
+        self,
+        max_retries: int = 3,
+        retry_statuses: set[int] = None,
+    ):
+        self.max_retries = max_retries
+        self.retry_statuses = retry_statuses or {500, 502, 503, 504}
+
+    def should_retry(self, request, response=None, exception=None) -> bool:
+        if request.dont_retry:
+            return False
+
+        if request.max_retries is not None and request.retries >= request.max_retries:
+            return False
+        
+        if request.retries >= self.max_retries:
+            return False
+
+        if response is not None and response.status in self.retry_statuses:
+            return True
+        
+        if exception is not None:
+            return True
+
+        return False
+
+    def get_delay(self, request) -> float:
+        return exponential_backoff(request.retries)

wxpath/http/policy/throttler.py

@@ -0,0 +1,114 @@
+import asyncio
+from abc import ABC, abstractmethod
+from collections import defaultdict
+
+from wxpath.util.logging import get_logger
+
+log = get_logger(__name__)
+
+
+# Abstract Base Class
+class AbstractThrottler(ABC):
+    @abstractmethod
+    async def wait(self, host: str):
+        pass
+
+    @abstractmethod
+    def record_latency(self, host: str, latency: float):
+        pass
+
+
+class AutoThrottler(AbstractThrottler):
+    """
+    Scrapy-inspired auto-throttle, simplified:
+    - increases delay when latency increases
+    - decreases delay when responses are fast
+
+    Explanation: 
+    - target_concurrency is the desired number of concurrent requests
+    - start_delay is the initial delay
+    - max_delay is the maximum delay
+    - smoothing is the exponential smoothing factor
+    """
+
+    def __init__(
+        self,
+        start_delay: float = 0.25,
+        max_delay: float = 10.0,
+        target_concurrency: float = 1.0,
+        smoothing: float = 0.7,
+    ):
+        self.start_delay = start_delay
+        self.max_delay = max_delay
+        self.target_concurrency = target_concurrency
+        self.smoothing = smoothing
+
+        self._delay = defaultdict(lambda: start_delay)
+        self._latency = defaultdict(lambda: None)
+
+    def record_latency(self, host: str, latency: float):
+        prev = self._latency[host]
+        if prev is None:
+            self._latency[host] = latency
+        else:
+            self._latency[host] = (
+                # exponential smoothing
+                self.smoothing * prev + (1 - self.smoothing) * latency
+            )
+
+        self._recalculate_delay(host)
+
+    def _recalculate_delay(self, host: str):
+        latency = self._latency[host]
+        if not latency:
+            return
+
+        target_delay = latency / self.target_concurrency
+        delay = min(self.max_delay, max(0.0, target_delay))
+        self._delay[host] = delay
+
+        log.debug(
+            "auto-throttle",
+            extra={"host": host, "latency": latency, "delay": delay},
+        )
+
+    async def wait(self, host: str):
+        delay = self._delay[host]
+        if delay > 0:
+            await asyncio.sleep(delay)
+
+
+class ImpoliteThrottle(AbstractThrottler):
+    """
+    Zero delay throttler
+    """
+
+    async def wait(self, host: str):
+        pass
+
+    def record_latency(self, host: str, latency: float):
+        pass
+
+
+ZeroWaitThrottler = ImpoliteThrottle
+
+
+class SimpleThrottler(AbstractThrottler):
+    """
+    Fixed delay throttler. Optionally provide per-host delays via `per_host_delays`.
+    """
+    def __init__(self, delay: float, per_host_delays: dict[str, float] = None):
+        self.delay = delay
+        self._delays = per_host_delays or defaultdict(lambda: delay)
+
+    async def wait(self, host: str):
+        if host in self._delays:
+            await asyncio.sleep(self._delays[host])
+        else:
+            await asyncio.sleep(self.delay)
+    
+    def record_latency(self, host: str, latency: float):
+        pass
+
+
+FixedDelayThrottler = SimpleThrottler