wxpath 0.1.1__py3-none-any.whl → 0.3.0__py3-none-any.whl

wxpath/http/stats.py

@@ -0,0 +1,96 @@
+"""
+aiohttp request statistics and tracing hooks.
+"""
+
+import time
+from collections import defaultdict
+from dataclasses import dataclass, field
+from typing import Optional
+
+from aiohttp import TraceConfig
+
+
+@dataclass
+class CrawlerStats:
+    # ---- Lifecycle counts ----
+    requests_enqueued: int = 0
+    requests_started: int = 0
+    requests_completed: int = 0
+
+    # ---- Concurrency ----
+    in_flight_global: int = 0
+    in_flight_per_host: defaultdict[str, int] = field(default_factory=lambda: defaultdict(int))
+
+    # ---- Queueing ----
+    queue_size: int = 0
+    queue_wait_time_total: float = 0.0
+
+    # ---- Throttling ----
+    throttle_waits: int = 0
+    throttle_wait_time: float = 0.0
+    throttle_waits_by_host: defaultdict[str, int] = field(default_factory=lambda: defaultdict(int))
+
+    # ---- Latency feedback ----
+    latency_samples: int = 0
+    latency_ewma: float = 0.0
+    min_latency: Optional[float] = None
+    max_latency: Optional[float] = None
+
+    # ---- Errors / retries ----
+    retries_scheduled: int = 0
+    retries_executed: int = 0
+    errors_by_host: defaultdict[str, int] = field(default_factory=lambda: defaultdict(int))
+
+
+def build_trace_config(stats: CrawlerStats) -> TraceConfig:
+    """
+    Returns an aiohttp TraceConfig wired to the given stats instance.
+    Tracks detailed per-request, per-host, and queue/throttle metrics.
+    """
+    trace = TraceConfig()
+
+    async def on_request_start(session, context, params):
+        stats.requests_started += 1
+        stats.in_flight_global += 1
+        host = params.url.host
+        stats.in_flight_per_host[host] += 1
+        context._start_time = time.monotonic()
+
+    async def on_request_end(session, context, params):
+        host = params.url.host
+        stats.in_flight_global -= 1
+        stats.in_flight_per_host[host] -= 1
+
+        latency = time.monotonic() - context._start_time
+        stats.latency_samples += 1
+        # EWMA update: alpha = 0.3
+        alpha = 0.3
+        stats.latency_ewma = (alpha * latency) + ((1 - alpha) * stats.latency_ewma)
+        stats.min_latency = latency if stats.min_latency is None \
+            else min(stats.min_latency, latency)
+        stats.max_latency = latency if stats.max_latency is None \
+            else max(stats.max_latency, latency)
+
+        status = getattr(params.response, "status", None)
+        if status is not None:
+            if not hasattr(stats, "status_counts"):
+                stats.status_counts = defaultdict(int)
+            stats.status_counts[status] += 1
+
+        content_length = getattr(params.response, "content_length", None)
+        if content_length:
+            if not hasattr(stats, "bytes_received"):
+                stats.bytes_received = 0
+            stats.bytes_received += content_length
+
+    async def on_request_exception(session, context, params):
+        host = params.url.host
+        stats.in_flight_global -= 1
+        stats.in_flight_per_host[host] -= 1
+        stats.errors_by_host[host] += 1
+
+    trace.on_request_start.append(on_request_start)
+    trace.on_request_end.append(on_request_end)
+    trace.on_request_exception.append(on_request_exception)
+
+    return trace

wxpath/patches.py

@@ -0,0 +1,63 @@
+import elementpath
+from elementpath.xpath3 import XPath3Parser
+from lxml import etree, html
+
+
+def html_element_repr(self):
+    return (f"HtmlElement(tag={self.tag}, "
+            f"depth={self.get('depth', -1)}, "
+            f"base_url={getattr(self, 'base_url', None)!r})")
+
+# Patch lxml.html.HtmlElement.__repr__ to improve debugging with base_url.
+html.HtmlElement.__repr__ = html_element_repr
+
+
+class XPath3Element(etree.ElementBase):
+    def xpath3(self, expr, **kwargs):
+        """
+        Evaluate an XPath 3 expression using elementpath library,
+        returning the results as a list.
+        """
+        kwargs.setdefault("parser", XPath3Parser)
+        kwargs.setdefault(
+            "uri", 
+            getattr(self.getroottree().docinfo, "URL", None) or self.get("base_url")
+        )
+        return elementpath.select(self, expr, **kwargs)
+
+    # --- Convenience property for backward‑compatibility -----------------
+    @property
+    def base_url(self):
+        # 1) Per-element override (keeps our “multiple base URLs” feature)
+        url = self.get("base_url")
+        if url is not None:
+            return url
+        # 2) Fall back to document URL (O(1))
+        return self.getroottree().docinfo.URL
+
+    @base_url.setter
+    def base_url(self, value):
+        # Keep the per-element attribute (used by our crawler)
+        self.set("base_url", value)
+        # Set xml:base attribute so XPath base-uri() picks it up
+        self.set("{http://www.w3.org/XML/1998/namespace}base", value)
+        # Also store on the document so descendants can fetch it quickly
+        self.getroottree().docinfo.URL = value
+
+    @property
+    def depth(self):
+        return int(self.get("depth", -1))
+
+    @depth.setter
+    def depth(self, value):
+        self.set("depth", str(value))
+    
+# Create and register custom parser that returns XPath3Element instances
+lookup = etree.ElementDefaultClassLookup(element=XPath3Element)
+parser = etree.HTMLParser()
+parser.set_element_class_lookup(lookup)
+
+
+# Expose parser for use in parse_html
+html_parser_with_xpath3 = parser
+html.HtmlElement.xpath3 = XPath3Element.xpath3

wxpath/util/logging.py

@@ -0,0 +1,91 @@
+import logging
+from logging.config import dictConfig
+from typing import Any, Mapping
+
+
+class KeyValueFormatter(logging.Formatter):
+    """
+    Formatter that automatically renders any 'extra' context added to the record
+    as key=value pairs at the end of the log line.
+    """
+    # Reserved keys that already exist in LogRecord and shouldn't be printed again
+    _RESERVED = {
+        'args', 'asctime', 'created', 'exc_info', 'exc_text', 'filename',
+        'funcName', 'levelname', 'levelno', 'lineno', 'message', 'module',
+        'msecs', 'msg', 'name', 'pathname', 'process', 'processName',
+        'relativeCreated', 'stack_info', 'thread', 'threadName', 'taskName'
+    }
+
+    def format(self, record: logging.LogRecord) -> str:
+        # 1. Format the standard message first
+        s = super().format(record)
+        
+        # 2. Find all 'extra' keys
+        extras = {k: v for k, v in record.__dict__.items() if k not in self._RESERVED}
+        
+        # 3. Append them as key=value
+        if extras:
+            # Sort for deterministic logs
+            context_str = " ".join(f"{k}={v}" for k, v in sorted(extras.items()))
+            s = f"{s} | {context_str}"
+            
+        return s
+
+
+_DEFAULT_LOGGING_CONF = {
+    "version": 1,
+    "disable_existing_loggers": False,
+    "formatters": {
+        "kv": {
+            # Note: We use the class path to our custom class
+            "()": KeyValueFormatter, 
+            "format": "%(asctime)s [%(levelname).1s] %(name)s | %(funcName)s | %(message)s"
+        }
+    },
+    "handlers": {
+        "stderr": {
+            "class": "logging.StreamHandler",
+            "formatter": "kv",
+        }
+    },
+    "loggers": {
+        "wxpath": {"level": "INFO", "handlers": ["stderr"]},
+    },
+}
+
+def configure_logging(level: str | int = "INFO", **overrides) -> None:
+    """
+    Configure wxpath's logger.
+
+    Call this once in an application entry-point **or** rely on defaults.
+
+    Parameters
+    ----------
+    level
+        "DEBUG"|"INFO"|... or `logging.DEBUG`, overrides the root wxpath logger.
+    overrides
+        Dict that is merged (shallow) into the default dictConfig.
+        Lets advanced users swap formatters/handlers.
+    """
+    conf = {**_DEFAULT_LOGGING_CONF, **overrides}
+    conf["loggers"]["wxpath"]["level"] = level
+    dictConfig(conf)
+    
+    
+class CrawlAdapter(logging.LoggerAdapter):
+    """
+    Inject crawl context (depth, op, url) so the handler/formatter
+    never needs to know scraping internals.
+    """
+    def process(self, msg: str, kwargs: Mapping[str, Any]):
+        extra = self.extra.copy()
+        extra.update(kwargs.pop("extra", {}))
+        kwargs["extra"] = extra
+        return msg, kwargs
+
+def get_logger(name: str, **ctx) -> CrawlAdapter:
+    base = logging.getLogger(name)
+    # default placeholders so formatter never blows up
+    defaults = {"depth": "-", "op": "-", "url": "-"}
+    defaults.update(ctx)
+    return CrawlAdapter(base, defaults)

wxpath/util/serialize.py

@@ -0,0 +1,22 @@
+from wxpath.core.ops import WxStr
+
+
+def simplify(obj):
+    """
+    Recursively convert custom wrapper types (e.g., WxStr / ExtractedStr,
+    lxml elements) into plain built-in Python types so that printing or
+    JSON serialising shows clean values.
+    """
+    # Scalars
+    if isinstance(obj, WxStr):
+        return str(obj)
+
+    # Mapping
+    if isinstance(obj, dict):
+        return {k: simplify(v) for k, v in obj.items()}
+
+    # Sequence (but not str/bytes)
+    if isinstance(obj, (list, tuple, set)):
+        return type(obj)(simplify(v) for v in obj)
+
+    return obj