wxpath 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl

wxpath/__init__.py

@@ -0,0 +1,9 @@
+from .core.runtime.engine import wxpath_async, wxpath_async_blocking, wxpath_async_blocking_iter
+from .util.logging import configure_logging
+
+__all__ = [
+    'wxpath_async', 
+    'wxpath_async_blocking',
+    'wxpath_async_blocking_iter',
+    'configure_logging',
+]

wxpath/cli.py

@@ -0,0 +1,52 @@
+import argparse
+import json
+import sys
+
+from wxpath.core.parser import parse_wxpath_expr
+from wxpath.core.runtime.engine import WXPathEngine, wxpath_async_blocking_iter
+from wxpath.hooks import builtin, registry
+from wxpath.util.serialize import simplify
+
+
+def main():
+    registry.register(builtin.SerializeXPathMapAndNodeHook)
+    parser = argparse.ArgumentParser(description="Run wxpath expression.")
+    parser.add_argument("expression", help="The wxpath expression")
+    parser.add_argument("--depth", type=int, default=1, help="Recursion depth")
+    # debug
+    parser.add_argument("--debug", action="store_true", help="Debug mode")
+    # verbose
+    parser.add_argument("--verbose", action="store_true", help="Verbose mode")
+    
+    parser.add_argument("--concurrency", type=int, default=16, help="Number of concurrent fetches")
+    parser.add_argument(
+        "--concurrency-per-host", 
+        type=int,
+        default=8,
+        help="Number of concurrent fetches per host"
+    )
+
+    args = parser.parse_args()
+
+    if args.verbose:
+        print("wxpath expression:", args.expression)
+        print("parsed expression:", parse_wxpath_expr(args.expression))
+
+    if args.debug:
+        from wxpath import configure_logging
+        configure_logging('DEBUG')
+
+    engine = WXPathEngine(
+        concurrency=args.concurrency,
+        per_host=args.concurrency_per_host,
+    )
+    try:
+        for r in wxpath_async_blocking_iter(args.expression, args.depth, engine):
+            clean = simplify(r)
+            print(json.dumps(clean, ensure_ascii=False), flush=True)
+    except BrokenPipeError:
+        sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()

wxpath/core/__init__.py

@@ -0,0 +1,13 @@
+from wxpath.core.runtime.engine import (
+    WXPathEngine,
+    wxpath_async,
+    wxpath_async_blocking,
+    wxpath_async_blocking_iter,
+)
+
+__all__ = [
+    'wxpath_async', 
+    'wxpath_async_blocking',
+    'wxpath_async_blocking_iter',
+    'WXPathEngine',
+]

wxpath/core/dom.py

@@ -0,0 +1,22 @@
+from urllib.parse import urljoin
+
+
+def _make_links_absolute(links: list[str], base_url: str) -> list[str]:
+    """
+    Convert relative links to absolute links based on the base URL.
+
+    Args:
+        links (list): List of link strings.
+        base_url (str): The base URL to resolve relative links against.
+
+    Returns:
+        List of absolute URLs.
+    """
+    if base_url is None:
+        raise ValueError("base_url must not be None when making links absolute.")
+    return [urljoin(base_url, link) for link in links if link]
+
+
+def get_absolute_links_from_elem_and_xpath(elem, xpath):
+    base_url = getattr(elem, 'base_url', None)
+    return _make_links_absolute(elem.xpath3(xpath), base_url)

wxpath/core/errors.py

@@ -0,0 +1,134 @@
+
+import collections.abc as cabc
+import functools
+import inspect
+import types
+from contextlib import contextmanager
+from contextvars import ContextVar
+from enum import Enum, auto
+from typing import AsyncGenerator
+
+from wxpath.util.logging import get_logger
+
+log = get_logger(__name__)
+
+class ErrorPolicy(Enum):
+    IGNORE  = auto()   # swallow completely
+    LOG     = auto()   # just log at ERROR
+    COLLECT = auto()   # yield {"_error": ..., "_ctx": ...}
+    RAISE   = auto()   # re-raise
+
+
+_GLOBAL_DEFAULT = ErrorPolicy.LOG
+
+# Task-local override (None => fall back to _GLOBAL_DEFAULT)
+_CURRENT: ContextVar[ErrorPolicy | None] = ContextVar("wx_err_policy", default=None)
+
+
+def get_current_error_policy() -> ErrorPolicy:
+    return _CURRENT.get() or _GLOBAL_DEFAULT
+
+
+def set_default_error_policy(policy: ErrorPolicy) -> None:
+    global _GLOBAL_DEFAULT
+    _GLOBAL_DEFAULT = policy
+
+
+@contextmanager
+def use_error_policy(policy: ErrorPolicy):
+    token = _CURRENT.set(policy)
+    try:
+        yield
+    finally:
+        _CURRENT.reset(token)
+        
+
+def handle_error(exc: Exception, policy: ErrorPolicy, ctx: dict):
+    if policy is ErrorPolicy.IGNORE:
+        return None
+
+    if policy is ErrorPolicy.LOG:
+        log.exception("processing error", extra=ctx)
+        return None
+
+    if policy is ErrorPolicy.COLLECT:
+        return {"_error": str(exc), "_ctx": ctx}
+
+    # RAISE (safe default)
+    raise exc
+
+
+def _is_gen(obj):     # helper
+    return isinstance(obj, (types.GeneratorType, cabc.Generator))
+
+
+def with_errors():
+    """
+    Apply the current ErrorPolicy at call time while preserving the callable kind:
+      - async generator -> async generator wrapper
+      - coroutine       -> async wrapper
+      - sync generator  -> sync generator wrapper
+      - plain function  -> plain wrapper
+    """
+    def decorator(fn):
+        # --- ASYNC GENERATOR ---
+        if inspect.isasyncgenfunction(fn):
+            @functools.wraps(fn)
+            async def asyncgen_wrapped(*a, **kw) -> AsyncGenerator:
+                try:
+                    async for item in fn(*a, **kw):
+                        yield item
+                except Exception as exc:
+                    collected = handle_error(exc, get_current_error_policy(),
+                                             _ctx_from_sig(fn, a, kw))
+                    if collected is not None:
+                        yield collected
+            return asyncgen_wrapped
+
+        # --- COROUTINE ---
+        if inspect.iscoroutinefunction(fn):
+            @functools.wraps(fn)
+            async def coro_wrapped(*a, **kw):
+                try:
+                    return await fn(*a, **kw)
+                except Exception as exc:
+                    return handle_error(exc, get_current_error_policy(),
+                                        _ctx_from_sig(fn, a, kw))
+            return coro_wrapped
+
+        # --- SYNC GENERATOR ---
+        if inspect.isgeneratorfunction(fn):
+            @functools.wraps(fn)
+            def gen_wrapped(*a, **kw):
+                try:
+                    for item in fn(*a, **kw):
+                        yield item
+                except Exception as exc:
+                    collected = handle_error(exc, get_current_error_policy(),
+                                             _ctx_from_sig(fn, a, kw))
+                    if collected is not None:
+                        yield collected
+            return gen_wrapped
+
+        # --- PLAIN SYNC FUNCTION ---
+        @functools.wraps(fn)
+        def plain_wrapped(*a, **kw):
+            try:
+                return fn(*a, **kw)
+            except Exception as exc:
+                return handle_error(exc, get_current_error_policy(),
+                                    _ctx_from_sig(fn, a, kw))
+        return plain_wrapped
+    return decorator
+
+
+def _ctx_from_sig(fn, a, kw):
+    """Best-effort extraction of depth/url/op for logging."""
+    # you already pass these in every handler, so pull by position
+    try:
+        elem, segs, depth, *_ = a
+        op, val = segs[0] if segs else ("?", "?")
+        url = getattr(elem, "base_url", None)
+        return {"op": op, "depth": depth, "url": url}
+    except Exception:
+        return {}

wxpath/core/models.py

@@ -0,0 +1,74 @@
+from dataclasses import dataclass, field
+from typing import Any, List, Optional, Tuple
+
+
+@dataclass(slots=True)
+class CrawlTask:
+    """A unit of work for the crawler."""
+    elem: Any
+    url: str
+    segments: List[Tuple[str, str]]
+    depth: int
+    backlink: Optional[str] = None
+    base_url: Optional[str] = None
+    
+    # Priority for the queue (lower number = higher priority)
+    # Useful if you want Depth-First behavior in a shared queue
+    priority: int = field(default=0)
+
+    def __post_init__(self):
+        # Automatically sync priority with depth for BFS behavior
+        self.priority = self.depth
+
+    def __lt__(self, other):
+        return self.priority < other.priority
+
+    def __iter__(self):
+        return iter((self.elem, self.segments, self.depth, self.backlink))
+
+
+@dataclass(slots=True)
+class Intent:
+    pass
+
+
+@dataclass(slots=True)
+class Result(Intent):
+    """A container for an extracted item or error."""
+    value: Any
+    url: str
+    depth: int
+    error: Optional[Exception] = None
+    backlink: Optional[str] = None
+
+
+@dataclass(slots=True)
+class CrawlIntent(Intent):
+    url: str             # "I found this link"
+    next_segments: list  # "Here is what to do next if you go there"
+
+
+@dataclass(slots=True)
+class ProcessIntent(Intent):
+    elem: Any
+    next_segments: list
+
+
+@dataclass(slots=True)
+class InfiniteCrawlIntent(ProcessIntent):
+    pass
+
+
+@dataclass(slots=True)
+class ExtractIntent(ProcessIntent):
+    pass
+
+
+@dataclass(slots=True)
+class CrawlFromAttributeIntent(ProcessIntent):
+    pass
+
+
+@dataclass(slots=True)
+class DataIntent(Intent):
+    value: Any

wxpath/core/ops.py

@@ -0,0 +1,244 @@
+"""
+`ops` for "operations". This module contains side-effect-free functions (operators) 
+for handling each segment of a wxpath expression.
+"""
+from typing import Callable, Iterable
+from urllib.parse import urljoin
+
+import elementpath
+from elementpath.datatypes import AnyAtomicType
+from elementpath.xpath3 import XPath3Parser
+from lxml import html
+
+from wxpath.core.dom import get_absolute_links_from_elem_and_xpath
+from wxpath.core.models import (
+    CrawlIntent,
+    DataIntent,
+    ExtractIntent,
+    InfiniteCrawlIntent,
+    Intent,
+    ProcessIntent,
+)
+from wxpath.core.parser import OPS, Segment, UrlInfAndXpathValue, XpathValue
+from wxpath.util.logging import get_logger
+
+log = get_logger(__name__)
+
+
+class WxStr(str):
+    """
+    A string that has a base_url and depth associated with it. Purely for debugging.
+    """
+    def __new__(cls, value, base_url=None, depth=-1):
+        obj = super().__new__(cls, value)
+        obj.base_url = base_url
+        obj.depth = depth
+        return obj
+
+    def __repr__(self):
+        return f"WxStr({super().__repr__()}, base_url={self.base_url!r}, depth={self.depth})"
+
+
+HANDLERS: dict[str, Callable] = {}
+
+def _op(name: OPS):
+    def reg(fn): 
+        if name in HANDLERS:
+            raise ValueError(f"Duplicate operation: {name}")
+        HANDLERS[name] = fn
+        return fn
+    return reg
+
+
+def get_operator(name: OPS) -> Callable[[html.HtmlElement, list[Segment], int], Iterable[Intent]]:
+    if name not in HANDLERS:
+        raise ValueError(f"Unknown operation: {name}")
+    return HANDLERS[name]
+
+
+@_op(OPS.URL_STR_LIT)
+def _handle_url_str_lit(curr_elem: html.HtmlElement, 
+                        curr_segments: list[Segment], 
+                        curr_depth: int, **kwargs) -> Iterable[Intent]:
+    op, value = curr_segments[0]
+
+    log.debug("queueing", extra={"depth": curr_depth, "op": op, "url": value.target})
+
+    next_segments = curr_segments[1:]
+
+    if value.follow:
+        _segments = [
+            (OPS.URL_INF_AND_XPATH, UrlInfAndXpathValue('', value.target, value.follow))
+        ] + next_segments
+        
+        yield CrawlIntent(url=value.target, next_segments=_segments)
+    else:
+        yield CrawlIntent(url=value.target, next_segments=next_segments)
+
+
+@_op(OPS.URL_EVAL)
+def _handle_url_eval(curr_elem: html.HtmlElement | str, 
+                     curr_segments: list[Segment], 
+                     curr_depth: int, 
+                     **kwargs) -> Iterable[Intent]:
+    op, value = curr_segments[0]
+
+    _path_exp = value.expr
+
+    if isinstance(curr_elem, str):
+        # TODO: IMO, ideally, wxpath grammar should not be checked/validated/enforced 
+        # in ops.py. It should instead be validated in the parser.
+        if _path_exp not in {'.', 'self::node()'}:
+            raise ValueError("Only '.' or 'self::node()' is supported in url() segments "
+                             f"when prior xpath operation results in a string. Got: {_path_exp}")
+
+        urls = [urljoin(getattr(curr_elem, 'base_url', None) or '', curr_elem)]
+    else:
+        # TODO: If prior xpath operation is XPATH_FN_MAP_FRAG, then this will likely fail.
+        # It should be handled in the parser.
+        urls = get_absolute_links_from_elem_and_xpath(curr_elem, _path_exp)
+        urls = dict.fromkeys(urls)
+
+    next_segments = curr_segments[1:]
+    for url in urls:
+        log.debug("queueing", extra={"depth": curr_depth, "op": op, "url": url})
+        yield CrawlIntent(url=url, next_segments=next_segments)
+
+
+@_op(OPS.URL_INF)
+def _handle_url_inf(curr_elem: html.HtmlElement, 
+                    curr_segments: list[Segment], 
+                    curr_depth: int, 
+                    **kwargs) -> Iterable[CrawlIntent]:
+    """
+    Handles the ///url() segment of a wxpath expression. This operation is also 
+    generated internally by the parser when a `///<xpath>/[/]url()` segment is
+    encountered by the parser.
+    This operation does not fetch URLs; instead, it XPaths the current element
+    for URLs, then queues them for further processing (see 
+    _handle_url_inf_and_xpath).
+    """
+    op, value = curr_segments[0]
+
+    _path_exp = value.expr
+
+    urls = get_absolute_links_from_elem_and_xpath(curr_elem, _path_exp)
+
+    log.debug("found urls", 
+              extra={"depth": curr_depth, "op": op, "url": getattr(curr_elem, 'base_url', None)})
+
+    tail_segments = curr_segments[1:]
+    for url in dict.fromkeys(urls):
+        _segments = [
+            (OPS.URL_INF_AND_XPATH, UrlInfAndXpathValue('', url, _path_exp))
+        ] + tail_segments
+        
+        log.debug("queueing", extra={"depth": curr_depth, "op": op, "url": url})
+
+        yield CrawlIntent(url=url, next_segments=_segments)
+
+
+@_op(OPS.URL_INF_AND_XPATH)
+def _handle_url_inf_and_xpath(curr_elem: html.HtmlElement, 
+                              curr_segments: list[Segment], 
+                              curr_depth: int, **kwargs) \
+                                -> Iterable[DataIntent | ProcessIntent | InfiniteCrawlIntent]:
+    """
+    This is an operation that is generated internally by the parser. There is
+    no explicit wxpath expression that generates this operation.
+    """
+    op, value = curr_segments[0]
+
+    try:
+        if curr_elem is None:
+            raise ValueError("Missing element when op is 'url_inf_and_xpath'.")
+
+        next_segments = curr_segments[1:]
+        if not next_segments:
+            yield DataIntent(value=curr_elem)
+        else:
+            yield ExtractIntent(elem=curr_elem, next_segments=next_segments)
+
+        # For url_inf, also re-enqueue for further infinite expansion
+        _segments = [(OPS.URL_INF, XpathValue('', value.expr))] + next_segments
+        crawl_intent = InfiniteCrawlIntent(elem=curr_elem, next_segments=_segments)
+        log.debug("queueing InfiniteCrawlIntent", 
+                  extra={"depth": curr_depth, "op": op, 
+                         "url": value.target, "crawl_intent": crawl_intent})
+        yield crawl_intent
+
+    except Exception:
+        log.exception("error fetching url", 
+                      extra={"depth": curr_depth, "op": op, "url": value.target})
+
+
+@_op(OPS.XPATH)
+def _handle_xpath(curr_elem: html.HtmlElement, 
+                  curr_segments: list[Segment], 
+                  curr_depth: int, 
+                  **kwargs) -> Iterable[DataIntent | ProcessIntent]:
+    """
+    Handles the [/|//]<xpath> segment of a wxpath expression. This is a plain XPath expression.
+    Also handles wxpath-specific macro expansions like wx:backlink() or wx:depth().
+    """
+    _, value = curr_segments[0]
+    expr = value.expr
+    if curr_elem is None:
+        raise ValueError("Element must be provided when path_expr does not start with 'url()'.")
+    base_url = getattr(curr_elem, 'base_url', None)
+    log.debug("base url", extra={"depth": curr_depth, "op": 'xpath', "base_url": base_url})
+
+    _backlink_str = f"string('{curr_elem.get('backlink')}')"
+    # We use the root tree's depth and not curr_depth because curr_depth accounts for a +1 
+    # increment after each url*() hop
+    _depth_str = f"number({curr_elem.getroottree().getroot().get('depth')})"
+    expr = expr.replace('wx:backlink()', _backlink_str)
+    expr = expr.replace('wx:backlink(.)', _backlink_str)
+    expr = expr.replace('wx:depth()', _depth_str)
+    expr = expr.replace('wx:depth(.)', _depth_str)
+
+    elems = curr_elem.xpath3(expr)
+    
+    next_segments = curr_segments[1:]
+    for elem in elems:
+        value_or_elem = WxStr(
+            elem, base_url=base_url, 
+            depth=curr_depth
+        ) if isinstance(elem, str) else elem
+        if len(curr_segments) == 1:
+            yield DataIntent(value=value_or_elem)
+        else:
+            yield ProcessIntent(elem=value_or_elem, next_segments=next_segments)
+
+
+@_op(OPS.XPATH_FN_MAP_FRAG)
+def _handle_xpath_fn_map_frag(curr_elem: html.HtmlElement | str, 
+                              curr_segments: list[Segment], 
+                              curr_depth: int, 
+                              **kwargs) -> Iterable[DataIntent | ProcessIntent]:
+    """
+    Handles the execution of XPath functions that were initially suffixed with a 
+    '!' (map) operator.
+    """
+    _, value = curr_segments[0]
+
+    base_url = getattr(curr_elem, 'base_url', None)
+    next_segments = curr_segments[1:]
+
+    result = elementpath.select(
+        curr_elem,
+        value.expr,
+        parser=XPath3Parser,
+        item='' if curr_elem is None else None
+    )
+
+    if isinstance(result, AnyAtomicType):
+        result = [result]
+
+    for r in result:
+        value_or_elem = WxStr(r, base_url=base_url, depth=curr_depth) if isinstance(r, str) else r
+        if len(curr_segments) == 1:
+            # XPATH_FN_MAP_FRAG is not a terminal operation
+            raise ValueError("XPATH_FN_MAP_FRAG is not a terminal operation")
+        else:
+            yield ProcessIntent(elem=value_or_elem, next_segments=next_segments)