wxpath 0.4.0__py3-none-any.whl

wxpath/core/runtime/engine.py

@@ -0,0 +1,444 @@
+import asyncio
+import contextlib
+import inspect
+from collections import deque
+from typing import Any, AsyncGenerator, Iterator
+
+from lxml.html import HtmlElement
+from tqdm import tqdm
+
+from wxpath import patches  # noqa: F401
+from wxpath.core import parser
+from wxpath.core.models import (
+    CrawlIntent,
+    CrawlTask,
+    DataIntent,
+    ExtractIntent,
+    InfiniteCrawlIntent,
+    ProcessIntent,
+)
+from wxpath.core.ops import get_operator
+from wxpath.core.parser import Binary, Segment, Segments
+from wxpath.core.runtime.helpers import parse_html
+from wxpath.hooks.registry import FetchContext, get_hooks
+from wxpath.http.client.crawler import Crawler
+from wxpath.http.client.request import Request
+from wxpath.util.logging import get_logger
+
+log = get_logger(__name__)
+
+
+class HookedEngineBase:
+    """Common hook invocation helpers shared by engine variants."""
+
+    async def post_fetch_hooks(self, body: bytes | str, task: CrawlTask) -> bytes | str | None:
+        """Run registered `post_fetch` hooks over a fetched response body.
+
+        Hooks may be synchronous or asynchronous and can transform or drop the
+        response payload entirely.
+
+        Args:
+            body: Raw response body bytes from the crawler.
+            task: The `CrawlTask` that produced the response.
+
+        Returns:
+            The transformed body, or `None` if any hook chooses to drop it.
+        """
+        for hook in get_hooks():
+            hook_method = getattr(hook, "post_fetch", lambda _, b: b)
+            if inspect.iscoroutinefunction(hook_method):
+                body = await hook_method(
+                    FetchContext(task.url, task.backlink, task.depth, task.segments), 
+                    body
+                )
+            else:
+                body = hook_method(
+                    FetchContext(task.url, task.backlink, task.depth, task.segments), 
+                    body
+                )
+            if not body:
+                log.debug(f"hook {type(hook).__name__} dropped {task.url}")
+                break
+        return body
+    
+    async def post_parse_hooks(
+        self, elem: HtmlElement | None, task: CrawlTask
+    ) -> HtmlElement | None:
+        """Run registered `post_parse` hooks on a parsed DOM element.
+
+        Args:
+            elem: Parsed `lxml` element to process.
+            task: The originating `CrawlTask`.
+
+        Returns:
+            The transformed element, or `None` if a hook drops the branch.
+        """
+        for hook in get_hooks():
+            hook_method = getattr(hook, "post_parse", lambda _, e: e)
+            if inspect.iscoroutinefunction(hook_method):
+                elem = await hook_method(
+                    FetchContext(
+                        url=task.url, 
+                        backlink=task.backlink, 
+                        depth=task.depth, 
+                        segments=task.segments
+                    ),
+                    elem,
+                )
+            else:
+                elem = hook_method(
+                    FetchContext(
+                        url=task.url, 
+                        backlink=task.backlink, 
+                        depth=task.depth, 
+                        segments=task.segments
+                    ),
+                    elem,
+                )
+            if elem is None:
+                log.debug(f"hook {type(hook).__name__} dropped {task.url}")
+                break
+        return elem
+    
+    async def post_extract_hooks(self, value: Any) -> Any | None:
+        """Run registered `post_extract` hooks on extracted values.
+
+        Args:
+            value: The extracted datum to post-process.
+
+        Returns:
+            The transformed value, or `None` if a hook drops it.
+        """
+        for hook in get_hooks():
+            hook_method = getattr(hook, "post_extract", lambda v: v)
+            if inspect.iscoroutinefunction(hook_method):
+                value = await hook_method(value)
+            else:
+                value = hook_method(value)
+            if value is None:
+                log.debug(f"hook {type(hook).__name__} dropped value")
+                break
+        return value
+
+
+class WXPathEngine(HookedEngineBase):
+    """Main class for executing wxpath expressions.
+
+    The core pattern is to build a queue of CrawlTasks that are crawled and
+    processed FIFO. Traversal of the queue (and therefore the web graph) is
+    done concurrently in BFS-ish order.
+
+    Args:
+        crawler: Crawler instance to use for HTTP requests.
+        concurrency: Number of concurrent fetches at the Crawler level.
+        per_host: Number of concurrent fetches per host.
+        respect_robots: Whether to respect robots.txt directives.
+        allowed_response_codes: Set of allowed HTTP response codes. Defaults
+            to ``{200}``. Responses may still be filtered and dropped.
+        allow_redirects: Whether to follow HTTP redirects. Defaults to ``True``.
+    """
+    def __init__(
+            self, 
+            crawler: Crawler | None = None,
+            concurrency: int = 16, 
+            per_host: int = 8,
+            respect_robots: bool = True,
+            allowed_response_codes: set[int] = None,
+            allow_redirects: bool = True,
+        ):
+        # NOTE: Will grow unbounded in large crawls. Consider a LRU cache, or bloom filter.
+        self.seen_urls: set[str] = set()
+        self.crawler = crawler or Crawler(
+            concurrency=concurrency, 
+            per_host=per_host,
+            respect_robots=respect_robots
+        )
+        self.allowed_response_codes = allowed_response_codes or {200}
+        self.allow_redirects = allow_redirects
+        if allow_redirects:
+            self.allowed_response_codes |= {301, 302, 303, 307, 308}
+
+    async def run(
+            self, 
+            expression: str, 
+            max_depth: int, 
+            progress: bool = False
+        ) -> AsyncGenerator[Any, None]:
+        """Execute a wxpath expression concurrently and yield results.
+
+        Builds and drives a BFS-like crawl pipeline that honors robots rules,
+        throttling, and hook callbacks while walking the web graph.
+
+        Args:
+            expression: WXPath expression string to evaluate.
+            max_depth: Maximum crawl depth to follow for url hops.
+            progress: Whether to display a progress bar.
+
+        Yields:
+            Extracted values produced by the expression (HTML elements or
+            wxpath-specific value types).
+        """
+        segments = parser.parse(expression)
+
+        queue: asyncio.Queue[CrawlTask] = asyncio.Queue()
+        inflight: dict[str, CrawlTask] = {}
+        pending_tasks = 0
+
+        def is_terminal():
+            # NOTE: consider adopting state machine pattern for determining 
+            #       the current state of the engine.
+            return queue.empty() and pending_tasks <= 0
+
+        total_yielded = 0
+        if progress:
+            pbar = tqdm(total=0)
+        else:
+            pbar = None
+
+        async with self.crawler as crawler:
+            async def submitter():
+                nonlocal pending_tasks
+                while True:
+                    task = await queue.get()
+
+                    if task is None:
+                        break
+
+                    if task.url in self.seen_urls or task.url in inflight:
+                        queue.task_done()
+                        continue
+
+                    # Mark URL as seen immediately
+                    self.seen_urls.add(task.url)
+                    inflight[task.url] = task
+
+                    pending_tasks += 1
+                    crawler.submit(Request(task.url, max_retries=0))
+                    queue.task_done()
+
+            submit_task = asyncio.create_task(submitter())
+
+            # Seed the pipeline with a dummy task
+            seed_task = CrawlTask(
+                elem=None,
+                url=None,
+                segments=segments,
+                depth=-1,
+                backlink=None,
+            )
+            async for output in self._process_pipeline(
+                task=seed_task,
+                elem=None,
+                depth=seed_task.depth,
+                max_depth=max_depth,
+                queue=queue,
+                pbar=pbar,
+            ):
+                yield await self.post_extract_hooks(output)
+
+            # While looping asynchronous generators, you MUST make sure 
+            # to check terminal conditions before re-iteration.
+            async for resp in crawler:
+                if pbar is not None:
+                    pbar.update(1)
+                    pbar.refresh()
+
+                task = inflight.pop(resp.request.url, None)
+                pending_tasks -= 1
+
+                if task is None:
+                    log.warning(f"Got unexpected response from {resp.request.url}")
+                    if is_terminal():
+                        break
+                    continue
+
+                if resp.error:
+                    log.warning(f"Got error from {resp.request.url}: {resp.error}")
+                    if is_terminal():
+                        break
+                    continue
+
+                # NOTE: Consider allowing redirects
+                if resp.status not in self.allowed_response_codes or not resp.body:
+                    log.warning(f"Got non-200 response from {resp.request.url}")
+                    if is_terminal():
+                        break
+                    continue
+
+                body = await self.post_fetch_hooks(resp.body, task)
+                if not body:
+                    if is_terminal():
+                        break
+                    continue
+
+                elem = parse_html(
+                    body,
+                    base_url=task.url,
+                    backlink=task.backlink,
+                    depth=task.depth,
+                )
+
+                elem = await self.post_parse_hooks(elem, task)
+                if elem is None:
+                    if is_terminal():
+                        break
+                    continue
+
+                if task.segments:
+                    async for output in self._process_pipeline(
+                        task=task,
+                        elem=elem,
+                        depth=task.depth,
+                        max_depth=max_depth,
+                        queue=queue,
+                        pbar=pbar
+                    ):  
+                        total_yielded += 1
+                        if pbar is not None:
+                            pbar.set_postfix(yielded=total_yielded, depth=task.depth,)
+
+                        yield await self.post_extract_hooks(output)
+                else:
+                    total_yielded += 1
+                    if pbar is not None:
+                        pbar.set_postfix(yielded=total_yielded, depth=task.depth,)
+
+                    yield await self.post_extract_hooks(elem)
+
+                # Termination condition
+                if is_terminal():
+                    break
+
+            submit_task.cancel()
+            with contextlib.suppress(asyncio.CancelledError):
+                await submit_task
+
+        if pbar is not None:
+            pbar.close()
+
+    async def _process_pipeline(
+        self,
+        task: CrawlTask,
+        elem: Any, 
+        depth: int,
+        max_depth: int,
+        queue: asyncio.Queue[CrawlTask],
+        pbar: tqdm = None
+    ) -> AsyncGenerator[Any, None]:
+        """Process a queue of intents for a single crawl branch.
+
+        Traverses wxpath segments depth-first within a page while coordinating
+        newly discovered crawl intents back to the shared queue.
+
+        Args:
+            task: The originating crawl task for this branch.
+            elem: Current DOM element (or extracted value) being processed.
+            depth: Current traversal depth.
+            max_depth: Maximum permitted crawl depth.
+            queue: Shared crawl queue for enqueuing downstream URLs.
+
+        Yields:
+            object: Extracted values or processed elements as produced by operators.
+        """
+        mini_queue: deque[tuple[HtmlElement | Any, list[Binary | Segment] | Segments]] = deque(
+            [(elem, task.segments)]
+        )
+
+        while mini_queue:
+            elem, bin_or_segs = mini_queue.popleft()
+
+            binary_or_segment = bin_or_segs if isinstance(bin_or_segs, Binary) else bin_or_segs[0]
+            operator = get_operator(binary_or_segment)
+            intents = operator(elem, bin_or_segs, depth)
+
+            if not intents:
+                return
+
+            for intent in intents:
+                if isinstance(intent, DataIntent):
+                    yield intent.value
+
+                elif isinstance(intent, CrawlIntent):
+                    next_depth = task.depth + 1
+                    # if intent.url not in self.seen_urls and next_depth <= max_depth:
+                    if next_depth <= max_depth and intent.url not in self.seen_urls:
+                        # self.seen_urls.add(intent.url)
+                        log.debug(f"Depth: {next_depth}; Enqueuing {intent.url}")
+                        
+                        queue.put_nowait(
+                            CrawlTask(
+                                elem=None,
+                                url=intent.url,
+                                segments=intent.next_segments,
+                                depth=next_depth,
+                                backlink=task.url,
+                            )
+                        )
+                        if pbar is not None:
+                            pbar.total += 1
+                            pbar.refresh()
+
+                elif isinstance(intent, (ExtractIntent, ProcessIntent, InfiniteCrawlIntent)):
+                    # immediately traverse the extraction
+                    elem = intent.elem
+                    next_segments = intent.next_segments
+                    mini_queue.append((elem, next_segments))
+
+
+def wxpath_async(path_expr: str,
+                 max_depth: int,
+                 progress: bool = False,
+                 engine: WXPathEngine | None = None) -> AsyncGenerator[Any, None]:
+    if engine is None:
+        engine = WXPathEngine()
+    return engine.run(path_expr, max_depth, progress=progress)
+
+
+##### ASYNC IN SYNC #####
+def wxpath_async_blocking_iter(
+    path_expr: str,
+    max_depth: int = 1,
+    progress: bool = False,
+    engine: WXPathEngine | None = None,
+) -> Iterator[Any]:
+    """Evaluate a wxpath expression using concurrent breadth-first traversal.
+
+    Warning:
+        Spins up its own event loop therefore this function must **not** be
+        invoked from within an active asyncio event loop.
+
+    Args:
+        path_expr: A wxpath expression.
+        max_depth: Maximum crawl depth. Must be at least the number of
+            ``url*`` segments minus one.
+        engine: Optional pre-configured WXPathEngine instance.
+
+    Yields:
+        object: Extracted objects (HtmlElement, WxStr, dict, or other values)
+        produced by the expression evaluator.
+    """
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
+    agen = wxpath_async(path_expr, max_depth=max_depth, progress=progress, engine=engine)
+
+    try:
+        while True:
+            try:
+                yield loop.run_until_complete(agen.__anext__())
+            except StopAsyncIteration:
+                break
+    finally:
+        loop.run_until_complete(loop.shutdown_asyncgens())
+        loop.close()
+
+
+def wxpath_async_blocking(
+    path_expr: str,
+    max_depth: int = 1,
+    progress: bool = False,
+    engine: WXPathEngine | None = None,
+) -> list[Any]:
+    return list(wxpath_async_blocking_iter(path_expr, 
+                                           max_depth=max_depth, 
+                                           progress=progress, 
+                                           engine=engine))

wxpath/core/runtime/helpers.py

@@ -0,0 +1,41 @@
+from lxml import etree, html
+
+from wxpath import patches
+from wxpath.util.logging import get_logger
+
+log = get_logger(__name__)
+
+
+def parse_html(content, base_url=None, **elem_kv_pairs) -> html.HtmlElement:
+    elem = etree.HTML(content, parser=patches.html_parser_with_xpath3, base_url=base_url)
+    if base_url:
+        elem.getroottree().docinfo.URL = base_url  # make base-uri() work
+        # Also set xml:base on the root element for XPath base-uri()
+        elem.set("{http://www.w3.org/XML/1998/namespace}base", base_url)
+        elem.base_url = base_url  # sets both attribute and doc-level URL
+    
+    # NOTE: some pages may have multiple root elements, i.e. 
+    # len(elem.itersiblings()) > 0 AND elem.getparent() is None. 
+    # This breaks elementpath. If elem has siblings, recreate the 
+    # root element and only the root element.
+    if len(list(elem.itersiblings())) > 0:
+        elem = detach_html_root(elem, base_url) 
+
+    for k, v in elem_kv_pairs.items():
+        elem.set(k, str(v))
+    return elem
+
+
+def detach_html_root(elem, base_url=None):
+    new_root = etree.HTML(
+        etree.tostring(elem, encoding="utf-8"),
+        parser=patches.html_parser_with_xpath3, 
+        base_url=base_url
+    )
+
+    if base_url:
+        new_root.getroottree().docinfo.URL = base_url
+        new_root.set("{http://www.w3.org/XML/1998/namespace}base", base_url)
+        new_root.base_url = base_url
+
+    return new_root

wxpath/hooks/__init__.py

@@ -0,0 +1,9 @@
+from wxpath.hooks.builtin import JSONLWriter as JSONLWriter
+from wxpath.hooks.builtin import SerializeXPathMapAndNodeHook as SerializeXPathMapAndNodeHook
+from wxpath.hooks.registry import register as register
+
+__all__ = [
+    "JSONLWriter",
+    "SerializeXPathMapAndNodeHook",
+    "register",
+]

wxpath/hooks/builtin.py

@@ -0,0 +1,113 @@
+import atexit
+import json
+import os
+import queue
+import threading
+import time
+
+from elementpath.serialization import XPathMap, XPathNode
+
+from wxpath.util.logging import get_logger
+
+log = get_logger(__name__)
+
+
+class SerializeXPathMapAndNodeHook:
+    """
+    Serialize XPathMap and XPathNode objects to plain Python types.
+    This is enabled by default (once this module is imported).
+    """
+    def post_extract(self, value):
+        if isinstance(value, (list, tuple, set)):
+            return type(value)(self.post_extract(v) for v in value)
+        if isinstance(value, XPathMap):
+            return {k: self.post_extract(v) for k, v in value.items()}
+        if isinstance(value, XPathNode):
+            return self.post_extract(value.obj)
+        return value
+
+
+class JSONLWriter:
+    """
+    Efficient writer that mirrors items to an NDJSON file.
+    - Non-blocking: post_extract enqueues and returns immediately.
+    - Background thread flushes to disk.
+    - Skips non-JSONable values (e.g., raw HtmlElement) by default.
+      Customize _jsonable() to change behavior.
+    """
+    def __init__(self, path=None):
+        self.path = path or os.getenv("WXPATH_OUT", "extractions.ndjson")
+        self._q: "queue.Queue[str]" = queue.Queue(maxsize=10000)
+        self._written = 0
+        self._dropped = 0
+        self._stop = False
+        self._t = threading.Thread(target=self._writer, name="wxpath-ndjson-writer", daemon=True)
+        self._t.start()
+        atexit.register(self._shutdown)
+
+    # ---- hook API ----
+    def post_extract(self, value):
+        js = self._jsonable(value)
+        if js is not None:
+            line = json.dumps(js, ensure_ascii=False, separators=(",", ":"))
+            try:
+                self._q.put_nowait(line)
+            except queue.Full:
+                self._dropped += 1
+                if self._dropped in (1, 100, 1000) or self._dropped % 10000 == 0:
+                    log.warning("NDJSON queue full; dropping items",
+                                extra={"dropped": self._dropped, "written": self._written})
+        return value  # always pass-through
+
+    # ---- internals ----
+    def _writer(self):
+        # Open lazily to avoid creating files when nothing is produced.
+        f = None
+        try:
+            last_flush = time.time()
+            while not self._stop or not self._q.empty():
+                try:
+                    line = self._q.get(timeout=0.5)
+                except queue.Empty:
+                    line = None
+                if line is not None:
+                    if f is None:
+                        f = open(self.path, "a", buffering=1, encoding="utf-8")  # line-buffered
+                    f.write(line)
+                    f.write("\n")
+                    self._written += 1
+                # periodic flush guard for OS buffers even with line buffering
+                if f and (time.time() - last_flush) > 1.0:
+                    f.flush()
+                    last_flush = time.time()
+        finally:
+            if f:
+                f.flush()
+                f.close()
+            if self._dropped:
+                log.warning("NDJSON writer finished with drops",
+                            extra={"dropped": self._dropped, "written": self._written})
+
+    def _shutdown(self):
+        self._stop = True
+        if self._t.is_alive():
+            self._t.join(timeout=2)
+
+    def _jsonable(self, v):
+        # Keep it conservative: only write JSON-friendly shapes by default.
+        # You can relax this if you want to serialize HtmlElement metadata, etc.
+        if v is None or isinstance(v, (bool, int, float, str, list, dict)):
+            return v
+        # Handle common wxpath types gently:
+        # - WxStr: stringify
+        if v.__class__.__name__ == "WxStr":
+            return str(v)
+        # - lxml HtmlElement: record minimal metadata instead of the whole DOM
+        base_url = getattr(v, "base_url", None)
+        tag = getattr(v, "tag", None)
+        if base_url or tag:
+            return {"_element": tag, "url": base_url}
+        return None  # skip unknowns
+    
+    
+NDJSONWriter = JSONLWriter