wxpath 0.1.1__py3-none-any.whl → 0.2.0__py3-none-any.whl

wxpath/core/parser.py

@@ -0,0 +1,319 @@
+"""
+This module contains mainly two kinds of functions: 
+
+1. functions for parsing wxpath expressions.
+2. functions for extracting information from wxpath expressions or subexpressions.
+
+"""
+import re
+from dataclasses import dataclass, fields
+from typing import NamedTuple, Optional, TypeAlias
+
+try:
+    from enum import StrEnum
+except ImportError:
+    from enum import Enum
+
+    class StrEnum(str, Enum):
+        pass
+
+
+@dataclass(frozen=True, slots=True)
+class ValueBase:
+    _value: str
+
+
+@dataclass(frozen=True, slots=True)
+class UrlValue(ValueBase):
+    target: str
+    follow: str | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class XpathValue(ValueBase):
+    expr: str
+
+
+@dataclass(frozen=True, slots=True)
+class UrlInfAndXpathValue(ValueBase):
+    target: str
+    expr: str
+
+
+Value: TypeAlias = UrlValue | XpathValue | UrlInfAndXpathValue
+
+
+class Segment(NamedTuple):
+    op: str
+    value: Value
+
+
+class OPS(StrEnum):
+    URL_STR_LIT       = "url_str_lit"
+    URL_EVAL          = "url_eval"
+    URL_INF           = "url_inf"
+    URL_INF_AND_XPATH = "url_inf_and_xpath"
+    XPATH             = "xpath"
+    XPATH_FN_MAP_FRAG = "xpath_fn_map_frag" # XPath function ending with map operator '!'
+    INF_XPATH         = "inf_xpath" # Experimental
+    OBJECT            = "object" # Deprecated
+    URL_FROM_ATTR     = "url_from_attr" # Deprecated
+    URL_OPR_AND_ARG   = "url_opr_and_arg" # Deprecated
+
+
+def _scan_path_expr(path_expr: str) -> list[str]:
+    """
+    Provided a wxpath expression, produce a list of all xpath and url() partitions
+    
+    :param path_expr: Description
+    """
+    # remove newlines
+    path_expr = path_expr.replace('\n', '')
+    partitions = []  # type: list[str]
+    i = 0
+    n = len(path_expr)
+    while i < n:
+        # Detect ///url(, //url(, /url(, or url(
+        match = re.match(r'/{0,3}url\(', path_expr[i:])
+        if match:
+            seg_start = i
+            i += match.end()  # Move past the matched "url("
+            paren_depth = 1
+            while i < n and paren_depth > 0:
+                if path_expr[i] == '(':
+                    paren_depth += 1
+                elif path_expr[i] == ')':
+                    paren_depth -= 1
+                i += 1
+            partitions.append(path_expr[seg_start:i])
+        else:
+            # Grab until the next /url(
+            next_url = re.search(r'/{0,3}url\(', path_expr[i:])
+            next_pos = next_url.start() + i if next_url else n
+            if i != next_pos:
+                partitions.append(path_expr[i:next_pos])
+            i = next_pos
+
+    return partitions
+
+
+def parse_wxpath_expr(path_expr):
+    partitions = _scan_path_expr(path_expr)
+
+    # Lex and parse
+    segments = []  # type: list[Segment]
+    for s in partitions:
+        s = s.strip()
+        if not s:
+            continue
+        if s.startswith('url("') or s.startswith("url('"):
+            segments.append(
+                Segment(
+                    OPS.URL_STR_LIT, 
+                    UrlValue(s, *parse_url_value(_extract_arg_from_url_op(s))),
+                )
+            )
+        elif s.startswith('///url('):
+            segments.append(
+                Segment(
+                    OPS.URL_INF, 
+                    # XpathValue(extract_url_op_arg(s))
+                    XpathValue(_value=s, expr=_extract_arg_from_url_xpath_op(s))
+                    )
+                )
+        elif s.startswith('/url("') or s.startswith('//url("'):
+            raise ValueError("url() segment cannot have string literal "
+                             f"argument and preceding navigation slashes (/|//): {s}")
+        elif s.startswith("/url('") or s.startswith("//url('"):
+            raise ValueError("url() segment cannot have string literal "
+                             f"argument and preceding navigation slashes (/|//): {s}")
+        elif s.startswith('/url(') or s.startswith("//url("):
+            segments.append(Segment(OPS.URL_EVAL, XpathValue(s, _extract_arg_from_url_xpath_op(s))))
+        elif s.startswith('url('):
+            segments.append(Segment(OPS.URL_EVAL, XpathValue(s, _extract_arg_from_url_xpath_op(s))))
+        elif s.startswith('///'):
+            raise ValueError(f"xpath segment cannot have preceding triple slashes : {s}")
+            # segments.append(Segment(OPS.INF_XPATH, XpathValue(s, "//" + s[3:])))
+        elif s.endswith('!'):
+            segments.append(Segment(OPS.XPATH_FN_MAP_FRAG, XpathValue(s, s[:-1])))
+        else:
+            segments.append(Segment(OPS.XPATH, XpathValue(s, s)))
+    
+    ## EXPERIMENTAL
+    ## Disabled for now
+    ## Collapes inf_xpath segment and the succeeding url_eval segment into a single url_inf segment
+    # for i in range(len(segments) - 1, 0, -1):
+    #     if segments[i - 1][0] == OPS.INF_XPATH and segments[i][0] == OPS.URL_EVAL:
+    #         inf_xpath_value = segments[i - 1][1]
+    #         url_eval_value = segments[i][1]
+    #         url_eval_traveral_fragment = url_eval_value._value.split('url')[0]
+    #         segments[i - 1] = Segment(
+    #             OPS.URL_INF,
+    #             XpathValue(
+    #                 _value='', 
+    #                 expr=(f'{inf_xpath_value.expr}'
+    #                       f'{url_eval_traveral_fragment}'
+    #                       f'{url_eval_value.expr}')
+    #             )
+    #         )
+    #         segments.pop(i)
+    
+    #### RAISE ERRORS FROM INVALID SEGMENTS ####
+    # Raises if multiple ///url() are present
+    if len([op for op, val in segments if op == OPS.URL_INF]) > 1:
+        raise ValueError("Only one ///url() is allowed")
+    
+    # Raises if multiple url() with string literals are present
+    if len([op for op, _ in segments if op == OPS.URL_STR_LIT]) > 1:
+        raise ValueError("Only one url() with string literal argument is allowed")
+    
+    # Raises when expr starts with //url(@<attr>)
+    if segments and segments[0][0] == OPS.URL_EVAL:
+        raise ValueError("Path expr cannot start with [//]url(<xpath>)")
+    
+    # Raises if expr ends with INF_XPATH
+    if segments and segments[-1][0] == OPS.INF_XPATH:
+        raise ValueError("Path expr cannot end with ///<xpath>")
+    
+    # Raises if expr ends with XPATH_FN_MAP_FRAG
+    if segments and segments[-1][0] == OPS.XPATH_FN_MAP_FRAG:
+        raise ValueError("Path expr cannot end with !")
+    return segments
+
+
+def parse_url_value(src: str) -> tuple[str, Optional[str]]:
+    """
+    Parse the contents of url(...).
+
+    Examples of src:
+      "'https://example.com'"
+      "//a/@href"
+      "'https://x', follow=//a/@href"
+    """
+
+    parts = _split_top_level_commas(src)
+
+    if not parts:
+        raise SyntaxError("url() requires at least one argument")
+
+    # ---- positional argument (target) ----
+    target_src = parts[0].strip()
+    if not target_src:
+        raise SyntaxError("url() target cannot be empty")
+
+    target = _parse_url_target(target_src)
+
+    follow = None
+
+    # ---- keyword arguments ----
+    for part in parts[1:]:
+        name, value = _split_kwarg(part)
+
+        if name == "follow":
+            if follow is not None:
+                raise SyntaxError("duplicate follow= in url()")
+            follow = value.strip()
+        else:
+            raise SyntaxError(f"unknown url() argument: {name}")
+
+    return target, follow
+
+
+def extract_url_op_arg(url_op_and_arg: str) -> str:
+    url_op_arg = _extract_arg_from_url_xpath_op(url_op_and_arg)
+    if url_op_arg.startswith('@'):
+        return ".//" + url_op_arg
+    elif url_op_arg.startswith('.'):
+        return url_op_arg
+    elif url_op_arg.startswith('//'):
+        return '.' + url_op_arg
+    elif not url_op_arg.startswith('.//'):
+        return './/' + url_op_arg
+    else:
+        return url_op_arg
+
+
+def _extract_arg_from_url_xpath_op(url_subsegment):
+    match = re.search(r"url\((.+)\)", url_subsegment)
+    if not match:
+        raise ValueError(f"Invalid url() segment: {url_subsegment}")
+    return match.group(1).strip("'\"")  # Remove surrounding quotes if any
+
+
+def _extract_arg_from_url_op(url_subsegment):
+    match = re.search(r"url\((.+)\)", url_subsegment)
+    if not match:
+        raise ValueError(f"Invalid url() segment: {url_subsegment}")
+    return match.group(1)  # Remove surrounding quotes if any
+
+
+def _split_top_level_commas(src: str) -> list[str]:
+    parts = []
+    buf = []
+    depth = 0
+    in_string = False
+    quote = None
+
+    for ch in src:
+        if in_string:
+            buf.append(ch)
+            if ch == quote:
+                in_string = False
+            continue
+
+        if ch in ("'", '"'):
+            in_string = True
+            quote = ch
+            buf.append(ch)
+            continue
+
+        if ch in "([{":
+            depth += 1
+        elif ch in ")]}":
+            depth -= 1
+            if depth < 0:
+                raise SyntaxError("unbalanced parentheses in url()")
+
+        if ch == "," and depth == 0:
+            parts.append("".join(buf).strip())
+            buf.clear()
+        else:
+            buf.append(ch)
+
+    if in_string or depth != 0:
+        raise SyntaxError("unbalanced expression in url()")
+
+    if buf:
+        parts.append("".join(buf).strip())
+
+    return parts
+
+
+def _split_kwarg(src: str) -> tuple[str, str]:
+    if "=" not in src:
+        raise SyntaxError(f"expected keyword argument, got: {src}")
+
+    name, value = src.split("=", 1)
+    name = name.strip()
+    value = value.strip()
+
+    if not name or not value:
+        raise SyntaxError(f"invalid keyword argument: {src}")
+
+    return name, value
+
+
+def _parse_url_target(src: str):
+    src = src.strip()
+    # string literal
+    if (src.startswith("'") and src.endswith("'")) or \
+       (src.startswith('"') and src.endswith('"')):
+        return src[1:-1]
+
+    return src
+
+
+def _get_shallow_dict(instance: Value):
+    return {field.name: getattr(instance, field.name) 
+            for field in fields(instance) if field.name not in {'_value'}}
+

wxpath/core/runtime/__init__.py

@@ -0,0 +1,5 @@
+from wxpath.core.runtime.engine import WXPathEngine
+
+__all__ = [
+    'WXPathEngine',
+]

wxpath/core/runtime/engine.py

@@ -0,0 +1,315 @@
+import asyncio
+import contextlib
+import inspect
+from collections import deque
+from typing import Any, AsyncGenerator
+
+from lxml.html import HtmlElement
+
+from wxpath import patches  # noqa: F401
+from wxpath.core.models import (
+    CrawlIntent,
+    CrawlTask,
+    DataIntent,
+    ExtractIntent,
+    InfiniteCrawlIntent,
+    ProcessIntent,
+)
+from wxpath.core.ops import get_operator
+from wxpath.core.parser import parse_wxpath_expr
+from wxpath.core.runtime.helpers import parse_html
+from wxpath.hooks.registry import FetchContext, get_hooks
+from wxpath.http.client.crawler import Crawler
+from wxpath.http.client.request import Request
+from wxpath.util.logging import get_logger
+
+log = get_logger(__name__)
+
+
+class HookedEngineBase:
+    async def post_fetch_hooks(self, body, task):
+        for hook in get_hooks():
+            hook_method = getattr(hook, "post_fetch", lambda _, b: b)
+            if inspect.iscoroutinefunction(hook_method):
+                body = await hook_method(
+                    FetchContext(task.url, task.backlink, task.depth, task.segments), 
+                    body
+                )
+            else:
+                body = hook_method(
+                    FetchContext(task.url, task.backlink, task.depth, task.segments), 
+                    body
+                )
+            if not body:
+                log.debug(f"hook {type(hook).__name__} dropped {task.url}")
+                break
+        return body
+    
+    async def post_parse_hooks(self, elem, task):
+        for hook in get_hooks():
+            hook_method = getattr(hook, "post_parse", lambda _, e: e)
+            if inspect.iscoroutinefunction(hook_method):
+                elem = await hook_method(
+                    FetchContext(
+                        url=task.url, 
+                        backlink=task.backlink, 
+                        depth=task.depth, 
+                        segments=task.segments
+                    ),
+                    elem,
+                )
+            else:
+                elem = hook_method(
+                    FetchContext(
+                        url=task.url, 
+                        backlink=task.backlink, 
+                        depth=task.depth, 
+                        segments=task.segments
+                    ),
+                    elem,
+                )
+            if elem is None:
+                log.debug(f"hook {type(hook).__name__} dropped {task.url}")
+                break
+        return elem
+    
+    async def post_extract_hooks(self, value):
+        for hook in get_hooks():
+            hook_method = getattr(hook, "post_extract", lambda v: v)
+            if inspect.iscoroutinefunction(hook_method):
+                value = await hook_method(value)
+            else:
+                value = hook_method(value)
+            if value is None:
+                log.debug(f"hook {type(hook).__name__} dropped value")
+                break
+        return value
+
+
+class WXPathEngine(HookedEngineBase):
+    """
+    Main class for executing wxpath expressions.
+
+    The core pattern and directive for this engine is to build a queue of CrawlTasks, 
+    which is crawled and processed FIFO. The traversal of the queue (and therefore 
+    the web graph) is done concurrently and in BFS-ish order.
+
+    Args:
+        crawler: Crawler instance
+        concurrency: number of concurrent fetches at the Crawler (request engine) level
+        per_host: number of concurrent fetches per host
+    """
+    def __init__(
+            self, 
+            crawler: Crawler | None = None,
+            concurrency: int = 16, 
+            per_host: int = 8
+        ):
+        self.seen_urls: set[str] = set()
+        self.crawler = crawler or Crawler(concurrency=concurrency, per_host=per_host)
+
+    async def run(self, expression: str, max_depth: int):
+        segments = parse_wxpath_expr(expression)
+
+        queue: asyncio.Queue[CrawlTask] = asyncio.Queue()
+        inflight: dict[str, CrawlTask] = {}
+        pending_tasks = 0
+
+        def is_terminal():
+            return queue.empty() and pending_tasks <= 0
+
+        async with self.crawler as crawler:
+            async def submitter():
+                nonlocal pending_tasks
+                while True:
+                    task = await queue.get()
+
+                    if task is None:
+                        break
+
+                    if task.url in self.seen_urls or task.url in inflight:
+                        queue.task_done()
+                        continue
+
+                    # Mark URL as seen immediately
+                    self.seen_urls.add(task.url)
+                    inflight[task.url] = task
+
+                    pending_tasks += 1
+                    crawler.submit(Request(task.url, max_retries=0))
+                    queue.task_done()
+
+            submit_task = asyncio.create_task(submitter())
+
+            # Seed the pipeline with a dummy task
+            seed_task = CrawlTask(
+                elem=None,
+                url=None,
+                segments=segments,
+                depth=-1,
+                backlink=None,
+            )
+            async for output in self._process_pipeline(
+                task=seed_task,
+                elem=None,
+                depth=seed_task.depth,
+                max_depth=max_depth,
+                queue=queue,
+            ):
+                yield await self.post_extract_hooks(output)
+
+            # While looping asynchronous generators, you MUST make sure 
+            # to check terminal conditions before re-iteration.
+            async for resp in crawler:
+                task = inflight.pop(resp.request.url, None)
+                pending_tasks -= 1
+
+                if task is None:
+                    log.warning(f"Got unexpected response from {resp.request.url}")
+                    if is_terminal():
+                        break
+                    continue
+
+                if resp.error:
+                    log.warning(f"Got error from {resp.request.url}: {resp.error}")
+                    if is_terminal():
+                        break
+                    continue
+
+                # NOTE: Consider allowing redirects
+                if resp.status != 200 or not resp.body:
+                    log.warning(f"Got non-200 response from {resp.request.url}")
+                    if is_terminal():
+                        break
+                    continue
+
+                body = await self.post_fetch_hooks(resp.body, task)
+                if not body:
+                    if is_terminal():
+                        break
+                    continue
+
+                elem = parse_html(
+                    body,
+                    base_url=task.url,
+                    backlink=task.backlink,
+                    depth=task.depth,
+                )
+
+                elem = await self.post_parse_hooks(elem, task)
+                if elem is None:
+                    if is_terminal():
+                        break
+                    continue
+
+                if task.segments:
+                    async for output in self._process_pipeline(
+                        task=task,
+                        elem=elem,
+                        depth=task.depth,
+                        max_depth=max_depth,
+                        queue=queue,
+                    ):
+
+                        yield await self.post_extract_hooks(output)
+                else:
+                    yield await self.post_extract_hooks(elem)
+
+                # Termination condition
+                if is_terminal():
+                    break
+
+            submit_task.cancel()
+            with contextlib.suppress(asyncio.CancelledError):
+                await submit_task
+
+    async def _process_pipeline(
+        self,
+        task: CrawlTask,
+        elem, 
+        depth: int,
+        max_depth: int,
+        queue: asyncio.Queue[CrawlTask],
+    ):
+        mini_queue: deque[(HtmlElement, list[tuple[str, str]])] = deque([(elem, task.segments)])
+
+        while mini_queue:
+            elem, segments = mini_queue.popleft()
+            
+            op, _ = segments[0]
+            operator = get_operator(op)
+
+            intents = operator(elem, segments, depth)
+
+            if not intents:
+                return
+
+            for intent in intents:
+                if isinstance(intent, DataIntent):
+                    yield intent.value
+
+                elif isinstance(intent, CrawlIntent):
+                    next_depth = task.depth + 1
+                    # if intent.url not in self.seen_urls and next_depth <= max_depth:
+                    if next_depth <= max_depth:
+                        # self.seen_urls.add(intent.url)
+                        queue.put_nowait(
+                            CrawlTask(
+                                elem=None,
+                                url=intent.url,
+                                segments=intent.next_segments,
+                                depth=next_depth,
+                                backlink=task.url,
+                            )
+                        )
+
+                elif isinstance(intent, (ExtractIntent, ProcessIntent, InfiniteCrawlIntent)):
+                    # immediately traverse the extraction
+                    elem = intent.elem
+                    next_segments = intent.next_segments
+                    mini_queue.append((elem, next_segments))
+
+
+def wxpath_async(path_expr: str, 
+                 max_depth: int, 
+                 engine: WXPathEngine = None) -> AsyncGenerator[Any, None]:
+    if engine is None:
+        engine = WXPathEngine()
+    return engine.run(path_expr, max_depth)
+
+
+##### ASYNC IN SYNC #####
+def wxpath_async_blocking_iter(path_expr, max_depth=1, engine: WXPathEngine = None):
+    """
+    Evaluate a wxpath expression using concurrent breadth-first traversal.
+    
+    Args:
+        path_expr (str): A wxpath expression.
+        max_depth (int, optional): Maximum crawl depth. Must be at least the
+            number of `url*` segments minus one. Defaults to `1`.
+
+    Yields:
+        lxml.html.HtmlElement | wxpath.models.WxStr | dict | Any: The same objects
+        produced by the sequential evaluator.
+
+    Warning:
+        Spins up its own event loop therefore this function must **not** be
+        invoked from within an active asyncio event loop.
+    """
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
+    agen = wxpath_async(path_expr, max_depth=max_depth, engine=engine)
+
+    try:
+        while True:
+            try:
+                yield loop.run_until_complete(agen.__anext__())
+            except StopAsyncIteration:
+                break
+    finally:
+        loop.run_until_complete(loop.shutdown_asyncgens())
+        loop.close()
+
+
+def wxpath_async_blocking(path_expr, max_depth=1, engine: WXPathEngine = None):
+    return list(wxpath_async_blocking_iter(path_expr, max_depth=max_depth, engine=engine))

wxpath/core/runtime/helpers.py

@@ -0,0 +1,48 @@
+import requests
+from lxml import etree, html
+
+from wxpath import patches
+from wxpath.util.logging import get_logger
+
+log = get_logger(__name__)
+
+
+def parse_html(content, base_url=None, **elem_kv_pairs) -> html.HtmlElement:
+    elem = etree.HTML(content, parser=patches.html_parser_with_xpath3, base_url=base_url)
+    if base_url:
+        elem.getroottree().docinfo.URL = base_url  # make base-uri() work
+        # Also set xml:base on the root element for XPath base-uri()
+        elem.set("{http://www.w3.org/XML/1998/namespace}base", base_url)
+        elem.base_url = base_url  # sets both attribute and doc-level URL
+    
+    # NOTE: some pages may have multiple root elements, i.e. 
+    # len(elem.itersiblings()) > 0 AND elem.getparent() is None. 
+    # This breaks elementpath. If elem has siblings, recreate the 
+    # root element and only the root element.
+    if len(list(elem.itersiblings())) > 0:
+        elem = detach_html_root(elem, base_url) 
+
+    for k, v in elem_kv_pairs.items():
+        elem.set(k, str(v))
+    return elem
+
+
+def detach_html_root(elem, base_url=None):
+    new_root = etree.HTML(
+        etree.tostring(elem, encoding="utf-8"),
+        parser=patches.html_parser_with_xpath3, 
+        base_url=base_url
+    )
+
+    if base_url:
+        new_root.getroottree().docinfo.URL = base_url
+        new_root.set("{http://www.w3.org/XML/1998/namespace}base", base_url)
+        new_root.base_url = base_url
+
+    return new_root
+
+
+def fetch_html(url):
+    response = requests.get(url, timeout=10)
+    response.raise_for_status()
+    return response.content

wxpath/hooks/__init__.py

@@ -0,0 +1,9 @@
+from wxpath.hooks.builtin import JSONLWriter as JSONLWriter
+from wxpath.hooks.builtin import SerializeXPathMapAndNodeHook as SerializeXPathMapAndNodeHook
+from wxpath.hooks.registry import register as register
+
+__all__ = [
+    "JSONLWriter",
+    "SerializeXPathMapAndNodeHook",
+    "register",
+]