wxpath 0.5.0__py3-none-any.whl → 0.5.1__py3-none-any.whl

wxpath/core/exceptions.py

@@ -0,0 +1,53 @@
+class XPathEvaluationError(Exception):
+    """Errors during XPath evaluation with elementpath."""
+    
+    def __init__(
+        self, 
+        message: str, 
+        xpath: str, 
+        base_url: str | None = None,
+        element_tag: str | None = None,
+        error_code: str | None = None,  # XPath error codes like XPST0003
+        position: tuple[int, int] | None = None,  # (line, column)
+        original_error: Exception | None = None
+    ):
+        context = {
+            "xpath": xpath,
+            "base_url": base_url,
+            "element_tag": element_tag,
+            "error_code": error_code,
+            "position": position,
+        }
+        if original_error:
+            context["original_error"] = str(original_error)
+            # Extract XPath error code if present (e.g., [err:XPST0003])
+            if hasattr(original_error, 'code'):
+                context["error_code"] = original_error.code
+        
+        super().__init__(message, context)
+
+    def to_dict(self) -> dict:
+        return {
+            "message": self.message,
+            "xpath": self.xpath,
+            "base_url": self.base_url,
+            "element_tag": self.element_tag,
+            "error_code": self.error_code,
+            "position": self.position,
+            "original_error": self.original_error,
+        }
+
+
+class XPathSyntaxError(XPathEvaluationError):
+    """Invalid XPath syntax."""
+    pass
+
+
+class XPathTypeError(XPathEvaluationError):
+    """Type error in XPath expression."""
+    pass
+
+
+class XPathRuntimeError(XPathEvaluationError):
+    """Runtime error during XPath evaluation."""
+    pass

wxpath/core/ops.py

@@ -2,11 +2,25 @@ from typing import Callable, Iterable
 from urllib.parse import urljoin
 
 import elementpath
+from elementpath import (
+    ElementPathError,
+    ElementPathSyntaxError as EPSyntaxError,
+    ElementPathTypeError as EPTypeError,
+    ElementPathZeroDivisionError,
+    ElementPathRuntimeError as EPRuntimeError,
+    MissingContextError,
+)
 from elementpath.datatypes import AnyAtomicType
 from elementpath.xpath3 import XPath3Parser
 from lxml import html
 
 from wxpath.core.dom import get_absolute_links_from_elem_and_xpath
+from wxpath.core.exceptions import (
+    XPathEvaluationError,
+    XPathSyntaxError, 
+    XPathTypeError, 
+    XPathRuntimeError,
+)
 from wxpath.core.models import (
     CrawlIntent,
     DataIntent,
@@ -119,7 +133,52 @@ def _handle_xpath(curr_elem: html.HtmlElement,
         raise ValueError("Element must be provided when path_expr does not start with 'url()'.")
     base_url = getattr(curr_elem, 'base_url', None)
     log.debug("base url", extra={"depth": curr_depth, "op": 'xpath', "base_url": base_url})
-    elems = curr_elem.xpath3(expr)
+    
+    try:
+        elems = curr_elem.xpath3(expr)
+    except EPSyntaxError as e:
+        # Parse the error message to extract line/column if available
+        # elementpath format: "... at line 1, column 7: [err:XPST0003] ..."
+        raise XPathSyntaxError(
+            f"Invalid XPath syntax: {str(e).split(': ', 1)[-1]}",
+            xpath=expr,
+            base_url=base_url,
+            element_tag=curr_elem.tag,
+            original_error=e
+        ) from e
+    except EPTypeError as e:
+        raise XPathTypeError(
+            f"XPath type error: {str(e).split(': ', 1)[-1]}",
+            xpath=expr,
+            base_url=base_url,
+            element_tag=curr_elem.tag,
+            original_error=e
+        ) from e
+    except ElementPathZeroDivisionError as e:
+        raise XPathRuntimeError(
+            f"Division by zero in XPath: {expr}",
+            xpath=expr,
+            base_url=base_url,
+            element_tag=curr_elem.tag,
+            original_error=e
+        ) from e
+    except MissingContextError as e:
+        raise XPathRuntimeError(
+            f"XPath requires context but none provided: {expr}",
+            xpath=expr,
+            base_url=base_url,
+            element_tag=curr_elem.tag,
+            original_error=e
+        ) from e
+    except ElementPathError as e:
+        # Catch-all for other elementpath errors
+        raise XPathEvaluationError(
+            f"XPath evaluation failed: {e}",
+            xpath=expr,
+            base_url=base_url,
+            element_tag=curr_elem.tag,
+            original_error=e
+        ) from e
     
     next_segments = curr_segments[1:]
     for elem in elems:
@@ -256,12 +315,37 @@ def _handle_binary(curr_elem: html.HtmlElement | str,
     base_url = getattr(curr_elem, 'base_url', None)
     next_segments = right
 
-    results = elementpath.select(
-        curr_elem,
-        left.value,
-        parser=XPath3Parser,
-        item='' if curr_elem is None else None
-    )
+    try:
+        results = elementpath.select(
+            curr_elem,
+            left.value,
+            parser=XPath3Parser,
+            item='' if curr_elem is None else None
+        )
+    except EPSyntaxError as e:
+        raise XPathSyntaxError(
+            f"Invalid XPath in binary operation: {str(e).split(': ', 1)[-1]}",
+            xpath=left.value,
+            base_url=base_url,
+            element_tag=getattr(curr_elem, 'tag', None),
+            original_error=e
+        ) from e
+    except EPTypeError as e:
+        raise XPathTypeError(
+            f"XPath type error in binary operation: {str(e).split(': ', 1)[-1]}",
+            xpath=left.value,
+            base_url=base_url,
+            element_tag=getattr(curr_elem, 'tag', None),
+            original_error=e
+        ) from e
+    except ElementPathError as e:
+        raise XPathEvaluationError(
+            f"XPath evaluation failed in binary operation: {e}",
+            xpath=left.value,
+            base_url=base_url,
+            element_tag=getattr(curr_elem, 'tag', None),
+            original_error=e
+        ) from e
 
     if isinstance(results, AnyAtomicType):
         results = [results]

wxpath/core/parser.py

@@ -13,6 +13,7 @@ except ImportError:
 
 
 TOKEN_SPEC = [
+    ("WXLOOP",  r"wx:loop"),
     ("NUMBER",   r"\d+\.\d+"),
     ("INTEGER",  r"\d+"),
     ("STRING",   r"'([^'\\]|\\.)*'|\"([^\"\\]|\\.)*\""), # TODO: Rename to URL Literal
@@ -180,7 +181,7 @@ class Parser:
 
     def parse_binary(self, min_prec: int) -> object:
         """Parse a binary expression chain honoring operator precedence."""
-        if self.token.type == "WXPATH":
+        if self.token.type == "WXPATH" or self.token.type == "WXLOOP":
             left = self.parse_segments()
         else:
             left = self.nud()

wxpath/core/runtime/engine.py

@@ -2,13 +2,14 @@ import asyncio
 import contextlib
 import inspect
 from collections import deque
-from typing import Any, AsyncGenerator, Iterator
+from typing import Any, AsyncGenerator, Iterator, Iterable
 
 from lxml.html import HtmlElement
 from tqdm import tqdm
 
 from wxpath import patches  # noqa: F401
 from wxpath.core import parser
+from wxpath.core.exceptions import XPathEvaluationError
 from wxpath.core.models import (
     CrawlIntent,
     CrawlTask,
@@ -145,6 +146,7 @@ class WXPathEngine(HookedEngineBase):
             respect_robots: bool = True,
             allowed_response_codes: set[int] = None,
             allow_redirects: bool = True,
+            yield_errors: bool = False,
         ):
         # NOTE: Will grow unbounded in large crawls. Consider a LRU cache, or bloom filter.
         self.seen_urls: set[str] = set()
@@ -157,6 +159,7 @@ class WXPathEngine(HookedEngineBase):
         self.allow_redirects = allow_redirects
         if allow_redirects:
             self.allowed_response_codes |= {301, 302, 303, 307, 308}
+        self.yield_errors = yield_errors
 
     def _get_max_depth(self, bin_or_segs: Binary | Segments, max_depth: int) -> int:
         """Get the maximum crawl depth for a given expression. Will find a Depth
@@ -182,7 +185,6 @@ class WXPathEngine(HookedEngineBase):
             expression: str, 
             max_depth: int, 
             progress: bool = False,
-            yield_errors: bool = False,
         ) -> AsyncGenerator[Any, None]:
         """Execute a wxpath expression concurrently and yield results.
 
@@ -267,7 +269,10 @@ class WXPathEngine(HookedEngineBase):
                 queue=queue,
                 pbar=pbar,
             ):
-                yield await self.post_extract_hooks(output)
+                if isinstance(output, dict) and output.get("__type__") == "error":
+                    yield output
+                else:
+                    yield await self.post_extract_hooks(output)
 
             # While looping asynchronous generators, you MUST make sure 
             # to check terminal conditions before re-iteration.
@@ -282,7 +287,7 @@ class WXPathEngine(HookedEngineBase):
                 if task is None:
                     log.warning(f"Got unexpected response from {resp.request.url}")
 
-                    if yield_errors:
+                    if self.yield_errors:
                         yield {
                             "__type__": "error",
                             "url": resp.request.url,
@@ -298,7 +303,7 @@ class WXPathEngine(HookedEngineBase):
                 if resp.error:
                     log.warning(f"Got error from {resp.request.url}: {resp.error}")
 
-                    if yield_errors:
+                    if self.yield_errors:
                         yield {
                             "__type__": "error",
                             "url": resp.request.url,
@@ -315,7 +320,7 @@ class WXPathEngine(HookedEngineBase):
                 if resp.status not in self.allowed_response_codes or not resp.body:
                     log.warning(f"Got non-200 response from {resp.request.url}")
 
-                    if yield_errors:
+                    if self.yield_errors:
                         yield {
                             "__type__": "error",
                             "url": resp.request.url,
@@ -413,7 +418,11 @@ class WXPathEngine(HookedEngineBase):
 
             binary_or_segment = bin_or_segs if isinstance(bin_or_segs, Binary) else bin_or_segs[0]
             operator = get_operator(binary_or_segment)
-            intents = operator(elem, bin_or_segs, depth)
+
+            if self.yield_errors:
+                intents = _safe_iterator(operator(elem, bin_or_segs, depth))
+            else:
+                intents = operator(elem, bin_or_segs, depth)
 
             if not intents:
                 return
@@ -449,6 +458,28 @@ class WXPathEngine(HookedEngineBase):
                     mini_queue.append((elem, next_segments))
 
 
+def _safe_iterator(iterable: Iterable[Any]) -> Iterator[Any]:
+    """Wrap an iterable in a try/except block and return an iterator that yields the result or the error."""
+    it = iter(iterable)
+    while True:
+        try:
+            yield next(it)
+        except StopIteration:
+            break
+        except XPathEvaluationError as e:
+            yield {
+                "__type__": "error",
+                "reason": "xpath_evaluation_error",
+                "exception": str(e),
+            }
+        except Exception as e:
+            yield {
+                "__type__": "error",
+                "reason": "iterator_error",
+                "exception": str(e),
+            }
+
+
 def wxpath_async(path_expr: str,
                  max_depth: int,
                  progress: bool = False,

wxpath/tui.py

@@ -20,6 +20,7 @@ Example:
 import asyncio
 import csv
 import json
+import traceback
 from datetime import datetime
 from pathlib import Path
 from typing import Any, Iterable
@@ -27,7 +28,7 @@ from typing import Any, Iterable
 from elementpath.xpath_tokens import XPathMap
 from lxml.html import HtmlElement, tostring
 from rich.console import RenderableType
-from textual import work
+from textual import events, work
 from textual.app import App, ComposeResult
 from textual.containers import Container, Horizontal, Vertical, VerticalScroll
 from textual.reactive import reactive
@@ -558,6 +559,7 @@ class WXPathTUI(App):
         ("ctrl+r", "execute", "Execute"),
         ("escape", "cancel_crawl", "Cancel Crawl"),
         ("ctrl+c", "clear", "Clear"),
+        ("ctrl+shift+backspace", "clear_editor", "Clear Editor"),
         ("ctrl+d", "clear_debug", "Clear Debug"),
         ("ctrl+shift+d", "toggle_debug", "Toggle Debug"),
         ("ctrl+e", "export", "Export"),
@@ -629,6 +631,7 @@ class WXPathTUI(App):
             "  • Press [bold]Escape[/bold] to cancel a running crawl\n"
             "  • Press [bold]Ctrl+E[/bold] to export table (CSV/JSON)\n"
             "  • Press [bold]Ctrl+C[/bold] to clear output\n"
+            "  • Press [bold]Ctrl+Shift+Backspace[/bold] to clear expression editor\n"
             "  • Press [bold]Ctrl+Shift+D[/bold] to toggle debug panel\n"
             "  • Press [bold]Ctrl+H[/bold] to configure HTTP headers\n"
             "  • Press [bold]Ctrl+Shift+S[/bold] to edit persistent settings (concurrency, robots)\n" # noqa: E501
@@ -645,7 +648,7 @@ class WXPathTUI(App):
         """Update global settings and subtitle when cache setting changes."""
         # Update the global settings - this is what the HTTP crawler will read
         SETTINGS.http.client.cache.enabled = bool(new_value)
-        print(f"Cache enabled: {SETTINGS.http.client.cache.enabled}")
+        self._debug(f"Cache enabled: {SETTINGS.http.client.cache.enabled}")
         self._update_subtitle()
     
     def watch_custom_headers(self, new_value: dict) -> None:
@@ -658,7 +661,8 @@ class WXPathTUI(App):
     
     def _update_subtitle(self) -> None:
         """Update subtitle with current cache, headers, and persistent settings."""
-        cache_state = "ON" if self.cache_enabled else "OFF"
+        # cache_state = "ON" if self.cache_enabled else "OFF"
+        cache_state = SETTINGS.http.client.cache.enabled
         headers_count = len(self.custom_headers)
         headers_info = f"{headers_count} custom" if headers_count > 0 else "default"
         conc = self.tui_settings.get("concurrency", 16)
@@ -940,7 +944,10 @@ class WXPathTUI(App):
             columns_initialized = False
             column_keys: list[str] = []
 
-            async for result in engine.run(expression, max_depth=1, progress=False):
+            async for result in engine.run(expression, max_depth=1, progress=False, yield_errors=True):
+                if isinstance(result, dict) and result.get("__type__") == "error":
+                    self._debug(f"Error: {result.get('reason')}: {result}")
+                    continue
                 count += 1
                 if count % 100 == 0:
                     self._debug(f"Received result {count} of type {type(result).__name__}")
@@ -990,8 +997,16 @@ class WXPathTUI(App):
             self._executing = False
             return
         except Exception as e:
-            # Handle execution errors separately
-            self._update_output(f"[red]Execution Error:[/red] {type(e).__name__}: {e}")
+            # Log full stack trace to debug panel
+            self._debug(traceback.format_exc())
+            # Append error as next row of table (do not clear output panel)
+            err_msg = f"Execution Error: {type(e).__name__}: {e}"
+            if columns_initialized and column_keys:
+                row = [err_msg] + [""] * (len(column_keys) - 1)
+                data_table.add_row(*row, key=f"error-{count}")
+            else:
+                data_table.add_column("error", key="error")
+                data_table.add_row(err_msg, key="error-0")
             self._executing = False
             return
         finally:
@@ -1086,7 +1101,13 @@ class WXPathTUI(App):
         """Clear the output panel."""
         self._update_output("Waiting for expression...")
         self._debug("Cleared output panel.")
-    
+
+    def action_clear_editor(self) -> None:
+        """Clear the expression editor (all text)."""
+        editor = self.query_one("#expression-editor", )
+        editor.clear()
+        self._debug("Expression editor cleared.")
+
     def _update_output(self, content: str | RenderableType) -> None:
         """Update the output panel with new content."""
         # output_panel = self.query_one("#output-panel", OutputPanel)

{wxpath-0.5.0.dist-info → wxpath-0.5.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: wxpath
-Version: 0.5.0
+Version: 0.5.1
 Summary: wxpath - a declarative web crawler and data extractor
 Author-email: Rodrigo Palacios <rodrigopala91@gmail.com>
 License-Expression: MIT
@@ -14,7 +14,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: lxml>=4.0
 Requires-Dist: elementpath<=5.0.3,>=5.0.0
-Requires-Dist: aiohttp<=3.12.15,>=3.8.0
+Requires-Dist: aiohttp<=4.0.0,>=3.8.0
 Requires-Dist: tqdm>=4.0.0
 Provides-Extra: cache
 Requires-Dist: aiohttp-client-cache>=0.14.0; extra == "cache"
@@ -35,6 +35,7 @@ Requires-Dist: pytest>=7.0; extra == "test"
 Requires-Dist: pytest-asyncio>=0.23; extra == "test"
 Provides-Extra: dev
 Requires-Dist: ruff; extra == "dev"
+Requires-Dist: tox; extra == "dev"
 Provides-Extra: docs
 Requires-Dist: mkdocs>=1.5; extra == "docs"
 Requires-Dist: mkdocs-material>=9.0; extra == "docs"
@@ -54,7 +55,7 @@ Dynamic: license-file
 [![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/release/python-3100/) [![Documentation Status](https://img.shields.io/badge/documentation-green.svg)](https://rodricios.github.io/wxpath)
 
 
-> NEW: [TUI](https://rodricios.github.io/wxpath/tui/quickstart.md) - Interactive terminal interface (powered by Textual) for testing wxpath expressions and exporting data.
+> NEW: [TUI](https://rodricios.github.io/wxpath/tui/quickstart) - Interactive terminal interface (powered by Textual) for testing wxpath expressions and exporting data.
 
 ![Wxpath TUI Demo screenshot](docs/assets/images/demo1.jpg)
 
@@ -65,7 +66,7 @@ Requires Python 3.10+.
 ```
 pip install wxpath
 # For TUI support
-pip install wxpath[tui]
+pip install "wxpath[tui]"
 ```
 ---
 
@@ -356,7 +357,7 @@ Command line options:
 
 **wxpath** provides a terminal interface (TUI) for interactive expression testing and data extraction.
 
-See [TUI Quickstart](https://rodricios.github.io/wxpath/tui/quickstart.md) for more details.
+See [TUI Quickstart](https://rodricios.github.io/wxpath/tui/quickstart) for more details.
 
 ## Persistence and Caching
 

{wxpath-0.5.0.dist-info → wxpath-0.5.1.dist-info}/RECORD

@@ -2,15 +2,16 @@ wxpath/__init__.py,sha256=nKo2ggHdlNUhTPzZlPOW-XssyYjVar6XcqaFsZLxsKU,304
 wxpath/cli.py,sha256=P2MU6UzWHiN-5roo-GdEb4OpTlCp0XA3AlFmjL7fI1o,4544
 wxpath/patches.py,sha256=nNFpNuapF30aVMVz6K1iJDX2RWmiyiECx2_UprcwiT4,8417
 wxpath/settings.py,sha256=dBSIxVPlcsP3IYuhuHG0uxZaNXLzcNUdBxEaZsZHcAc,3862
-wxpath/tui.py,sha256=CG8xvGnYNbruD4lw50Agu8RKKUJEpEl0WG0SyLBW4c8,42786
+wxpath/tui.py,sha256=0fq8fZhqHMf7hX8tM75HmAjLIfD5VLVBN5mo_E3Q1GQ,43880
 wxpath/tui_settings.py,sha256=rM2IBeOzQUIzjk2Ds1Jlnvb7IUtdJdKMN2j3GHk7Z9M,5051
 wxpath/core/__init__.py,sha256=U9_In2iRaZrpiIVavIli1M59gCB6Kn1en-1Fza-qIiI,257
 wxpath/core/dom.py,sha256=X0L3n8jRfO5evEypDaJTD-NQ3cLXWvnEUVERAHo3vV0,701
+wxpath/core/exceptions.py,sha256=BwVoBWhulv24m13hZaQ3hlVYF8foiufiXZMAh7gESE0,1615
 wxpath/core/models.py,sha256=xsNY9ZmUILB5_O1GHRkn3cLBtPs3-krguU5NlqFe0bM,1664
-wxpath/core/ops.py,sha256=4vzLOqRM_LbXc1cAnWCuKGt2m_pbvyHO0p5ee2Upjog,9569
-wxpath/core/parser.py,sha256=ufUSEfyR6aO10pV_E39-uSiLQfYvngNQnHcs1GJlpbA,21392
+wxpath/core/ops.py,sha256=0hyfAyBWTy4uNIW4oec6j1y3H-HC6msm5MreL2cIAvY,12428
+wxpath/core/parser.py,sha256=S6Wg1lxgqY2btEJxbmxbBrOVVL0GOqvS3qivdPY2eOQ,21452
 wxpath/core/runtime/__init__.py,sha256=_iCgkIWxXvxzQcenHOsjYGsk74HboTIYWOtgM8GtCyc,86
-wxpath/core/runtime/engine.py,sha256=ocGBTIHdFgOh3LzkgEUKZ59Ozn3nKqvBBAloj4Ln5D4,19229
+wxpath/core/runtime/engine.py,sha256=vXRahRyz8iII0Y7dESk5zTU-NJNalMHJfN77nw2wi_o,20313
 wxpath/core/runtime/helpers.py,sha256=RFLonAjRsL_CHFV0biUsgk0lOL8MKvHXdFg7p65xEP8,1554
 wxpath/hooks/__init__.py,sha256=9JG63e4z_8CZLWugFcY786hebaEEPZ5FmZhyDHat-98,294
 wxpath/hooks/builtin.py,sha256=GJ4w1C9djWNzAmAA3U0qI9OoCOeC5R8tEGtWXJVHSYs,4125
@@ -36,9 +37,9 @@ wxpath/util/cleaners.py,sha256=JtUwCKjSJV-qw2CBrcB1oYswBDeXiqndGiz3-MlxeG0,946
 wxpath/util/common_paths.py,sha256=Y-0yq6IMjlSl1t4GbmK9TeJFTQ-MVvJOINhglvD4djA,980
 wxpath/util/logging.py,sha256=hgN4OC1y2oZWewtL-O-Ei_1lOaadH9eSyo0Iz2t_s1c,2858
 wxpath/util/serialize.py,sha256=uUs4C9VErpFd97smBM2bRWo2nW25kCgKdsMrVtVxhg8,575
-wxpath-0.5.0.dist-info/licenses/LICENSE,sha256=AVBZLhdWmqxm-f-dy5prVB1E-solHWoP2EXEIV_o-00,1076
-wxpath-0.5.0.dist-info/METADATA,sha256=uBaoDrnCdTvzmt3yZ2ywCImWCeg_zOTQKPIgDMA3LI4,22002
-wxpath-0.5.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-wxpath-0.5.0.dist-info/entry_points.txt,sha256=CSr67nPxU_tZ_XdAdDmvW9b9VRUhFAhGhEC41YNJEfE,72
-wxpath-0.5.0.dist-info/top_level.txt,sha256=uFCcveG78mnefxRGvYsR2OexDlKR_Z1UD4vZijUcex8,7
-wxpath-0.5.0.dist-info/RECORD,,
+wxpath-0.5.1.dist-info/licenses/LICENSE,sha256=AVBZLhdWmqxm-f-dy5prVB1E-solHWoP2EXEIV_o-00,1076
+wxpath-0.5.1.dist-info/METADATA,sha256=ao3G5gyrVZ17uJCnm0ld5taVl4dHDCPsFStEbw3gn4U,22031
+wxpath-0.5.1.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+wxpath-0.5.1.dist-info/entry_points.txt,sha256=CSr67nPxU_tZ_XdAdDmvW9b9VRUhFAhGhEC41YNJEfE,72
+wxpath-0.5.1.dist-info/top_level.txt,sha256=uFCcveG78mnefxRGvYsR2OexDlKR_Z1UD4vZijUcex8,7
+wxpath-0.5.1.dist-info/RECORD,,