wxpath 0.4.0__tar.gz → 0.5.0__tar.gz

{wxpath-0.4.0 → wxpath-0.5.0}/PKG-INFO

@@ -1,9 +1,14 @@
 Metadata-Version: 2.4
 Name: wxpath
-Version: 0.4.0
+Version: 0.5.0
 Summary: wxpath - a declarative web crawler and data extractor
 Author-email: Rodrigo Palacios <rodrigopala91@gmail.com>
 License-Expression: MIT
+Project-URL: Homepage, https://rodricios.github.io/wxpath
+Project-URL: Documentation, https://rodricios.github.io/wxpath
+Project-URL: Repository, https://github.com/rodricios/wxpath
+Project-URL: Issues, https://github.com/rodricios/wxpath/issues
+Project-URL: Changelog, https://github.com/rodricios/wxpath/blob/main/CHANGELOG.md
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
@@ -17,27 +22,112 @@ Provides-Extra: cache-sqlite
 Requires-Dist: aiohttp-client-cache[sqlite]; extra == "cache-sqlite"
 Provides-Extra: cache-redis
 Requires-Dist: aiohttp-client-cache[redis]; extra == "cache-redis"
+Provides-Extra: llm
+Requires-Dist: langchain>=1.0.0; extra == "llm"
+Requires-Dist: langchain-core>=1.0.0; extra == "llm"
+Requires-Dist: langchain-ollama>=1.0.0; extra == "llm"
+Requires-Dist: langchain-community>=0.4.0; extra == "llm"
+Requires-Dist: langchain-chroma>=1.0.0; extra == "llm"
+Requires-Dist: chromadb>=1.0.0; extra == "llm"
+Requires-Dist: langchain-text-splitters>=1.1.0; extra == "llm"
 Provides-Extra: test
 Requires-Dist: pytest>=7.0; extra == "test"
 Requires-Dist: pytest-asyncio>=0.23; extra == "test"
 Provides-Extra: dev
 Requires-Dist: ruff; extra == "dev"
+Provides-Extra: docs
+Requires-Dist: mkdocs>=1.5; extra == "docs"
+Requires-Dist: mkdocs-material>=9.0; extra == "docs"
+Requires-Dist: mkdocstrings[python]>=0.24; extra == "docs"
+Requires-Dist: mkdocs-macros-plugin>=1.0; extra == "docs"
+Requires-Dist: mkdocs-resize-images>=1.0; extra == "docs"
+Requires-Dist: mkdocs-glightbox; extra == "docs"
+Requires-Dist: pyyaml>=6.0; extra == "docs"
+Provides-Extra: tui
+Requires-Dist: textual>=1.0.0; extra == "tui"
+Requires-Dist: aiohttp-client-cache>=0.14.0; extra == "tui"
+Requires-Dist: aiohttp-client-cache[sqlite]; extra == "tui"
 Dynamic: license-file
 
-# **wxpath** - declarative web crawling with XPath 
+# **wxpath** - declarative web graph traversal with XPath 
 
-[![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/release/python-3100/)
+[![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/release/python-3100/) [![Documentation Status](https://img.shields.io/badge/documentation-green.svg)](https://rodricios.github.io/wxpath)
+
+
+> NEW: [TUI](https://rodricios.github.io/wxpath/tui/quickstart.md) - Interactive terminal interface (powered by Textual) for testing wxpath expressions and exporting data.
+
+![Wxpath TUI Demo screenshot](docs/assets/images/demo1.jpg)
+
+## Install
+
+Requires Python 3.10+.
+
+```
+pip install wxpath
+# For TUI support
+pip install wxpath[tui]
+```
+---
+
+
+## What is wxpath?
 
 **wxpath** is a declarative web crawler where traversal is expressed directly in XPath. Instead of writing imperative crawl loops, wxpath lets you describe what to follow and what to extract in a single expression. **wxpath** executes that expression concurrently, breadth-first-*ish*, and streams results as they are discovered.
 
-By introducing the `url(...)` operator and the `///` syntax, wxpath's engine is able to perform deep (or paginated) web crawling and extraction.
+This expression fetches a page, extracts links, and streams them concurrently - no crawl loop required:
+
+```python
+import wxpath
+
+expr = "url('https://quotes.toscrape.com')//a/@href"
+
+for link in wxpath.wxpath_async_blocking_iter(expr):
+    print(link)
+```
+
+
+By introducing the `url(...)` operator and the `///` syntax, wxpath's engine is able to perform recursive (or paginated) web crawling and extraction:
+
+```python
+import wxpath
+
+path_expr = """
+url('https://quotes.toscrape.com')
+  ///url(//a/@href)
+    //a/@href
+"""
+
+for item in wxpath.wxpath_async_blocking_iter(path_expr, max_depth=1):
+    print(item)
+```
+
+
+## Why wxpath?
+
+Most web scrapers force you to write crawl control flow first, and extraction second.
+
+**wxpath** converges those two steps into one:
+- **You describe traversal declaratively**
+- **Extraction is expressed inline**
+- **The engine handles scheduling, concurrency, and deduplication**
+
+
+### RAG-Ready Output 
+
+Extract clean, structured JSON hierarchies directly from the graph - feed your LLMs signal, not noise. Refer to [LangChain Integration](https://rodricios.github.io/wxpath/api/integrations/langchain/) for more details.
+
 
-NOTE: This project is in early development. Core concepts are stable, but the API and features may change. Please report issues - in particular, deadlocked crawls or unexpected behavior - and any features you'd like to see (no guarantee they'll be implemented).
+### Deterministic
 
+**wxpath** is deterministic (read: not powered by LLMs). While we can't guarantee the network is stable, we can guarantee the traversal is.
+
+## Documentation (WIP)
+
+Documentation is now available [here](https://rodricios.github.io/wxpath/).
 
 ## Contents
 
-- [Example](#example)
+- [Example: Knowledge Graph](#example)
 - [Language Design](DESIGN.md)
 - [`url(...)` and `///url(...)` Explained](#url-and-url-explained)
 - [General flow](#general-flow)
@@ -47,6 +137,7 @@ NOTE: This project is in early development. Core concepts are stable, but the AP
 - [XPath 3.1](#xpath-31-by-default)
 - [Progress Bar](#progress-bar)
 - [CLI](#cli)
+- [TUI](#tui)
 - [Persistence and Caching](#persistence-and-caching)
 - [Settings](#settings)
 - [Hooks (Experimental)](#hooks-experimental)
@@ -56,7 +147,7 @@ NOTE: This project is in early development. Core concepts are stable, but the AP
 - [Advanced: Engine & Crawler Configuration](#advanced-engine--crawler-configuration)
 - [Project Philosophy](#project-philosophy)
 - [Warnings](#warnings)
-- [Commercial support / consulting](#commercial-support--consulting)
+- [Commercial support/consulting](#commercial-supportconsulting)
 - [Versioning](#versioning)
 - [License](#license)
 
@@ -73,7 +164,11 @@ CRAWLER_SETTINGS.headers = {'User-Agent': 'my-app/0.4.0 (contact: you@example.co
 # Crawl, extract fields, build a knowledge graph
 path_expr = """
 url('https://en.wikipedia.org/wiki/Expression_language')
-  ///url(//main//a/@href[starts-with(., '/wiki/') and not(contains(., ':'))])
+  ///url(
+        //main//a/@href[
+            starts-with(., '/wiki/') and not(contains(., ':'))
+        ]
+    )
     /map{
         'title': (//span[contains(@class, "mw-page-title-main")]/text())[1] ! string(.),
         'url': string(base-uri(.)),
@@ -86,15 +181,6 @@ for item in wxpath.wxpath_async_blocking_iter(path_expr, max_depth=1):
     print(item)
 ```
 
-Output:
-
-```python
-map{'title': 'Computer language', 'url': 'https://en.wikipedia.org/wiki/Computer_language', 'short_description': 'Formal language for communicating with a computer', 'forward_links': ['/wiki/Formal_language', '/wiki/Communication', ...]}
-map{'title': 'Advanced Boolean Expression Language', 'url': 'https://en.wikipedia.org/wiki/Advanced_Boolean_Expression_Language', 'short_description': 'Hardware description language and software', 'forward_links': ['/wiki/File:ABEL_HDL_example_SN74162.png', '/wiki/Hardware_description_language', ...]}
-map{'title': 'Machine-readable medium and data', 'url': 'https://en.wikipedia.org/wiki/Machine_readable', 'short_description': 'Medium capable of storing data in a format readable by a machine', 'forward_links': ['/wiki/File:EAN-13-ISBN-13.svg', '/wiki/ISBN', ...]}
-...
-```
-
 **Note:** Some sites (including Wikipedia) may block requests without proper headers.  
 See [Advanced: Engine & Crawler Configuration](#advanced-engine--crawler-configuration) to set a custom `User-Agent`.
 
@@ -266,12 +352,17 @@ Command line options:
 --cache                [true|false] (Default: False) Persist crawl results to a local database
 ```
 
+## TUI
+
+**wxpath** provides a terminal interface (TUI) for interactive expression testing and data extraction.
+
+See [TUI Quickstart](https://rodricios.github.io/wxpath/tui/quickstart.md) for more details.
 
 ## Persistence and Caching
 
 **wxpath** optionally persists crawl results to a local database. This is especially useful when you're crawling a large number of URLs, and you decide to pause the crawl, change extraction expressions, or otherwise need to restart the crawl. 
 
-**wxpath** supports two backends: sqlite and redis. SQLite is great for small-scale crawls, with a single worker (i.e., `engine.crawler.concurrency == 1`). Redis is great for large-scale crawls, with multiple workers. You will be encounter a warning if you `min(engine.crawler.concurrency, engine.crawler.per_host) > 1` when using the sqlite backend.
+**wxpath** supports two backends: sqlite and redis. SQLite is great for small-scale crawls, with a single worker (i.e., `engine.crawler.concurrency == 1`). Redis is great for large-scale crawls, with multiple workers. You will encounter a warning if `min(engine.crawler.concurrency, engine.crawler.per_host) > 1` when using the sqlite backend.
 
 To use, you must install the appropriate optional dependency:
 
@@ -406,6 +497,17 @@ path_expr = "url('https://en.wikipedia.org/wiki/Expression_language')//url(//mai
 items = list(wxpath_async_blocking_iter(path_expr, max_depth=1, engine=engine))
 ```
 
+### Runtime API (`wxpath_async*`) options
+
+- `max_depth`: int = 1
+- `progress`: bool = False
+- `engine`: WXPathEngine | None = None
+- `yield_errors`: bool = False
+
+
+### Settings
+You can also use [settings.py](src/wxpath/settings.py) to enable caching, throttling, concurrency and more.
+
 
 ## Project Philosophy
 
@@ -433,13 +535,15 @@ The following features are not yet supported:
 
 ## WARNINGS!!!
 
+This project is in early development. Core concepts are stable, but the API and features may change. Please report issues - in particular, deadlocked crawls or unexpected behavior - and any features you'd like to see (no guarantee they'll be implemented).
+
 - Be respectful when crawling websites. A scrapy-inspired throttler is enabled by default.
 - Deep crawls (`///`) require user discipline to avoid unbounded expansion (traversal explosion).
 - Deadlocks and hangs are possible in certain situations (e.g., all tasks waiting on blocked requests). Please report issues if you encounter such behavior.
 - Consider using timeouts, `max_depth`, and XPath predicates and filters to limit crawl scope.
 
 
-## Commercial support / consulting
+## Commercial support/consulting
 
 If you want help building or operating crawlers/data feeds with wxpath (extraction, scheduling, monitoring, breakage fixes) or other web-scraping needs, please contact me at: rodrigopala91@gmail.com.
 

wxpath-0.4.0/src/wxpath.egg-info/PKG-INFO → wxpath-0.5.0/README.md

@@ -1,43 +1,82 @@
-Metadata-Version: 2.4
-Name: wxpath
-Version: 0.4.0
-Summary: wxpath - a declarative web crawler and data extractor
-Author-email: Rodrigo Palacios <rodrigopala91@gmail.com>
-License-Expression: MIT
-Requires-Python: >=3.10
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: lxml>=4.0
-Requires-Dist: elementpath<=5.0.3,>=5.0.0
-Requires-Dist: aiohttp<=3.12.15,>=3.8.0
-Requires-Dist: tqdm>=4.0.0
-Provides-Extra: cache
-Requires-Dist: aiohttp-client-cache>=0.14.0; extra == "cache"
-Provides-Extra: cache-sqlite
-Requires-Dist: aiohttp-client-cache[sqlite]; extra == "cache-sqlite"
-Provides-Extra: cache-redis
-Requires-Dist: aiohttp-client-cache[redis]; extra == "cache-redis"
-Provides-Extra: test
-Requires-Dist: pytest>=7.0; extra == "test"
-Requires-Dist: pytest-asyncio>=0.23; extra == "test"
-Provides-Extra: dev
-Requires-Dist: ruff; extra == "dev"
-Dynamic: license-file
-
-# **wxpath** - declarative web crawling with XPath 
-
-[![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/release/python-3100/)
+# **wxpath** - declarative web graph traversal with XPath 
+
+[![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/release/python-3100/) [![Documentation Status](https://img.shields.io/badge/documentation-green.svg)](https://rodricios.github.io/wxpath)
+
+
+> NEW: [TUI](https://rodricios.github.io/wxpath/tui/quickstart.md) - Interactive terminal interface (powered by Textual) for testing wxpath expressions and exporting data.
+
+![Wxpath TUI Demo screenshot](docs/assets/images/demo1.jpg)
+
+## Install
+
+Requires Python 3.10+.
+
+```
+pip install wxpath
+# For TUI support
+pip install wxpath[tui]
+```
+---
+
+
+## What is wxpath?
 
 **wxpath** is a declarative web crawler where traversal is expressed directly in XPath. Instead of writing imperative crawl loops, wxpath lets you describe what to follow and what to extract in a single expression. **wxpath** executes that expression concurrently, breadth-first-*ish*, and streams results as they are discovered.
 
-By introducing the `url(...)` operator and the `///` syntax, wxpath's engine is able to perform deep (or paginated) web crawling and extraction.
+This expression fetches a page, extracts links, and streams them concurrently - no crawl loop required:
 
-NOTE: This project is in early development. Core concepts are stable, but the API and features may change. Please report issues - in particular, deadlocked crawls or unexpected behavior - and any features you'd like to see (no guarantee they'll be implemented).
+```python
+import wxpath
 
+expr = "url('https://quotes.toscrape.com')//a/@href"
+
+for link in wxpath.wxpath_async_blocking_iter(expr):
+    print(link)
+```
+
+
+By introducing the `url(...)` operator and the `///` syntax, wxpath's engine is able to perform recursive (or paginated) web crawling and extraction:
+
+```python
+import wxpath
+
+path_expr = """
+url('https://quotes.toscrape.com')
+  ///url(//a/@href)
+    //a/@href
+"""
+
+for item in wxpath.wxpath_async_blocking_iter(path_expr, max_depth=1):
+    print(item)
+```
+
+
+## Why wxpath?
+
+Most web scrapers force you to write crawl control flow first, and extraction second.
+
+**wxpath** converges those two steps into one:
+- **You describe traversal declaratively**
+- **Extraction is expressed inline**
+- **The engine handles scheduling, concurrency, and deduplication**
+
+
+### RAG-Ready Output 
+
+Extract clean, structured JSON hierarchies directly from the graph - feed your LLMs signal, not noise. Refer to [LangChain Integration](https://rodricios.github.io/wxpath/api/integrations/langchain/) for more details.
+
+
+### Deterministic
+
+**wxpath** is deterministic (read: not powered by LLMs). While we can't guarantee the network is stable, we can guarantee the traversal is.
+
+## Documentation (WIP)
+
+Documentation is now available [here](https://rodricios.github.io/wxpath/).
 
 ## Contents
 
-- [Example](#example)
+- [Example: Knowledge Graph](#example)
 - [Language Design](DESIGN.md)
 - [`url(...)` and `///url(...)` Explained](#url-and-url-explained)
 - [General flow](#general-flow)
@@ -47,6 +86,7 @@ NOTE: This project is in early development. Core concepts are stable, but the AP
 - [XPath 3.1](#xpath-31-by-default)
 - [Progress Bar](#progress-bar)
 - [CLI](#cli)
+- [TUI](#tui)
 - [Persistence and Caching](#persistence-and-caching)
 - [Settings](#settings)
 - [Hooks (Experimental)](#hooks-experimental)
@@ -56,7 +96,7 @@ NOTE: This project is in early development. Core concepts are stable, but the AP
 - [Advanced: Engine & Crawler Configuration](#advanced-engine--crawler-configuration)
 - [Project Philosophy](#project-philosophy)
 - [Warnings](#warnings)
-- [Commercial support / consulting](#commercial-support--consulting)
+- [Commercial support/consulting](#commercial-supportconsulting)
 - [Versioning](#versioning)
 - [License](#license)
 
@@ -73,7 +113,11 @@ CRAWLER_SETTINGS.headers = {'User-Agent': 'my-app/0.4.0 (contact: you@example.co
 # Crawl, extract fields, build a knowledge graph
 path_expr = """
 url('https://en.wikipedia.org/wiki/Expression_language')
-  ///url(//main//a/@href[starts-with(., '/wiki/') and not(contains(., ':'))])
+  ///url(
+        //main//a/@href[
+            starts-with(., '/wiki/') and not(contains(., ':'))
+        ]
+    )
     /map{
         'title': (//span[contains(@class, "mw-page-title-main")]/text())[1] ! string(.),
         'url': string(base-uri(.)),
@@ -86,15 +130,6 @@ for item in wxpath.wxpath_async_blocking_iter(path_expr, max_depth=1):
     print(item)
 ```
 
-Output:
-
-```python
-map{'title': 'Computer language', 'url': 'https://en.wikipedia.org/wiki/Computer_language', 'short_description': 'Formal language for communicating with a computer', 'forward_links': ['/wiki/Formal_language', '/wiki/Communication', ...]}
-map{'title': 'Advanced Boolean Expression Language', 'url': 'https://en.wikipedia.org/wiki/Advanced_Boolean_Expression_Language', 'short_description': 'Hardware description language and software', 'forward_links': ['/wiki/File:ABEL_HDL_example_SN74162.png', '/wiki/Hardware_description_language', ...]}
-map{'title': 'Machine-readable medium and data', 'url': 'https://en.wikipedia.org/wiki/Machine_readable', 'short_description': 'Medium capable of storing data in a format readable by a machine', 'forward_links': ['/wiki/File:EAN-13-ISBN-13.svg', '/wiki/ISBN', ...]}
-...
-```
-
 **Note:** Some sites (including Wikipedia) may block requests without proper headers.  
 See [Advanced: Engine & Crawler Configuration](#advanced-engine--crawler-configuration) to set a custom `User-Agent`.
 
@@ -266,12 +301,17 @@ Command line options:
 --cache                [true|false] (Default: False) Persist crawl results to a local database
 ```
 
+## TUI
+
+**wxpath** provides a terminal interface (TUI) for interactive expression testing and data extraction.
+
+See [TUI Quickstart](https://rodricios.github.io/wxpath/tui/quickstart.md) for more details.
 
 ## Persistence and Caching
 
 **wxpath** optionally persists crawl results to a local database. This is especially useful when you're crawling a large number of URLs, and you decide to pause the crawl, change extraction expressions, or otherwise need to restart the crawl. 
 
-**wxpath** supports two backends: sqlite and redis. SQLite is great for small-scale crawls, with a single worker (i.e., `engine.crawler.concurrency == 1`). Redis is great for large-scale crawls, with multiple workers. You will be encounter a warning if you `min(engine.crawler.concurrency, engine.crawler.per_host) > 1` when using the sqlite backend.
+**wxpath** supports two backends: sqlite and redis. SQLite is great for small-scale crawls, with a single worker (i.e., `engine.crawler.concurrency == 1`). Redis is great for large-scale crawls, with multiple workers. You will encounter a warning if `min(engine.crawler.concurrency, engine.crawler.per_host) > 1` when using the sqlite backend.
 
 To use, you must install the appropriate optional dependency:
 
@@ -406,6 +446,17 @@ path_expr = "url('https://en.wikipedia.org/wiki/Expression_language')//url(//mai
 items = list(wxpath_async_blocking_iter(path_expr, max_depth=1, engine=engine))
 ```
 
+### Runtime API (`wxpath_async*`) options
+
+- `max_depth`: int = 1
+- `progress`: bool = False
+- `engine`: WXPathEngine | None = None
+- `yield_errors`: bool = False
+
+
+### Settings
+You can also use [settings.py](src/wxpath/settings.py) to enable caching, throttling, concurrency and more.
+
 
 ## Project Philosophy
 
@@ -433,13 +484,15 @@ The following features are not yet supported:
 
 ## WARNINGS!!!
 
+This project is in early development. Core concepts are stable, but the API and features may change. Please report issues - in particular, deadlocked crawls or unexpected behavior - and any features you'd like to see (no guarantee they'll be implemented).
+
 - Be respectful when crawling websites. A scrapy-inspired throttler is enabled by default.
 - Deep crawls (`///`) require user discipline to avoid unbounded expansion (traversal explosion).
 - Deadlocks and hangs are possible in certain situations (e.g., all tasks waiting on blocked requests). Please report issues if you encounter such behavior.
 - Consider using timeouts, `max_depth`, and XPath predicates and filters to limit crawl scope.
 
 
-## Commercial support / consulting
+## Commercial support/consulting
 
 If you want help building or operating crawlers/data feeds with wxpath (extraction, scheduling, monitoring, breakage fixes) or other web-scraping needs, please contact me at: rodrigopala91@gmail.com.
 

{wxpath-0.4.0 → wxpath-0.5.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "wxpath"
-version = "0.4.0"
+version = "0.5.0"
 description = "wxpath - a declarative web crawler and data extractor"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -13,6 +13,7 @@ authors = [
 ]
 license = "MIT"
 license-files = ["LICENSE"]
+
 dependencies = [
     "lxml>=4.0",
     "elementpath>=5.0.0,<=5.0.3",
@@ -20,16 +21,32 @@ dependencies = [
     "tqdm>=4.0.0"
 ]
 
+[project.urls]
+Homepage = "https://rodricios.github.io/wxpath"
+Documentation = "https://rodricios.github.io/wxpath"
+Repository = "https://github.com/rodricios/wxpath"
+Issues = "https://github.com/rodricios/wxpath/issues"
+Changelog = "https://github.com/rodricios/wxpath/blob/main/CHANGELOG.md"
+
+
 [project.optional-dependencies]
 cache = ["aiohttp-client-cache>=0.14.0"]
 cache-sqlite   = ["aiohttp-client-cache[sqlite]"]
 cache-redis    = ["aiohttp-client-cache[redis]"]
 
+# langchain langchain-ollama langchain-chroma chromadb
+llm = ["langchain>=1.0.0", "langchain-core>=1.0.0", "langchain-ollama>=1.0.0", 
+       "langchain-community>=0.4.0", "langchain-chroma>=1.0.0", "chromadb>=1.0.0",
+       "langchain-text-splitters>=1.1.0"]
+
 test = ["pytest>=7.0", "pytest-asyncio>=0.23"]
 dev = ["ruff"]
+docs = ["mkdocs>=1.5", "mkdocs-material>=9.0", "mkdocstrings[python]>=0.24", "mkdocs-macros-plugin>=1.0", "mkdocs-resize-images>=1.0", "mkdocs-glightbox", "pyyaml>=6.0"]
+tui = ["textual>=1.0.0", "aiohttp-client-cache>=0.14.0", "aiohttp-client-cache[sqlite]"]
 
 [project.scripts]
 wxpath = "wxpath.cli:main"
+wxpath-tui = "wxpath.tui:main"
 
 [tool.pytest.ini_options]
 minversion = "6.0"

{wxpath-0.4.0 → wxpath-0.5.0}/src/wxpath/__init__.py

@@ -1,3 +1,4 @@
+from . import settings
 from .core.runtime.engine import wxpath_async, wxpath_async_blocking, wxpath_async_blocking_iter
 from .util.logging import configure_logging
 
@@ -6,4 +7,5 @@ __all__ = [
     'wxpath_async_blocking',
     'wxpath_async_blocking_iter',
     'configure_logging',
+    'settings',
 ]

{wxpath-0.4.0 → wxpath-0.5.0}/src/wxpath/cli.py

@@ -47,6 +47,11 @@ def main():
         help="Respect robots.txt",
         default=True
     )
+    arg_parser.add_argument(
+        "--insecure",
+        action="store_true",
+        help="Disable SSL certificate verification (use for sites with broken chains)",
+    )
     arg_parser.add_argument(
         "--cache",
         action="store_true",
@@ -112,6 +117,7 @@ def main():
         concurrency=args.concurrency,
         per_host=args.concurrency_per_host,
         respect_robots=args.respect_robots,
+        verify_ssl=not args.insecure,
         headers=custom_headers
     )
     engine = WXPathEngine(crawler=crawler)

{wxpath-0.4.0 → wxpath-0.5.0}/src/wxpath/core/models.py

@@ -61,6 +61,7 @@ class InfiniteCrawlIntent(ProcessIntent):
 
 @dataclass(slots=True)
 class ExtractIntent(ProcessIntent):
+    """TODO: May be redundant with ProcessIntent?"""
     pass
 
 

{wxpath-0.4.0 → wxpath-0.5.0}/src/wxpath/core/ops.py

@@ -19,6 +19,7 @@ from wxpath.core.parser import (
     Binary,
     Call,
     ContextItem,
+    Depth,
     Segment,
     Segments,
     String,
@@ -78,7 +79,10 @@ def get_operator(
 
 
 @register('url', (String,))
+@register('url', (String, Depth))
 @register('url', (String, Xpath))
+@register('url', (String, Depth, Xpath))
+@register('url', (String, Xpath, Depth))
 def _handle_url_str_lit(curr_elem: html.HtmlElement, 
                         curr_segments: list[Url | Xpath], 
                         curr_depth: int, **kwargs) -> Iterable[Intent]:
@@ -87,9 +91,12 @@ def _handle_url_str_lit(curr_elem: html.HtmlElement,
 
     next_segments = curr_segments[1:]
 
-    if len(url_call.args) == 2:
+    # NOTE: Expects parser to produce UrlCrawl node in expressions
+    # that look like `url('...', follow=//a/@href)`
+    if isinstance(url_call, UrlCrawl):
+        xpath_arg = [arg for arg in url_call.args if isinstance(arg, Xpath)][0]
         _segments = [
-            UrlCrawl('///url', [url_call.args[1], url_call.args[0].value])
+            UrlCrawl('///url', [xpath_arg, url_call.args[0].value])
         ] + next_segments
         
         yield CrawlIntent(url=url_call.args[0].value, next_segments=_segments)
@@ -112,16 +119,6 @@ def _handle_xpath(curr_elem: html.HtmlElement,
         raise ValueError("Element must be provided when path_expr does not start with 'url()'.")
     base_url = getattr(curr_elem, 'base_url', None)
     log.debug("base url", extra={"depth": curr_depth, "op": 'xpath', "base_url": base_url})
-
-    _backlink_str = f"string('{curr_elem.get('backlink')}')"
-    # We use the root tree's depth and not curr_depth because curr_depth accounts for a +1 
-    # increment after each url*() hop
-    _depth_str = f"number({curr_elem.getroottree().getroot().get('depth')})"
-    expr = expr.replace('wx:backlink()', _backlink_str)
-    expr = expr.replace('wx:backlink(.)', _backlink_str)
-    expr = expr.replace('wx:depth()', _depth_str)
-    expr = expr.replace('wx:depth(.)', _depth_str)
-
     elems = curr_elem.xpath3(expr)
     
     next_segments = curr_segments[1:]