wxpath 0.4.1__py3-none-any.whl → 0.5.0__py3-none-any.whl

wxpath/__init__.py

@@ -1,3 +1,4 @@
+from . import settings
 from .core.runtime.engine import wxpath_async, wxpath_async_blocking, wxpath_async_blocking_iter
 from .util.logging import configure_logging
 
@@ -6,4 +7,5 @@ __all__ = [
     'wxpath_async_blocking',
     'wxpath_async_blocking_iter',
     'configure_logging',
+    'settings',
 ]

wxpath/cli.py

@@ -47,6 +47,11 @@ def main():
         help="Respect robots.txt",
         default=True
     )
+    arg_parser.add_argument(
+        "--insecure",
+        action="store_true",
+        help="Disable SSL certificate verification (use for sites with broken chains)",
+    )
     arg_parser.add_argument(
         "--cache",
         action="store_true",
@@ -112,6 +117,7 @@ def main():
         concurrency=args.concurrency,
         per_host=args.concurrency_per_host,
         respect_robots=args.respect_robots,
+        verify_ssl=not args.insecure,
         headers=custom_headers
     )
     engine = WXPathEngine(crawler=crawler)

wxpath/core/models.py

@@ -61,6 +61,7 @@ class InfiniteCrawlIntent(ProcessIntent):
 
 @dataclass(slots=True)
 class ExtractIntent(ProcessIntent):
+    """TODO: May be redundant with ProcessIntent?"""
     pass
 
 

wxpath/core/ops.py

@@ -19,6 +19,7 @@ from wxpath.core.parser import (
     Binary,
     Call,
     ContextItem,
+    Depth,
     Segment,
     Segments,
     String,
@@ -78,7 +79,10 @@ def get_operator(
 
 
 @register('url', (String,))
+@register('url', (String, Depth))
 @register('url', (String, Xpath))
+@register('url', (String, Depth, Xpath))
+@register('url', (String, Xpath, Depth))
 def _handle_url_str_lit(curr_elem: html.HtmlElement, 
                         curr_segments: list[Url | Xpath], 
                         curr_depth: int, **kwargs) -> Iterable[Intent]:
@@ -87,9 +91,12 @@ def _handle_url_str_lit(curr_elem: html.HtmlElement,
 
     next_segments = curr_segments[1:]
 
-    if len(url_call.args) == 2:
+    # NOTE: Expects parser to produce UrlCrawl node in expressions
+    # that look like `url('...', follow=//a/@href)`
+    if isinstance(url_call, UrlCrawl):
+        xpath_arg = [arg for arg in url_call.args if isinstance(arg, Xpath)][0]
         _segments = [
-            UrlCrawl('///url', [url_call.args[1], url_call.args[0].value])
+            UrlCrawl('///url', [xpath_arg, url_call.args[0].value])
         ] + next_segments
         
         yield CrawlIntent(url=url_call.args[0].value, next_segments=_segments)
@@ -112,16 +119,6 @@ def _handle_xpath(curr_elem: html.HtmlElement,
         raise ValueError("Element must be provided when path_expr does not start with 'url()'.")
     base_url = getattr(curr_elem, 'base_url', None)
     log.debug("base url", extra={"depth": curr_depth, "op": 'xpath', "base_url": base_url})
-
-    _backlink_str = f"string('{curr_elem.get('backlink')}')"
-    # We use the root tree's depth and not curr_depth because curr_depth accounts for a +1 
-    # increment after each url*() hop
-    _depth_str = f"number({curr_elem.getroottree().getroot().get('depth')})"
-    expr = expr.replace('wx:backlink()', _backlink_str)
-    expr = expr.replace('wx:backlink(.)', _backlink_str)
-    expr = expr.replace('wx:depth()', _depth_str)
-    expr = expr.replace('wx:depth(.)', _depth_str)
-
     elems = curr_elem.xpath3(expr)
     
     next_segments = curr_segments[1:]

wxpath/core/parser.py

@@ -13,7 +13,8 @@ except ImportError:
 
 
 TOKEN_SPEC = [
-    ("NUMBER",   r"\d+(\.\d+)?"),
+    ("NUMBER",   r"\d+\.\d+"),
+    ("INTEGER",  r"\d+"),
     ("STRING",   r"'([^'\\]|\\.)*'|\"([^\"\\]|\\.)*\""), # TODO: Rename to URL Literal
     ("WXPATH",   r"/{0,3}\s*url"),  # Must come before NAME to match 'url' as WXPATH
     # ("///URL",   r"/{3}\s*url"),
@@ -22,6 +23,7 @@ TOKEN_SPEC = [
     ("URL",      r"\s*url"),  # Must come before NAME to match 'url' as WXPATH
     # ("NAME",     r"[a-zA-Z_][a-zA-Z0-9_]*"),
     ("FOLLOW",   r",?\s{,}follow="),
+    ("DEPTH",    r",?\s{,}depth="),
     ("OP",       r"\|\||<=|>=|!=|=|<|>|\+|-|\*|/|!"),  # Added || for string concat
     ("LPAREN",   r"\("),
     ("RPAREN",   r"\)"),
@@ -63,6 +65,14 @@ def tokenize(src: str):
 class Number:
     value: float
 
+@dataclass
+class Integer:
+    value: int
+
+@dataclass
+class Depth(Integer):
+    pass
+
 @dataclass
 class String:
     value: str
@@ -273,6 +283,10 @@ class Parser:
         if tok.type == "NUMBER":
             self.advance()
             return Number(float(tok.value))
+
+        if tok.type == "INTEGER":
+            self.advance()
+            return Integer(int(tok.value))
         
         if tok.type == "STRING":
             self.advance()
@@ -358,18 +372,18 @@ class Parser:
             self.advance()
         
         return result
-    
 
     def capture_url_arg_content(self) -> list[Call | Xpath | ContextItem]:
         """Capture content inside a url() call, handling nested wxpath expressions.
 
         Supports patterns like::
 
-            url('...')                      -> [String]
-            url('...' follow=//a/@href)     -> [String, Xpath]
-            url(//a/@href)                  -> [Xpath]
-            url( url('..')//a/@href )       -> [Call, Xpath]
-            url( url( url('..')//a )//b )   -> [Call, Xpath]
+            url('...')                          -> [String]
+            url('...' follow=//a/@href)         -> [String, Xpath]
+            url('...' follow=//a/@href depth=2) -> [String, Xpath, Integer]
+            url(//a/@href depth=2)              -> [Xpath, Integer]
+            url( url('..')//a/@href )           -> [Call, Xpath]
+            url( url( url('..')//a )//b )       -> [Call, Xpath]
 
         Returns:
             A list of parsed elements: Xpath nodes for xpath content and Call
@@ -380,7 +394,10 @@ class Parser:
         paren_balance = 1  # We're already inside the opening paren of url()
         brace_balance = 0  # Track braces for map constructors
         reached_follow_token = False
+        reached_depth_token = False
         follow_xpath = ""
+        depth_number = ""
+
         while paren_balance > 0 and self.token.type != "EOF":
             if self.token.type == "WXPATH":
                 # Found nested wxpath: save any accumulated xpath content first
@@ -396,13 +413,22 @@ class Parser:
 
             elif self.token.type == "FOLLOW":
                 reached_follow_token = True
+                reached_depth_token = False
+                self.advance()
+
+            elif self.token.type == "DEPTH":
+                reached_depth_token = True
+                reached_follow_token = False
                 self.advance()
 
             elif self.token.type == "LPAREN":
                 # Opening paren that's NOT part of a url() call
                 # (it's part of an xpath function like contains(), starts-with(), etc.)
                 paren_balance += 1
-                current_xpath += self.token.value
+                if not reached_follow_token:
+                    current_xpath += self.token.value
+                else:
+                    follow_xpath += self.token.value
                 self.advance()
                 
             elif self.token.type == "RPAREN":
@@ -410,26 +436,37 @@ class Parser:
                 if paren_balance == 0:
                     # This is the closing paren of the outer url()
                     break
-                current_xpath += self.token.value
+                if not reached_follow_token:
+                    current_xpath += self.token.value
+                else:
+                    follow_xpath += self.token.value
                 self.advance()
 
             elif self.token.type == "LBRACE":
                 # Opening brace for map constructors
                 brace_balance += 1
-                current_xpath += self.token.value
+                if not reached_follow_token:
+                    current_xpath += self.token.value
+                else:
+                    follow_xpath += self.token.value
                 self.advance()
 
             elif self.token.type == "RBRACE":
                 brace_balance -= 1
-                current_xpath += self.token.value
+                if not reached_follow_token:
+                        current_xpath += self.token.value
+                else:
+                    follow_xpath += self.token.value
                 self.advance()
                 
             else:
                 # Accumulate all other tokens as xpath content
-                if not reached_follow_token:
-                    current_xpath += self.token.value
-                else: 
+                if reached_follow_token:
                     follow_xpath += self.token.value
+                elif reached_depth_token:
+                    depth_number += self.token.value
+                else:
+                    current_xpath += self.token.value
 
                 self.advance()
         
@@ -447,6 +484,9 @@ class Parser:
         if follow_xpath.strip():
             elements.append(Xpath(follow_xpath.strip()))
 
+        if depth_number.strip():
+            elements.append(Depth(int(depth_number.strip())))
+
         return elements
 
     def parse_call(self, func_name: str) -> Call | Segments:
@@ -462,13 +502,16 @@ class Parser:
                 self.advance()
                 # Handle follow=...
                 if self.token.type == "FOLLOW":
-                    self.advance()
                     follow_arg = self.capture_url_arg_content()
                     args.extend(follow_arg)
+                if self.token.type == "DEPTH":
+                    depth_arg = self.capture_url_arg_content()
+                    args.extend(depth_arg)
             elif self.token.type == "WXPATH":
                 # Nested wxpath: url( url('...')//a/@href ) or url( /url(...) )
-                # Use capture_url_arg_content to handle nested wxpath and xpath
-                args = self.capture_url_arg_content()
+                # NOTE: We used to use capture_url_arg_content to handle nested wxpath and xpath
+                # args = self.capture_url_arg_content()
+                args = self.nud()
             else:
                 # Simple xpath argument: url(//a/@href)
                 # Could still contain nested wxpath, so use capture_url_arg_content
@@ -489,8 +532,18 @@ class Parser:
 
         return _specify_call_types(func_name, args)
 
-
 def _specify_call_types(func_name: str, args: list) -> Call | Segments:
+    """
+    Specify the type of a call based on the function name and arguments.
+    TODO: Provide example wxpath expressions for each call type.
+    
+    Args:
+        func_name: The name of the function.
+        args: The arguments of the function.
+
+    Returns:
+        Call | Segments: The type of the call.
+    """
     if func_name == "url":
         if len(args) == 1:
             if isinstance(args[0], String):
@@ -500,17 +553,33 @@ def _specify_call_types(func_name: str, args: list) -> Call | Segments:
             else:
                 raise ValueError(f"Unknown argument type: {type(args[0])}")
         elif len(args) == 2:
-            if isinstance(args[0], String) and isinstance(args[1], Xpath):
+            arg0, arg1 = args
+            if isinstance(arg0, String) and isinstance(arg1, Xpath):
+                # Example: url('...', follow=//a/@href)
                 return UrlCrawl(func_name, args)
-            elif isinstance(args[0], UrlLiteral) and isinstance(args[1], Xpath):
+            elif isinstance(arg0, String) and isinstance(arg1, Integer):
+                # Example: url('...', depth=2)
+                return UrlLiteral(func_name, args)
+            elif isinstance(arg0, UrlLiteral) and isinstance(arg1, Xpath):
                 args.append(UrlQuery('url', [ContextItem()]))
                 return Segments(args)
-            elif isinstance(args[0], (Segments, list)) and isinstance(args[1], Xpath):
-                segs = args[0]
-                segs.append(args[1])
+            elif isinstance(arg0, (Segments, list)) and isinstance(arg1, Xpath):
+                segs = arg0
+                segs.append(arg1)
                 return Segments(segs)
             else:
                 raise ValueError(f"Unknown arguments: {args}")
+        elif len(args) == 3:
+            arg0, arg1, arg2 = args
+            if (isinstance(arg0, String) and (
+                (isinstance(arg1, Xpath) and isinstance(arg2, Integer)) or
+                (isinstance(arg1, Integer) and isinstance(arg2, Xpath))
+            )):
+                # Example: url('...', follow=//a/@href, depth=2)
+                # Example: url('...', depth=2, follow=//a/@href)
+                return UrlCrawl(func_name, args)
+            else:
+                raise ValueError(f"Unknown arguments: {args}")
         else:
             raise ValueError(f"Unknown arguments: {args}")
     elif func_name == "/url" or func_name == "//url":

wxpath/core/runtime/engine.py

@@ -18,7 +18,7 @@ from wxpath.core.models import (
     ProcessIntent,
 )
 from wxpath.core.ops import get_operator
-from wxpath.core.parser import Binary, Segment, Segments
+from wxpath.core.parser import Binary, Depth, Segment, Segments
 from wxpath.core.runtime.helpers import parse_html
 from wxpath.hooks.registry import FetchContext, get_hooks
 from wxpath.http.client.crawler import Crawler
@@ -158,6 +158,25 @@ class WXPathEngine(HookedEngineBase):
         if allow_redirects:
             self.allowed_response_codes |= {301, 302, 303, 307, 308}
 
+    def _get_max_depth(self, bin_or_segs: Binary | Segments, max_depth: int) -> int:
+        """Get the maximum crawl depth for a given expression. Will find a Depth
+        argument at the beginning of the expression and return its value. Otherwise, returns the
+        max_depth value provided.
+        TODO: There has to be a better way to do this.
+        """
+        if isinstance(bin_or_segs, Binary):
+            if hasattr(bin_or_segs.left, 'func') == 'url': 
+                depth_arg = [arg for arg in bin_or_segs.left.args if isinstance(arg, Depth)][0]
+                return int(depth_arg.value)
+            elif hasattr(bin_or_segs.right, 'func') == 'url':
+                depth_arg = [arg for arg in bin_or_segs.right.args if isinstance(arg, Depth)][0]
+                return int(depth_arg.value)
+        elif isinstance(bin_or_segs, Segments):
+            depth_arg = [arg for arg in bin_or_segs[0].args if isinstance(arg, Depth)]
+            if depth_arg:
+                return int(depth_arg[0].value)
+        return max_depth
+
     async def run(
             self, 
             expression: str, 
@@ -170,6 +189,17 @@ class WXPathEngine(HookedEngineBase):
         Builds and drives a BFS-like crawl pipeline that honors robots rules,
         throttling, and hook callbacks while walking the web graph.
 
+        NOTES ON max_depth:
+        If depth is provided in the expression, it will be used to limit the depth of the
+        crawl. If depth is provided in the expression and max_depth is provided as an argument
+        to `run`, the inline depth in the expression will take precedence.
+
+        Currently, max_depth control flow logic is detected and executed in the
+        engine. In the future, the operation handlers (ops.py) could be responsible for 
+        detecting max_depth, and sending a terminal intent to the engine. It's also possible 
+        that the depth terminals are relative to the current depth (i.e. `url(//xpath, depth=2)`
+        implies crawling only the next 2 levels). This is not yet supported.
+
         Args:
             expression: WXPath expression string to evaluate.
             max_depth: Maximum crawl depth to follow for url hops.
@@ -179,7 +209,9 @@ class WXPathEngine(HookedEngineBase):
             Extracted values produced by the expression (HTML elements or
             wxpath-specific value types).
         """
-        segments = parser.parse(expression)
+        bin_or_segs = parser.parse(expression)
+
+        max_depth = self._get_max_depth(bin_or_segs, max_depth)
 
         queue: asyncio.Queue[CrawlTask] = asyncio.Queue()
         inflight: dict[str, CrawlTask] = {}
@@ -223,7 +255,7 @@ class WXPathEngine(HookedEngineBase):
             seed_task = CrawlTask(
                 elem=None,
                 url=None,
-                segments=segments,
+                segments=bin_or_segs,
                 depth=-1,
                 backlink=None,
             )
@@ -307,6 +339,7 @@ class WXPathEngine(HookedEngineBase):
                     base_url=task.url,
                     backlink=task.backlink,
                     depth=task.depth,
+                    response=resp
                 )
 
                 elem = await self.post_parse_hooks(elem, task)

wxpath/core/runtime/helpers.py

@@ -6,7 +6,7 @@ from wxpath.util.logging import get_logger
 log = get_logger(__name__)
 
 
-def parse_html(content, base_url=None, **elem_kv_pairs) -> html.HtmlElement:
+def parse_html(content, base_url=None, response=None, **elem_kv_pairs) -> html.HtmlElement:
     elem = etree.HTML(content, parser=patches.html_parser_with_xpath3, base_url=base_url)
     if base_url:
         elem.getroottree().docinfo.URL = base_url  # make base-uri() work
@@ -14,12 +14,15 @@ def parse_html(content, base_url=None, **elem_kv_pairs) -> html.HtmlElement:
         elem.set("{http://www.w3.org/XML/1998/namespace}base", base_url)
         elem.base_url = base_url  # sets both attribute and doc-level URL
     
-    # NOTE: some pages may have multiple root elements, i.e. 
+    if response:
+        elem.response = response
+        elem.getroottree().getroot().response = response
+    # NOTE: some pages may have multiple root elements, i.e.
     # len(elem.itersiblings()) > 0 AND elem.getparent() is None. 
     # This breaks elementpath. If elem has siblings, recreate the 
     # root element and only the root element.
     if len(list(elem.itersiblings())) > 0:
-        elem = detach_html_root(elem, base_url) 
+        elem = detach_html_root(elem, base_url)
 
     for k, v in elem_kv_pairs.items():
         elem.set(k, str(v))

wxpath/http/client/__init__.py

@@ -5,5 +5,5 @@ from wxpath.http.client.response import Response
 __all__ = [
     "Crawler",
     "Request",
-    "Response"
+    "Response",
 ]

wxpath/http/client/crawler.py

@@ -71,6 +71,7 @@ class Crawler:
         *,
         headers: dict | None = None,
         proxies: dict | None = None,
+        verify_ssl: bool | None = None,
         retry_policy: RetryPolicy | None = None,
         throttler: AbstractThrottler | None = None,
         auto_throttle_target_concurrency: float = None,
@@ -82,6 +83,9 @@ class Crawler:
 
         self.concurrency = concurrency if concurrency is not None else cfg.concurrency
         self.per_host = per_host if per_host is not None else cfg.per_host
+        self._verify_ssl = verify_ssl if verify_ssl is not None else getattr(
+            cfg, "verify_ssl", True
+        )
 
         timeout = timeout if timeout is not None else cfg.timeout
         self._timeout = aiohttp.ClientTimeout(total=timeout)
@@ -141,7 +145,11 @@ class Crawler:
         """Construct an `aiohttp.ClientSession` with tracing and pooling."""
         trace_config = build_trace_config(self._stats)
         # Need to build the connector as late as possible as it requires the loop
-        connector = aiohttp.TCPConnector(limit=self.concurrency*2, ttl_dns_cache=300)
+        connector = aiohttp.TCPConnector(
+            limit=self.concurrency * 2,
+            ttl_dns_cache=300,
+            ssl=self._verify_ssl,
+        )
         return get_async_session(
             headers=self._headers,
             timeout=self._timeout,
@@ -274,22 +282,26 @@ class Crawler:
                     else:
                         log.info("[CACHE MISS]", extra={"req.url": req.url, "resp.url": resp.url})
 
+                    _start = time.monotonic()
                     body = await resp.read()
 
-                    latency = time.monotonic() - start
+                    end = time.monotonic()
+                    latency = end - _start
                     self.throttler.record_latency(host, latency)
 
                     if self.retry_policy.should_retry(req, response=resp):
                         await self._retry(req)
                         return None
 
-                    return Response(req, resp.status, body, dict(resp.headers))
+                    return Response(req, resp.status, body, dict(resp.headers),
+                                    request_start=_start, response_end=end)
             except asyncio.CancelledError:
                 # Normal during shutdown / timeout propagation
                 log.debug("cancelled error", extra={"url": req.url})
                 raise
             except Exception as exc:
-                latency = time.monotonic() - start
+                end = time.monotonic()
+                latency = end - start
                 self.throttler.record_latency(host, latency)
 
                 if self.retry_policy.should_retry(req, exception=exc):
@@ -297,7 +309,7 @@ class Crawler:
                     return None
                 
                 log.error("request failed", extra={"url": req.url}, exc_info=exc)
-                return Response(req, 0, b"", error=exc)
+                return Response(req, 0, b"", error=exc, request_start=start, response_end=end)
 
     async def _retry(self, req: Request) -> None:
         """Reschedule a request according to the retry policy."""

wxpath/http/client/response.py

@@ -1,4 +1,3 @@
-# wxpath/http/response.py
 from dataclasses import dataclass, field
 from typing import Optional
 
@@ -12,3 +11,10 @@ class Response:
     body: bytes
     headers: dict[str, str] | None = None
     error: Optional[Exception] = field(default=None, kw_only=True)
+
+    request_start: float | None = None
+    response_end: float | None = None
+
+    @property
+    def latency(self) -> float:
+        return self.response_end - self.request_start

wxpath/http/policy/retry.py

@@ -19,13 +19,13 @@ class RetryPolicy:
 
         if request.max_retries is not None and request.retries >= request.max_retries:
             return False
-        
+
         if request.retries >= self.max_retries:
             return False
 
         if response is not None and response.status in self.retry_statuses:
             return True
-        
+
         if exception is not None:
             return True
 

wxpath/integrations/langchain/examples/basic_rag.py

@@ -0,0 +1,85 @@
+
+# pip install langchain langchain-ollama langchain-chroma chromadb 
+from langchain_chroma import Chroma
+from langchain_core.output_parsers import StrOutputParser
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.runnables import RunnablePassthrough
+from langchain_ollama import ChatOllama, OllamaEmbeddings
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+
+from wxpath.integrations.langchain.loader import WXPathLoader
+
+# ------------------------------------------------------------------
+# STEP 1: Load & Embed (Same as before)
+# ------------------------------------------------------------------
+print("🕷️  Crawling with wxpath...")
+loader = WXPathLoader(
+    expression="""
+    url('https://docs.python.org/3/library/argparse.html', 
+    follow=//a/@href[contains(., 'argparse')])
+      /map{
+          'text': string-join(//div[@role='main']//text()),
+          'source': string(base-uri(.))
+      }
+    """,
+    max_depth=1
+)
+docs = loader.load()
+
+print("🔪 Splitting and Embedding...")
+text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
+splits = text_splitter.split_documents(docs)
+
+vectorstore = Chroma.from_documents(
+    documents=splits, 
+    # Must use model that support embeddings (`ollama pull nomic-embed-text`)
+    embedding=OllamaEmbeddings(model="nomic-embed-text"),
+    collection_name="wxpath"
+)
+retriever = vectorstore.as_retriever()
+
+# ------------------------------------------------------------------
+# STEP 2: Define Components 
+# ------------------------------------------------------------------
+
+# A helper to join retrieved documents into a single string
+def format_docs(docs):
+    return "\n\n".join(doc.page_content for doc in docs)
+
+# The Prompt (Standard RAG template)
+template = """You are an assistant for question-answering tasks. 
+Use the following pieces of retrieved context to answer the question. 
+If you don't know the answer, just say that you don't know. 
+Use three sentences maximum and keep the answer concise.
+
+Context: {context}
+
+Question: {question}
+
+Answer:"""
+prompt = ChatPromptTemplate.from_template(template)
+
+# The Model
+llm = ChatOllama(model="gemma3")
+
+# ------------------------------------------------------------------
+# STEP 3: Build the Chain with LCEL
+# ------------------------------------------------------------------
+# The pipe operator (|) passes output from one component to the next.
+rag_chain = (
+    {"context": retriever | format_docs, "question": RunnablePassthrough()}
+    | prompt
+    | llm
+    | StrOutputParser()
+)
+
+# ------------------------------------------------------------------
+# STEP 4: Invoke
+# ------------------------------------------------------------------
+query = "How do I add arguments in argparse?"
+print(f"\n❓ Question: {query}")
+
+# The chain returns a string directly because of StrOutputParser
+response = rag_chain.invoke(query)
+
+print(f"\n🤖 Ollama Answer:\n{response}")