wxpath 0.4.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

wxpath/integrations/langchain/examples/basic_rag.py

@@ -0,0 +1,85 @@
+
+# pip install langchain langchain-ollama langchain-chroma chromadb 
+from langchain_chroma import Chroma
+from langchain_core.output_parsers import StrOutputParser
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.runnables import RunnablePassthrough
+from langchain_ollama import ChatOllama, OllamaEmbeddings
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+
+from wxpath.integrations.langchain.loader import WXPathLoader
+
+# ------------------------------------------------------------------
+# STEP 1: Load & Embed (Same as before)
+# ------------------------------------------------------------------
+print("🕷️  Crawling with wxpath...")
+loader = WXPathLoader(
+    expression="""
+    url('https://docs.python.org/3/library/argparse.html', 
+    follow=//a/@href[contains(., 'argparse')])
+      /map{
+          'text': string-join(//div[@role='main']//text()),
+          'source': string(base-uri(.))
+      }
+    """,
+    max_depth=1
+)
+docs = loader.load()
+
+print("🔪 Splitting and Embedding...")
+text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
+splits = text_splitter.split_documents(docs)
+
+vectorstore = Chroma.from_documents(
+    documents=splits, 
+    # Must use model that support embeddings (`ollama pull nomic-embed-text`)
+    embedding=OllamaEmbeddings(model="nomic-embed-text"),
+    collection_name="wxpath"
+)
+retriever = vectorstore.as_retriever()
+
+# ------------------------------------------------------------------
+# STEP 2: Define Components 
+# ------------------------------------------------------------------
+
+# A helper to join retrieved documents into a single string
+def format_docs(docs):
+    return "\n\n".join(doc.page_content for doc in docs)
+
+# The Prompt (Standard RAG template)
+template = """You are an assistant for question-answering tasks. 
+Use the following pieces of retrieved context to answer the question. 
+If you don't know the answer, just say that you don't know. 
+Use three sentences maximum and keep the answer concise.
+
+Context: {context}
+
+Question: {question}
+
+Answer:"""
+prompt = ChatPromptTemplate.from_template(template)
+
+# The Model
+llm = ChatOllama(model="gemma3")
+
+# ------------------------------------------------------------------
+# STEP 3: Build the Chain with LCEL
+# ------------------------------------------------------------------
+# The pipe operator (|) passes output from one component to the next.
+rag_chain = (
+    {"context": retriever | format_docs, "question": RunnablePassthrough()}
+    | prompt
+    | llm
+    | StrOutputParser()
+)
+
+# ------------------------------------------------------------------
+# STEP 4: Invoke
+# ------------------------------------------------------------------
+query = "How do I add arguments in argparse?"
+print(f"\n❓ Question: {query}")
+
+# The chain returns a string directly because of StrOutputParser
+response = rag_chain.invoke(query)
+
+print(f"\n🤖 Ollama Answer:\n{response}")

wxpath/integrations/langchain/examples/rolling_window_rag.py

@@ -0,0 +1,218 @@
+"""
+Rolling Window RAG Example
+
+This examples demonstrates how to use a rolling window of news articles as context.
+
+More importantly, it demonstrates complex string cleanup, metadata extraction, and other
+real-world challenges of building a RAG application.
+
+This script assumes you have gemma3 installed and your machine is capable of running a 32k
+token model.
+"""
+import asyncio
+import datetime
+import threading
+from collections import deque
+from operator import itemgetter
+from typing import List
+
+from langchain_core.callbacks import CallbackManagerForRetrieverRun
+from langchain_core.documents import Document
+from langchain_core.retrievers import BaseRetriever
+
+from wxpath import wxpath_async
+
+# If you have the cache dependency installed, you can enable it:
+# wxpath.settings.CACHE_SETTINGS.enabled = True
+
+# ------------------------------------------------------------------
+# 1. The Rolling Buffer (The "Context Window")
+# ------------------------------------------------------------------
+class RollingNewsBuffer(BaseRetriever):
+    capacity: int = 100
+    
+    # Define as PrivateAttrs so Pydantic ignores them for validation
+    _buffer: deque
+    _seen_urls: set 
+    _lock: threading.Lock 
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self._buffer = deque(maxlen=self.capacity)
+        self._seen_urls = set()
+        self._lock = threading.Lock()
+    
+    def add_document(self, doc: Document):
+        """Thread-safe add with url cleanup on eviction."""
+        with self._lock:
+            # Check if we are about to evict an item (buffer full)
+            if len(self._buffer) == self._buffer.maxlen:
+                # We must manually find what is being removed to clean up seen_urls
+                # Note: deque[0] is the one about to be popped when appending
+                oldest_doc = self._buffer[0] 
+                oldest_url = oldest_doc.metadata.get("url")
+                if oldest_url in self._seen_urls:
+                    self._seen_urls.remove(oldest_url)
+
+            self._buffer.append(doc)
+            self._seen_urls.add(doc.metadata["url"])
+
+    def is_seen(self, url: str) -> bool:
+        """Thread-safe check."""
+        with self._lock:
+            return url in self._seen_urls
+
+    def _get_relevant_documents(
+        self, query: str, *, run_manager: CallbackManagerForRetrieverRun = None
+    ) -> List[Document]:
+        """
+        Thread-safe read.
+        """
+        with self._lock:
+            # Create a snapshot list while locked to prevent iteration crash
+            snapshot = list(self._buffer)
+            
+        print(f"📰 Context Retrieval: Returning {len(snapshot)} docs for query: {query}")
+        return snapshot
+    
+# ------------------------------------------------------------------
+# 2. The Background Crawler (The Producer)
+# ------------------------------------------------------------------
+async def continuous_crawl(buffer: RollingNewsBuffer):
+    """
+    Constantly crawls Newsweek and feeds the buffer.
+    """
+    print("🕷️  Crawler started...")
+    
+    # Example Expression: deep crawl of newsweek
+    expression = """
+    url('https://www.newsweek.com/')
+      ///url(
+          //a/@href[starts-with(., '/') or starts-with(., './') or contains(., 'newsweek.com')]
+      )
+      /map{
+          'title': //h1/text()[1] ! string(.),
+          'text': string-join(//article//p/text()),
+          'url': string(base-uri(.)),
+          'pubDate': //meta[@name='article:modified_time']/@content[1] ! string(.)
+      }
+    """
+
+    # Infinite loop to restart crawl if it finishes, or run continuously
+    while True:
+        try:
+            # We use the async generator to stream results as they are found
+            async for item in wxpath_async(expression, max_depth=1):
+                item = item._map
+                url = item.get('url')
+                # Check seen status safely before doing processing work
+                if not url or buffer.is_seen(url):
+                    continue
+
+                # Convert wxpath dict to LangChain Document
+                text_content = item.get('text', '')
+                # Basic cleaning (optional)
+                if isinstance(text_content, list):
+                    text_content = " ".join(text_content)
+                
+                if not text_content: 
+                    continue
+
+                title = item.get('title')
+                if not title:
+                    title = ''
+
+                if isinstance(title, list):
+                    title = " ".join(title)
+
+                pub_date = item.get('pubDate')
+                if not pub_date:
+                    pub_date = str(datetime.date.today())
+
+                text_content = ("Title: " + title + 
+                                "\nPublished: " + pub_date + "\n" + 
+                                text_content)
+    
+                doc = Document(
+                    page_content=text_content,
+                    metadata={"title": item.get('title'), 
+                              "url": item.get('url'), 
+                              "pubDate": item.get('pubDate')}
+                )
+                
+                # PUSH TO BUFFER (Oldest gets evicted automatically if full)
+                buffer.add_document(doc)
+                print(f"📰 Added: {title[:30]}... (Buffer size: {len(buffer._buffer)})")
+                print(f"\tArticle text: {doc.page_content[:100]}...")
+                print()
+            # Rate limit slightly to be polite
+            await asyncio.sleep(60) 
+            
+        except Exception as e:
+            print(f"⚠️ Crawler error: {e}. Restarting in 10s...")
+            await asyncio.sleep(10)
+
+
+def debug_print_prompt(prompt_value):
+    print("\n" + "="*40)
+    print("📢 FULL PROMPT SENT TO LLM:")
+    print("="*40)
+    print(prompt_value.to_string()) # This prints the exact text
+    print("="*40 + "\n")
+    return prompt_value
+
+if __name__ == "__main__":
+    # Initialize the Rolling Buffer
+    retriever = RollingNewsBuffer(capacity=100)
+
+    # Start Crawler in a background thread so it doesn't block the Chat
+    def start_background_loop(loop):
+        asyncio.set_event_loop(loop)
+        loop.run_until_complete(continuous_crawl(retriever))
+
+    crawler_loop = asyncio.new_event_loop()
+    t = threading.Thread(target=start_background_loop, args=(crawler_loop,), daemon=True)
+    t.start()
+
+    import time
+
+    from langchain_core.prompts import ChatPromptTemplate
+    from langchain_ollama import ChatOllama
+
+    # Setup standard RAG chain
+    llm = ChatOllama(model="gemma3", num_ctx=32768)
+    prompt = ChatPromptTemplate.from_template(
+        "Answer based ONLY on the following news:\n\n{context}\n\nQuestion: {question}\n\n"
+        "DO NOT include generic Newsweek-administrative articles like 'Corrections', "
+        "'Company Info', 'Subscribe', Opinions', 'Press Releases', 'Editorials', etc. in your "
+        "analysis or answers. Answer the question using the non-Newsweek-related news provided. "
+        "You will be penalized for including old or undated news in your answer. If asked for "
+        "overviews or summaries, split news items into paragraphs and provide a summary of each "
+        "news item."
+    )
+    
+    def format_docs(docs):
+        slice_of_news = "\n\n".join([d.page_content[:1000] for d in docs]) # Truncate for demo
+        print(f"📰 Latest news char length: {len(slice_of_news)}")
+        return slice_of_news
+
+    chain = (
+        {
+            # FIX: Use itemgetter so retriever gets a string, not a dict
+            "context": itemgetter("question") | retriever | format_docs, 
+            "question": itemgetter("question")
+        }
+        | prompt
+        | debug_print_prompt
+        | llm
+    )
+
+    # Simulate querying constantly while buffer fills in background
+    print("⏳ Waiting for crawler to gather some data...")
+    time.sleep(10) 
+    
+    while True:
+        query = input("Press Enter to ask about current news (or Ctrl+C to quit)...")
+        print(f"\nQuery: {query}\nThinking... 🤔")
+        response = chain.invoke({"question": query})
+        print(response.content)

wxpath/integrations/langchain/loader.py

@@ -0,0 +1,60 @@
+from typing import Iterator
+
+from elementpath.xpath_tokens import XPathMap
+from langchain_core.document_loaders import BaseLoader
+from langchain_core.documents import Document
+
+import wxpath
+
+
+class WXPathLoader(BaseLoader):
+    """A LangChain loader for wxpath queries. 
+    
+    For more complex examples, see the examples directory.
+    Best practice would be to subclass the loader and override the _prep_doc method.
+    For example:
+    ```python
+    class MyWXPathLoader(WXPathLoader):
+        def _prep_doc(self, item: (XPathMap | dict)) -> Document:
+            # Custom processing here
+            return super()._prep_doc(item)
+    ```
+    """
+    
+    def __init__(self, expression: str, max_depth: int = 1):
+        self.expression = expression
+        self.max_depth = max_depth
+
+    def _prep_doc(self, item: (XPathMap | dict)) -> Document:
+        
+        if isinstance(item, dict):
+            content = item.pop("text", str(item)) # Fallback if no "text" key
+        else:
+            content = item._map.pop("text", str(item._map)) # Fallback if no "text" key
+            item = item._map
+
+        return Document(
+            page_content=content,
+            metadata=item # Remaining keys go here (url, title, etc.)
+        )
+
+    def lazy_load(self) -> Iterator[Document]:
+        """
+        Lazy load documents from the wxpath query.
+        Each item yielded by wxpath becomes a LangChain Document.
+        """
+        # wxpath_async_blocking_iter allows iteration in sync environments
+        results = wxpath.wxpath_async_blocking_iter(
+            self.expression, 
+            max_depth=self.max_depth
+        )
+
+        for item in results:
+            yield self._prep_doc(item)
+
+    async def alazy_load(self):
+        async for item in wxpath.wxpath_async(
+            self.expression, 
+            max_depth=self.max_depth
+        ):
+            yield self._prep_doc(item)

wxpath/patches.py

@@ -1,7 +1,17 @@
+import urllib.parse
+
 import elementpath
+from elementpath import XPathContext, XPathFunction
 from elementpath.xpath3 import XPath3Parser
 from lxml import etree, html
 
+from wxpath.http.client import Response as Response
+from wxpath.util.cleaners import main_text_extractor
+from wxpath.util.common_paths import XPATH_PATH_TO_EXTERNAL_LINKS, XPATH_PATH_TO_INTERNAL_LINKS
+from wxpath.util.logging import get_logger
+
+log = get_logger(__name__)
+
 
 def html_element_repr(self):
     return (f"HtmlElement(tag={self.tag}, "
@@ -13,14 +23,18 @@ html.HtmlElement.__repr__ = html_element_repr
 
 
 class XPath3Element(etree.ElementBase):
-    def xpath3(self, expr, **kwargs):
+    def __init__(self, tag, attrib=None, nsmap=None, **extra):
+        super().__init__(tag, attrib, nsmap, **extra)
+        self.response = None  # type: Response | None
+        
+    def xpath3(self, expr, request=None, **kwargs):
         """
         Evaluate an XPath 3 expression using elementpath library,
         returning the results as a list.
         """
-        kwargs.setdefault("parser", XPath3Parser)
+        kwargs.setdefault("parser", WXPathParser)
         kwargs.setdefault(
-            "uri", 
+            "uri",
             getattr(self.getroottree().docinfo, "URL", None) or self.get("base_url")
         )
         return elementpath.select(self, expr, **kwargs)
@@ -51,7 +65,8 @@ class XPath3Element(etree.ElementBase):
     @depth.setter
     def depth(self, value):
         self.set("depth", str(value))
-    
+
+
 # Create and register custom parser that returns XPath3Element instances
 lookup = etree.ElementDefaultClassLookup(element=XPath3Element)
 parser = etree.HTMLParser()
@@ -60,4 +75,199 @@ parser.set_element_class_lookup(lookup)
 
 # Expose parser for use in parse_html
 html_parser_with_xpath3 = parser
-html.HtmlElement.xpath3 = XPath3Element.xpath3
+html.HtmlElement.xpath3 = XPath3Element.xpath3
+
+# --- WXPATH functions ---
+WX_NAMESPACE = "http://wxpath.dev/ns"
+
+class WXPathParser(XPath3Parser):
+    """Custom parser that includes wxpath-specific functions."""
+    pass
+
+# 2. Register the namespace mapping globally on the parser class
+WXPathParser.DEFAULT_NAMESPACES['wx'] = WX_NAMESPACE
+
+# 2. Helper to register functions easily
+def register_wxpath_function(name, nargs=None, **kwargs):
+    """Registers a function token on the custom parser."""
+    
+    # Define the token on the class (this registers the symbol)
+    # Check if this is a prefixed function (e.g. 'wx:depth')
+    if ':' in name:
+        prefix, local_name = name.split(':', 1)
+        kwargs['prefix'] = prefix
+        # kwargs['namespace'] = WX_NAMESPACE
+        name = local_name
+        
+    # Register the token symbol
+    # WXPathParser.function(name, nargs=nargs, **kwargs)
+    # Register the token symbol and capture the created class
+    token_class = WXPathParser.function(name, nargs=nargs, **kwargs)
+    # Return a decorator to define the 'evaluate' method
+    def decorator(func):
+        # @WXPathParser.method(name)
+        # def evaluate(self, context=None):
+        #     # 'self' is the Token instance. 
+        #     # 'self.get_argument(context, index)' evaluates arguments.
+        #     return func(self, context)
+        # return evaluate
+        token_class.evaluate = func
+        return func
+    return decorator
+
+
+class XPathContextRequired(Exception):
+    message = ('XPathContext is required. This usually arises when you call '
+               'the function without a preceding axes expression ("/")')
+    def __init__(self, *args):
+        super().__init__(self.message, *args)
+   
+
+def _get_root(context: XPathContext):
+    if context is None:
+        raise XPathContextRequired
+    
+    if not hasattr(context.item, 'elem'):
+        return context.item.parent.elem.getroottree().getroot()
+    return context.item.elem.getroottree().getroot()
+
+
+@register_wxpath_function('wx:depth', nargs=0)
+def wx_depth(_: XPathFunction, context: XPathContext):
+    if context is None:
+        raise XPathContextRequired
+
+    root = _get_root(context)
+
+    depth = root.get('depth')
+    return int(depth) if depth is not None else 0
+
+
+@register_wxpath_function('wx:backlink', nargs=0)
+def wx_backlink(_: XPathFunction, context: XPathContext):
+    if context is None:
+        raise XPathContextRequired
+
+    item = context.item.elem
+    if item is None:
+        return ''
+    return item.get('backlink') or ''
+
+
+@register_wxpath_function('wx:current-url', nargs=0)
+def wx_current_url(_: XPathFunction, context: XPathContext):
+    if context is None:
+        raise XPathContextRequired
+
+    item = context.item.elem
+    if item is None:
+        return ''
+    return item.base_url
+
+
+@register_wxpath_function('wx:elapsed', nargs=0)
+@register_wxpath_function('wx:fetch-time', nargs=0)
+def wx_fetch_time(_: XPathFunction, context: XPathContext):
+    if context is None:
+        raise XPathContextRequired
+
+    item = context.item.elem
+    if item is None:
+        return ''
+    resp = item.response # type: Response
+    return resp.latency
+
+
+# @register_wxpath_function('wx:status-code', nargs=0)
+@register_wxpath_function('wx:status-code', nargs=0)
+def wx_status_code(_: XPathFunction, context: XPathContext) -> int:
+    if context is None:
+        raise XPathContextRequired
+
+    item = context.item.elem
+    if item is None:
+        return ''
+    
+    resp = item.response # type: Response
+    return resp.status
+
+
+@register_wxpath_function('wx:elem', nargs=0)
+def wx_elem(_: XPathFunction, context: XPathContext):
+    if context is None:
+        raise XPathContextRequired
+
+    item = context.item.elem
+    if item is None:
+        return ''
+    return item
+
+
+def _get_root_domain(base_url: str) -> str:
+    parsed_url = urllib.parse.urlparse(base_url)
+
+    netloc = parsed_url.netloc
+    parts = netloc.split('.')
+    root_domain = netloc
+
+    if len(parts) > 2:
+        # Heuristic: If the last part is 2 chars (uk, au) and 2nd to last is < 4 (co, com, org)
+        # It's likely a compound TLD like co.uk. This isn't perfect but better than [-2:].
+        if len(parts[-1]) == 2 and len(parts[-2]) <= 3:
+             root_domain = ".".join(parts[-3:]) # grab bbc.co.uk
+        else:
+             # grab books.toscrape.com -> toscrape.com
+             root_domain = ".".join(parts[-2:]) 
+
+    return root_domain
+
+
+@register_wxpath_function('wx:internal-links', nargs=0)
+def wx_internal_links(_: XPathFunction, context: XPathContext):
+    """
+    Returns a list of internal links.
+    Allows for false positives.
+    """
+    if context is None:
+        raise XPathContextRequired
+
+    item = context.item.elem
+    if item is None:
+        return ''
+    
+    root_domain = _get_root_domain(item.base_url)
+    _path = XPATH_PATH_TO_INTERNAL_LINKS.format(root_domain)
+    return item.xpath3(_path)
+
+
+@register_wxpath_function('wx:external-links', nargs=0)
+def wx_external_links(_: XPathFunction, context: XPathContext):
+    """
+    Returns a list of external links.
+    """
+    if context is None:
+        raise XPathContextRequired
+
+    item = context.item.elem
+    if item is None:
+        return ''
+    
+    root_domain = _get_root_domain(item.base_url)
+    _path = XPATH_PATH_TO_EXTERNAL_LINKS.format(root_domain)
+    return item.xpath3(_path)
+
+
+@register_wxpath_function('wx:main-article-text', nargs=0)
+def wx_main_article_text(_: XPathFunction, context: XPathContext):
+    if context is None:
+        raise XPathContextRequired
+    
+    item = context.item.elem
+    if item is None:
+        return ''
+    
+    try:
+        return main_text_extractor(item)
+    except Exception:
+        log.exception('Failed to extract main article text')
+        return ''

wxpath/settings.py

@@ -54,10 +54,12 @@ SETTINGS = {
                 'concurrency': 16,
                 'per_host': 8,
                 'timeout': 15,
+                'verify_ssl': True,
                 'headers': {
                     "User-Agent": ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)" 
                                    "AppleWebKit/537.36 (KHTML, like Gecko) "
-                                   "Chrome/142.0.0.0 Safari/537.36")},
+                                   "Chrome/142.0.0.0 Safari/537.36")
+                },
                 'proxies': None,
                 'auto_throttle_target_concurrency': None,
                 'auto_throttle_start_delay': 0.25,