langroid 0.31.1__py3-none-any.whl → 0.33.3__py3-none-any.whl

langroid/parsing/routing.py

@@ -1,36 +0,0 @@
-import re
-from typing import Optional, Tuple
-
-
-def parse_addressed_message(
-    content: str, addressing: str = "@"
-) -> Tuple[Optional[str], str]:
-    """In a message-string containing possibly multiple @<recipient> occurrences,
-    find the last addressee and extract their name,
-    and the message content following it.
-
-    E.g. "thank you @bob, now I will ask @alice again. @alice, where is the mirror?" =>
-    ("alice", "where is the mirror?")
-
-    Args:
-        content (str): The message content.
-        addressing (str, optional): The addressing character. Defaults to "@".
-
-    Returns:
-        Tuple[Optional[str], str]:
-        A tuple containing the last addressee and the subsequent message content.
-    """
-    # Regex to find all occurrences of the pattern
-    pattern = re.compile(rf"{re.escape(addressing)}(\w+)[^\w]")
-    matches = list(pattern.finditer(content))
-
-    if not matches:
-        return None, content  # No addressee found, return None and original content
-
-    # Get the last match
-    last_match = matches[-1]
-    last_addressee = last_match.group(1)
-    # Extract content after the last addressee
-    content_after = content[last_match.end() :].strip()
-
-    return last_addressee, content_after

langroid/parsing/search.py

@@ -1,275 +0,0 @@
-"""
-Utils to search for close matches in (a list of) strings.
-Useful for retrieval of docs/chunks relevant to a query, in the context of
-Retrieval-Augmented Generation (RAG), and SQLChat (e.g., to pull relevant parts of a
-large schema).
-See tests for examples: tests/main/test_string_search.py
-"""
-
-import difflib
-from typing import List, Tuple
-
-from nltk.corpus import stopwords
-from nltk.stem import WordNetLemmatizer
-from nltk.tokenize import RegexpTokenizer
-from rank_bm25 import BM25Okapi
-from thefuzz import fuzz, process
-
-from langroid.mytypes import Document
-
-from .utils import download_nltk_resource
-
-
-def find_fuzzy_matches_in_docs(
-    query: str,
-    docs: List[Document],
-    docs_clean: List[Document],
-    k: int,
-    words_before: int | None = None,
-    words_after: int | None = None,
-) -> List[Tuple[Document, float]]:
-    """
-    Find approximate matches of the query in the docs and return surrounding
-    characters.
-
-    Args:
-        query (str): The search string.
-        docs (List[Document]): List of Document objects to search through.
-        docs_clean (List[Document]): List of Document objects with cleaned content.
-        k (int): Number of best matches to return.
-        words_before (int|None): Number of words to include before each match.
-            Default None => return max
-        words_after (int|None): Number of words to include after each match.
-            Default None => return max
-
-    Returns:
-        List[Tuple[Document,float]]: List of (Document, score) tuples.
-    """
-    if len(docs) == 0:
-        return []
-    best_matches = process.extract(
-        query,
-        [d.content for d in docs_clean],
-        limit=k,
-        scorer=fuzz.partial_ratio,
-    )
-
-    real_matches = [(m, score) for m, score in best_matches if score > 50]
-    # find the original docs that corresponding to the matches
-    orig_doc_matches = []
-    for i, (m, s) in enumerate(real_matches):
-        for j, doc_clean in enumerate(docs_clean):
-            if m in doc_clean.content:
-                orig_doc_matches.append((docs[j], s))
-                break
-    if words_after is None and words_before is None:
-        return orig_doc_matches
-    if len(orig_doc_matches) == 0:
-        return []
-    if set(orig_doc_matches[0][0].__fields__) != {"content", "metadata"}:
-        # If there are fields beyond just content and metadata,
-        # we do NOT want to create new document objects with content fields
-        # based on words_before and words_after, since we don't know how to
-        # set those other fields.
-        return orig_doc_matches
-
-    contextual_matches = []
-    for match, score in orig_doc_matches:
-        choice_text = match.content
-        contexts = []
-        while choice_text != "":
-            context, start_pos, end_pos = get_context(
-                query, choice_text, words_before, words_after
-            )
-            if context == "" or end_pos == 0:
-                break
-            contexts.append(context)
-            words = choice_text.split()
-            end_pos = min(end_pos, len(words))
-            choice_text = " ".join(words[end_pos:])
-        if len(contexts) > 0:
-            contextual_matches.append(
-                (
-                    Document(
-                        content=" ... ".join(contexts),
-                        metadata=match.metadata,
-                    ),
-                    score,
-                )
-            )
-
-    return contextual_matches
-
-
-def preprocess_text(text: str) -> str:
-    """
-    Preprocesses the given text by:
-    1. Lowercasing all words.
-    2. Tokenizing (splitting the text into words).
-    3. Removing punctuation.
-    4. Removing stopwords.
-    5. Lemmatizing words.
-
-    Args:
-        text (str): The input text.
-
-    Returns:
-        str: The preprocessed text.
-    """
-    # Ensure the NLTK resources are available
-    for resource in ["punkt", "wordnet", "stopwords"]:
-        download_nltk_resource(resource)
-
-    # Lowercase the text
-    text = text.lower()
-
-    # Tokenize the text and remove punctuation
-    tokenizer = RegexpTokenizer(r"\w+")
-    tokens = tokenizer.tokenize(text)
-
-    # Remove stopwords
-    stop_words = set(stopwords.words("english"))
-    tokens = [t for t in tokens if t not in stop_words]
-
-    # Lemmatize words
-    lemmatizer = WordNetLemmatizer()
-    tokens = [lemmatizer.lemmatize(t) for t in tokens]
-
-    # Join the words back into a string
-    text = " ".join(tokens)
-
-    return text
-
-
-def find_closest_matches_with_bm25(
-    docs: List[Document],
-    docs_clean: List[Document],
-    query: str,
-    k: int = 5,
-) -> List[Tuple[Document, float]]:
-    """
-    Finds the k closest approximate matches using the BM25 algorithm.
-
-    Args:
-        docs (List[Document]): List of Documents to search through.
-        docs_clean (List[Document]): List of cleaned Documents
-        query (str): The search query.
-        k (int, optional): Number of matches to retrieve. Defaults to 5.
-
-    Returns:
-        List[Tuple[Document,float]]: List of (Document, score) tuples.
-    """
-    if len(docs) == 0:
-        return []
-    texts = [doc.content for doc in docs_clean]
-    query = preprocess_text(query)
-
-    text_words = [text.split() for text in texts]
-
-    bm25 = BM25Okapi(text_words)
-    query_words = query.split()
-    doc_scores = bm25.get_scores(query_words)
-
-    # Get indices of top k scores
-    top_indices = sorted(range(len(doc_scores)), key=lambda i: -doc_scores[i])[:k]
-
-    # return the original docs, based on the scores from cleaned docs
-    return [(docs[i], doc_scores[i]) for i in top_indices]
-
-
-def get_context(
-    query: str,
-    text: str,
-    words_before: int | None = 100,
-    words_after: int | None = 100,
-) -> Tuple[str, int, int]:
-    """
-    Returns a portion of text containing the best approximate match of the query,
-    including b words before and a words after the match.
-
-    Args:
-    query (str): The string to search for.
-    text (str): The body of text in which to search.
-    b (int): The number of words before the query to return.
-    a (int): The number of words after the query to return.
-
-    Returns:
-    str: A string containing b words before, the match, and a words after
-        the best approximate match position of the query in the text. If no
-        match is found, returns empty string.
-    int: The start position of the match in the text.
-    int: The end position of the match in the text.
-
-    Example:
-    >>> get_context("apple", "The quick brown fox jumps over the apple.", 3, 2)
-    # 'fox jumps over the apple.'
-    """
-    if words_after is None and words_before is None:
-        # return entire text since we're not asked to return a bounded context
-        return text, 0, 0
-
-    # make sure there is a good enough match to the query
-    if fuzz.partial_ratio(query, text) < 40:
-        return "", 0, 0
-
-    sequence_matcher = difflib.SequenceMatcher(None, text, query)
-    match = sequence_matcher.find_longest_match(0, len(text), 0, len(query))
-
-    if match.size == 0:
-        return "", 0, 0
-
-    segments = text.split()
-    n_segs = len(segments)
-
-    start_segment_pos = len(text[: match.a].split())
-
-    words_before = words_before or n_segs
-    words_after = words_after or n_segs
-    start_pos = max(0, start_segment_pos - words_before)
-    end_pos = min(len(segments), start_segment_pos + words_after + len(query.split()))
-
-    return " ".join(segments[start_pos:end_pos]), start_pos, end_pos
-
-
-def eliminate_near_duplicates(passages: List[str], threshold: float = 0.8) -> List[str]:
-    """
-    Eliminate near duplicate text passages from a given list using MinHash and LSH.
-    TODO: this has not been tested and the datasketch lib is not a dependency.
-    Args:
-        passages (List[str]): A list of text passages.
-        threshold (float, optional): Jaccard similarity threshold to consider two
-                                     passages as near-duplicates. Default is 0.8.
-
-    Returns:
-        List[str]: A list of passages after eliminating near duplicates.
-
-    Example:
-        passages = ["Hello world", "Hello, world!", "Hi there", "Hello world!"]
-        print(eliminate_near_duplicates(passages))
-        # ['Hello world', 'Hi there']
-    """
-
-    from datasketch import MinHash, MinHashLSH
-
-    # Create LSH index
-    lsh = MinHashLSH(threshold=threshold, num_perm=128)
-
-    # Create MinHash objects for each passage and insert to LSH
-    minhashes = {}
-    for idx, passage in enumerate(passages):
-        m = MinHash(num_perm=128)
-        for word in passage.split():
-            m.update(word.encode("utf-8"))
-        lsh.insert(idx, m)
-        minhashes[idx] = m
-
-    unique_idxs = set()
-    for idx in minhashes.keys():
-        # Query for similar passages (including itself)
-        result = lsh.query(minhashes[idx])
-
-        # If only the passage itself is returned, it's unique
-        if len(result) == 1 and idx in result:
-            unique_idxs.add(idx)
-
-    return [passages[idx] for idx in unique_idxs]

langroid/parsing/spider.py

@@ -1,102 +0,0 @@
-from typing import List, Set, no_type_check
-from urllib.parse import urlparse
-
-from langroid.exceptions import LangroidImportError
-
-try:
-    from pydispatch import dispatcher
-    from scrapy import signals
-    from scrapy.crawler import CrawlerRunner
-    from scrapy.http import Response
-    from scrapy.linkextractors import LinkExtractor
-    from scrapy.spiders import CrawlSpider, Rule
-    from twisted.internet import defer, reactor
-except ImportError:
-    raise LangroidImportError("scrapy", "scrapy")
-
-
-@no_type_check
-class DomainSpecificSpider(CrawlSpider):  # type: ignore
-    name = "domain_specific_spider"
-
-    custom_settings = {"DEPTH_LIMIT": 1, "CLOSESPIDER_ITEMCOUNT": 20}
-
-    rules = (Rule(LinkExtractor(), callback="parse_item", follow=True),)
-
-    def __init__(self, start_url: str, k: int = 20, *args, **kwargs):  # type: ignore
-        """Initialize the spider with start_url and k.
-
-        Args:
-            start_url (str): The starting URL.
-            k (int, optional): The max desired final URLs. Defaults to 20.
-        """
-        super(DomainSpecificSpider, self).__init__(*args, **kwargs)
-        self.start_urls = [start_url]
-        self.allowed_domains = [urlparse(start_url).netloc]
-        self.k = k
-        self.visited_urls: Set[str] = set()
-
-    def parse_item(self, response: Response):  # type: ignore
-        """Extracts URLs that are within the same domain.
-
-        Args:
-            response: The scrapy response object.
-        """
-        for link in LinkExtractor(allow_domains=self.allowed_domains).extract_links(
-            response
-        ):
-            if len(self.visited_urls) < self.k:
-                self.visited_urls.add(link.url)
-                yield {"url": link.url}
-
-
-@no_type_check
-def scrapy_fetch_urls(url: str, k: int = 20) -> List[str]:
-    """Fetches up to k URLs reachable from the input URL using Scrapy.
-
-    Args:
-        url (str): The starting URL.
-        k (int, optional): The max desired final URLs. Defaults to 20.
-
-    Returns:
-        List[str]: List of URLs within the same domain as the input URL.
-    """
-    urls = []
-
-    def _collect_urls(spider):
-        """Handler for the spider_closed signal. Collects the visited URLs."""
-        nonlocal urls
-        urls.extend(list(spider.visited_urls))
-
-    # Connect the spider_closed signal with our handler
-    dispatcher.connect(_collect_urls, signal=signals.spider_closed)
-
-    runner = CrawlerRunner(
-        {
-            "USER_AGENT": "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
-        }
-    )
-
-    d = runner.crawl(DomainSpecificSpider, start_url=url, k=k)
-
-    # Block until crawling is done and then stop the reactor
-    crawl_deferred = defer.Deferred()
-
-    def _crawl_done(_):
-        reactor.stop()
-        crawl_deferred.callback(urls)
-
-    d.addBoth(_crawl_done)
-
-    # Start the reactor, it will stop once the crawl is done
-    reactor.run(installSignalHandlers=0)
-
-    # This will block until the deferred gets a result
-    return crawl_deferred.result
-
-
-# Test the function
-if __name__ == "__main__":
-    fetched_urls = scrapy_fetch_urls("https://example.com", 5)
-    for url in fetched_urls:
-        print(url)

langroid/parsing/table_loader.py

@@ -1,94 +0,0 @@
-from csv import Sniffer
-from typing import List
-
-import pandas as pd
-
-
-def read_tabular_data(path_or_url: str, sep: None | str = None) -> pd.DataFrame:
-    """
-    Reads tabular data from a file or URL and returns a pandas DataFrame.
-    The separator is auto-detected if not specified.
-
-    Args:
-        path_or_url (str): Path or URL to the file to be read.
-
-    Returns:
-        pd.DataFrame: Data from file or URL as a pandas DataFrame.
-
-    Raises:
-        ValueError: If the data cannot be read or is misformatted.
-    """
-    try:
-        if sep is None:
-            # Read the first few lines to guess the separator
-            with pd.io.common.get_handle(path_or_url, "r") as file_handler:
-                first_lines = "".join(file_handler.handle.readlines(5))
-                sep = Sniffer().sniff(first_lines).delimiter
-                # If it's a local file, reset to the beginning
-                if hasattr(file_handler.handle, "seek"):
-                    file_handler.handle.seek(0)
-
-        # Read the data
-
-        # get non-blank column names
-        with pd.io.common.get_handle(path_or_url, "r") as f:
-            header_line = f.handle.readline().strip()
-            valid_cols = [col for col in header_line.split(sep) if col]
-            valid_cols = [c.replace('"', "").replace("'", "") for c in valid_cols]
-            if hasattr(f.handle, "seek"):
-                f.handle.seek(0)
-
-        # use only those columns
-        data = pd.read_csv(path_or_url, sep=sep, usecols=valid_cols)
-        data.columns = data.columns.str.strip()  # e.g. "  column 1  " -> "column 1"
-
-        return data
-
-    except Exception as e:
-        raise ValueError(
-            "Unable to read data. "
-            "Please ensure it is correctly formatted. Error: " + str(e)
-        )
-
-
-def describe_dataframe(
-    df: pd.DataFrame, filter_fields: List[str] = [], n_vals: int = 10
-) -> str:
-    """
-    Generates a description of the columns in the dataframe,
-    along with a listing of up to `n_vals` unique values for each column.
-    Intended to be used to insert into an LLM context so it can generate
-    appropriate queries or filters on the df.
-
-    Args:
-    df (pd.DataFrame): The dataframe to describe.
-    filter_fields (list): A list of fields that can be used for filtering.
-        When non-empty, the values-list will be restricted to these.
-    n_vals (int): How many unique values to show for each column.
-
-    Returns:
-    str: A description of the dataframe.
-    """
-    description = []
-    for column in df.columns.to_list():
-        unique_values = df[column].dropna().unique()
-        unique_count = len(unique_values)
-        if column not in filter_fields:
-            values_desc = f"{unique_count} unique values"
-        else:
-            if unique_count > n_vals:
-                displayed_values = unique_values[:n_vals]
-                more_count = unique_count - n_vals
-                values_desc = f" Values - {displayed_values}, ... {more_count} more"
-            else:
-                values_desc = f" Values - {unique_values}"
-        col_type = "string" if df[column].dtype == "object" else df[column].dtype
-        col_desc = f"* {column} ({col_type}); {values_desc}"
-        description.append(col_desc)
-
-    all_cols = "\n".join(description)
-
-    return f"""
-        Name of each field, its type and unique values (up to {n_vals}):
-        {all_cols}
-        """

langroid/parsing/url_loader.py

@@ -1,111 +0,0 @@
-import logging
-import os
-from tempfile import NamedTemporaryFile
-from typing import List, no_type_check
-
-import requests
-import trafilatura
-from trafilatura.downloads import (
-    add_to_compressed_dict,
-    buffered_downloads,
-    load_download_buffer,
-)
-
-from langroid.mytypes import DocMetaData, Document
-from langroid.parsing.document_parser import DocumentParser, ImagePdfParser
-from langroid.parsing.parser import Parser, ParsingConfig
-
-logging.getLogger("trafilatura").setLevel(logging.ERROR)
-
-
-class URLLoader:
-    """
-    Load a list of URLs and extract the text content.
-    Alternative approaches could use `bs4` or `scrapy`.
-
-    TODO - this currently does not handle cookie dialogs,
-     i.e. if there is a cookie pop-up, most/all of the extracted
-     content could be cookie policy text.
-     We could use `playwright` to simulate a user clicking
-     the "accept" button on the cookie dialog.
-    """
-
-    def __init__(self, urls: List[str], parser: Parser = Parser(ParsingConfig())):
-        self.urls = urls
-        self.parser = parser
-
-    @no_type_check
-    def load(self) -> List[Document]:
-        docs = []
-        threads = 4
-        # converted the input list to an internal format
-        dl_dict = add_to_compressed_dict(self.urls)
-        # processing loop
-        while not dl_dict.done:
-            buffer, dl_dict = load_download_buffer(
-                dl_dict,
-                sleep_time=5,
-            )
-            for url, result in buffered_downloads(buffer, threads):
-                if (
-                    url.lower().endswith(".pdf")
-                    or url.lower().endswith(".docx")
-                    or url.lower().endswith(".doc")
-                ):
-                    doc_parser = DocumentParser.create(
-                        url,
-                        self.parser.config,
-                    )
-                    new_chunks = doc_parser.get_doc_chunks()
-                    if len(new_chunks) == 0:
-                        # If the document is empty, try to extract images
-                        img_parser = ImagePdfParser(url, self.parser.config)
-                        new_chunks = img_parser.get_doc_chunks()
-                    docs.extend(new_chunks)
-                else:
-                    # Try to detect content type and handle accordingly
-                    headers = requests.head(url).headers
-                    content_type = headers.get("Content-Type", "").lower()
-                    temp_file_suffix = None
-                    if "application/pdf" in content_type:
-                        temp_file_suffix = ".pdf"
-                    elif (
-                        "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
-                        in content_type
-                    ):
-                        temp_file_suffix = ".docx"
-                    elif "application/msword" in content_type:
-                        temp_file_suffix = ".doc"
-
-                    if temp_file_suffix:
-                        # Download the document content
-                        response = requests.get(url)
-                        with NamedTemporaryFile(
-                            delete=False, suffix=temp_file_suffix
-                        ) as temp_file:
-                            temp_file.write(response.content)
-                            temp_file_path = temp_file.name
-                        # Process the downloaded document
-                        doc_parser = DocumentParser.create(
-                            temp_file_path, self.parser.config
-                        )
-                        docs.extend(doc_parser.get_doc_chunks())
-                        # Clean up the temporary file
-                        os.remove(temp_file_path)
-                    else:
-                        text = trafilatura.extract(
-                            result,
-                            no_fallback=False,
-                            favor_recall=True,
-                        )
-                        if (
-                            text is None
-                            and result is not None
-                            and isinstance(result, str)
-                        ):
-                            text = result
-                        if text is not None and text != "":
-                            docs.append(
-                                Document(content=text, metadata=DocMetaData(source=url))
-                            )
-        return docs

langroid/parsing/url_loader_cookies.py

@@ -1,73 +0,0 @@
-import logging
-from typing import List, no_type_check
-
-import trafilatura
-from playwright.sync_api import sync_playwright
-
-from langroid.mytypes import DocMetaData, Document
-
-logging.getLogger("trafilatura").setLevel(logging.ERROR)
-
-
-def accept_cookies_and_extract_content(url: str) -> str:
-    with sync_playwright() as playwright:
-        browser = playwright.chromium.launch(headless=True)
-        context = browser.new_context()
-        page = context.new_page()
-        page.goto(url)
-
-        # List of possible selectors or texts on the cookie consent buttons
-        possible_selectors = [
-            'text="Accept"',
-            'text="Agree"',
-            'text="OK"',
-            'text="Continue"',
-        ]
-
-        # Try to click each possible consent button
-        for selector in possible_selectors:
-            try:
-                page.click(selector)
-                print(f"Clicked {selector}")
-                break  # If click is successful, break out of the loop
-            except Exception:
-                print(f"Could not click {selector}")
-
-        # Extract and return the page's text content
-        content = page.content()
-
-        context.close()
-        browser.close()
-        content_str: str = content if isinstance(content, str) else ""
-        return content_str
-
-
-class URLLoader:
-    """
-    Load a list of URLs and extract the text content.
-    Alternative approaches could use `bs4` or `scrapy`.
-
-    TODO - this currently does not handle cookie dialogs,
-     i.e. if there is a cookie pop-up, most/all of the extracted
-     content could be cookie policy text.
-     We could use `playwright` to simulate a user clicking
-     the "accept" button on the cookie dialog.
-    """
-
-    def __init__(self, urls: List[str]):
-        self.urls = urls
-
-    @no_type_check
-    def load(self) -> List[Document]:
-        docs = []
-        # converted the input list to an internal format
-        for url in self.urls:
-            html_content = accept_cookies_and_extract_content(url)
-            text = trafilatura.extract(
-                html_content,
-                no_fallback=False,
-                favor_recall=True,
-            )
-            if text is not None and text != "":
-                docs.append(Document(content=text, metadata=DocMetaData(source=url)))
-        return docs