langroid 0.33.6__py3-none-any.whl → 0.33.7__py3-none-any.whl

langroid/parsing/urls.py

@@ -0,0 +1,273 @@
+import logging
+import os
+import tempfile
+import urllib.parse
+import urllib.robotparser
+from typing import List, Optional, Set, Tuple
+from urllib.parse import urldefrag, urljoin, urlparse
+
+import fire
+import requests
+from bs4 import BeautifulSoup
+from rich import print
+from rich.prompt import Prompt
+from trafilatura.spider import focused_crawler
+
+from langroid.pydantic_v1 import BaseModel, HttpUrl, ValidationError, parse_obj_as
+
+logger = logging.getLogger(__name__)
+
+
+def url_to_tempfile(url: str) -> str:
+    """
+    Fetch content from the given URL and save it to a temporary local file.
+
+    Args:
+        url (str): The URL of the content to fetch.
+
+    Returns:
+        str: The path to the temporary file where the content is saved.
+
+    Raises:
+        HTTPError: If there's any issue fetching the content.
+    """
+
+    response = requests.get(url)
+    response.raise_for_status()  # Raise an exception for HTTP errors
+
+    # Create a temporary file and write the content
+    with tempfile.NamedTemporaryFile(delete=False, suffix=".tmp") as temp_file:
+        temp_file.write(response.content)
+        return temp_file.name
+
+
+def get_user_input(msg: str, color: str = "blue") -> str:
+    """
+    Prompt the user for input.
+    Args:
+        msg: printed prompt
+        color: color of the prompt
+    Returns:
+        user input
+    """
+    color_str = f"[{color}]{msg} " if color else msg + " "
+    print(color_str, end="")
+    return input("")
+
+
+def get_list_from_user(
+    prompt: str = "Enter input (type 'done' or hit return to finish)",
+    n: int | None = None,
+) -> List[str]:
+    """
+    Prompt the user for inputs.
+    Args:
+        prompt: printed prompt
+        n: how many inputs to prompt for. If None, then prompt until done, otherwise
+            quit after n inputs.
+    Returns:
+        list of input strings
+    """
+    # Create an empty set to store the URLs.
+    input_set = set()
+
+    # Use a while loop to continuously ask the user for URLs.
+    for _ in range(n or 1000):
+        # Prompt the user for input.
+        input_str = Prompt.ask(f"[blue]{prompt}")
+
+        # Check if the user wants to exit the loop.
+        if input_str.lower() == "done" or input_str == "":
+            break
+
+        # if it is a URL, ask how many to crawl
+        if is_url(input_str):
+            url = input_str
+            input_str = Prompt.ask("[blue] How many new URLs to crawl?", default="0")
+            max_urls = int(input_str) + 1
+            tot_urls = list(find_urls(url, max_links=max_urls, max_depth=2))
+            tot_urls_str = "\n".join(tot_urls)
+            print(
+                f"""
+                Found these {len(tot_urls)} links upto depth 2:
+                {tot_urls_str}
+                """
+            )
+
+            input_set.update(tot_urls)
+        else:
+            input_set.add(input_str.strip())
+
+    return list(input_set)
+
+
+class Url(BaseModel):
+    url: HttpUrl
+
+
+def is_url(s: str) -> bool:
+    try:
+        Url(url=parse_obj_as(HttpUrl, s))
+        return True
+    except ValidationError:
+        return False
+
+
+def get_urls_paths_bytes_indices(
+    inputs: List[str | bytes],
+) -> Tuple[List[int], List[int], List[int]]:
+    """
+    Given a list of inputs, return a
+    list of indices of URLs, list of indices of paths, list of indices of byte-contents.
+    Args:
+        inputs: list of strings or bytes
+    Returns:
+        list of Indices of URLs,
+        list of indices of paths,
+        list of indices of byte-contents
+    """
+    urls = []
+    paths = []
+    byte_list = []
+    for i, item in enumerate(inputs):
+        if isinstance(item, bytes):
+            byte_list.append(i)
+            continue
+        try:
+            Url(url=parse_obj_as(HttpUrl, item))
+            urls.append(i)
+        except ValidationError:
+            if os.path.exists(item):
+                paths.append(i)
+            else:
+                logger.warning(f"{item} is neither a URL nor a path.")
+    return urls, paths, byte_list
+
+
+def crawl_url(url: str, max_urls: int = 1) -> List[str]:
+    """
+    Crawl starting at the url and return a list of URLs to be parsed,
+    up to a maximum of `max_urls`.
+    This has not been tested to work as intended. Ignore.
+    """
+    if max_urls == 1:
+        # no need to crawl, just return the original list
+        return [url]
+
+    to_visit = None
+    known_urls = None
+
+    # Create a RobotFileParser object
+    robots = urllib.robotparser.RobotFileParser()
+    while True:
+        if known_urls is not None and len(known_urls) >= max_urls:
+            break
+        # Set the RobotFileParser object to the website's robots.txt file
+        robots.set_url(url + "/robots.txt")
+        robots.read()
+
+        if robots.can_fetch("*", url):
+            # Start or resume the crawl
+            to_visit, known_urls = focused_crawler(
+                url,
+                max_seen_urls=max_urls,
+                max_known_urls=max_urls,
+                todo=to_visit,
+                known_links=known_urls,
+                rules=robots,
+            )
+        if to_visit is None:
+            break
+
+    if known_urls is None:
+        return [url]
+    final_urls = [s.strip() for s in known_urls]
+    return list(final_urls)[:max_urls]
+
+
+def find_urls(
+    url: str = "https://en.wikipedia.org/wiki/Generative_pre-trained_transformer",
+    max_links: int = 20,
+    visited: Optional[Set[str]] = None,
+    depth: int = 0,
+    max_depth: int = 2,
+    match_domain: bool = True,
+) -> Set[str]:
+    """
+    Recursively find all URLs on a given page.
+
+    Args:
+        url (str): The URL to start from.
+        max_links (int): The maximum number of links to find.
+        visited (set): A set of URLs that have already been visited.
+        depth (int): The current depth of the recursion.
+        max_depth (int): The maximum depth of the recursion.
+        match_domain (bool): Whether to only return URLs that are on the same domain.
+
+    Returns:
+        set: A set of URLs found on the page.
+    """
+
+    if visited is None:
+        visited = set()
+
+    if url in visited or depth > max_depth:
+        return visited
+
+    visited.add(url)
+    base_domain = urlparse(url).netloc
+
+    try:
+        response = requests.get(url, timeout=5)
+        response.raise_for_status()
+        soup = BeautifulSoup(response.text, "html.parser")
+        links = [urljoin(url, a["href"]) for a in soup.find_all("a", href=True)]
+
+        # Defrag links: discard links that are to portions of same page
+        defragged_links = list(set(urldefrag(link).url for link in links))
+
+        # Filter links based on domain matching requirement
+        domain_matching_links = [
+            link for link in defragged_links if urlparse(link).netloc == base_domain
+        ]
+
+        # ensure url is first, since below we are taking first max_links urls
+        domain_matching_links = [url] + [x for x in domain_matching_links if x != url]
+
+        # If found links exceed max_links, return immediately
+        if len(domain_matching_links) >= max_links:
+            return set(domain_matching_links[:max_links])
+
+        for link in domain_matching_links:
+            if len(visited) >= max_links:
+                break
+
+            if link not in visited:
+                visited.update(
+                    find_urls(
+                        link,
+                        max_links,
+                        visited,
+                        depth + 1,
+                        max_depth,
+                        match_domain,
+                    )
+                )
+
+    except (requests.RequestException, Exception) as e:
+        print(f"Error fetching {url}. Error: {e}")
+
+    return set(list(visited)[:max_links])
+
+
+def org_user_from_github(url: str) -> str:
+    parsed = urllib.parse.urlparse(url)
+    org, user = parsed.path.lstrip("/").split("/")
+    return f"{org}-{user}"
+
+
+if __name__ == "__main__":
+    # Example usage
+    found_urls = set(fire.Fire(find_urls))
+    for url in found_urls:
+        print(url)

langroid/parsing/utils.py

@@ -0,0 +1,373 @@
+import difflib
+import logging
+import random
+import re
+from functools import cache
+from itertools import islice
+from typing import Iterable, List, Sequence, TypeVar
+
+import nltk
+from faker import Faker
+
+from langroid.mytypes import Document
+from langroid.parsing.document_parser import DocumentType
+from langroid.parsing.parser import Parser, ParsingConfig
+from langroid.parsing.repo_loader import RepoLoader
+from langroid.parsing.url_loader import URLLoader
+from langroid.parsing.urls import get_urls_paths_bytes_indices
+
+Faker.seed(23)
+random.seed(43)
+
+logger = logging.getLogger(__name__)
+
+
+# Ensures the NLTK resource is available
+@cache
+def download_nltk_resource(resource: str) -> None:
+    try:
+        nltk.data.find(resource)
+    except LookupError:
+        nltk.download(resource, quiet=True)
+
+
+# Download punkt_tab resource at module import
+download_nltk_resource("punkt_tab")
+download_nltk_resource("gutenberg")
+
+T = TypeVar("T")
+
+
+def batched(iterable: Iterable[T], n: int) -> Iterable[Sequence[T]]:
+    """Batch data into tuples of length n. The last batch may be shorter."""
+    # batched('ABCDEFG', 3) --> ABC DEF G
+    if n < 1:
+        raise ValueError("n must be at least one")
+    it = iter(iterable)
+    while batch := tuple(islice(it, n)):
+        yield batch
+
+
+def generate_random_sentences(k: int) -> str:
+    # Load the sample text
+
+    from nltk.corpus import gutenberg
+
+    text = gutenberg.raw("austen-emma.txt")
+
+    # Split the text into sentences
+    sentences = nltk.tokenize.sent_tokenize(text)
+
+    # Generate k random sentences
+    random_sentences = random.choices(sentences, k=k)
+    return " ".join(random_sentences)
+
+
+def generate_random_text(num_sentences: int) -> str:
+    fake = Faker()
+    text = ""
+    for _ in range(num_sentences):
+        text += fake.sentence() + " "
+    return text
+
+
+def closest_string(query: str, string_list: List[str]) -> str:
+    """Find the closest match to the query in a list of strings.
+
+    This function is case-insensitive and ignores leading and trailing whitespace.
+    If no match is found, it returns 'No match found'.
+
+    Args:
+        query (str): The string to match.
+        string_list (List[str]): The list of strings to search.
+
+    Returns:
+        str: The closest match to the query from the list, or 'No match found'
+             if no match is found.
+    """
+    # Create a dictionary where the keys are the standardized strings and
+    # the values are the original strings.
+    str_dict = {s.lower().strip(): s for s in string_list}
+
+    # Standardize the query and find the closest match in the list of keys.
+    closest_match = difflib.get_close_matches(
+        query.lower().strip(), str_dict.keys(), n=1
+    )
+
+    # Retrieve the original string from the value in the dictionary.
+    original_closest_match = (
+        str_dict[closest_match[0]] if closest_match else "No match found"
+    )
+
+    return original_closest_match
+
+
+def split_paragraphs(text: str) -> List[str]:
+    """
+    Split the input text into paragraphs using "\n\n" as the delimiter.
+
+    Args:
+        text (str): The input text.
+
+    Returns:
+        list: A list of paragraphs.
+    """
+    # Split based on a newline, followed by spaces/tabs, then another newline.
+    paras = re.split(r"\n[ \t]*\n", text)
+    return [para.strip() for para in paras if para.strip()]
+
+
+def split_newlines(text: str) -> List[str]:
+    """
+    Split the input text into lines using "\n" as the delimiter.
+
+    Args:
+        text (str): The input text.
+
+    Returns:
+        list: A list of lines.
+    """
+    lines = re.split(r"\n", text)
+    return [line.strip() for line in lines if line.strip()]
+
+
+def number_segments(s: str, granularity: int = 1) -> str:
+    """
+    Number the segments in a given text, preserving paragraph structure.
+    A segment is a sequence of `len` consecutive "sentences", where a "sentence"
+    is either a normal sentence, or if there isn't enough punctuation to properly
+    identify sentences, then we use a pseudo-sentence via heuristics (split by newline
+    or failing that, just split every 40 words). The goal here is simply to number
+    segments at a reasonable granularity so the LLM can identify relevant segments,
+    in the RelevanceExtractorAgent.
+
+    Args:
+        s (str): The input text.
+        granularity (int): The number of sentences in a segment.
+            If this is -1, then the entire text is treated as a single segment,
+            and is numbered as <#1#>.
+
+    Returns:
+        str: The text with segments numbered in the style <#1#>, <#2#> etc.
+
+    Example:
+        >>> number_segments("Hello world! How are you? Have a good day.")
+        '<#1#> Hello world! <#2#> How are you? <#3#> Have a good day.'
+    """
+    if granularity < 0:
+        return "<#1#> " + s
+    numbered_text = []
+    count = 0
+
+    paragraphs = split_paragraphs(s)
+    for paragraph in paragraphs:
+        sentences = nltk.sent_tokenize(paragraph)
+        # Some docs are problematic (e.g. resumes) and have no (or too few) periods,
+        # so we can't split usefully into sentences.
+        # We try a series of heuristics to split into sentences,
+        # until the avg num words per sentence is less than 40.
+        avg_words_per_sentence = sum(
+            len(nltk.word_tokenize(sentence)) for sentence in sentences
+        ) / len(sentences)
+        if avg_words_per_sentence > 40:
+            sentences = split_newlines(paragraph)
+        avg_words_per_sentence = sum(
+            len(nltk.word_tokenize(sentence)) for sentence in sentences
+        ) / len(sentences)
+        if avg_words_per_sentence > 40:
+            # Still too long, just split on every 40 words
+            sentences = []
+            for sentence in nltk.sent_tokenize(paragraph):
+                words = nltk.word_tokenize(sentence)
+                for i in range(0, len(words), 40):
+                    # if there are less than 20 words left after this,
+                    # just add them to the last sentence and break
+                    if len(words) - i < 20:
+                        sentences.append(" ".join(words[i:]))
+                        break
+                    else:
+                        sentences.append(" ".join(words[i : i + 40]))
+        for i, sentence in enumerate(sentences):
+            num = count // granularity + 1
+            number_prefix = f"<#{num}#>" if count % granularity == 0 else ""
+            sentence = f"{number_prefix} {sentence}"
+            count += 1
+            sentences[i] = sentence
+        numbered_paragraph = " ".join(sentences)
+        numbered_text.append(numbered_paragraph)
+
+    return "  \n\n  ".join(numbered_text)
+
+
+def number_sentences(s: str) -> str:
+    return number_segments(s, granularity=1)
+
+
+def parse_number_range_list(specs: str) -> List[int]:
+    """
+    Parse a specs string like "3,5,7-10" into a list of integers.
+
+    Args:
+        specs (str): A string containing segment numbers and/or ranges
+                     (e.g., "3,5,7-10").
+
+    Returns:
+        List[int]: List of segment numbers.
+
+    Example:
+        >>> parse_number_range_list("3,5,7-10")
+        [3, 5, 7, 8, 9, 10]
+    """
+    spec_indices = set()  # type: ignore
+    for part in specs.split(","):
+        # some weak LLMs may generate <#1#> instead of 1, so extract just the digits
+        # or the "-"
+        part = "".join(char for char in part if char.isdigit() or char == "-")
+        if "-" in part:
+            start, end = map(int, part.split("-"))
+            spec_indices.update(range(start, end + 1))
+        else:
+            spec_indices.add(int(part))
+
+    return sorted(list(spec_indices))
+
+
+def strip_k(s: str, k: int = 2) -> str:
+    """
+    Strip any leading and trailing whitespaces from the input text beyond length k.
+    This is useful for removing leading/trailing whitespaces from a text while
+    preserving paragraph structure.
+
+    Args:
+        s (str): The input text.
+        k (int): The number of leading and trailing whitespaces to retain.
+
+    Returns:
+        str: The text with leading and trailing whitespaces removed beyond length k.
+    """
+
+    # Count leading and trailing whitespaces
+    leading_count = len(s) - len(s.lstrip())
+    trailing_count = len(s) - len(s.rstrip())
+
+    # Determine how many whitespaces to retain
+    leading_keep = min(leading_count, k)
+    trailing_keep = min(trailing_count, k)
+
+    # Use slicing to get the desired output
+    return s[leading_count - leading_keep : len(s) - (trailing_count - trailing_keep)]
+
+
+def clean_whitespace(text: str) -> str:
+    """Remove extra whitespace from the input text, while preserving
+    paragraph structure.
+    """
+    paragraphs = split_paragraphs(text)
+    cleaned_paragraphs = [" ".join(p.split()) for p in paragraphs if p]
+    return "\n\n".join(cleaned_paragraphs)  # Join the cleaned paragraphs.
+
+
+def extract_numbered_segments(s: str, specs: str) -> str:
+    """
+    Extract specified segments from a numbered text, preserving paragraph structure.
+
+    Args:
+        s (str): The input text containing numbered segments.
+        specs (str): A string containing segment numbers and/or ranges
+                     (e.g., "3,5,7-10").
+
+    Returns:
+        str: Extracted segments, keeping original paragraph structures.
+
+    Example:
+        >>> text = "(1) Hello world! (2) How are you? (3) Have a good day."
+        >>> extract_numbered_segments(text, "1,3")
+        'Hello world! Have a good day.'
+    """
+    # Use the helper function to get the list of indices from specs
+    if specs.strip() == "":
+        return ""
+    spec_indices = parse_number_range_list(specs)
+
+    # Regular expression to identify numbered segments like
+    # <#1#> Hello world! This is me. <#2#> How are you? <#3#> Have a good day.
+    # Note we match any character between segment markers, including newlines.
+    segment_pattern = re.compile(r"<#(\d+)#>([\s\S]*?)(?=<#\d+#>|$)")
+
+    # Split the text into paragraphs while preserving their boundaries
+    paragraphs = split_paragraphs(s)
+
+    extracted_paragraphs = []
+
+    for paragraph in paragraphs:
+        segments_with_numbers = segment_pattern.findall(paragraph)
+
+        # Extract the desired segments from this paragraph
+        extracted_segments = [
+            segment
+            for num, segment in segments_with_numbers
+            if int(num) in spec_indices
+        ]
+
+        # If we extracted any segments from this paragraph,
+        # join them and append to results
+        if extracted_segments:
+            extracted_paragraphs.append(" ".join(extracted_segments))
+
+    return "\n\n".join(extracted_paragraphs)
+
+
+def extract_content_from_path(
+    path: bytes | str | List[bytes | str],
+    parsing: ParsingConfig,
+    doc_type: str | DocumentType | None = None,
+) -> str | List[str]:
+    """
+    Extract the content from a file path or URL, or a list of file paths or URLs.
+
+    Args:
+        path (bytes | str | List[str]): The file path or URL, or a list of file paths or
+            URLs, or bytes content. The bytes option is meant to support cases
+            where upstream code may have already loaded the content (e.g., from a
+            database or API) and we want to avoid having to copy the content to a
+            temporary file.
+        parsing (ParsingConfig): The parsing configuration.
+        doc_type (str | DocumentType | None): The document type if known.
+            If multiple paths are given, this MUST apply to ALL docs.
+
+    Returns:
+        str | List[str]: The extracted content if a single file path or URL is provided,
+                or a list of extracted contents if a
+                list of file paths or URLs is provided.
+    """
+    if isinstance(path, str) or isinstance(path, bytes):
+        paths = [path]
+    elif isinstance(path, list) and len(path) == 0:
+        return ""
+    else:
+        paths = path
+
+    url_idxs, path_idxs, byte_idxs = get_urls_paths_bytes_indices(paths)
+    urls = [paths[i] for i in url_idxs]
+    path_list = [paths[i] for i in path_idxs]
+    byte_list = [paths[i] for i in byte_idxs]
+    path_list.extend(byte_list)
+    parser = Parser(parsing)
+    docs: List[Document] = []
+    try:
+        if len(urls) > 0:
+            loader = URLLoader(urls=urls, parser=parser)  # type: ignore
+            docs = loader.load()
+        if len(path_list) > 0:
+            for p in path_list:
+                path_docs = RepoLoader.get_documents(
+                    p, parser=parser, doc_type=doc_type
+                )
+                docs.extend(path_docs)
+    except Exception as e:
+        logger.warning(f"Error loading path {paths}: {e}")
+        return ""
+    if len(docs) == 1:
+        return docs[0].content
+    else:
+        return [d.content for d in docs]