langroid 0.1.85__py3-none-any.whl → 0.1.219__py3-none-any.whl

langroid/parsing/utils.py

@@ -1,19 +1,40 @@
 import difflib
+import logging
 import random
+import re
+from functools import cache
 from itertools import islice
-from typing import Any, Iterable, List
+from typing import Iterable, List, Sequence, TypeVar
 
 import nltk
 from faker import Faker
 
-nltk.download("punkt", quiet=True)
-nltk.download("gutenberg", quiet=True)
+from langroid.mytypes import Document
+from langroid.parsing.document_parser import DocumentType
+from langroid.parsing.parser import Parser, ParsingConfig
+from langroid.parsing.repo_loader import RepoLoader
+from langroid.parsing.url_loader import URLLoader
+from langroid.parsing.urls import get_urls_paths_bytes_indices
 
 Faker.seed(23)
 random.seed(43)
 
+logger = logging.getLogger(__name__)
 
-def batched(iterable: Iterable[Any], n: int) -> Iterable[Any]:
+
+# Ensures the NLTK resource is available
+@cache
+def download_nltk_resource(resource: str) -> None:
+    try:
+        nltk.data.find(resource)
+    except LookupError:
+        nltk.download(resource, quiet=True)
+
+
+T = TypeVar("T")
+
+
+def batched(iterable: Iterable[T], n: int) -> Iterable[Sequence[T]]:
     """Batch data into tuples of length n. The last batch may be shorter."""
     # batched('ABCDEFG', 3) --> ABC DEF G
     if n < 1:
@@ -25,6 +46,8 @@ def batched(iterable: Iterable[Any], n: int) -> Iterable[Any]:
 
 def generate_random_sentences(k: int) -> str:
     # Load the sample text
+    download_nltk_resource("gutenberg")
+
     from nltk.corpus import gutenberg
 
     text = gutenberg.raw("austen-emma.txt")
@@ -74,3 +97,274 @@ def closest_string(query: str, string_list: List[str]) -> str:
     )
 
     return original_closest_match
+
+
+def split_paragraphs(text: str) -> List[str]:
+    """
+    Split the input text into paragraphs using "\n\n" as the delimiter.
+
+    Args:
+        text (str): The input text.
+
+    Returns:
+        list: A list of paragraphs.
+    """
+    # Split based on a newline, followed by spaces/tabs, then another newline.
+    paras = re.split(r"\n[ \t]*\n", text)
+    return [para.strip() for para in paras if para.strip()]
+
+
+def split_newlines(text: str) -> List[str]:
+    """
+    Split the input text into lines using "\n" as the delimiter.
+
+    Args:
+        text (str): The input text.
+
+    Returns:
+        list: A list of lines.
+    """
+    lines = re.split(r"\n", text)
+    return [line.strip() for line in lines if line.strip()]
+
+
+def number_segments(s: str, granularity: int = 1) -> str:
+    """
+    Number the segments in a given text, preserving paragraph structure.
+    A segment is a sequence of `len` consecutive "sentences", where a "sentence"
+    is either a normal sentence, or if there isn't enough punctuation to properly
+    identify sentences, then we use a pseudo-sentence via heuristics (split by newline
+    or failing that, just split every 40 words). The goal here is simply to number
+    segments at a reasonable granularity so the LLM can identify relevant segments,
+    in the RelevanceExtractorAgent.
+
+    Args:
+        s (str): The input text.
+        granularity (int): The number of sentences in a segment.
+            If this is -1, then the entire text is treated as a single segment,
+            and is numbered as <#1#>.
+
+    Returns:
+        str: The text with segments numbered in the style <#1#>, <#2#> etc.
+
+    Example:
+        >>> number_segments("Hello world! How are you? Have a good day.")
+        '<#1#> Hello world! <#2#> How are you? <#3#> Have a good day.'
+    """
+    if granularity < 0:
+        return "<#1#> " + s
+    numbered_text = []
+    count = 0
+
+    paragraphs = split_paragraphs(s)
+    for paragraph in paragraphs:
+        sentences = nltk.sent_tokenize(paragraph)
+        # Some docs are problematic (e.g. resumes) and have no (or too few) periods,
+        # so we can't split usefully into sentences.
+        # We try a series of heuristics to split into sentences,
+        # until the avg num words per sentence is less than 40.
+        avg_words_per_sentence = sum(
+            len(nltk.word_tokenize(sentence)) for sentence in sentences
+        ) / len(sentences)
+        if avg_words_per_sentence > 40:
+            sentences = split_newlines(paragraph)
+        avg_words_per_sentence = sum(
+            len(nltk.word_tokenize(sentence)) for sentence in sentences
+        ) / len(sentences)
+        if avg_words_per_sentence > 40:
+            # Still too long, just split on every 40 words
+            sentences = []
+            for sentence in nltk.sent_tokenize(paragraph):
+                words = nltk.word_tokenize(sentence)
+                for i in range(0, len(words), 40):
+                    # if there are less than 20 words left after this,
+                    # just add them to the last sentence and break
+                    if len(words) - i < 20:
+                        sentences.append(" ".join(words[i:]))
+                        break
+                    else:
+                        sentences.append(" ".join(words[i : i + 40]))
+        for i, sentence in enumerate(sentences):
+            num = count // granularity + 1
+            number_prefix = f"<#{num}#>" if count % granularity == 0 else ""
+            sentence = f"{number_prefix} {sentence}"
+            count += 1
+            sentences[i] = sentence
+        numbered_paragraph = " ".join(sentences)
+        numbered_text.append(numbered_paragraph)
+
+    return "  \n\n  ".join(numbered_text)
+
+
+def number_sentences(s: str) -> str:
+    return number_segments(s, granularity=1)
+
+
+def parse_number_range_list(specs: str) -> List[int]:
+    """
+    Parse a specs string like "3,5,7-10" into a list of integers.
+
+    Args:
+        specs (str): A string containing segment numbers and/or ranges
+                     (e.g., "3,5,7-10").
+
+    Returns:
+        List[int]: List of segment numbers.
+
+    Example:
+        >>> parse_number_range_list("3,5,7-10")
+        [3, 5, 7, 8, 9, 10]
+    """
+    spec_indices = set()  # type: ignore
+    for part in specs.split(","):
+        # some weak LLMs may generate <#1#> instead of 1, so extract just the digits
+        # or the "-"
+        part = "".join(char for char in part if char.isdigit() or char == "-")
+        if "-" in part:
+            start, end = map(int, part.split("-"))
+            spec_indices.update(range(start, end + 1))
+        else:
+            spec_indices.add(int(part))
+
+    return sorted(list(spec_indices))
+
+
+def strip_k(s: str, k: int = 2) -> str:
+    """
+    Strip any leading and trailing whitespaces from the input text beyond length k.
+    This is useful for removing leading/trailing whitespaces from a text while
+    preserving paragraph structure.
+
+    Args:
+        s (str): The input text.
+        k (int): The number of leading and trailing whitespaces to retain.
+
+    Returns:
+        str: The text with leading and trailing whitespaces removed beyond length k.
+    """
+
+    # Count leading and trailing whitespaces
+    leading_count = len(s) - len(s.lstrip())
+    trailing_count = len(s) - len(s.rstrip())
+
+    # Determine how many whitespaces to retain
+    leading_keep = min(leading_count, k)
+    trailing_keep = min(trailing_count, k)
+
+    # Use slicing to get the desired output
+    return s[leading_count - leading_keep : len(s) - (trailing_count - trailing_keep)]
+
+
+def clean_whitespace(text: str) -> str:
+    """Remove extra whitespace from the input text, while preserving
+    paragraph structure.
+    """
+    paragraphs = split_paragraphs(text)
+    cleaned_paragraphs = [" ".join(p.split()) for p in paragraphs if p]
+    return "\n\n".join(cleaned_paragraphs)  # Join the cleaned paragraphs.
+
+
+def extract_numbered_segments(s: str, specs: str) -> str:
+    """
+    Extract specified segments from a numbered text, preserving paragraph structure.
+
+    Args:
+        s (str): The input text containing numbered segments.
+        specs (str): A string containing segment numbers and/or ranges
+                     (e.g., "3,5,7-10").
+
+    Returns:
+        str: Extracted segments, keeping original paragraph structures.
+
+    Example:
+        >>> text = "(1) Hello world! (2) How are you? (3) Have a good day."
+        >>> extract_numbered_segments(text, "1,3")
+        'Hello world! Have a good day.'
+    """
+    # Use the helper function to get the list of indices from specs
+    if specs.strip() == "":
+        return ""
+    spec_indices = parse_number_range_list(specs)
+
+    # Regular expression to identify numbered segments like
+    # <#1#> Hello world! This is me. <#2#> How are you? <#3#> Have a good day.
+    # Note we match any character between segment markers, including newlines.
+    segment_pattern = re.compile(r"<#(\d+)#>([\s\S]*?)(?=<#\d+#>|$)")
+
+    # Split the text into paragraphs while preserving their boundaries
+    paragraphs = split_paragraphs(s)
+
+    extracted_paragraphs = []
+
+    for paragraph in paragraphs:
+        segments_with_numbers = segment_pattern.findall(paragraph)
+
+        # Extract the desired segments from this paragraph
+        extracted_segments = [
+            segment
+            for num, segment in segments_with_numbers
+            if int(num) in spec_indices
+        ]
+
+        # If we extracted any segments from this paragraph,
+        # join them and append to results
+        if extracted_segments:
+            extracted_paragraphs.append(" ".join(extracted_segments))
+
+    return "\n\n".join(extracted_paragraphs)
+
+
+def extract_content_from_path(
+    path: bytes | str | List[bytes | str],
+    parsing: ParsingConfig,
+    doc_type: str | DocumentType | None = None,
+) -> str | List[str]:
+    """
+    Extract the content from a file path or URL, or a list of file paths or URLs.
+
+    Args:
+        path (bytes | str | List[str]): The file path or URL, or a list of file paths or
+            URLs, or bytes content. The bytes option is meant to support cases
+            where upstream code may have already loaded the content (e.g., from a
+            database or API) and we want to avoid having to copy the content to a
+            temporary file.
+        parsing (ParsingConfig): The parsing configuration.
+        doc_type (str | DocumentType | None): The document type if known.
+            If multiple paths are given, this MUST apply to ALL docs.
+
+    Returns:
+        str | List[str]: The extracted content if a single file path or URL is provided,
+                or a list of extracted contents if a
+                list of file paths or URLs is provided.
+    """
+    if isinstance(path, str) or isinstance(path, bytes):
+        paths = [path]
+    elif isinstance(path, list) and len(path) == 0:
+        return ""
+    else:
+        paths = path
+
+    url_idxs, path_idxs, byte_idxs = get_urls_paths_bytes_indices(paths)
+    urls = [paths[i] for i in url_idxs]
+    path_list = [paths[i] for i in path_idxs]
+    byte_list = [paths[i] for i in byte_idxs]
+    path_list.extend(byte_list)
+    parser = Parser(parsing)
+    docs: List[Document] = []
+    try:
+        if len(urls) > 0:
+            loader = URLLoader(urls=urls, parser=parser)  # type: ignore
+            docs = loader.load()
+        if len(path_list) > 0:
+            for p in path_list:
+                path_docs = RepoLoader.get_documents(
+                    p, parser=parser, doc_type=doc_type
+                )
+                docs.extend(path_docs)
+    except Exception as e:
+        logger.warning(f"Error loading path {paths}: {e}")
+        return ""
+    if len(docs) == 1:
+        return docs[0].content
+    else:
+        return [d.content for d in docs]

langroid/parsing/web_search.py

@@ -12,6 +12,7 @@ from typing import Dict, List
 import requests
 from bs4 import BeautifulSoup
 from dotenv import load_dotenv
+from duckduckgo_search import DDGS
 from googleapiclient.discovery import Resource, build
 from requests.models import Response
 
@@ -77,3 +78,75 @@ def google_search(query: str, num_results: int = 5) -> List[WebSearchResult]:
         WebSearchResult(result["title"], result["link"], 3500, 300)
         for result in raw_results
     ]
+
+
+def metaphor_search(query: str, num_results: int = 5) -> List[WebSearchResult]:
+    """
+    Method that makes an API call by Metaphor client that queries
+    the top num_results links that matches the query. Returns a list
+    of WebSearchResult objects.
+
+    Args:
+        query (str): The query body that users wants to make.
+        num_results (int): Number of top matching results that we want
+            to grab
+    """
+
+    load_dotenv()
+
+    api_key = os.getenv("METAPHOR_API_KEY") or os.getenv("EXA_API_KEY")
+    if not api_key:
+        raise ValueError(
+            """
+            Neither METAPHOR_API_KEY nor EXA_API_KEY environment variables are set. 
+            Please set one of them to your API key, and try again.
+            """
+        )
+
+    try:
+        from metaphor_python import Metaphor
+    except ImportError:
+        raise ImportError(
+            "You are attempting to use the `metaphor_python` library;"
+            "To use it, please install langroid with the `metaphor` extra, e.g. "
+            "`pip install langroid[metaphor]` or `poetry add langroid[metaphor]` "
+            "(it installs the `metaphor_python` package from pypi)."
+        )
+
+    client = Metaphor(api_key=api_key)
+
+    response = client.search(
+        query=query,
+        num_results=num_results,
+    )
+    raw_results = response.results
+
+    return [
+        WebSearchResult(result.title, result.url, 3500, 300) for result in raw_results
+    ]
+
+
+def duckduckgo_search(query: str, num_results: int = 5) -> List[WebSearchResult]:
+    """
+    Method that makes an API call by DuckDuckGo client that queries
+    the top `num_results` links that matche the query. Returns a list
+    of WebSearchResult objects.
+
+    Args:
+        query (str): The query body that users wants to make.
+        num_results (int): Number of top matching results that we want
+            to grab
+    """
+
+    with DDGS() as ddgs:
+        search_results = [r for r in ddgs.text(query, max_results=num_results)]
+
+    return [
+        WebSearchResult(
+            title=result["title"],
+            link=result["href"],
+            max_content_length=3500,
+            max_summary_length=300,
+        )
+        for result in search_results
+    ]

langroid/prompts/__init__.py

@@ -0,0 +1,11 @@
+from . import dialog
+from . import prompts_config
+from . import templates
+from . import transforms
+
+__all__ = [
+    "dialog",
+    "prompts_config",
+    "templates",
+    "transforms",
+]

langroid/prompts/chat-gpt4-system-prompt.md

@@ -0,0 +1,68 @@
+Image input capabilities: Enabled
+
+Tools
+python
+When you send a message containing Python code to python, it will be executed in a
+stateful Jupyter notebook environment. python will respond with the output of the execution or time out after 60.0
+seconds. The drive at '/mnt/data' can be used to save and persist user files. Internet access for this session is disabled. Do not make external web requests or API calls as they will fail.
+
+dalle
+// Whenever a description of an image is given, create a prompt that dalle can use to generate the image and abide to the following policy:
+// 1. The prompt must be in English. Translate to English if needed.
+// 2. DO NOT ask for permission to generate the image, just do it!
+// 3. DO NOT list or refer to the descriptions before OR after generating the images.
+// 4. Do not create more than 1 image, even if the user requests more.
+// 5. Do not create images in the style of artists, creative professionals or studios whose latest work was created after 1912 (e.g. Picasso, Kahlo).
+// - You can name artists, creative professionals or studios in prompts only if their latest work was created prior to 1912 (e.g. Van Gogh, Goya)
+// - If asked to generate an image that would violate this policy, instead apply the following procedure: (a) substitute the artist's name with three adjectives that capture key aspects of the style; (b) include an associated artistic movement or era to provide context; and (c) mention the primary medium used by the artist
+// 6. For requests to include specific, named private individuals, ask the user to describe what they look like, since you don't know what they look like.
+// 7. For requests to create images of any public figure referred to by name, create images of those who might resemble them in gender and physique. But they shouldn't look like them. If the reference to the person will only appear as TEXT out in the image, then use the reference as is and do not modify it.
+// 8. Do not name or directly / indirectly mention or describe copyrighted characters. Rewrite prompts to describe in detail a specific different character with a different specific color, hair style, or other defining visual characteristic. Do not discuss copyright policies in responses.
+// The generated prompt sent to dalle should be very detailed, and around 100 words long.
+// Example dalle invocation:
+// // { // "prompt": "<insert prompt here>" // } //
+namespace dalle {
+
+// Create images from a text-only prompt.
+type text2im = (_: {
+// The size of the requested image. Use 1024x1024 (square) as the default, 1792x1024 if the user requests a wide image, and 1024x1792 for full-body portraits. Always include this parameter in the request.
+size?: "1792x1024" | "1024x1024" | "1024x1792",
+// The number of images to generate. If the user does not specify a number, generate 1 image.
+n?: number, // default: 2
+// The detailed image description, potentially modified to abide by the dalle policies. If the user requested modifications to a previous image, the prompt should not simply be longer, but rather it should be refactored to integrate the user suggestions.
+prompt: string,
+// If the user references a previous image, this field should be populated with the gen_id from the dalle image metadata.
+referenced_image_ids?: string[],
+}) => any;
+
+} // namespace dalle
+
+voice_mode
+// Voice mode functions are not available in text conversations.
+namespace voice_mode {
+
+} // namespace voice_mode
+
+browser
+You have the tool browser. Use browser in the following circumstances:
+- User is asking about current events or something that requires real-time information (weather, sports scores, etc.)
+- User is asking about some term you are totally unfamiliar with (it might be new)
+- User explicitly asks you to browse or provide links to references
+
+Given a query that requires retrieval, your turn will consist of three steps:
+
+Call the search function to get a list of results.
+Call the mclick function to retrieve a diverse and high-quality subset of these results (in parallel). Remember to SELECT AT LEAST 3 sources when using mclick.
+Write a response to the user based on these results. In your response, cite sources using the citation format below.
+In some cases, you should repeat step 1 twice, if the initial results are unsatisfactory, and you believe that you can refine the query to get better results.
+
+You can also open a url directly if one is provided by the user. Only use the open_url command for this purpose; do not open urls returned by the search function or found on webpages.
+
+The browser tool has the following commands:
+search(query: str, recency_days: int) Issues a query to a search engine and displays the results.
+mclick(ids: list[str]). Retrieves the contents of the webpages with provided IDs (indices). You should ALWAYS SELECT AT LEAST 3 and at most 10 pages. Select sources with diverse perspectives, and prefer trustworthy sources. Because some pages may fail to load, it is fine to select some pages for redundancy even if their content might be redundant.
+open_url(url: str) Opens the given URL and displays it.
+
+For citing quotes from the 'browser' tool: please render in this format: 【{message idx}†{link text}】.
+For long citations: please render in this format: [link text](message idx).
+Otherwise do not render links.

langroid/prompts/prompts_config.py

@@ -2,4 +2,4 @@ from pydantic import BaseSettings
 
 
 class PromptsConfig(BaseSettings):
-    max_tokens: int = 1000
+    max_tokens: int = 1000  # for output; NOT USED ANYWHERE

langroid/utils/__init__.py

@@ -0,0 +1,17 @@
+from . import configuration
+from . import globals
+from . import constants
+from . import logging
+from . import pydantic_utils
+from . import system
+from . import output
+
+__all__ = [
+    "configuration",
+    "globals",
+    "constants",
+    "logging",
+    "pydantic_utils",
+    "system",
+    "output",
+]

langroid/utils/algorithms/__init__.py

@@ -0,0 +1,3 @@
+from . import graph
+
+__all__ = ["graph"]

langroid/utils/algorithms/graph.py

@@ -0,0 +1,103 @@
+"""
+Graph algos.
+"""
+
+from typing import Dict, List, no_type_check
+
+import numpy as np
+
+
+@no_type_check
+def topological_sort(order: np.array) -> List[int]:
+    """
+    Given a directed adjacency matrix, return a topological sort of the nodes.
+    order[i,j] = -1 means there is an edge from i to j.
+    order[i,j] = 0 means there is no edge from i to j.
+    order[i,j] = 1 means there is an edge from j to i.
+
+    Args:
+        order (np.array): The adjacency matrix.
+
+    Returns:
+        List[int]: The topological sort of the nodes.
+
+    """
+    n = order.shape[0]
+
+    # Calculate the in-degrees
+    in_degree = [0] * n
+    for i in range(n):
+        for j in range(n):
+            if order[i, j] == -1:
+                in_degree[j] += 1
+
+    # Initialize the queue with nodes of in-degree 0
+    queue = [i for i in range(n) if in_degree[i] == 0]
+    result = []
+
+    while queue:
+        node = queue.pop(0)
+        result.append(node)
+
+        for i in range(n):
+            if order[node, i] == -1:
+                in_degree[i] -= 1
+                if in_degree[i] == 0:
+                    queue.append(i)
+
+    assert len(result) == n, "Cycle detected"
+    return result
+
+
+@no_type_check
+def components(order: np.ndarray) -> List[List[int]]:
+    """
+    Find the connected components in an undirected graph represented by a matrix.
+
+    Args:
+        order (np.ndarray): A matrix with values 0 or 1 indicating
+            undirected graph edges. `order[i][j] = 1` means an edge between `i`
+            and `j`, and `0` means no edge.
+
+    Returns:
+        List[List[int]]: A list of List where each List contains the indices of
+            nodes in the same connected component.
+
+    Example:
+        order = np.array([
+            [1, 1, 0, 0],
+            [1, 1, 1, 0],
+            [0, 1, 1, 0],
+            [0, 0, 0, 1]
+        ])
+        components(order)
+        # [[0, 1, 2], [3]]
+    """
+
+    i2g: Dict[int, int] = {}  # index to group mapping
+    next_group = 0
+    n = order.shape[0]
+    for i in range(n):
+        connected_groups = {i2g[j] for j in np.nonzero(order[i, :])[0] if j in i2g}
+
+        # If the node is not part of any group
+        # and is not connected to any groups, assign a new group
+        if not connected_groups:
+            i2g[i] = next_group
+            next_group += 1
+        else:
+            # If the node is connected to multiple groups, we merge them
+            main_group = min(connected_groups)
+            for j in np.nonzero(order[i, :])[0]:
+                if i2g.get(j) in connected_groups:
+                    i2g[j] = main_group
+            i2g[i] = main_group
+
+    # Convert i2g to a list of Lists
+    groups: Dict[int, List[int]] = {}
+    for index, group in i2g.items():
+        if group not in groups:
+            groups[group] = []
+        groups[group].append(index)
+
+    return list(groups.values())