langroid 0.33.6__py3-none-any.whl → 0.33.8__py3-none-any.whl

langroid/parsing/web_search.py

@@ -0,0 +1,156 @@
+"""
+Utilities for web search.
+
+NOTE: Using Google Search requires setting the GOOGLE_API_KEY and GOOGLE_CSE_ID
+environment variables in your `.env` file, as explained in the
+[README](https://github.com/langroid/langroid#gear-installation-and-setup).
+"""
+
+import os
+from typing import Dict, List
+
+import requests
+from bs4 import BeautifulSoup
+from dotenv import load_dotenv
+from duckduckgo_search import DDGS
+from googleapiclient.discovery import Resource, build
+from requests.models import Response
+
+
+class WebSearchResult:
+    """
+    Class representing a Web Search result, containing the title, link,
+    summary and full content of the result.
+    """
+
+    def __init__(
+        self,
+        title: str,
+        link: str,
+        max_content_length: int = 3500,
+        max_summary_length: int = 300,
+    ):
+        """
+        Args:
+            title (str): The title of the search result.
+            link (str): The link to the search result.
+            max_content_length (int): The maximum length of the full content.
+            max_summary_length (int): The maximum length of the summary.
+        """
+        self.title = title
+        self.link = link
+        self.max_content_length = max_content_length
+        self.max_summary_length = max_summary_length
+        self.full_content = self.get_full_content()
+        self.summary = self.get_summary()
+
+    def get_summary(self) -> str:
+        return self.full_content[: self.max_summary_length]
+
+    def get_full_content(self) -> str:
+        try:
+            response: Response = requests.get(self.link)
+            soup: BeautifulSoup = BeautifulSoup(response.text, "lxml")
+            text = " ".join(soup.stripped_strings)
+            return text[: self.max_content_length]
+        except Exception as e:
+            return f"Error fetching content from {self.link}: {e}"
+
+    def __str__(self) -> str:
+        return f"Title: {self.title}\nLink: {self.link}\nSummary: {self.summary}"
+
+    def to_dict(self) -> Dict[str, str]:
+        return {
+            "title": self.title,
+            "link": self.link,
+            "summary": self.summary,
+            "full_content": self.full_content,
+        }
+
+
+def google_search(query: str, num_results: int = 5) -> List[WebSearchResult]:
+    load_dotenv()
+    api_key = os.getenv("GOOGLE_API_KEY")
+    cse_id = os.getenv("GOOGLE_CSE_ID")
+    service: Resource = build("customsearch", "v1", developerKey=api_key)
+    raw_results = (
+        service.cse().list(q=query, cx=cse_id, num=num_results).execute()["items"]
+    )
+
+    return [
+        WebSearchResult(result["title"], result["link"], 3500, 300)
+        for result in raw_results
+    ]
+
+
+def metaphor_search(query: str, num_results: int = 5) -> List[WebSearchResult]:
+    """
+    Method that makes an API call by Metaphor client that queries
+    the top num_results links that matches the query. Returns a list
+    of WebSearchResult objects.
+
+    Args:
+        query (str): The query body that users wants to make.
+        num_results (int): Number of top matching results that we want
+            to grab
+    """
+
+    load_dotenv()
+
+    api_key = os.getenv("METAPHOR_API_KEY") or os.getenv("EXA_API_KEY")
+    if not api_key:
+        raise ValueError(
+            """
+            Neither METAPHOR_API_KEY nor EXA_API_KEY environment variables are set. 
+            Please set one of them to your API key, and try again.
+            """
+        )
+
+    try:
+        from metaphor_python import Metaphor
+    except ImportError:
+        raise ImportError(
+            "You are attempting to use the `metaphor_python` library;"
+            "To use it, please install langroid with the `metaphor` extra, e.g. "
+            "`pip install langroid[metaphor]` or `poetry add langroid[metaphor]` "
+            "or `uv add langroid[metaphor]`"
+            "(it installs the `metaphor_python` package from pypi)."
+        )
+
+    client = Metaphor(api_key=api_key)
+
+    response = client.search(
+        query=query,
+        num_results=num_results,
+    )
+    raw_results = response.results
+
+    return [
+        WebSearchResult(result.title, result.url, 3500, 300) for result in raw_results
+    ]
+
+
+def duckduckgo_search(query: str, num_results: int = 5) -> List[WebSearchResult]:
+    """
+    Method that makes an API call by DuckDuckGo client that queries
+    the top `num_results` links that matche the query. Returns a list
+    of WebSearchResult objects.
+
+    Args:
+        query (str): The query body that users wants to make.
+        num_results (int): Number of top matching results that we want
+            to grab
+    """
+
+    with DDGS() as ddgs:
+        search_results = [r for r in ddgs.text(query, max_results=num_results)]
+
+    return [
+        WebSearchResult(
+            title=result["title"],
+            link=result["href"],
+            max_content_length=3500,
+            max_summary_length=300,
+        )
+        for result in search_results
+    ]

langroid/prompts/__init__.py

@@ -0,0 +1,9 @@
+from . import dialog
+from . import prompts_config
+from . import templates
+
+__all__ = [
+    "dialog",
+    "prompts_config",
+    "templates",
+]

langroid/prompts/dialog.py

@@ -0,0 +1,17 @@
+from typing import List
+
+
+def collate_chat_history(inputs: List[tuple[str, str]]) -> str:
+    """
+    Collate (human, ai) pairs into a single, string
+    Args:
+        inputs:
+    Returns:
+    """
+    pairs = [
+        f"""Human:{human}
+        AI:{ai}
+        """
+        for human, ai in inputs
+    ]
+    return "\n".join(pairs)

langroid/prompts/prompts_config.py

@@ -0,0 +1,5 @@
+from langroid.pydantic_v1 import BaseSettings
+
+
+class PromptsConfig(BaseSettings):
+    max_tokens: int = 1000  # for output; NOT USED ANYWHERE

langroid/prompts/templates.py

@@ -0,0 +1,141 @@
+from langroid.utils.constants import NO_ANSWER
+
+EXTRACT_RELEVANT = """
+    Here is a passage from a long document, followed by a question. 
+    In case the passage contains any text relevant to answer the question, return it 
+    verbatim.
+    {passage}    
+    Question:{question}
+    Relevant text, if any: """.strip()
+
+EXTRACTION_PROMPT_GPT4 = f"""
+Given the content and question below, extract COMPLETE SENTENCES OR PHRASES 
+VERBATIM from the content, that are relevant to answering the question (if such text 
+exists), even if it contradicts your knowledge, and even if it is factually incorrect.
+Do not  make up an answer that is not supported by the content. 
+When you answer, be concise, no need to explain anything. If there is no relevant text,
+simply say {NO_ANSWER}.
+
+Content: {{content}}
+Question: {{question}}
+Relevant text, if any:
+"""
+
+EXTRACTION_PROMPT = f"""
+    Given the content and question below, extract a COMPLETE SENTENCE verbatim from the 
+    content that is relevant to answering the question (if such text exists). Do not 
+    make up an answer.
+    
+    Content: The Eiffel Tower is a wrought-iron lattice tower on the Champ de Mars in
+    Paris, France. It is named after Gustave Eiffel, whose company designed and built
+    the tower. It is a recognizable landmark.
+    Question: Where is the Eiffel Tower located?
+    Relevant text, if any: on the Champ de Mars in Paris, France.
+    
+    Content: Apples and oranges are both fruits, but differ in taste and texture.
+    Apples are sweet and crisp, while oranges are citrusy and juicy. Both are
+    nutritious and commonly consumed worldwide.
+    Question: What are the similarities between apples and oranges?
+    Relevant text, if any: both fruits
+    
+    Content: The sun rises in the east and sets in the west. It is a source of light
+    and warmth for the Earth.
+    Question: What is the color of the sun?
+    Relevant text, if any: {NO_ANSWER}
+    
+    Content: {{content}}
+    Question: {{question}}
+    Relevant text (COMPLETE SENTENCE), if any:
+    """.strip()
+
+SUMMARY_ANSWER_PROMPT_GPT4 = f"""
+
+        Use the provided NUMBERED extracts (with sources)  to answer the QUESTION. 
+        If there's not enough information, respond with {NO_ANSWER}. Use only the 
+        information in these extracts, even if your answer is factually incorrect, 
+        and even if the answer contradicts other parts of the document. The only 
+        important thing is that your answer is consistent with and supported by the 
+        extracts. Compose your complete answer, inserting CITATIONS in MARKDOWN format
+        [^i][^j] where i,j,... are the extract NUMBERS you are 
+        citing.
+        For example your answer might look like this (NOTE HOW multiple citations
+        are grouped as [^2][^5]):
+        
+        Beethoven composed the 9th symphony in 1824.[^1] After that he became deaf
+        and could not hear his own music. [^2][^5]. He was a prolific composer and
+        wrote many famous pieces.
+        
+        NUMBERED EXTRACTS:
+        
+        {{extracts}}
+        
+        QUESTION:
+        {{question}}
+
+""".strip()
+
+ANSWER_PROMPT_USE_HISTORY_GPT4 = f"""
+
+        Use ANY of the information earlier, as well as the extracts provided below 
+        (with sources)  to answer the question. If there's not 
+        enough information, respond with {NO_ANSWER}.
+        Use only the information in this conversation or these extracts, 
+        even if your answer is factually incorrect, and even 
+        if the answer contracts other parts of the document.
+        The only important thing is that your answer is 
+        consistent with information provided here or earlier.
+        Compose your complete answer and cite all supporting sources 
+        on a separate separate line as "SOURCE:". 
+        When citing a SOURCE: be concise, whether it refers to a source in these 
+        extracts, or info provided earlier.
+        
+        {{extracts}}
+        {{question}}
+        Answer:   
+""".strip()
+
+
+SUMMARY_ANSWER_PROMPT = f"""
+        Use the provided extracts (with sources)  to answer the question. 
+        If there's not enough information, respond with {NO_ANSWER}.
+        Use only the information in these extracts, even if it contradicts your prior 
+        knowledge. Justify your answer by citing your sources, as in these examples:
+        
+        Extract: The tree species in the garden include oak, maple, and birch.
+        Source: https://en.wikipedia.org/wiki/Tree
+        Extract: The oak trees are known for their longevity and strength.
+        Source: https://en.wikipedia.org/wiki/Oak
+        Question: What types of trees are in the garden?
+        Answer: The types of trees in the garden include oak, maple, and birch.
+        SOURCE: https://en.wikipedia.org/wiki/Tree
+        TEXT: The tree species in the garden include oak, maple, and birch.
+        
+        Extract: The experiment involved three groups: control, low dose, and high 
+        dose. 
+        Source: https://en.wikipedia.org/wiki/Experiment
+        Extract: The high dose group showed significant improvement in symptoms.
+        Source: https://en.wikipedia.org/wiki/Experiment
+        Extract: The control group did not receive any 
+        treatment and served as a baseline.
+        Source: https://en.wikipedia.org/wiki/Experiment
+        Question: How many groups were involved which group showed significant 
+        improvement? 
+        Answer: There were three groups and the high dose group showed significant 
+        improvement in symptoms.
+        SOURCE: https://en.wikipedia.org/wiki/Experiment
+        TEXT: The experiment involved three groups: control, low dose, and high dose.
+        SOURCE: https://en.wikipedia.org/wiki/Experiment
+        TEXT: The high dose group showed significant improvement in symptoms.
+        
+        
+        Extract: The CEO announced several new initiatives during the company meeting.
+        Source: https://en.wikipedia.org/wiki/CEO
+        Extract: The financial performance of the company has been strong this year.
+        Source: https://en.wikipedia.org/wiki/CEO
+        Question: What new initiatives did the CEO announce?
+        Answer: {NO_ANSWER}
+        
+        {{extracts}}
+        {{question}}
+        Answer:
+        """.strip()

langroid/pydantic_v1/__init__.py

@@ -0,0 +1,10 @@
+"""
+If we're on Pydantic v2, use the v1 namespace, else just use the main namespace.
+
+This allows compatibility with both Pydantic v1 and v2
+"""
+
+try:
+    from pydantic.v1 import *  # noqa: F403, F401
+except ImportError:
+    from pydantic import *  # type: ignore # noqa: F403, F401

langroid/pydantic_v1/main.py

@@ -0,0 +1,4 @@
+try:
+    from pydantic.v1.main import *  # noqa: F403, F401
+except ImportError:
+    from pydantic.main import *  # type: ignore # noqa: F403, F401

langroid/utils/__init__.py

@@ -0,0 +1,19 @@
+from . import configuration
+from . import globals
+from . import constants
+from . import logging
+from . import pydantic_utils
+from . import system
+from . import output
+from . import object_registry
+
+__all__ = [
+    "configuration",
+    "globals",
+    "constants",
+    "logging",
+    "pydantic_utils",
+    "system",
+    "output",
+    "object_registry",
+]

langroid/utils/algorithms/__init__.py

@@ -0,0 +1,3 @@
+from . import graph
+
+__all__ = ["graph"]

langroid/utils/algorithms/graph.py

@@ -0,0 +1,103 @@
+"""
+Graph algos.
+"""
+
+from typing import Dict, List, no_type_check
+
+import numpy as np
+
+
+@no_type_check
+def topological_sort(order: np.array) -> List[int]:
+    """
+    Given a directed adjacency matrix, return a topological sort of the nodes.
+    order[i,j] = -1 means there is an edge from i to j.
+    order[i,j] = 0 means there is no edge from i to j.
+    order[i,j] = 1 means there is an edge from j to i.
+
+    Args:
+        order (np.array): The adjacency matrix.
+
+    Returns:
+        List[int]: The topological sort of the nodes.
+
+    """
+    n = order.shape[0]
+
+    # Calculate the in-degrees
+    in_degree = [0] * n
+    for i in range(n):
+        for j in range(n):
+            if order[i, j] == -1:
+                in_degree[j] += 1
+
+    # Initialize the queue with nodes of in-degree 0
+    queue = [i for i in range(n) if in_degree[i] == 0]
+    result = []
+
+    while queue:
+        node = queue.pop(0)
+        result.append(node)
+
+        for i in range(n):
+            if order[node, i] == -1:
+                in_degree[i] -= 1
+                if in_degree[i] == 0:
+                    queue.append(i)
+
+    assert len(result) == n, "Cycle detected"
+    return result
+
+
+@no_type_check
+def components(order: np.ndarray) -> List[List[int]]:
+    """
+    Find the connected components in an undirected graph represented by a matrix.
+
+    Args:
+        order (np.ndarray): A matrix with values 0 or 1 indicating
+            undirected graph edges. `order[i][j] = 1` means an edge between `i`
+            and `j`, and `0` means no edge.
+
+    Returns:
+        List[List[int]]: A list of List where each List contains the indices of
+            nodes in the same connected component.
+
+    Example:
+        order = np.array([
+            [1, 1, 0, 0],
+            [1, 1, 1, 0],
+            [0, 1, 1, 0],
+            [0, 0, 0, 1]
+        ])
+        components(order)
+        # [[0, 1, 2], [3]]
+    """
+
+    i2g: Dict[int, int] = {}  # index to group mapping
+    next_group = 0
+    n = order.shape[0]
+    for i in range(n):
+        connected_groups = {i2g[j] for j in np.nonzero(order[i, :])[0] if j in i2g}
+
+        # If the node is not part of any group
+        # and is not connected to any groups, assign a new group
+        if not connected_groups:
+            i2g[i] = next_group
+            next_group += 1
+        else:
+            # If the node is connected to multiple groups, we merge them
+            main_group = min(connected_groups)
+            for j in np.nonzero(order[i, :])[0]:
+                if i2g.get(j) in connected_groups:
+                    i2g[j] = main_group
+            i2g[i] = main_group
+
+    # Convert i2g to a list of Lists
+    groups: Dict[int, List[int]] = {}
+    for index, group in i2g.items():
+        if group not in groups:
+            groups[group] = []
+        groups[group].append(index)
+
+    return list(groups.values())

langroid/utils/configuration.py

@@ -0,0 +1,98 @@
+import copy
+import os
+from contextlib import contextmanager
+from typing import Iterator, List, Literal
+
+from dotenv import find_dotenv, load_dotenv
+
+from langroid.pydantic_v1 import BaseSettings
+
+
+class Settings(BaseSettings):
+    # NOTE all of these can be overridden in your .env file with upper-case names,
+    # for example CACHE_TYPE=momento
+    debug: bool = False  # show debug messages?
+    max_turns: int = -1  # maximum number of turns in a task (to avoid inf loop)
+    progress: bool = False  # show progress spinners/bars?
+    stream: bool = True  # stream output?
+    cache: bool = True  # use cache?
+    cache_type: Literal["redis", "fakeredis", "momento", "none"] = "redis"  # cache type
+    chat_model: str = ""  # language model name, e.g. litellm/ollama/llama2
+    quiet: bool = False  # quiet mode (i.e. suppress all output)?
+    notebook: bool = False  # running in a notebook?
+
+    class Config:
+        extra = "forbid"
+
+
+load_dotenv(find_dotenv(usecwd=True))  # get settings from .env file
+settings = Settings()
+
+
+def update_global_settings(cfg: BaseSettings, keys: List[str]) -> None:
+    """
+    Update global settings so modules can access them via (as an example):
+    ```
+    from langroid.utils.configuration import settings
+    if settings.debug...
+    ```
+    Caution we do not want to have too many such global settings!
+    Args:
+        cfg: pydantic config, typically from a main script
+        keys: which keys from cfg to use, to update the global settings object
+    """
+    config_dict = cfg.dict()
+
+    # Filter the config_dict based on the keys
+    filtered_config = {key: config_dict[key] for key in keys if key in config_dict}
+
+    # create a new Settings() object to let pydantic validate it
+    new_settings = Settings(**filtered_config)
+
+    # Update the unique global settings object
+    settings.__dict__.update(new_settings.__dict__)
+
+
+def set_global(key_vals: Settings) -> None:
+    """Update the unique global settings object"""
+    settings.__dict__.update(key_vals.__dict__)
+
+
+@contextmanager
+def temporary_settings(temp_settings: Settings) -> Iterator[None]:
+    """Temporarily update the global settings and restore them afterward."""
+    original_settings = copy.deepcopy(settings)
+
+    set_global(temp_settings)
+
+    try:
+        yield
+    finally:
+        settings.__dict__.update(original_settings.__dict__)
+
+
+@contextmanager
+def quiet_mode(quiet: bool = True) -> Iterator[None]:
+    """Temporarily set quiet=True in global settings and restore afterward."""
+    original_settings = copy.deepcopy(settings)
+    if quiet:
+        temp_settings = original_settings.copy(update={"quiet": True})
+        set_global(temp_settings)
+
+    try:
+        yield
+    finally:
+        if quiet:
+            settings.__dict__.update(original_settings.__dict__)
+
+
+def set_env(settings: BaseSettings) -> None:
+    """
+    Set environment variables from a BaseSettings instance
+    Args:
+        settings (BaseSettings): desired settings
+    Returns:
+    """
+    for field_name, field in settings.__class__.__fields__.items():
+        env_var_name = field.field_info.extra.get("env", field_name).upper()
+        os.environ[env_var_name] = str(settings.dict()[field_name])

langroid/utils/constants.py

@@ -0,0 +1,30 @@
+from langroid.pydantic_v1 import BaseModel
+
+
+# Define the ANSI escape sequences for various colors and reset
+class Colors(BaseModel):
+    RED: str = "\033[31m"
+    BLUE: str = "\033[34m"
+    GREEN: str = "\033[32m"
+    ORANGE: str = "\033[33m"  # no standard ANSI color for orange; using yellow
+    CYAN: str = "\033[36m"
+    MAGENTA: str = "\033[35m"
+    YELLOW: str = "\033[33m"
+    RESET: str = "\033[0m"
+
+
+NO_ANSWER = "DO-NOT-KNOW"
+DONE = "DONE"
+USER_QUIT_STRINGS = ["q", "x", "quit", "exit", "bye", DONE]
+PASS = "__PASS__"
+PASS_TO = PASS + ":"
+SEND_TO = "__SEND__:"
+TOOL = "TOOL"
+# This is a recommended setting for TaskConfig.addressing_prefix if using it at all;
+# prefer to use `RecipientTool` to allow agents addressing others.
+# Caution the AT string should NOT contain any 'word' characters, i.e.
+# it no letters, digits or underscores.
+# See tests/main/test_msg_routing for example usage
+AT = "|@|"
+TOOL_BEGIN = "TOOL_BEGIN"
+TOOL_END = "TOOL_END"