langroid 0.33.6__py3-none-any.whl → 0.33.7__py3-none-any.whl

langroid/language_models/prompt_formatter/__init__.py

@@ -0,0 +1,16 @@
+from . import base
+from . import llama2_formatter
+from .base import PromptFormatter
+from .llama2_formatter import Llama2Formatter
+from ..config import PromptFormatterConfig
+from ..config import Llama2FormatterConfig
+
+
+__all__ = [
+    "PromptFormatter",
+    "Llama2Formatter",
+    "PromptFormatterConfig",
+    "Llama2FormatterConfig",
+    "base",
+    "llama2_formatter",
+]

langroid/language_models/prompt_formatter/base.py

@@ -0,0 +1,40 @@
+import logging
+from abc import ABC, abstractmethod
+from typing import List
+
+from langroid.language_models.base import LLMMessage
+from langroid.language_models.config import PromptFormatterConfig
+
+logger = logging.getLogger(__name__)
+
+
+class PromptFormatter(ABC):
+    """
+    Abstract base class for a prompt formatter
+    """
+
+    def __init__(self, config: PromptFormatterConfig):
+        self.config = config
+
+    @staticmethod
+    def create(formatter: str) -> "PromptFormatter":
+        from langroid.language_models.config import HFPromptFormatterConfig
+        from langroid.language_models.prompt_formatter.hf_formatter import HFFormatter
+
+        return HFFormatter(HFPromptFormatterConfig(model_name=formatter))
+
+    @abstractmethod
+    def format(self, messages: List[LLMMessage]) -> str:
+        """
+        Convert sequence of messages (system, user, assistant, user, assistant...user)
+            to a single prompt formatted according to the specific format type,
+            to be used in a /completions endpoint.
+
+        Args:
+            messages (List[LLMMessage]): chat history as a sequence of messages
+
+        Returns:
+            (str): formatted version of chat history
+
+        """
+        pass

langroid/language_models/prompt_formatter/hf_formatter.py

@@ -0,0 +1,132 @@
+"""
+Prompt formatter based on HuggingFace `AutoTokenizer.apply_chat_template` method
+from their Transformers library. It searches the hub for a model matching the
+specified name, and uses the first one it finds. We assume that all matching
+models will have the same tokenizer, so we just use the first one.
+"""
+
+import logging
+import re
+from typing import Any, List, Set, Tuple, Type
+
+from jinja2.exceptions import TemplateError
+
+from langroid.language_models.base import LanguageModel, LLMMessage, Role
+from langroid.language_models.config import HFPromptFormatterConfig
+from langroid.language_models.prompt_formatter.base import PromptFormatter
+
+logger = logging.getLogger(__name__)
+
+
+def try_import_hf_modules() -> Tuple[Type[Any], Type[Any]]:
+    """
+    Attempts to import the AutoTokenizer class from the transformers package.
+    Returns:
+        The AutoTokenizer class if successful.
+    Raises:
+        ImportError: If the transformers package is not installed.
+    """
+    try:
+        from huggingface_hub import HfApi
+        from transformers import AutoTokenizer
+
+        return AutoTokenizer, HfApi
+    except ImportError:
+        raise ImportError(
+            """
+            You are trying to use some/all of:
+            HuggingFace transformers.AutoTokenizer,
+            huggingface_hub.HfApi,
+            but these are not not installed 
+            by default with Langroid. Please install langroid using the 
+            `transformers` extra, like so:
+            pip install "langroid[transformers]"
+            or equivalent.
+            """
+        )
+
+
+def find_hf_formatter(model_name: str) -> str:
+    AutoTokenizer, HfApi = try_import_hf_modules()
+    hf_api = HfApi()
+    # try to find a matching model, with progressivly shorter prefixes of model_name
+    model_name = model_name.lower().split("/")[-1]
+    parts = re.split("[:\\-_]", model_name)
+    parts = [p.lower() for p in parts if p != ""]
+    for i in range(len(parts), 0, -1):
+        prefix = "-".join(parts[:i])
+        models = hf_api.list_models(
+            task="text-generation",
+            model_name=prefix,
+        )
+        try:
+            mdl = next(models)
+            tokenizer = AutoTokenizer.from_pretrained(mdl.id)
+            if tokenizer.chat_template is not None:
+                return str(mdl.id)
+            else:
+                continue
+        except Exception:
+            continue
+
+    return ""
+
+
+class HFFormatter(PromptFormatter):
+    models: Set[str] = set()  # which models have been used for formatting
+
+    def __init__(self, config: HFPromptFormatterConfig):
+        super().__init__(config)
+        AutoTokenizer, HfApi = try_import_hf_modules()
+        self.config: HFPromptFormatterConfig = config
+        hf_api = HfApi()
+        models = hf_api.list_models(
+            task="text-generation",
+            model_name=config.model_name,
+        )
+        try:
+            mdl = next(models)
+        except StopIteration:
+            raise ValueError(f"Model {config.model_name} not found on HuggingFace Hub")
+
+        self.tokenizer = AutoTokenizer.from_pretrained(mdl.id)
+        if self.tokenizer.chat_template is None:
+            raise ValueError(
+                f"Model {config.model_name} does not support chat template"
+            )
+        elif mdl.id not in HFFormatter.models:
+            # only warn if this is the first time we've used this mdl.id
+            logger.warning(
+                f"""
+            Using HuggingFace {mdl.id} for prompt formatting: 
+            This is the CHAT TEMPLATE. If this is not what you intended,
+            consider specifying a more complete model name for the formatter.
+             
+            {self.tokenizer.chat_template}
+            """
+            )
+        HFFormatter.models.add(mdl.id)
+
+    def format(self, messages: List[LLMMessage]) -> str:
+        sys_msg, chat_msgs, user_msg = LanguageModel.get_chat_history_components(
+            messages
+        )
+        # build msg dicts expected by AutoTokenizer.apply_chat_template
+        sys_msg_dict = dict(role=Role.SYSTEM.value, content=sys_msg)
+        chat_dicts = []
+        for user, assistant in chat_msgs:
+            chat_dicts.append(dict(role=Role.USER.value, content=user))
+            chat_dicts.append(dict(role=Role.ASSISTANT.value, content=assistant))
+        chat_dicts.append(dict(role=Role.USER.value, content=user_msg))
+        all_dicts = [sys_msg_dict] + chat_dicts
+        try:
+            # apply chat template
+            result = self.tokenizer.apply_chat_template(all_dicts, tokenize=False)
+        except TemplateError:
+            # this likely means the model doesn't support a system msg,
+            # so combine it with the first user msg
+            first_user_msg = chat_msgs[0][0] if len(chat_msgs) > 0 else user_msg
+            first_user_msg = sys_msg + "\n\n" + first_user_msg
+            chat_dicts[0] = dict(role=Role.USER.value, content=first_user_msg)
+            result = self.tokenizer.apply_chat_template(chat_dicts, tokenize=False)
+        return str(result)

langroid/language_models/prompt_formatter/llama2_formatter.py

@@ -0,0 +1,75 @@
+import logging
+from typing import List, Tuple
+
+from langroid.language_models.base import LanguageModel, LLMMessage
+from langroid.language_models.config import Llama2FormatterConfig
+from langroid.language_models.prompt_formatter.base import PromptFormatter
+
+logger = logging.getLogger(__name__)
+
+
+BOS: str = "<s>"
+EOS: str = "</s>"
+B_INST: str = "[INST]"
+E_INST: str = "[/INST]"
+B_SYS: str = "<<SYS>>\n"
+E_SYS: str = "\n<</SYS>>\n\n"
+SPECIAL_TAGS: List[str] = [B_INST, E_INST, BOS, EOS, "<<SYS>>", "<</SYS>>"]
+
+
+class Llama2Formatter(PromptFormatter):
+    def __int__(self, config: Llama2FormatterConfig) -> None:
+        super().__init__(config)
+        self.config: Llama2FormatterConfig = config
+
+    def format(self, messages: List[LLMMessage]) -> str:
+        sys_msg, chat_msgs, user_msg = LanguageModel.get_chat_history_components(
+            messages
+        )
+        return self._get_prompt_from_components(sys_msg, chat_msgs, user_msg)
+
+    def _get_prompt_from_components(
+        self,
+        system_prompt: str,
+        chat_history: List[Tuple[str, str]],
+        user_message: str,
+    ) -> str:
+        """
+        For llama2 models, convert chat history into a single
+        prompt for Llama2 models, for use in the /completions endpoint
+        (as opposed to the /chat/completions endpoint).
+        See:
+        https://www.reddit.com/r/LocalLLaMA/comments/155po2p/get_llama_2_prompt_format_right/
+        https://github.com/facebookresearch/llama/blob/main/llama/generation.py#L44
+
+        Args:
+            system_prompt (str): system prompt, typically specifying role/task.
+            chat_history (List[Tuple[str,str]]): List of (user, assistant) pairs
+            user_message (str): user message, at the end of the chat, i.e. the message
+                for which we want to generate a response.
+
+        Returns:
+            str: Prompt for Llama2 models
+
+        Typical structure of the formatted prompt:
+        Note important that the first [INST], [/INST] surrounds the system prompt,
+        together with the first user message. A lot of libs seem to miss this detail.
+
+        <s>[INST] <<SYS>>
+        You are are a helpful... bla bla.. assistant
+        <</SYS>>
+
+        Hi there! [/INST] Hello! How can I help you today? </s><s>[INST]
+        What is a neutron star? [/INST] A neutron star is a ... </s><s>
+        [INST] Okay cool, thank you! [/INST] You're welcome! </s><s>
+        [INST] Ah, I have one more question.. [/INST]
+        """
+        bos = BOS if self.config.use_bos_eos else ""
+        eos = EOS if self.config.use_bos_eos else ""
+        text = f"{bos}{B_INST} {B_SYS}{system_prompt}{E_SYS}"
+        for user_input, response in chat_history:
+            text += (
+                f"{user_input.strip()} {E_INST} {response.strip()} {eos}{bos} {B_INST} "
+            )
+        text += f"{user_message.strip()} {E_INST}"
+        return text

langroid/language_models/utils.py

@@ -0,0 +1,151 @@
+# from openai-cookbook
+import asyncio
+import logging
+import random
+import time
+from typing import Any, Callable, Dict, List
+
+import aiohttp
+import openai
+import requests
+
+logger = logging.getLogger(__name__)
+# setlevel to warning
+logger.setLevel(logging.WARNING)
+
+
+# define a retry decorator
+def retry_with_exponential_backoff(
+    func: Callable[..., Any],
+    initial_delay: float = 1,
+    exponential_base: float = 1.3,
+    jitter: bool = True,
+    max_retries: int = 5,
+    errors: tuple = (  # type: ignore
+        requests.exceptions.RequestException,
+        openai.APITimeoutError,
+        openai.RateLimitError,
+        openai.AuthenticationError,
+        openai.APIError,
+        aiohttp.ServerTimeoutError,
+        asyncio.TimeoutError,
+    ),
+) -> Callable[..., Any]:
+    """Retry a function with exponential backoff."""
+
+    def wrapper(*args: List[Any], **kwargs: Dict[Any, Any]) -> Any:
+        # Initialize variables
+        num_retries = 0
+        delay = initial_delay
+
+        # Loop until a successful response or max_retries is hit or exception is raised
+        while True:
+            try:
+                return func(*args, **kwargs)
+
+            except openai.BadRequestError as e:
+                # do not retry when the request itself is invalid,
+                # e.g. when context is too long
+                logger.error(f"OpenAI API request failed with error: {e}.")
+                raise e
+            except openai.AuthenticationError as e:
+                # do not retry when there's an auth error
+                logger.error(f"OpenAI API request failed with error: {e}.")
+                raise e
+
+            except openai.UnprocessableEntityError as e:
+                logger.error(f"OpenAI API request failed with error: {e}.")
+                raise e
+
+            # Retry on specified errors
+            except errors as e:
+                # Increment retries
+                num_retries += 1
+
+                # Check if max retries has been reached
+                if num_retries > max_retries:
+                    raise Exception(
+                        f"Maximum number of retries ({max_retries}) exceeded."
+                        f" Last error: {str(e)}."
+                    )
+
+                # Increment the delay
+                delay *= exponential_base * (1 + jitter * random.random())
+                logger.warning(
+                    f"""OpenAI API request failed with error: 
+                    {e}. 
+                    Retrying in {delay} seconds..."""
+                )
+                # Sleep for the delay
+                time.sleep(delay)
+
+            # Raise exceptions for any errors not specified
+            except Exception as e:
+                raise e
+
+    return wrapper
+
+
+def async_retry_with_exponential_backoff(
+    func: Callable[..., Any],
+    initial_delay: float = 1,
+    exponential_base: float = 1.3,
+    jitter: bool = True,
+    max_retries: int = 5,
+    errors: tuple = (  # type: ignore
+        openai.APITimeoutError,
+        openai.RateLimitError,
+        openai.AuthenticationError,
+        openai.APIError,
+        aiohttp.ServerTimeoutError,
+        asyncio.TimeoutError,
+    ),
+) -> Callable[..., Any]:
+    """Retry a function with exponential backoff."""
+
+    async def wrapper(*args: List[Any], **kwargs: Dict[Any, Any]) -> Any:
+        # Initialize variables
+        num_retries = 0
+        delay = initial_delay
+
+        # Loop until a successful response or max_retries is hit or exception is raised
+        while True:
+            try:
+                result = await func(*args, **kwargs)
+                return result
+
+            except openai.BadRequestError as e:
+                # do not retry when the request itself is invalid,
+                # e.g. when context is too long
+                logger.error(f"OpenAI API request failed with error: {e}.")
+                raise e
+            except openai.AuthenticationError as e:
+                # do not retry when there's an auth error
+                logger.error(f"OpenAI API request failed with error: {e}.")
+                raise e
+            # Retry on specified errors
+            except errors as e:
+                # Increment retries
+                num_retries += 1
+
+                # Check if max retries has been reached
+                if num_retries > max_retries:
+                    raise Exception(
+                        f"Maximum number of retries ({max_retries}) exceeded."
+                        f" Last error: {str(e)}."
+                    )
+
+                # Increment the delay
+                delay *= exponential_base * (1 + jitter * random.random())
+                logger.warning(
+                    f"""OpenAI API request failed with error{e}. 
+                    Retrying in {delay} seconds..."""
+                )
+                # Sleep for the delay
+                time.sleep(delay)
+
+            # Raise exceptions for any errors not specified
+            except Exception as e:
+                raise e
+
+    return wrapper

langroid/mytypes.py

@@ -0,0 +1,84 @@
+from enum import Enum
+from textwrap import dedent
+from typing import Any, Callable, Dict, List, Union
+from uuid import uuid4
+
+from langroid.pydantic_v1 import BaseModel, Extra, Field
+
+Number = Union[int, float]
+Embedding = List[Number]
+Embeddings = List[Embedding]
+EmbeddingFunction = Callable[[List[str]], Embeddings]
+
+
+class Entity(str, Enum):
+    """
+    Enum for the different types of entities that can respond to the current message.
+    """
+
+    AGENT = "Agent"
+    LLM = "LLM"
+    USER = "User"
+    SYSTEM = "System"
+
+    def __eq__(self, other: object) -> bool:
+        """Allow case-insensitive equality (==) comparison with strings."""
+        if other is None:
+            return False
+        if isinstance(other, str):
+            return self.value.lower() == other.lower()
+        return super().__eq__(other)
+
+    def __ne__(self, other: object) -> bool:
+        """Allow case-insensitive non-equality (!=) comparison with strings."""
+        return not self.__eq__(other)
+
+    def __hash__(self) -> int:
+        """Override this to ensure hashability of the enum,
+        so it can be used sets and dictionary keys.
+        """
+        return hash(self.value.lower())
+
+
+class DocMetaData(BaseModel):
+    """Metadata for a document."""
+
+    source: str = "context"
+    is_chunk: bool = False  # if it is a chunk, don't split
+    id: str = Field(default_factory=lambda: str(uuid4()))
+    window_ids: List[str] = []  # for RAG: ids of chunks around this one
+
+    def dict_bool_int(self, *args: Any, **kwargs: Any) -> Dict[str, Any]:
+        """
+        Special dict method to convert bool fields to int, to appease some
+        downstream libraries,  e.g. Chroma which complains about bool fields in
+        metadata.
+        """
+        original_dict = super().dict(*args, **kwargs)
+
+        for key, value in original_dict.items():
+            if isinstance(value, bool):
+                original_dict[key] = 1 * value
+
+        return original_dict
+
+    class Config:
+        extra = Extra.allow
+
+
+class Document(BaseModel):
+    """Interface for interacting with a document."""
+
+    content: str
+    metadata: DocMetaData
+
+    def id(self) -> str:
+        return self.metadata.id
+
+    def __str__(self) -> str:
+        return dedent(
+            f"""
+        CONTENT: {self.content}         
+        SOURCE:{self.metadata.source}
+        """
+        )

langroid/parsing/__init__.py

@@ -0,0 +1,52 @@
+from . import parser
+from . import agent_chats
+from . import code_parser
+from . import document_parser
+from . import parse_json
+from . import para_sentence_split
+from . import repo_loader
+from . import url_loader
+from . import table_loader
+from . import urls
+from . import utils
+from . import search
+from . import web_search
+
+from .parser import (
+    Splitter,
+    PdfParsingConfig,
+    DocxParsingConfig,
+    DocParsingConfig,
+    ParsingConfig,
+    Parser,
+)
+
+__all__ = [
+    "parser",
+    "agent_chats",
+    "code_parser",
+    "document_parser",
+    "parse_json",
+    "para_sentence_split",
+    "repo_loader",
+    "url_loader",
+    "table_loader",
+    "urls",
+    "utils",
+    "search",
+    "web_search",
+    "Splitter",
+    "PdfParsingConfig",
+    "DocxParsingConfig",
+    "DocParsingConfig",
+    "ParsingConfig",
+    "Parser",
+]
+
+try:
+    from . import spider
+
+    spider
+    __all__.append("spider")
+except ImportError:
+    pass

langroid/parsing/agent_chats.py

@@ -0,0 +1,38 @@
+from typing import Tuple, no_type_check
+
+from pyparsing import Empty, Literal, ParseException, SkipTo, StringEnd, Word, alphanums
+
+
+@no_type_check
+def parse_message(msg: str) -> Tuple[str, str]:
+    """
+    Parse the intended recipient and content of a message.
+    Message format is assumed to be TO[<recipient>]:<message>.
+    The TO[<recipient>]: part is optional.
+
+    Args:
+        msg (str): message to parse
+
+    Returns:
+        str, str: task-name of intended recipient, and content of message
+            (if recipient is not specified, task-name is empty string)
+
+    """
+    if msg is None:
+        return "", ""
+
+    # Grammar definition
+    name = Word(alphanums)
+    to_start = Literal("TO[").suppress()
+    to_end = Literal("]:").suppress()
+    to_field = (to_start + name("name") + to_end) | Empty().suppress()
+    message = SkipTo(StringEnd())("text")
+
+    # Parser definition
+    parser = to_field + message
+
+    try:
+        parsed = parser.parseString(msg)
+        return parsed.name, parsed.text
+    except ParseException:
+        return "", msg