langroid 0.51.2__tar.gz → 0.52.1__tar.gz

{langroid-0.51.2 → langroid-0.52.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langroid
-Version: 0.51.2
+Version: 0.52.1
 Summary: Harness LLMs with Multi-Agent Programming
 Author-email: Prasad Chalasani <pchalasani@gmail.com>
 License: MIT

{langroid-0.51.2 → langroid-0.52.1}/langroid/agent/base.py

@@ -47,6 +47,7 @@ from langroid.language_models.base import (
 )
 from langroid.language_models.openai_gpt import OpenAIGPT, OpenAIGPTConfig
 from langroid.mytypes import Entity
+from langroid.parsing.file_attachment import FileAttachment
 from langroid.parsing.parse_json import extract_top_level_json
 from langroid.parsing.parser import Parser, ParsingConfig
 from langroid.prompts.prompts_config import PromptsConfig
@@ -440,6 +441,7 @@ class Agent(ABC):
     def create_agent_response(
         self,
         content: str | None = None,
+        files: List[FileAttachment] = [],
         content_any: Any = None,
         tool_messages: List[ToolMessage] = [],
         oai_tool_calls: Optional[List[OpenAIToolCall]] = None,
@@ -452,6 +454,7 @@ class Agent(ABC):
         return self.response_template(
             Entity.AGENT,
             content=content,
+            files=files,
             content_any=content_any,
             tool_messages=tool_messages,
             oai_tool_calls=oai_tool_calls,
@@ -689,6 +692,7 @@ class Agent(ABC):
         self,
         e: Entity,
         content: str | None = None,
+        files: List[FileAttachment] = [],
         content_any: Any = None,
         tool_messages: List[ToolMessage] = [],
         oai_tool_calls: Optional[List[OpenAIToolCall]] = None,
@@ -700,6 +704,7 @@ class Agent(ABC):
         """Template for response from entity `e`."""
         return ChatDocument(
             content=content or "",
+            files=files,
             content_any=content_any,
             tool_messages=tool_messages,
             oai_tool_calls=oai_tool_calls,
@@ -714,6 +719,7 @@ class Agent(ABC):
     def create_user_response(
         self,
         content: str | None = None,
+        files: List[FileAttachment] = [],
         content_any: Any = None,
         tool_messages: List[ToolMessage] = [],
         oai_tool_calls: List[OpenAIToolCall] | None = None,
@@ -726,6 +732,7 @@ class Agent(ABC):
         return self.response_template(
             e=Entity.USER,
             content=content,
+            files=files,
             content_any=content_any,
             tool_messages=tool_messages,
             oai_tool_calls=oai_tool_calls,

{langroid-0.51.2 → langroid-0.52.1}/langroid/agent/chat_document.py

@@ -19,6 +19,7 @@ from langroid.language_models.base import (
 )
 from langroid.mytypes import DocMetaData, Document, Entity
 from langroid.parsing.agent_chats import parse_message
+from langroid.parsing.file_attachment import FileAttachment
 from langroid.parsing.parse_json import extract_top_level_json, top_level_json_field
 from langroid.pydantic_v1 import BaseModel, Extra
 from langroid.utils.object_registry import ObjectRegistry
@@ -119,6 +120,7 @@ class ChatDocument(Document):
 
     reasoning: str = ""  # reasoning produced by a reasoning LLM
     content_any: Any = None  # to hold arbitrary data returned by responders
+    files: List[FileAttachment] = []  # list of file attachments
     oai_tool_calls: Optional[List[OpenAIToolCall]] = None
     oai_tool_id2result: Optional[OrderedDict[str, str]] = None
     oai_tool_choice: ToolChoiceTypes | Dict[str, Dict[str, str] | str] = "auto"
@@ -407,6 +409,7 @@ class ChatDocument(Document):
                         role=Role.TOOL,
                         tool_call_id=oai_tools[0].id,
                         content=content,
+                        files=message.files,
                         chat_document_id=chat_document_id,
                     )
                 ]
@@ -422,6 +425,7 @@ class ChatDocument(Document):
                         role=Role.TOOL,
                         tool_call_id=message.metadata.oai_tool_id,
                         content=content,
+                        files=message.files,
                         chat_document_id=chat_document_id,
                     )
                 ]
@@ -436,6 +440,7 @@ class ChatDocument(Document):
                         role=Role.TOOL,
                         tool_call_id=tool_id,
                         content=result or " ",
+                        files=message.files,
                         chat_document_id=chat_document_id,
                     )
                     for tool_id, result in message.oai_tool_id2result.items()
@@ -448,6 +453,7 @@ class ChatDocument(Document):
                 role=sender_role,
                 tool_id=tool_id,  # for OpenAI Assistant
                 content=content,
+                files=message.files,
                 function_call=fun_call,
                 tool_calls=oai_tool_calls,
                 name=sender_name,

{langroid-0.51.2 → langroid-0.52.1}/langroid/language_models/base.py

@@ -21,6 +21,7 @@ from langroid.cachedb.base import CacheDBConfig
 from langroid.cachedb.redis_cachedb import RedisCacheConfig
 from langroid.language_models.model_info import ModelInfo, get_model_info
 from langroid.parsing.agent_chats import parse_message
+from langroid.parsing.file_attachment import FileAttachment
 from langroid.parsing.parse_json import parse_imperfect_json, top_level_json_field
 from langroid.prompts.dialog import collate_chat_history
 from langroid.pydantic_v1 import BaseModel, BaseSettings, Field
@@ -53,6 +54,13 @@ class StreamEventType(Enum):
     TOOL_ARGS = 5
 
 
+class RetryParams(BaseSettings):
+    max_retries: int = 5
+    initial_delay: float = 1.0
+    exponential_base: float = 1.3
+    jitter: bool = True
+
+
 class LLMConfig(BaseSettings):
     """
     Common configuration for all language models.
@@ -87,6 +95,7 @@ class LLMConfig(BaseSettings):
     # Dict of model -> (input/prompt cost, output/completion cost)
     chat_cost_per_1k_tokens: Tuple[float, float] = (0.0, 0.0)
     completion_cost_per_1k_tokens: Tuple[float, float] = (0.0, 0.0)
+    retry_params: RetryParams = RetryParams()
 
     @property
     def model_max_output_tokens(self) -> int:
@@ -265,13 +274,14 @@ class LLMMessage(BaseModel):
     tool_call_id: Optional[str] = None  # which OpenAI LLM tool this is a response to
     tool_id: str = ""  # used by OpenAIAssistant
     content: str
+    files: List[FileAttachment] = []
     function_call: Optional[LLMFunctionCall] = None
     tool_calls: Optional[List[OpenAIToolCall]] = None
     timestamp: datetime = Field(default_factory=datetime.utcnow)
     # link to corresponding chat document, for provenance/rewind purposes
     chat_document_id: str = ""
 
-    def api_dict(self, has_system_role: bool = True) -> Dict[str, Any]:
+    def api_dict(self, model: str, has_system_role: bool = True) -> Dict[str, Any]:
         """
         Convert to dictionary for API request, keeping ONLY
         the fields that are expected in an API call!
@@ -285,6 +295,17 @@ class LLMMessage(BaseModel):
             dict: dictionary representation of LLM message
         """
         d = self.dict()
+        files: List[FileAttachment] = d.pop("files")
+        if len(files) > 0 and self.role == Role.USER:
+            # In there are files, then content is an array of
+            # different content-parts
+            d["content"] = [
+                dict(
+                    type="text",
+                    text=self.content,
+                )
+            ] + [f.to_dict(model) for f in self.files]
+
         # if there is a key k = "role" with value "system", change to "user"
         # in case has_system_role is False
         if not has_system_role and "role" in d and d["role"] == "system":

{langroid-0.51.2 → langroid-0.52.1}/langroid/language_models/model_info.py

@@ -31,6 +31,8 @@ class OpenAIChatModel(ModelName):
     O1 = "o1"
     O1_MINI = "o1-mini"
     O3_MINI = "o3-mini"
+    O3 = "o3"
+    O4_MINI = "o4-mini"
     GPT4_1 = "gpt-4.1"
     GPT4_1_MINI = "gpt-4.1-mini"
     GPT4_1_NANO = "gpt-4.1-nano"
@@ -226,6 +228,20 @@ MODEL_INFO: Dict[str, ModelInfo] = {
         has_tools=False,
         description="O1 Reasoning LM",
     ),
+    OpenAIChatModel.O3.value: ModelInfo(
+        name=OpenAIChatModel.O3.value,
+        provider=ModelProvider.OPENAI,
+        context_length=200_000,
+        max_output_tokens=100_000,
+        input_cost_per_million=10.0,
+        output_cost_per_million=40.0,
+        allows_streaming=True,
+        allows_system_message=False,
+        unsupported_params=["temperature"],
+        rename_params={"max_tokens": "max_completion_tokens"},
+        has_tools=False,
+        description="O1 Reasoning LM",
+    ),
     OpenAIChatModel.O1_MINI.value: ModelInfo(
         name=OpenAIChatModel.O1_MINI.value,
         provider=ModelProvider.OPENAI,
@@ -254,6 +270,20 @@ MODEL_INFO: Dict[str, ModelInfo] = {
         has_tools=False,
         description="O3 Mini Reasoning LM",
     ),
+    OpenAIChatModel.O4_MINI.value: ModelInfo(
+        name=OpenAIChatModel.O4_MINI.value,
+        provider=ModelProvider.OPENAI,
+        context_length=200_000,
+        max_output_tokens=100_000,
+        input_cost_per_million=1.10,
+        output_cost_per_million=4.40,
+        allows_streaming=False,
+        allows_system_message=False,
+        unsupported_params=["temperature", "stream"],
+        rename_params={"max_tokens": "max_completion_tokens"},
+        has_tools=False,
+        description="O3 Mini Reasoning LM",
+    ),
     # Anthropic Models
     AnthropicModel.CLAUDE_3_5_SONNET.value: ModelInfo(
         name=AnthropicModel.CLAUDE_3_5_SONNET.value,

{langroid-0.51.2 → langroid-0.52.1}/langroid/language_models/openai_gpt.py

@@ -1734,8 +1734,7 @@ class OpenAIGPT(LanguageModel):
             logging.error(friendly_error(e, "Error in OpenAIGPT.achat: "))
             raise e
 
-    @retry_with_exponential_backoff
-    def _chat_completions_with_backoff(self, **kwargs):  # type: ignore
+    def _chat_completions_with_backoff_body(self, **kwargs):  # type: ignore
         cached = False
         hashed_key, result = self._cache_lookup("Completion", **kwargs)
         if result is not None:
@@ -1784,8 +1783,17 @@ class OpenAIGPT(LanguageModel):
                 self._cache_store(hashed_key, result.model_dump())
         return cached, hashed_key, result
 
-    @async_retry_with_exponential_backoff
-    async def _achat_completions_with_backoff(self, **kwargs):  # type: ignore
+    def _chat_completions_with_backoff(self, **kwargs):  # type: ignore
+        retry_func = retry_with_exponential_backoff(
+            self._chat_completions_with_backoff_body,
+            initial_delay=self.config.retry_params.initial_delay,
+            max_retries=self.config.retry_params.max_retries,
+            exponential_base=self.config.retry_params.exponential_base,
+            jitter=self.config.retry_params.jitter,
+        )
+        return retry_func(**kwargs)
+
+    async def _achat_completions_with_backoff_body(self, **kwargs):  # type: ignore
         cached = False
         hashed_key, result = self._cache_lookup("Completion", **kwargs)
         if result is not None:
@@ -1839,6 +1847,16 @@ class OpenAIGPT(LanguageModel):
                 self._cache_store(hashed_key, result.model_dump())
         return cached, hashed_key, result
 
+    async def _achat_completions_with_backoff(self, **kwargs):  # type: ignore
+        retry_func = async_retry_with_exponential_backoff(
+            self._achat_completions_with_backoff_body,
+            initial_delay=self.config.retry_params.initial_delay,
+            max_retries=self.config.retry_params.max_retries,
+            exponential_base=self.config.retry_params.exponential_base,
+            jitter=self.config.retry_params.jitter,
+        )
+        return await retry_func(**kwargs)
+
     def _prep_chat_completion(
         self,
         messages: Union[str, List[LLMMessage]],
@@ -1879,7 +1897,10 @@ class OpenAIGPT(LanguageModel):
         args: Dict[str, Any] = dict(
             model=chat_model,
             messages=[
-                m.api_dict(has_system_role=self.info().allows_system_message)
+                m.api_dict(
+                    self.config.chat_model,
+                    has_system_role=self.info().allows_system_message,
+                )
                 for m in (llm_messages)
             ],
             max_completion_tokens=max_tokens,
@@ -2076,7 +2097,7 @@ class OpenAIGPT(LanguageModel):
             function_call,
             response_format,
         )
-        cached, hashed_key, response = self._chat_completions_with_backoff(**args)
+        cached, hashed_key, response = self._chat_completions_with_backoff(**args)  # type: ignore
         if self.get_stream() and not cached:
             llm_response, openai_response = self._stream_response(response, chat=True)
             self._cache_store(hashed_key, openai_response)
@@ -2109,7 +2130,7 @@ class OpenAIGPT(LanguageModel):
             function_call,
             response_format,
         )
-        cached, hashed_key, response = await self._achat_completions_with_backoff(
+        cached, hashed_key, response = await self._achat_completions_with_backoff(  # type: ignore
             **args
         )
         if self.get_stream() and not cached:

{langroid-0.51.2 → langroid-0.52.1}/langroid/parsing/document_parser.py

@@ -1242,7 +1242,7 @@ class LLMPdfParser(DocumentParser):
                             image_url=dict(url=data_uri),
                         )
                     elif "claude" in self.model_name.lower():
-                        # optimistrally try this: some API proxies like litellm
+                        # optimistically try this: some API proxies like litellm
                         # support this, and others may not.
                         file_content = dict(
                             type="file",
@@ -1259,27 +1259,32 @@ class LLMPdfParser(DocumentParser):
                                 file_data=data_uri,
                             ),
                         )
+                    prompt = (
+                        self.llm_parser_config.prompt
+                        or self.LLM_PDF_MD_SYSTEM_INSTRUCTION
+                    )
+                    system_prompt = (
+                        self.llm_parser_config.system_prompt
+                        or """
+                         You are an expert pdf -> markdown converter.
+                         Do NOT use any triple backquotes when you present the
+                         markdown content,like ```markdown etc.
+                         FAITHFULLY CONVERT THE PDF TO MARKDOWN,
+                         retaining ALL content as you find it.
+                        """
+                    )
 
                     # Send the request with PDF content and system instructions
                     response = await llm.async_client.chat.completions.create(  # type: ignore
                         model=self.model_name.split("/")[-1],
                         messages=[
-                            dict(
-                                role="system",
-                                content="""
-                                You are an expert pdf -> markdown converter.
-                                Do NOT use any triple backquotes when you present the 
-                                markdown content,like ```markdown etc.
-                                FAITHFULLY CONVERT THE PDF TO MARKDOWN, 
-                                retaining ALL content as you find it.
-                                """,
-                            ),
+                            dict(role="system", content=system_prompt),
                             dict(  # type: ignore
                                 role="user",
                                 content=[
                                     dict(
                                         type="text",
-                                        text=self.LLM_PDF_MD_SYSTEM_INSTRUCTION,
+                                        text=prompt,
                                     ),
                                     file_content,
                                 ],

langroid-0.52.1/langroid/parsing/file_attachment.py

@@ -0,0 +1,157 @@
+import base64
+import mimetypes
+import uuid
+from pathlib import Path
+from typing import Any, BinaryIO, Dict, Optional, Union
+
+from langroid.pydantic_v1 import BaseModel
+
+
+class FileAttachment(BaseModel):
+    """Represents a file attachment to be sent to an LLM API."""
+
+    content: bytes
+    filename: Optional[str] = None
+    mime_type: str = "application/octet-stream"
+
+    def __init__(self, **data: Any) -> None:
+        """Initialize with sensible defaults for filename if not provided."""
+        if "filename" not in data or data["filename"] is None:
+            # Generate a more readable unique filename
+            unique_id = str(uuid.uuid4())[:8]
+            data["filename"] = f"attachment_{unique_id}.bin"
+        super().__init__(**data)
+
+    @classmethod
+    def from_path(cls, file_path: Union[str, Path]) -> "FileAttachment":
+        """Create a FileAttachment from a file path.
+
+        Args:
+            file_path: Path to the file to attach
+
+        Returns:
+            FileAttachment instance
+        """
+        path = Path(file_path)
+        with open(path, "rb") as f:
+            content = f.read()
+
+        mime_type, _ = mimetypes.guess_type(path)
+        if mime_type is None:
+            mime_type = "application/octet-stream"
+
+        return cls(content=content, filename=path.name, mime_type=mime_type)
+
+    @classmethod
+    def from_bytes(
+        cls,
+        content: bytes,
+        filename: Optional[str] = None,
+        mime_type: Optional[str] = None,
+    ) -> "FileAttachment":
+        """Create a FileAttachment from bytes content.
+
+        Args:
+            content: Raw bytes content
+            filename: Optional name to use for the file
+            mime_type: MIME type of the content, guessed from filename if provided
+
+        Returns:
+            FileAttachment instance
+        """
+        if mime_type is None and filename is not None:
+            mime_type, _ = mimetypes.guess_type(filename)
+
+        return cls(
+            content=content,
+            filename=filename,
+            mime_type=mime_type or "application/octet-stream",
+        )
+
+    @classmethod
+    def from_io(
+        cls,
+        file_obj: BinaryIO,
+        filename: Optional[str] = None,
+        mime_type: Optional[str] = None,
+    ) -> "FileAttachment":
+        """Create a FileAttachment from a file-like object.
+
+        Args:
+            file_obj: File-like object with binary content
+            filename: Optional name to use for the file
+            mime_type: MIME type of the content, guessed from filename if provided
+
+        Returns:
+            FileAttachment instance
+        """
+        content = file_obj.read()
+        return cls.from_bytes(content, filename, mime_type)
+
+    @classmethod
+    def from_text(
+        cls,
+        text: str,
+        filename: Optional[str] = None,
+        mime_type: str = "text/plain",
+        encoding: str = "utf-8",
+    ) -> "FileAttachment":
+        """Create a FileAttachment from text content.
+
+        Args:
+            text: Text content to include
+            filename: Optional name to use for the file
+            mime_type: MIME type of the content
+            encoding: Text encoding to use
+
+        Returns:
+            FileAttachment instance
+        """
+        content = text.encode(encoding)
+        return cls(content=content, filename=filename, mime_type=mime_type)
+
+    def to_base64(self) -> str:
+        """Convert content to base64 encoding.
+
+        Returns:
+            Base64 encoded string
+        """
+        return base64.b64encode(self.content).decode("utf-8")
+
+    def to_data_uri(self) -> str:
+        """Convert content to a data URI.
+
+        Returns:
+            A data URI string containing the base64-encoded content with MIME type
+        """
+        base64_content = self.to_base64()
+        return f"data:{self.mime_type};base64,{base64_content}"
+
+    def to_dict(self, model: str) -> Dict[str, Any]:
+        """
+        Convert to a dictionary suitable for API requests.
+        Tested only for PDF files.
+
+        Returns:
+            Dictionary with file data
+        """
+        if "gemini" in model.lower():
+            return dict(type="image_url", image_url=dict(url=self.to_data_uri()))
+        elif "claude" in model.lower():
+            # optimistically try this: some API proxies like litellm
+            # support this, and others may not.
+            return dict(
+                type="file",
+                file=dict(
+                    file_data=self.to_data_uri(),
+                ),
+            )
+        else:
+            # fallback: assume file upload is similar to OpenAI API
+            return dict(
+                type="file",
+                file=dict(
+                    filename=self.filename,
+                    file_data=self.to_data_uri(),
+                ),
+            )

{langroid-0.51.2 → langroid-0.52.1}/langroid/parsing/parser.py

@@ -44,6 +44,8 @@ class LLMPdfParserConfig(BaseSettings):
     split_on_page: Optional[bool] = True
     requests_per_minute: Optional[int] = 5
     timeout: int = 60
+    prompt: str = ""  # override with a domain-specific prompt
+    system_prompt: str = ""  # override with a domain-specific system prompt
 
 
 class MarkerConfig(BaseSettings):

{langroid-0.51.2 → langroid-0.52.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "langroid"
-version = "0.51.2"
+version = "0.52.1"
 authors = [
     {name = "Prasad Chalasani", email = "pchalasani@gmail.com"},
 ]