langroid 0.38.0__py3-none-any.whl → 0.39.1__py3-none-any.whl

langroid/agent/base.py

@@ -333,6 +333,11 @@ class Agent(ABC):
         if hasattr(message_class, "handle_message_fallback") and (
             inspect.isfunction(message_class.handle_message_fallback)
         ):
+            # When a ToolMessage has a `handle_message_fallback` method,
+            # we inject it into the agent as a method, overriding the default
+            # `handle_message_fallback` method (which does nothing).
+            # It's possible multiple tool messages have a `handle_message_fallback`,
+            # in which case, the last one inserted will be used.
             setattr(
                 self,
                 "handle_message_fallback",
@@ -912,7 +917,7 @@ class Agent(ABC):
         else:
             prompt = message
 
-        output_len = self.config.llm.max_output_tokens
+        output_len = self.config.llm.model_max_output_tokens
         if self.num_tokens(prompt) + output_len > self.llm.completion_context_length():
             output_len = self.llm.completion_context_length() - self.num_tokens(prompt)
             if output_len < self.config.llm.min_output_tokens:
@@ -981,7 +986,7 @@ class Agent(ABC):
                 # show rich spinner only if not streaming!
                 cm = status("LLM responding to message...")
                 stack.enter_context(cm)
-            output_len = self.config.llm.max_output_tokens
+            output_len = self.config.llm.model_max_output_tokens
             if (
                 self.num_tokens(prompt) + output_len
                 > self.llm.completion_context_length()
@@ -1866,7 +1871,7 @@ class Agent(ABC):
             cumul_cost = format(tot_cost, ".4f")
             assert isinstance(self.llm, LanguageModel)
             context_length = self.llm.chat_context_length()
-            max_out = self.config.llm.max_output_tokens
+            max_out = self.config.llm.model_max_output_tokens
 
             llm_model = (
                 "no-LLM" if self.config.llm is None else self.llm.config.chat_model

langroid/agent/chat_agent.py

@@ -5,7 +5,7 @@ import logging
 import textwrap
 from contextlib import ExitStack
 from inspect import isclass
-from typing import Dict, List, Optional, Self, Set, Tuple, Type, Union, cast
+from typing import Any, Dict, List, Optional, Self, Set, Tuple, Type, Union, cast
 
 import openai
 from rich import print
@@ -31,6 +31,7 @@ from langroid.language_models.base import (
     ToolChoiceTypes,
 )
 from langroid.language_models.openai_gpt import OpenAIGPT
+from langroid.mytypes import Entity, NonToolAction
 from langroid.pydantic_v1 import BaseModel, ValidationError
 from langroid.utils.configuration import settings
 from langroid.utils.object_registry import ObjectRegistry
@@ -52,6 +53,7 @@ class ChatAgentConfig(AgentConfig):
         user_message: user message to include in message sequence.
              Used only if `task` is not specified in the constructor.
         use_tools: whether to use our own ToolMessages mechanism
+        handle_llm_no_tool (NonToolAction|str): routing when LLM generates non-tool msg.
         use_functions_api: whether to use functions/tools native to the LLM API
                 (e.g. OpenAI's `function_call` or `tool_call` mechanism)
         use_tools_api: When `use_functions_api` is True, if this is also True,
@@ -84,6 +86,7 @@ class ChatAgentConfig(AgentConfig):
 
     system_message: str = "You are a helpful assistant."
     user_message: Optional[str] = None
+    handle_llm_no_tool: NonToolAction | None = None
     use_tools: bool = False
     use_functions_api: bool = True
     use_tools_api: bool = False
@@ -579,6 +582,31 @@ class ChatAgent(Agent):
         # remove leading and trailing newlines and other whitespace
         return LLMMessage(role=Role.SYSTEM, content=content.strip())
 
+    def handle_message_fallback(self, msg: str | ChatDocument) -> Any:
+        """
+        Fallback method for the "no-tools" scenario.
+        Users the self.config.non_tool_routing to determine the action to take.
+
+        This method can be overridden by subclasses, e.g.,
+        to create a "reminder" message when a tool is expected but the LLM "forgot"
+        to generate one.
+
+        Args:
+            msg (str | ChatDocument): The input msg to handle
+        Returns:
+            Any: The result of the handler method
+        """
+        if self.config.handle_llm_no_tool is None:
+            return None
+        if isinstance(msg, ChatDocument) and msg.metadata.sender == Entity.LLM:
+            from langroid.agent.tools.orchestration import AgentDoneTool, ForwardTool
+
+            match self.config.handle_llm_no_tool:
+                case NonToolAction.FORWARD_USER:
+                    return ForwardTool(agent="User")
+                case NonToolAction.DONE:
+                    return AgentDoneTool(content=msg.content, tools=msg.tool_messages)
+
     def unhandled_tools(self) -> set[str]:
         """The set of tools that are known but not handled.
         Useful in task flow: an agent can refuse to accept an incoming msg
@@ -1460,11 +1488,11 @@ class ChatAgent(Agent):
                 self.message_history.extend(llm_msgs)
 
         hist = self.message_history
-        output_len = self.config.llm.max_output_tokens
+        output_len = self.config.llm.model_max_output_tokens
         if (
             truncate
             and self.chat_num_tokens(hist)
-            > self.llm.chat_context_length() - self.config.llm.max_output_tokens
+            > self.llm.chat_context_length() - self.config.llm.model_max_output_tokens
         ):
             # chat + output > max context length,
             # so first try to shorten requested output len to fit.
@@ -1489,7 +1517,7 @@ class ChatAgent(Agent):
                         The message history is longer than the max chat context 
                         length allowed, and we have run out of messages to drop.
                         HINT: In your `OpenAIGPTConfig` object, try increasing
-                        `chat_context_length` or decreasing `max_output_tokens`.
+                        `chat_context_length` or decreasing `model_max_output_tokens`.
                         """
                         )
                     # drop the second message, i.e. first msg after the sys msg
@@ -1638,12 +1666,12 @@ class ChatAgent(Agent):
         Args:
             messages: seq of messages (with role, content fields) sent to LLM
             output_len: max number of tokens expected in response.
-                    If None, use the LLM's default max_output_tokens.
+                    If None, use the LLM's default model_max_output_tokens.
         Returns:
             Document (i.e. with fields "content", "metadata")
         """
         assert self.config.llm is not None and self.llm is not None
-        output_len = output_len or self.config.llm.max_output_tokens
+        output_len = output_len or self.config.llm.model_max_output_tokens
         streamer = noop_fn
         if self.llm.get_stream():
             streamer = self.callbacks.start_llm_stream()
@@ -1713,7 +1741,7 @@ class ChatAgent(Agent):
         Async version of `llm_response_messages`. See there for details.
         """
         assert self.config.llm is not None and self.llm is not None
-        output_len = output_len or self.config.llm.max_output_tokens
+        output_len = output_len or self.config.llm.model_max_output_tokens
         functions, fun_call, tools, force_tool, output_format = self._function_args()
         assert self.llm is not None
 

langroid/agent/special/doc_chat_agent.py

@@ -1565,7 +1565,7 @@ class DocChatAgent(ChatAgent):
         tot_tokens = self.parser.num_tokens(full_text)
         MAX_INPUT_TOKENS = (
             self.llm.completion_context_length()
-            - self.config.llm.max_output_tokens
+            - self.config.llm.model_max_output_tokens
             - 100
         )
         if tot_tokens > MAX_INPUT_TOKENS:

langroid/language_models/__init__.py

@@ -15,14 +15,13 @@ from .base import (
     LLMTokenUsage,
     LLMResponse,
 )
-from .openai_gpt import (
+from .model_info import (
     OpenAIChatModel,
     AnthropicModel,
     GeminiModel,
     OpenAICompletionModel,
-    OpenAIGPTConfig,
-    OpenAIGPT,
 )
+from .openai_gpt import OpenAIGPTConfig, OpenAIGPT, OpenAICallParams
 from .mock_lm import MockLM, MockLMConfig
 from .azure_openai import AzureConfig, AzureGPT
 
@@ -32,6 +31,7 @@ __all__ = [
     "config",
     "base",
     "openai_gpt",
+    "model_info",
     "azure_openai",
     "prompt_formatter",
     "StreamEventType",
@@ -48,6 +48,7 @@ __all__ = [
     "OpenAICompletionModel",
     "OpenAIGPTConfig",
     "OpenAIGPT",
+    "OpenAICallParams",
     "AzureConfig",
     "AzureGPT",
     "MockLM",

langroid/language_models/base.py

@@ -19,6 +19,7 @@ from typing import (
 
 from langroid.cachedb.base import CacheDBConfig
 from langroid.cachedb.redis_cachedb import RedisCacheConfig
+from langroid.language_models.model_info import get_model_info
 from langroid.parsing.agent_chats import parse_message
 from langroid.parsing.parse_json import parse_imperfect_json, top_level_json_field
 from langroid.prompts.dialog import collate_chat_history
@@ -60,6 +61,7 @@ class LLMConfig(BaseSettings):
     streamer_async: Optional[Callable[..., Awaitable[None]]] = async_noop_fn
     api_base: str | None = None
     formatter: None | str = None
+    max_output_tokens: int | None = 8192  # specify None to use model_max_output_tokens
     timeout: int = 20  # timeout for API requests
     chat_model: str = ""
     completion_model: str = ""
@@ -67,7 +69,6 @@ class LLMConfig(BaseSettings):
     chat_context_length: int = 8000
     async_stream_quiet: bool = True  # suppress streaming output in async mode?
     completion_context_length: int = 8000
-    max_output_tokens: int = 1024  # generate at most this many tokens
     # if input length + max_output_tokens > context length of model,
     # we will try shortening requested output
     min_output_tokens: int = 64
@@ -84,6 +85,12 @@ class LLMConfig(BaseSettings):
     chat_cost_per_1k_tokens: Tuple[float, float] = (0.0, 0.0)
     completion_cost_per_1k_tokens: Tuple[float, float] = (0.0, 0.0)
 
+    @property
+    def model_max_output_tokens(self) -> int:
+        return (
+            self.max_output_tokens or get_model_info(self.chat_model).max_output_tokens
+        )
+
 
 class LLMFunctionCall(BaseModel):
     """

langroid/language_models/model_info.py

@@ -0,0 +1,307 @@
+from enum import Enum
+from typing import Dict, List, Optional
+
+from langroid.pydantic_v1 import BaseModel
+
+
+class ModelProvider(str, Enum):
+    """Enum for model providers"""
+
+    OPENAI = "openai"
+    ANTHROPIC = "anthropic"
+    DEEPSEEK = "deepseek"
+    GOOGLE = "google"
+    UNKNOWN = "unknown"
+
+
+class ModelName(str, Enum):
+    """Parent class for all model name enums"""
+
+    pass
+
+
+class OpenAIChatModel(ModelName):
+    """Enum for OpenAI Chat models"""
+
+    GPT3_5_TURBO = "gpt-3.5-turbo-1106"
+    GPT4 = "gpt-4"
+    GPT4_TURBO = "gpt-4-turbo"
+    GPT4o = "gpt-4o"
+    GPT4o_MINI = "gpt-4o-mini"
+    O1 = "o1"
+    O1_MINI = "o1-mini"
+    O3_MINI = "o3-mini"
+
+
+class OpenAICompletionModel(str, Enum):
+    """Enum for OpenAI Completion models"""
+
+    DAVINCI = "davinci-002"
+    BABBAGE = "babbage-002"
+
+
+class AnthropicModel(ModelName):
+    """Enum for Anthropic models"""
+
+    CLAUDE_3_5_SONNET = "claude-3-5-sonnet-latest"
+    CLAUDE_3_OPUS = "claude-3-opus-latest"
+    CLAUDE_3_SONNET = "claude-3-sonnet-20240229"
+    CLAUDE_3_HAIKU = "claude-3-haiku-20240307"
+
+
+class DeepSeekModel(ModelName):
+    """Enum for DeepSeek models direct from DeepSeek API"""
+
+    DEEPSEEK = "deepseek/deepseek-chat"
+    DEEPSEEK_R1 = "deepseek/deepseek-reasoner"
+
+
+class GeminiModel(ModelName):
+    """Enum for Gemini models"""
+
+    GEMINI_1_5_FLASH = "gemini/gemini-1.5-flash"
+    GEMINI_1_5_FLASH_8B = "gemini/gemini-1.5-flash-8b"
+    GEMINI_1_5_PRO = "gemini/gemini-1.5-pro"
+    GEMINI_2_FLASH = "gemini/gemini-2.0-flash-exp"
+    GEMINI_2_FLASH_THINKING = "gemini/gemini-2.0-flash-thinking-exp"
+
+
+class ModelInfo(BaseModel):
+    """
+    Consolidated information about LLM, related to capacity, cost and API
+    idiosyncrasies. Reasonable defaults for all params in case there's no
+    specific info available.
+    """
+
+    name: str = "unknown"
+    provider: ModelProvider = ModelProvider.UNKNOWN
+    context_length: int = 16_000
+    max_cot_tokens: int = 0  # max chain of thought (thinking) tokens where applicable
+    max_output_tokens: int = 8192  # Maximum number of output tokens - model dependent
+    input_cost_per_million: float = 0.0  # Cost in USD per million input tokens
+    output_cost_per_million: float = 0.0  # Cost in USD per million output tokens
+    allows_streaming: bool = True  # Whether model supports streaming output
+    allows_system_message: bool = True  # Whether model supports system messages
+    rename_params: Dict[str, str] = {}  # Rename parameters for OpenAI API
+    unsupported_params: List[str] = []
+    has_structured_output: bool = False  # Does model API support structured output?
+    has_tools: bool = True  # Does model API support tools/function-calling?
+    needs_first_user_message: bool = False  # Does API need first msg to be from user?
+    description: Optional[str] = None
+
+
+# Model information registry
+MODEL_INFO: Dict[str, ModelInfo] = {
+    # OpenAI Models
+    OpenAICompletionModel.DAVINCI.value: ModelInfo(
+        name=OpenAICompletionModel.DAVINCI.value,
+        provider=ModelProvider.OPENAI,
+        context_length=4096,
+        max_output_tokens=4096,
+        input_cost_per_million=2.0,
+        output_cost_per_million=2.0,
+        description="Davinci-002",
+    ),
+    OpenAICompletionModel.BABBAGE.value: ModelInfo(
+        name=OpenAICompletionModel.BABBAGE.value,
+        provider=ModelProvider.OPENAI,
+        context_length=4096,
+        max_output_tokens=4096,
+        input_cost_per_million=0.40,
+        output_cost_per_million=0.40,
+        description="Babbage-002",
+    ),
+    OpenAIChatModel.GPT3_5_TURBO.value: ModelInfo(
+        name=OpenAIChatModel.GPT3_5_TURBO.value,
+        provider=ModelProvider.OPENAI,
+        context_length=16_385,
+        max_output_tokens=4096,
+        input_cost_per_million=0.50,
+        output_cost_per_million=1.50,
+        description="GPT-3.5 Turbo",
+    ),
+    OpenAIChatModel.GPT4.value: ModelInfo(
+        name=OpenAIChatModel.GPT4.value,
+        provider=ModelProvider.OPENAI,
+        context_length=8192,
+        max_output_tokens=8192,
+        input_cost_per_million=30.0,
+        output_cost_per_million=60.0,
+        description="GPT-4 (8K context)",
+    ),
+    OpenAIChatModel.GPT4_TURBO.value: ModelInfo(
+        name=OpenAIChatModel.GPT4_TURBO.value,
+        provider=ModelProvider.OPENAI,
+        context_length=128_000,
+        max_output_tokens=4096,
+        input_cost_per_million=10.0,
+        output_cost_per_million=30.0,
+        description="GPT-4 Turbo",
+    ),
+    OpenAIChatModel.GPT4o.value: ModelInfo(
+        name=OpenAIChatModel.GPT4o.value,
+        provider=ModelProvider.OPENAI,
+        context_length=128_000,
+        max_output_tokens=16_384,
+        input_cost_per_million=2.5,
+        output_cost_per_million=10.0,
+        has_structured_output=True,
+        description="GPT-4o (128K context)",
+    ),
+    OpenAIChatModel.GPT4o_MINI.value: ModelInfo(
+        name=OpenAIChatModel.GPT4o_MINI.value,
+        provider=ModelProvider.OPENAI,
+        context_length=128_000,
+        max_output_tokens=16_384,
+        input_cost_per_million=0.15,
+        output_cost_per_million=0.60,
+        has_structured_output=True,
+        description="GPT-4o Mini",
+    ),
+    OpenAIChatModel.O1.value: ModelInfo(
+        name=OpenAIChatModel.O1.value,
+        provider=ModelProvider.OPENAI,
+        context_length=200_000,
+        max_output_tokens=100_000,
+        input_cost_per_million=15.0,
+        output_cost_per_million=60.0,
+        allows_streaming=False,
+        allows_system_message=False,
+        unsupported_params=["temperature", "stream"],
+        rename_params={"max_tokens": "max_completion_tokens"},
+        has_tools=False,
+        description="O1 Reasoning LM",
+    ),
+    OpenAIChatModel.O1_MINI.value: ModelInfo(
+        name=OpenAIChatModel.O1_MINI.value,
+        provider=ModelProvider.OPENAI,
+        context_length=128_000,
+        max_output_tokens=65_536,
+        input_cost_per_million=1.1,
+        output_cost_per_million=4.4,
+        allows_streaming=False,
+        allows_system_message=False,
+        unsupported_params=["temperature", "stream"],
+        rename_params={"max_tokens": "max_completion_tokens"},
+        has_tools=False,
+        description="O1 Mini Reasoning LM",
+    ),
+    OpenAIChatModel.O3_MINI.value: ModelInfo(
+        name=OpenAIChatModel.O3_MINI.value,
+        provider=ModelProvider.OPENAI,
+        context_length=200_000,
+        max_output_tokens=100_000,
+        input_cost_per_million=1.1,
+        output_cost_per_million=4.4,
+        allows_streaming=False,
+        allows_system_message=False,
+        unsupported_params=["temperature", "stream"],
+        rename_params={"max_tokens": "max_completion_tokens"},
+        has_tools=False,
+        description="O3 Mini Reasoning LM",
+    ),
+    # Anthropic Models
+    AnthropicModel.CLAUDE_3_5_SONNET.value: ModelInfo(
+        name=AnthropicModel.CLAUDE_3_5_SONNET.value,
+        provider=ModelProvider.ANTHROPIC,
+        context_length=200_000,
+        max_output_tokens=8192,
+        input_cost_per_million=3.0,
+        output_cost_per_million=15.0,
+        description="Claude 3.5 Sonnet",
+    ),
+    AnthropicModel.CLAUDE_3_OPUS.value: ModelInfo(
+        name=AnthropicModel.CLAUDE_3_OPUS.value,
+        provider=ModelProvider.ANTHROPIC,
+        context_length=200_000,
+        max_output_tokens=4096,
+        input_cost_per_million=15.0,
+        output_cost_per_million=75.0,
+        description="Claude 3 Opus",
+    ),
+    AnthropicModel.CLAUDE_3_SONNET.value: ModelInfo(
+        name=AnthropicModel.CLAUDE_3_SONNET.value,
+        provider=ModelProvider.ANTHROPIC,
+        context_length=200_000,
+        max_output_tokens=4096,
+        input_cost_per_million=3.0,
+        output_cost_per_million=15.0,
+        description="Claude 3 Sonnet",
+    ),
+    AnthropicModel.CLAUDE_3_HAIKU.value: ModelInfo(
+        name=AnthropicModel.CLAUDE_3_HAIKU.value,
+        provider=ModelProvider.ANTHROPIC,
+        context_length=200_000,
+        max_output_tokens=4096,
+        input_cost_per_million=0.25,
+        output_cost_per_million=1.25,
+        description="Claude 3 Haiku",
+    ),
+    # DeepSeek Models
+    DeepSeekModel.DEEPSEEK.value: ModelInfo(
+        name=DeepSeekModel.DEEPSEEK.value,
+        provider=ModelProvider.DEEPSEEK,
+        context_length=64_000,
+        max_output_tokens=8_000,
+        input_cost_per_million=0.27,
+        output_cost_per_million=1.10,
+        description="DeepSeek Chat",
+    ),
+    DeepSeekModel.DEEPSEEK_R1.value: ModelInfo(
+        name=DeepSeekModel.DEEPSEEK_R1.value,
+        provider=ModelProvider.DEEPSEEK,
+        context_length=64_000,
+        max_output_tokens=8_000,
+        input_cost_per_million=0.55,
+        output_cost_per_million=2.19,
+        description="DeepSeek-R1 Reasoning LM",
+    ),
+    # Gemini Models
+    GeminiModel.GEMINI_2_FLASH.value: ModelInfo(
+        name=GeminiModel.GEMINI_2_FLASH.value,
+        provider=ModelProvider.GOOGLE,
+        context_length=1_056_768,
+        max_output_tokens=8192,
+        rename_params={"max_tokens": "max_completion_tokens"},
+        description="Gemini 2.0 Flash",
+    ),
+    GeminiModel.GEMINI_1_5_FLASH.value: ModelInfo(
+        name=GeminiModel.GEMINI_1_5_FLASH.value,
+        provider=ModelProvider.GOOGLE,
+        context_length=1_056_768,
+        max_output_tokens=8192,
+        rename_params={"max_tokens": "max_completion_tokens"},
+        description="Gemini 1.5 Flash",
+    ),
+    GeminiModel.GEMINI_1_5_FLASH_8B.value: ModelInfo(
+        name=GeminiModel.GEMINI_1_5_FLASH_8B.value,
+        provider=ModelProvider.GOOGLE,
+        context_length=1_000_000,
+        max_output_tokens=8192,
+        rename_params={"max_tokens": "max_completion_tokens"},
+        description="Gemini 1.5 Flash 8B",
+    ),
+    GeminiModel.GEMINI_1_5_PRO.value: ModelInfo(
+        name=GeminiModel.GEMINI_1_5_PRO.value,
+        provider=ModelProvider.GOOGLE,
+        context_length=2_000_000,
+        max_output_tokens=8192,
+        rename_params={"max_tokens": "max_completion_tokens"},
+        description="Gemini 1.5 Pro",
+    ),
+    GeminiModel.GEMINI_2_FLASH_THINKING.value: ModelInfo(
+        name=GeminiModel.GEMINI_2_FLASH_THINKING.value,
+        provider=ModelProvider.GOOGLE,
+        context_length=1_000_000,
+        max_output_tokens=64_000,
+        rename_params={"max_tokens": "max_completion_tokens"},
+        description="Gemini 2.0 Flash Thinking",
+    ),
+}
+
+
+def get_model_info(model: str | ModelName) -> ModelInfo:
+    """Get model information by name or enum value"""
+    if isinstance(model, str):
+        return MODEL_INFO.get(model) or ModelInfo()
+    return MODEL_INFO.get(model.value) or ModelInfo()

langroid/language_models/openai_gpt.py

@@ -5,7 +5,6 @@ import os
 import sys
 import warnings
 from collections import defaultdict
-from enum import Enum
 from functools import cache
 from itertools import chain
 from typing import (
@@ -47,6 +46,17 @@ from langroid.language_models.base import (
     ToolChoiceTypes,
 )
 from langroid.language_models.config import HFPromptFormatterConfig
+from langroid.language_models.model_info import (
+    DeepSeekModel,
+    GeminiModel,
+    get_model_info,
+)
+from langroid.language_models.model_info import (
+    OpenAIChatModel as OpenAIChatModel,
+)
+from langroid.language_models.model_info import (
+    OpenAICompletionModel as OpenAICompletionModel,
+)
 from langroid.language_models.prompt_formatter.hf_formatter import (
     HFFormatter,
     find_hf_formatter,
@@ -79,118 +89,19 @@ VLLM_API_KEY = os.environ.get("VLLM_API_KEY", DUMMY_API_KEY)
 LLAMACPP_API_KEY = os.environ.get("LLAMA_API_KEY", DUMMY_API_KEY)
 
 
-class DeepSeekModel(str, Enum):
-    DEEPSEEK = "deepseek/deepseek-chat"
-
-
-class AnthropicModel(str, Enum):
-    """Enum for Anthropic models"""
-
-    CLAUDE_3_5_SONNET = "claude-3-5-sonnet-latest"
-    CLAUDE_3_OPUS = "claude-3-opus-20240229"
-    CLAUDE_3_SONNET = "claude-3-sonnet-20240229"
-    CLAUDE_3_HAIKU = "claude-3-turbo-20240307"
-
-
-class OpenAIChatModel(str, Enum):
-    """Enum for OpenAI Chat models"""
-
-    GPT3_5_TURBO = "gpt-3.5-turbo-1106"
-    GPT4 = "gpt-4"
-    GPT4_32K = "gpt-4-32k"
-    GPT4_TURBO = "gpt-4-turbo"
-    GPT4o = "gpt-4o"
-    GPT4o_MINI = "gpt-4o-mini"
-    O1_PREVIEW = "o1-preview"
-    O1_MINI = "o1-mini"
-
-
-class GeminiModel(str, Enum):
-    """Enum for Gemini models"""
-
-    GEMINI_1_5_FLASH = "gemini/gemini-1.5-flash"
-    GEMINI_1_5_FLASH_8B = "gemini/gemini-1.5-flash-8b"
-    GEMINI_1_5_PRO = "gemini/gemini-1.5-pro"
-    GEMINI_2_FLASH = "gemini/gemini-2.0-flash-exp"
-
-
-class OpenAICompletionModel(str, Enum):
-    """Enum for OpenAI Completion models"""
-
-    TEXT_DA_VINCI_003 = "text-davinci-003"  # deprecated
-    GPT3_5_TURBO_INSTRUCT = "gpt-3.5-turbo-instruct"
-
-
-_context_length: Dict[str, int] = {
-    # can add other non-openAI models here
-    OpenAIChatModel.GPT3_5_TURBO: 16_385,
-    OpenAIChatModel.GPT4: 8192,
-    OpenAIChatModel.GPT4_32K: 32_768,
-    OpenAIChatModel.GPT4_TURBO: 128_000,
-    OpenAIChatModel.GPT4o: 128_000,
-    OpenAIChatModel.GPT4o_MINI: 128_000,
-    OpenAIChatModel.O1_PREVIEW: 128_000,
-    OpenAIChatModel.O1_MINI: 128_000,
-    OpenAICompletionModel.TEXT_DA_VINCI_003: 4096,
-    AnthropicModel.CLAUDE_3_5_SONNET: 200_000,
-    AnthropicModel.CLAUDE_3_OPUS: 200_000,
-    AnthropicModel.CLAUDE_3_SONNET: 200_000,
-    AnthropicModel.CLAUDE_3_HAIKU: 200_000,
-    DeepSeekModel.DEEPSEEK: 64_000,
-    GeminiModel.GEMINI_2_FLASH: 1_000_000,
-    GeminiModel.GEMINI_1_5_FLASH: 1_000_000,
-    GeminiModel.GEMINI_1_5_FLASH_8B: 1_000_000,
-    GeminiModel.GEMINI_1_5_PRO: 2_000_000,
-}
-
-_cost_per_1k_tokens: Dict[str, Tuple[float, float]] = {
-    # can add other non-openAI models here.
-    # model => (prompt cost, generation cost) in USD
-    OpenAIChatModel.GPT3_5_TURBO: (0.001, 0.002),
-    OpenAIChatModel.GPT4: (0.03, 0.06),  # 8K context
-    OpenAIChatModel.GPT4_TURBO: (0.01, 0.03),  # 128K context
-    OpenAIChatModel.GPT4o: (0.0025, 0.010),  # 128K context
-    OpenAIChatModel.GPT4o_MINI: (0.00015, 0.0006),  # 128K context
-    OpenAIChatModel.O1_PREVIEW: (0.015, 0.060),  # 128K context
-    OpenAIChatModel.O1_MINI: (0.003, 0.012),  # 128K context
-    AnthropicModel.CLAUDE_3_5_SONNET: (0.003, 0.015),
-    AnthropicModel.CLAUDE_3_OPUS: (0.015, 0.075),
-    AnthropicModel.CLAUDE_3_SONNET: (0.003, 0.015),
-    AnthropicModel.CLAUDE_3_HAIKU: (0.00025, 0.00125),
-    DeepSeekModel.DEEPSEEK: (0.00014, 0.00028),
-    # Gemini models have complex pricing based on input-len
-}
-
-
-openAIChatModelPreferenceList = [
+openai_chat_model_pref_list = [
     OpenAIChatModel.GPT4o,
-    OpenAIChatModel.GPT4_TURBO,
-    OpenAIChatModel.GPT4,
     OpenAIChatModel.GPT4o_MINI,
     OpenAIChatModel.O1_MINI,
-    OpenAIChatModel.O1_PREVIEW,
+    OpenAIChatModel.O1,
     OpenAIChatModel.GPT3_5_TURBO,
 ]
 
-openAICompletionModelPreferenceList = [
-    OpenAICompletionModel.GPT3_5_TURBO_INSTRUCT,
-    OpenAICompletionModel.TEXT_DA_VINCI_003,
-]
-
-openAIStructuredOutputList = [
-    OpenAIChatModel.GPT4o_MINI,
-    OpenAIChatModel.GPT4o,
-]
-
-NON_STREAMING_MODELS = [
-    OpenAIChatModel.O1_MINI,
-    OpenAIChatModel.O1_PREVIEW,
+openai_completion_model_pref_list = [
+    OpenAICompletionModel.DAVINCI,
+    OpenAICompletionModel.BABBAGE,
 ]
 
-NON_SYSTEM_MESSAGE_MODELS = [
-    OpenAIChatModel.O1_MINI,
-    OpenAIChatModel.O1_PREVIEW,
-]
 
 if "OPENAI_API_KEY" in os.environ:
     try:
@@ -218,22 +129,22 @@ if "OPENAI_API_KEY" in os.environ:
 else:
     available_models = set()
 
-defaultOpenAIChatModel = next(
+default_openai_chat_model = next(
     chain(
         filter(
             lambda m: m.value in available_models,
-            openAIChatModelPreferenceList,
+            openai_chat_model_pref_list,
         ),
-        [OpenAIChatModel.GPT4_TURBO],
+        [OpenAIChatModel.GPT4o],
     )
 )
-defaultOpenAICompletionModel = next(
+default_openai_completion_model = next(
     chain(
         filter(
             lambda m: m.value in available_models,
-            openAICompletionModelPreferenceList,
+            openai_completion_model_pref_list,
         ),
-        [OpenAICompletionModel.GPT3_5_TURBO_INSTRUCT],
+        [OpenAICompletionModel.DAVINCI],
     )
 )
 
@@ -245,8 +156,9 @@ class AccessWarning(Warning):
 @cache
 def gpt_3_5_warning() -> None:
     warnings.warn(
-        """
-        GPT-4 is not available, falling back to GPT-3.5.
+        f"""
+        {OpenAIChatModel.GPT4o} is not available, 
+        falling back to {OpenAIChatModel.GPT3_5_TURBO}.
         Examples may not work properly and unexpected behavior may occur.
         Adjustments to prompts may be necessary.
         """,
@@ -285,6 +197,7 @@ class OpenAICallParams(BaseModel):
     logit_bias: Dict[int, float] | None = None  # token_id -> bias
     logprobs: bool = False
     top_p: float | None = 1.0
+    reasoning_effort: str | None = None  # or "low" or "high" or "medium"
     top_logprobs: int | None = None  # if int, requires logprobs=True
     n: int = 1  # how many completions to generate (n > 1 is NOT handled now)
     stop: str | List[str] | None = None  # (list of) stop sequence(s)
@@ -310,7 +223,6 @@ class OpenAIGPTConfig(LLMConfig):
     api_base: str | None = None  # used for local or other non-OpenAI models
     litellm: bool = False  # use litellm api?
     ollama: bool = False  # use ollama's OpenAI-compatible endpoint?
-    max_output_tokens: int = 1024
     min_output_tokens: int = 1
     use_chat_for_completion = True  # do not change this, for OpenAI models!
     timeout: int = 20
@@ -318,8 +230,8 @@ class OpenAIGPTConfig(LLMConfig):
     seed: int | None = 42
     params: OpenAICallParams | None = None
     # these can be any model name that is served at an OpenAI-compatible API end point
-    chat_model: str = defaultOpenAIChatModel
-    completion_model: str = defaultOpenAICompletionModel
+    chat_model: str = default_openai_chat_model
+    completion_model: str = default_openai_completion_model
     run_on_first_use: Callable[[], None] = noop
     parallel_tool_calls: Optional[bool] = None
     # Supports constrained decoding which enforces that the output of the LLM
@@ -345,7 +257,7 @@ class OpenAIGPTConfig(LLMConfig):
         warn_gpt_3_5 = (
             "chat_model" not in kwargs.keys()
             and not local_model
-            and defaultOpenAIChatModel == OpenAIChatModel.GPT3_5_TURBO
+            and default_openai_chat_model == OpenAIChatModel.GPT3_5_TURBO
         )
 
         if warn_gpt_3_5:
@@ -554,7 +466,7 @@ class OpenAIGPT(LanguageModel):
             self.supports_strict_tools = self.api_base is None
             self.supports_json_schema = (
                 self.api_base is None
-                and self.config.chat_model in openAIStructuredOutputList
+                and get_model_info(self.config.chat_model).has_structured_output
             )
 
         if settings.chat_model != "":
@@ -704,10 +616,10 @@ class OpenAIGPT(LanguageModel):
         return self.config.chat_model in openai_chat_models
 
     def supports_functions_or_tools(self) -> bool:
-        return self.is_openai_chat_model() and self.config.chat_model not in [
-            OpenAIChatModel.O1_MINI,
-            OpenAIChatModel.O1_PREVIEW,
-        ]
+        return (
+            self.is_openai_chat_model()
+            and get_model_info(self.config.chat_model).has_tools
+        )
 
     def is_openai_completion_model(self) -> bool:
         openai_completion_models = [e.value for e in OpenAICompletionModel]
@@ -726,40 +638,18 @@ class OpenAIGPT(LanguageModel):
             or self.chat_model_orig.startswith("deepseek/")
         )
 
-    def requires_first_user_message(self) -> bool:
-        """
-        Does the chat_model require a non-empty first user message?
-        TODO: Add other models here; we know gemini requires a non-empty
-        user message, after the system message.
-        """
-        return self.is_gemini_model()
-
     def unsupported_params(self) -> List[str]:
         """
         List of params that are not supported by the current model
         """
-        match self.chat_model_orig:
-            case OpenAIChatModel.O1_MINI | OpenAIChatModel.O1_PREVIEW:
-                return ["temperature", "stream"]
-            case _:
-                return []
+        return get_model_info(self.config.chat_model).unsupported_params
 
     def rename_params(self) -> Dict[str, str]:
         """
         Map of param name -> new name for specific models.
         Currently main troublemaker is o1* series.
         """
-        match self.config.chat_model:
-            case (
-                OpenAIChatModel.O1_MINI
-                | OpenAIChatModel.O1_PREVIEW
-                | GeminiModel.GEMINI_1_5_FLASH
-                | GeminiModel.GEMINI_1_5_FLASH_8B
-                | GeminiModel.GEMINI_1_5_PRO
-            ):
-                return {"max_tokens": "max_completion_tokens"}
-            case _:
-                return {}
+        return get_model_info(self.config.chat_model).rename_params
 
     def chat_context_length(self) -> int:
         """
@@ -771,7 +661,7 @@ class OpenAIGPT(LanguageModel):
             if self.config.use_completion_for_chat
             else self.config.chat_model
         )
-        return _context_length.get(model, super().chat_context_length())
+        return get_model_info(model).context_length
 
     def completion_context_length(self) -> int:
         """
@@ -783,7 +673,7 @@ class OpenAIGPT(LanguageModel):
             if self.config.use_chat_for_completion
             else self.config.completion_model
         )
-        return _context_length.get(model, super().completion_context_length())
+        return get_model_info(model).context_length
 
     def chat_cost(self) -> Tuple[float, float]:
         """
@@ -791,7 +681,8 @@ class OpenAIGPT(LanguageModel):
         models/endpoints.
         Get it from the dict, otherwise fail-over to general method
         """
-        return _cost_per_1k_tokens.get(self.chat_model_orig, super().chat_cost())
+        info = get_model_info(self.config.chat_model)
+        return (info.input_cost_per_million / 1000, info.output_cost_per_million / 1000)
 
     def set_stream(self, stream: bool) -> bool:
         """Enable or disable streaming output from API.
@@ -808,7 +699,7 @@ class OpenAIGPT(LanguageModel):
         return (
             self.config.stream
             and settings.stream
-            and self.config.chat_model not in NON_STREAMING_MODELS
+            and get_model_info(self.config.chat_model).allows_streaming
             and not settings.quiet
         )
 
@@ -1795,7 +1686,7 @@ class OpenAIGPT(LanguageModel):
                 and llm_messages[0].role == Role.SYSTEM
                 # TODO: we will unconditionally insert a dummy user msg
                 # if the only msg is a system msg.
-                # and self.requires_first_user_message()
+                # We could make this conditional on ModelInfo.needs_first_user_message
             ):
                 # some LLMs, notable Gemini as of 12/11/24,
                 # require the first message to be from the user,
@@ -1813,8 +1704,9 @@ class OpenAIGPT(LanguageModel):
             model=chat_model,
             messages=[
                 m.api_dict(
-                    has_system_role=self.config.chat_model
-                    not in NON_SYSTEM_MESSAGE_MODELS
+                    has_system_role=get_model_info(
+                        self.config.chat_model
+                    ).allows_system_message
                 )
                 for m in (llm_messages)
             ],

langroid/mytypes.py

@@ -93,3 +93,12 @@ class Document(BaseModel):
         SOURCE:{self.metadata.source}
         """
         )
+
+
+class NonToolAction(str, Enum):
+    """
+    Possible Routing options. Mainly used to handle non-tool msgs from LLM.
+    """
+
+    FORWARD_USER = "user"  # forward msg to user
+    DONE = "done"  # task done

{langroid-0.38.0.dist-info → langroid-0.39.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langroid
-Version: 0.38.0
+Version: 0.39.1
 Summary: Harness LLMs with Multi-Agent Programming
 Author-email: Prasad Chalasani <pchalasani@gmail.com>
 License: MIT

{langroid-0.38.0.dist-info → langroid-0.39.1.dist-info}/RECORD

@@ -1,11 +1,11 @@
 langroid/__init__.py,sha256=z_fCOLQJPOw3LLRPBlFB5-2HyCjpPgQa4m4iY5Fvb8Y,1800
 langroid/exceptions.py,sha256=OPjece_8cwg94DLPcOGA1ddzy5bGh65pxzcHMnssTz8,2995
-langroid/mytypes.py,sha256=h1eMq1ZwTLVezObPfCseWNWbEOzP7mAKu2XoS63W1cM,2647
+langroid/mytypes.py,sha256=RUMSf2i-qE2L2A5Bigvi-1jL5MwsAFfG59rEEkcq7h0,2854
 langroid/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 langroid/agent/__init__.py,sha256=ll0Cubd2DZ-fsCMl7e10hf9ZjFGKzphfBco396IKITY,786
-langroid/agent/base.py,sha256=oThlrYygKDu1-bKjAfygldJ511gMKT8Z0qCrD52DdDM,77834
+langroid/agent/base.py,sha256=CVPvy-bLI_6wHZxYf_spo4eq-utR373Ur7T3zKpe18U,78222
 langroid/agent/batch.py,sha256=vi1r5i1-vN80WfqHDSwjEym_KfGsqPGUtwktmiK1nuk,20635
-langroid/agent/chat_agent.py,sha256=_7vOhTauPpPiOih2hnec8hz0rytaxGN110ja9wRCLJ0,82276
+langroid/agent/chat_agent.py,sha256=DQkYAWQoHH4uaBsF9n7JSbfpyPn7Sr16bdblzZPx78U,83573
 langroid/agent/chat_document.py,sha256=xzMtrPbaW-Y-BnF7kuhr2dorsD-D5rMWzfOqJ8HAoo8,17885
 langroid/agent/openai_assistant.py,sha256=JkAcs02bIrgPNVvUWVR06VCthc5-ulla2QMBzux_q6o,34340
 langroid/agent/task.py,sha256=XrXUbSoiFasvpIsZPn_cBpdWaTCKljJPRimtLMrSZrs,90347
@@ -14,7 +14,7 @@ langroid/agent/xml_tool_message.py,sha256=6SshYZJKIfi4mkE-gIoSwjkEYekQ8GwcSiCv7a
 langroid/agent/callbacks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 langroid/agent/callbacks/chainlit.py,sha256=RH8qUXaZE5o2WQz3WJQ1SdFtASGlxWCA6_HYz_3meDQ,20822
 langroid/agent/special/__init__.py,sha256=gik_Xtm_zV7U9s30Mn8UX3Gyuy4jTjQe9zjiE3HWmEo,1273
-langroid/agent/special/doc_chat_agent.py,sha256=6Wz_i6lLwUcYIgaOrbP8oLebxHE0uAbo2_mbWXCT0-k,64686
+langroid/agent/special/doc_chat_agent.py,sha256=tMx-3cBEIHJM14P20cYvIzAc9z-skSnHOPVJ0eegxzc,64692
 langroid/agent/special/lance_doc_chat_agent.py,sha256=s8xoRs0gGaFtDYFUSIRchsgDVbS5Q3C2b2mr3V1Fd-Q,10419
 langroid/agent/special/lance_tools.py,sha256=qS8x4wi8mrqfbYV2ztFzrcxyhHQ0ZWOc-zkYiH7awj0,2105
 langroid/agent/special/relevance_extractor_agent.py,sha256=zIx8GUdVo1aGW6ASla0NPQjYYIpmriK_TYMijqAx3F8,4796
@@ -64,12 +64,13 @@ langroid/embedding_models/protoc/embeddings.proto,sha256=_O-SgFpTaylQeOTgSpxhEJ7
 langroid/embedding_models/protoc/embeddings_pb2.py,sha256=4Q57PhOunv-uZNJrxYrWBXAI0ZtfnVZXFRhRj5JuRSg,1662
 langroid/embedding_models/protoc/embeddings_pb2.pyi,sha256=UkNy7BrNsmQm0vLb3NtGXy8jVtz-kPWwwFsX-QbQBhQ,1475
 langroid/embedding_models/protoc/embeddings_pb2_grpc.py,sha256=9dYQqkW3JPyBpSEjeGXTNpSqAkC-6FPtBHyteVob2Y8,2452
-langroid/language_models/__init__.py,sha256=ps8nhRavCu2-Bv7IQ5hrzody6lzKjHkivJsblDZZIQ8,1020
+langroid/language_models/__init__.py,sha256=3aD2qC1lz8v12HX4B-dilv27gNxYdGdeu1QvDlkqqHs,1095
 langroid/language_models/azure_openai.py,sha256=zNQzzsERxNestq-hFfQZbvTzK43G2vjRWnTV3ktm1DQ,5845
-langroid/language_models/base.py,sha256=qxPcY-zBoP9xXVtytg4kfjTKLU9QOZLLAfDa_6skE94,24921
+langroid/language_models/base.py,sha256=mN6HAjLgF2xpHObz5uPZ3JDID7jdTiRLEkoGgGrqLM8,25177
 langroid/language_models/config.py,sha256=9Q8wk5a7RQr8LGMT_0WkpjY8S4ywK06SalVRjXlfCiI,378
 langroid/language_models/mock_lm.py,sha256=5BgHKDVRWFbUwDT_PFgTZXz9-k8wJSA2e3PZmyDgQ1k,4022
-langroid/language_models/openai_gpt.py,sha256=yzkv9AFNL1iKdIQitB9mUqLjVwNREt8RWPEwVTwxDTk,80306
+langroid/language_models/model_info.py,sha256=n60j5DrAlxCsHa8pl7PWNBPP7pSAEVMi2mUQ4JOqo7Q,10648
+langroid/language_models/openai_gpt.py,sha256=Vuxz3tZThUN_9MF16KW0HJe0k0qk1tv58PZ0qqwpGdg,76754
 langroid/language_models/utils.py,sha256=L4_CbihDMTGcsg0TOG1Yd5JFEto46--h7CX_14m89sQ,5016
 langroid/language_models/prompt_formatter/__init__.py,sha256=2-5cdE24XoFDhifOLl8yiscohil1ogbP1ECkYdBlBsk,372
 langroid/language_models/prompt_formatter/base.py,sha256=eDS1sgRNZVnoajwV_ZIha6cba5Dt8xjgzdRbPITwx3Q,1221
@@ -123,7 +124,7 @@ langroid/vector_store/meilisearch.py,sha256=6frB7GFWeWmeKzRfLZIvzRjllniZ1cYj3Hmh
 langroid/vector_store/momento.py,sha256=xOaU7Hlyyn_5ihb0ARS5JHtmrKrTCt2IdRA-ioMM5ek,10307
 langroid/vector_store/qdrantdb.py,sha256=v7TAsIoj_vxeKDYS9tpwJLBZA8fuTweTYxHo0X_uawM,17949
 langroid/vector_store/weaviatedb.py,sha256=FOzgvqLqvdN5jJebVtJ-8tu2CeBzBfSP3ih4_ODEOOw,10605
-langroid-0.38.0.dist-info/METADATA,sha256=PMVjo4UfoNgUyP9RWp2AAEIWA9amAnTHUvzrPPjt_ms,60634
-langroid-0.38.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-langroid-0.38.0.dist-info/licenses/LICENSE,sha256=EgVbvA6VSYgUlvC3RvPKehSg7MFaxWDsFuzLOsPPfJg,1065
-langroid-0.38.0.dist-info/RECORD,,
+langroid-0.39.1.dist-info/METADATA,sha256=4_I1_wyUNt5OCUpdKv2amypFdloe15-0pLndN-rwkew,60634
+langroid-0.39.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+langroid-0.39.1.dist-info/licenses/LICENSE,sha256=EgVbvA6VSYgUlvC3RvPKehSg7MFaxWDsFuzLOsPPfJg,1065
+langroid-0.39.1.dist-info/RECORD,,