langroid 0.50.2__py3-none-any.whl → 0.50.4__py3-none-any.whl

langroid/agent/base.py

@@ -1929,10 +1929,13 @@ class Agent(ABC):
         print_response_stats: bool = True,
     ) -> None:
         """
-        Updates `response.usage` obj (token usage and cost fields).the usage memebr
-        It updates the cost after checking the cache and updates the
-        tokens (prompts and completion) if the response stream is True, because OpenAI
-        doesn't returns these fields.
+        Updates `response.usage` obj (token usage and cost fields) if needed.
+        An update is needed only if:
+        - stream is True (i.e. streaming was enabled), and
+        - the response was NOT obtained from cached, and
+        - the API did NOT provide the usage/cost fields during streaming
+          (As of Sep 2024, the OpenAI API started providing these; for other APIs
+            this may not necessarily be the case).
 
         Args:
             response (LLMResponse): LLMResponse object
@@ -1945,10 +1948,11 @@ class Agent(ABC):
         if response is None or self.llm is None:
             return
 
+        no_usage_info = response.usage is None or response.usage.prompt_tokens == 0
         # Note: If response was not streamed, then
         # `response.usage` would already have been set by the API,
         # so we only need to update in the stream case.
-        if stream:
+        if stream and no_usage_info:
             # usage, cost = 0 when response is from cache
             prompt_tokens = 0
             completion_tokens = 0

langroid/agent/chat_agent.py

@@ -94,7 +94,7 @@ class ChatAgentConfig(AgentConfig):
     handle_llm_no_tool: Any = None
     use_tools: bool = False
     use_functions_api: bool = True
-    use_tools_api: bool = False
+    use_tools_api: bool = True
     strict_recovery: bool = True
     enable_orchestration_tool_handling: bool = True
     output_format: Optional[type] = None
@@ -308,12 +308,7 @@ class ChatAgent(Agent):
 
     def _fn_call_available(self) -> bool:
         """Does this agent's LLM support function calling?"""
-        return (
-            self.llm is not None
-            and isinstance(self.llm, OpenAIGPT)
-            and self.llm.is_openai_chat_model()
-            and self.llm.supports_functions_or_tools()
-        )
+        return self.llm is not None and self.llm.supports_functions_or_tools()
 
     def _strict_tools_available(self) -> bool:
         """Does this agent's LLM support strict tools?"""

langroid/language_models/base.py

@@ -216,7 +216,7 @@ class LLMTokenUsage(BaseModel):
     prompt_tokens: int = 0
     completion_tokens: int = 0
     cost: float = 0.0
-    calls: int = 0  # how many API calls
+    calls: int = 0  # how many API calls - not used as of 2025-04-04
 
     def reset(self) -> None:
         self.prompt_tokens = 0
@@ -626,6 +626,20 @@ class LanguageModel(ABC):
         )
         return get_model_info(orig_model, model)
 
+    def supports_functions_or_tools(self) -> bool:
+        """
+        Does this Model's API support "native" tool-calling, i.e.
+        can we call the API with arguments that contain a list of available tools,
+        and their schemas?
+        Note that, given the plethora of LLM provider APIs this determination is
+        imperfect at best, and leans towards returning True.
+        When the API calls fails with an error indicating tools are not supported,
+        then users are encouraged to use the Langroid-based prompt-based
+        ToolMessage mechanism, which works with ANY LLM. To enable this,
+        in your ChatAgentConfig, set `use_functions_api=False`, and `use_tools=True`.
+        """
+        return self.info().has_tools
+
     def chat_context_length(self) -> int:
         return self.config.chat_context_length or DEFAULT_CONTEXT_LENGTH
 

langroid/language_models/openai_gpt.py

@@ -682,9 +682,6 @@ class OpenAIGPT(LanguageModel):
         openai_chat_models = [e.value for e in OpenAIChatModel]
         return self.config.chat_model in openai_chat_models
 
-    def supports_functions_or_tools(self) -> bool:
-        return self.is_openai_chat_model() and self.info().has_tools
-
     def is_openai_completion_model(self) -> bool:
         openai_completion_models = [e.value for e in OpenAICompletionModel]
         return self.config.completion_model in openai_completion_models
@@ -780,22 +777,39 @@ class OpenAIGPT(LanguageModel):
         reasoning: str = "",
         function_args: str = "",
         function_name: str = "",
-    ) -> Tuple[bool, bool, str, str]:
+    ) -> Tuple[bool, bool, str, str, Dict[str, int]]:
         """Process state vars while processing a streaming API response.
             Returns a tuple consisting of:
         - is_break: whether to break out of the loop
         - has_function: whether the response contains a function_call
         - function_name: name of the function
         - function_args: args of the function
+        - completion: completion text
+        - reasoning: reasoning text
+        - usage: usage dict
         """
         # convert event obj (of type ChatCompletionChunk) to dict so rest of code,
         # which expects dicts, works as it did before switching to openai v1.x
         if not isinstance(event, dict):
             event = event.model_dump()
 
+        usage = event.get("usage", {}) or {}
         choices = event.get("choices", [{}])
-        if len(choices) == 0:
+        if choices is None or len(choices) == 0:
             choices = [{}]
+        if len(usage) > 0 and len(choices[0]) == 0:
+            # we have a "usage" chunk, and empty choices, so we're done
+            # ASSUMPTION: a usage chunk ONLY arrives AFTER all normal completion text!
+            # If any API does not follow this, we need to change this code.
+            return (
+                True,
+                has_function,
+                function_name,
+                function_args,
+                completion,
+                reasoning,
+                usage,
+            )
         event_args = ""
         event_fn_name = ""
         event_tool_deltas: Optional[List[Dict[str, Any]]] = None
@@ -876,23 +890,23 @@ class OpenAIGPT(LanguageModel):
                     self.config.streamer(tool_fn_args, StreamEventType.TOOL_ARGS)
 
         # show this delta in the stream
-        if finish_reason in [
+        is_break = finish_reason in [
             "stop",
             "function_call",
             "tool_calls",
-        ]:
-            # for function_call, finish_reason does not necessarily
-            # contain "function_call" as mentioned in the docs.
-            # So we check for "stop" or "function_call" here.
-            return (
-                True,
-                has_function,
-                function_name,
-                function_args,
-                completion,
-                reasoning,
-            )
-        return False, has_function, function_name, function_args, completion, reasoning
+        ]
+        # for function_call, finish_reason does not necessarily
+        # contain "function_call" as mentioned in the docs.
+        # So we check for "stop" or "function_call" here.
+        return (
+            is_break,
+            has_function,
+            function_name,
+            function_args,
+            completion,
+            reasoning,
+            usage,
+        )
 
     @no_type_check
     async def _process_stream_event_async(
@@ -912,15 +926,30 @@ class OpenAIGPT(LanguageModel):
         - has_function: whether the response contains a function_call
         - function_name: name of the function
         - function_args: args of the function
+        - completion: completion text
+        - reasoning: reasoning text
+        - usage: usage dict
         """
         # convert event obj (of type ChatCompletionChunk) to dict so rest of code,
         # which expects dicts, works as it did before switching to openai v1.x
         if not isinstance(event, dict):
             event = event.model_dump()
 
+        usage = event.get("usage", {}) or {}
         choices = event.get("choices", [{}])
         if len(choices) == 0:
             choices = [{}]
+        if len(usage) > 0 and len(choices[0]) == 0:
+            # we got usage chunk, and empty choices, so we're done
+            return (
+                True,
+                has_function,
+                function_name,
+                function_args,
+                completion,
+                reasoning,
+                usage,
+            )
         event_args = ""
         event_fn_name = ""
         event_tool_deltas: Optional[List[Dict[str, Any]]] = None
@@ -996,23 +1025,23 @@ class OpenAIGPT(LanguageModel):
                     )
 
         # show this delta in the stream
-        if choices[0].get("finish_reason", "") in [
+        is_break = choices[0].get("finish_reason", "") in [
             "stop",
             "function_call",
             "tool_calls",
-        ]:
-            # for function_call, finish_reason does not necessarily
-            # contain "function_call" as mentioned in the docs.
-            # So we check for "stop" or "function_call" here.
-            return (
-                True,
-                has_function,
-                function_name,
-                function_args,
-                completion,
-                reasoning,
-            )
-        return False, has_function, function_name, function_args, completion, reasoning
+        ]
+        # for function_call, finish_reason does not necessarily
+        # contain "function_call" as mentioned in the docs.
+        # So we check for "stop" or "function_call" here.
+        return (
+            is_break,
+            has_function,
+            function_name,
+            function_args,
+            completion,
+            reasoning,
+            usage,
+        )
 
     @retry_with_exponential_backoff
     def _stream_response(  # type: ignore
@@ -1038,6 +1067,8 @@ class OpenAIGPT(LanguageModel):
         sys.stdout.flush()
         has_function = False
         tool_deltas: List[Dict[str, Any]] = []
+        token_usage: Dict[str, int] = {}
+        done: bool = False
         try:
             for event in response:
                 (
@@ -1047,6 +1078,7 @@ class OpenAIGPT(LanguageModel):
                     function_args,
                     completion,
                     reasoning,
+                    usage,
                 ) = self._process_stream_event(
                     event,
                     chat=chat,
@@ -1057,8 +1089,17 @@ class OpenAIGPT(LanguageModel):
                     function_args=function_args,
                     function_name=function_name,
                 )
+                if len(usage) > 0:
+                    # capture the token usage when non-empty
+                    token_usage = usage
                 if is_break:
-                    break
+                    if not self.get_stream() or done:
+                        # if not streaming, then we don't wait for last "usage" chunk
+                        break
+                    else:
+                        # mark done, so we quit after the last "usage" chunk
+                        done = True
+
         except Exception as e:
             logging.warning("Error while processing stream response: %s", str(e))
 
@@ -1073,6 +1114,7 @@ class OpenAIGPT(LanguageModel):
             reasoning=reasoning,
             function_args=function_args,
             function_name=function_name,
+            usage=token_usage,
         )
 
     @async_retry_with_exponential_backoff
@@ -1100,6 +1142,8 @@ class OpenAIGPT(LanguageModel):
         sys.stdout.flush()
         has_function = False
         tool_deltas: List[Dict[str, Any]] = []
+        token_usage: Dict[str, int] = {}
+        done: bool = False
         try:
             async for event in response:
                 (
@@ -1109,6 +1153,7 @@ class OpenAIGPT(LanguageModel):
                     function_args,
                     completion,
                     reasoning,
+                    usage,
                 ) = await self._process_stream_event_async(
                     event,
                     chat=chat,
@@ -1119,8 +1164,17 @@ class OpenAIGPT(LanguageModel):
                     function_args=function_args,
                     function_name=function_name,
                 )
+                if len(usage) > 0:
+                    # capture the token usage when non-empty
+                    token_usage = usage
                 if is_break:
-                    break
+                    if not self.get_stream() or done:
+                        # if not streaming, then we don't wait for last "usage" chunk
+                        break
+                    else:
+                        # mark done, so we quit after the next "usage" chunk
+                        done = True
+
         except Exception as e:
             logging.warning("Error while processing stream response: %s", str(e))
 
@@ -1135,6 +1189,7 @@ class OpenAIGPT(LanguageModel):
             reasoning=reasoning,
             function_args=function_args,
             function_name=function_name,
+            usage=token_usage,
         )
 
     @staticmethod
@@ -1272,6 +1327,7 @@ class OpenAIGPT(LanguageModel):
         reasoning: str = "",
         function_args: str = "",
         function_name: str = "",
+        usage: Dict[str, int] = {},
     ) -> Tuple[LLMResponse, Dict[str, Any]]:
         """
         Create an LLMResponse object from the streaming API response.
@@ -1281,8 +1337,10 @@ class OpenAIGPT(LanguageModel):
             tool_deltas: list of tool deltas received from streaming API
             has_function: whether the response contains a function_call
             completion: completion text
+            reasoning: reasoning text
             function_args: string representing function args
             function_name: name of the function
+            usage: token usage dict
         Returns:
             Tuple consisting of:
                 LLMResponse object (with message, usage),
@@ -1347,6 +1405,14 @@ class OpenAIGPT(LanguageModel):
                 # don't allow empty list [] here
                 oai_tool_calls=tool_calls or None if len(tool_deltas) > 0 else None,
                 function_call=function_call if has_function else None,
+                usage=LLMTokenUsage(
+                    prompt_tokens=usage.get("prompt_tokens", 0),
+                    completion_tokens=usage.get("completion_tokens", 0),
+                    cost=self._cost_chat_model(
+                        usage.get("prompt_tokens", 0),
+                        usage.get("completion_tokens", 0),
+                    ),
+                ),
             ),
             openai_response.dict(),
         )
@@ -1575,17 +1641,6 @@ class OpenAIGPT(LanguageModel):
     ) -> LLMResponse:
         self.run_on_first_use()
 
-        if [functions, tools] != [None, None] and not self.is_openai_chat_model():
-            raise ValueError(
-                f"""
-                `functions` and `tools` can only be specified for OpenAI chat LLMs, 
-                or LLMs served via an OpenAI-compatible API.
-                {self.config.chat_model} does not support function-calling or tools.
-                Instead, please use Langroid's ToolMessages, which are equivalent.
-                In the ChatAgentConfig, set `use_functions_api=False` 
-                and `use_tools=True`, this will enable ToolMessages.
-                """
-            )
         if self.config.use_completion_for_chat and not self.is_openai_chat_model():
             # only makes sense for non-OpenAI models
             if self.config.formatter is None or self.config.hf_formatter is None:
@@ -1630,16 +1685,6 @@ class OpenAIGPT(LanguageModel):
     ) -> LLMResponse:
         self.run_on_first_use()
 
-        if [functions, tools] != [None, None] and not self.is_openai_chat_model():
-            raise ValueError(
-                f"""
-                `functions` and `tools` can only be specified for OpenAI chat models;
-                {self.config.chat_model} does not support function-calling or tools.
-                Instead, please use Langroid's ToolMessages, which are equivalent.
-                In the ChatAgentConfig, set `use_functions_api=False` 
-                and `use_tools=True`, this will enable ToolMessages.
-                """
-            )
         # turn off streaming for async calls
         if (
             self.config.use_completion_for_chat
@@ -1833,6 +1878,14 @@ class OpenAIGPT(LanguageModel):
             max_tokens=max_tokens,
             stream=self.get_stream(),
         )
+        if self.get_stream():
+            args.update(
+                dict(
+                    # get token-usage numbers in stream mode from OpenAI API,
+                    # and possibly other OpenAI-compatible APIs.
+                    stream_options=dict(include_usage=True),
+                )
+            )
         args.update(self._openai_api_call_params(args))
         # only include functions-related args if functions are provided
         # since the OpenAI API will throw an error if `functions` is None or []

{langroid-0.50.2.dist-info → langroid-0.50.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langroid
-Version: 0.50.2
+Version: 0.50.4
 Summary: Harness LLMs with Multi-Agent Programming
 Author-email: Prasad Chalasani <pchalasani@gmail.com>
 License: MIT

{langroid-0.50.2.dist-info → langroid-0.50.4.dist-info}/RECORD

@@ -3,9 +3,9 @@ langroid/exceptions.py,sha256=OPjece_8cwg94DLPcOGA1ddzy5bGh65pxzcHMnssTz8,2995
 langroid/mytypes.py,sha256=HIcYAqGeA9OK0Hlscym2FI5Oax9QFljDZoVgRlomhRk,4014
 langroid/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 langroid/agent/__init__.py,sha256=ll0Cubd2DZ-fsCMl7e10hf9ZjFGKzphfBco396IKITY,786
-langroid/agent/base.py,sha256=U-UjdpxIFqkzRIB5-LYwHrhMSNI3sDbfnNRqIhrtsyI,79568
+langroid/agent/base.py,sha256=bs5OLCf534mhsdR7Rgf27GqVNuSV2bOVbD46Y86mGFA,79829
 langroid/agent/batch.py,sha256=vi1r5i1-vN80WfqHDSwjEym_KfGsqPGUtwktmiK1nuk,20635
-langroid/agent/chat_agent.py,sha256=Z53oleOUcOXVs_UL90spttGoAooe0mrx3tDtOuhKVms,85214
+langroid/agent/chat_agent.py,sha256=m1alf-KNJSU6PeFF6ocwTHSG0cmTE-iy1o7UYAvRGQE,85081
 langroid/agent/chat_document.py,sha256=xzMtrPbaW-Y-BnF7kuhr2dorsD-D5rMWzfOqJ8HAoo8,17885
 langroid/agent/openai_assistant.py,sha256=JkAcs02bIrgPNVvUWVR06VCthc5-ulla2QMBzux_q6o,34340
 langroid/agent/task.py,sha256=HB6N-Jn80HFqCf0ZYOC1v3Bn3oO7NLjShHQJJFwW0q4,90557
@@ -15,6 +15,7 @@ langroid/agent/callbacks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZ
 langroid/agent/callbacks/chainlit.py,sha256=UHB6P_J40vsVnssosqkpkOVWRf9NK4TOY0_G2g_Arsg,20900
 langroid/agent/special/__init__.py,sha256=gik_Xtm_zV7U9s30Mn8UX3Gyuy4jTjQe9zjiE3HWmEo,1273
 langroid/agent/special/doc_chat_agent.py,sha256=dOL9Y0xAslkwepCdKU8Dc1m5Vk8qgk-gLbU4JzsmTII,65234
+langroid/agent/special/doc_chat_task.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 langroid/agent/special/lance_doc_chat_agent.py,sha256=s8xoRs0gGaFtDYFUSIRchsgDVbS5Q3C2b2mr3V1Fd-Q,10419
 langroid/agent/special/lance_tools.py,sha256=qS8x4wi8mrqfbYV2ztFzrcxyhHQ0ZWOc-zkYiH7awj0,2105
 langroid/agent/special/relevance_extractor_agent.py,sha256=zIx8GUdVo1aGW6ASla0NPQjYYIpmriK_TYMijqAx3F8,4796
@@ -68,11 +69,11 @@ langroid/embedding_models/protoc/embeddings_pb2.pyi,sha256=UkNy7BrNsmQm0vLb3NtGX
 langroid/embedding_models/protoc/embeddings_pb2_grpc.py,sha256=9dYQqkW3JPyBpSEjeGXTNpSqAkC-6FPtBHyteVob2Y8,2452
 langroid/language_models/__init__.py,sha256=3aD2qC1lz8v12HX4B-dilv27gNxYdGdeu1QvDlkqqHs,1095
 langroid/language_models/azure_openai.py,sha256=SW0Fp_y6HpERr9l6TtF6CYsKgKwjUf_hSL_2mhTV4wI,5034
-langroid/language_models/base.py,sha256=mDYmFCBCLdq8_Uvws4MiewwEgcOCP8Qb0e5yUXr3zpQ,26249
+langroid/language_models/base.py,sha256=A9A83SJOGh_elRScKNIDAcVT4mw9vKCou7txsPMVSv4,27023
 langroid/language_models/config.py,sha256=9Q8wk5a7RQr8LGMT_0WkpjY8S4ywK06SalVRjXlfCiI,378
 langroid/language_models/mock_lm.py,sha256=5BgHKDVRWFbUwDT_PFgTZXz9-k8wJSA2e3PZmyDgQ1k,4022
 langroid/language_models/model_info.py,sha256=tfBBxL0iUf2mVN6CjcvqflzFUVg2oZqOJZexZ8jHTYA,12216
-langroid/language_models/openai_gpt.py,sha256=M_jp97Ksp5r3U-d0jCLPLjVmn7IK1mC8Ry4t7k6A5tc,82906
+langroid/language_models/openai_gpt.py,sha256=t5oFJFxfm0ZwbD5kS73TY8Hal6aCFUWX7O7dBZc-7fw,84480
 langroid/language_models/utils.py,sha256=L4_CbihDMTGcsg0TOG1Yd5JFEto46--h7CX_14m89sQ,5016
 langroid/language_models/prompt_formatter/__init__.py,sha256=2-5cdE24XoFDhifOLl8yiscohil1ogbP1ECkYdBlBsk,372
 langroid/language_models/prompt_formatter/base.py,sha256=eDS1sgRNZVnoajwV_ZIha6cba5Dt8xjgzdRbPITwx3Q,1221
@@ -128,7 +129,7 @@ langroid/vector_store/pineconedb.py,sha256=otxXZNaBKb9f_H75HTaU3lMHiaR2NUp5MqwLZ
 langroid/vector_store/postgres.py,sha256=wHPtIi2qM4fhO4pMQr95pz1ZCe7dTb2hxl4VYspGZoA,16104
 langroid/vector_store/qdrantdb.py,sha256=O6dSBoDZ0jzfeVBd7LLvsXu083xs2fxXtPa9gGX3JX4,18443
 langroid/vector_store/weaviatedb.py,sha256=Yn8pg139gOy3zkaPfoTbMXEEBCiLiYa1MU5d_3UA1K4,11847
-langroid-0.50.2.dist-info/METADATA,sha256=ttDoi8hgjYIzOjj_bHVN8IMtJcN574iXw4ZN4_q1VUQ,63641
-langroid-0.50.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-langroid-0.50.2.dist-info/licenses/LICENSE,sha256=EgVbvA6VSYgUlvC3RvPKehSg7MFaxWDsFuzLOsPPfJg,1065
-langroid-0.50.2.dist-info/RECORD,,
+langroid-0.50.4.dist-info/METADATA,sha256=3HQH6x3pAFIDSUIt8TLI912CA-8EN_Jl_nKpQ09oli0,63641
+langroid-0.50.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+langroid-0.50.4.dist-info/licenses/LICENSE,sha256=EgVbvA6VSYgUlvC3RvPKehSg7MFaxWDsFuzLOsPPfJg,1065
+langroid-0.50.4.dist-info/RECORD,,