vectorvein 0.1.45__py3-none-any.whl → 0.1.47__py3-none-any.whl

vectorvein/chat_clients/anthropic_client.py

@@ -38,6 +38,7 @@ from ..types.llm_parameters import (
     ChatCompletionMessage,
     ChatCompletionToolParam,
     ChatCompletionDeltaMessage,
+    ChatCompletionStreamOptionsParam,
 )
 
 
@@ -214,6 +215,7 @@ class AnthropicChatClient(BaseChatClient):
         tools: Iterable[ToolParam] | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None = None,
         **kwargs,
     ) -> ChatCompletionMessage:
         pass
@@ -229,6 +231,7 @@ class AnthropicChatClient(BaseChatClient):
         tools: Iterable[ToolParam] | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None = None,
         **kwargs,
     ) -> Generator[ChatCompletionDeltaMessage, None, None]:
         pass
@@ -244,6 +247,7 @@ class AnthropicChatClient(BaseChatClient):
         tools: Iterable[ToolParam] | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None = None,
         **kwargs,
     ) -> ChatCompletionMessage | Generator[ChatCompletionDeltaMessage, Any, None]:
         pass
@@ -258,6 +262,7 @@ class AnthropicChatClient(BaseChatClient):
         tools: Iterable[ToolParam] | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None = None,
         **kwargs,
     ):
         if model is not None:
@@ -315,7 +320,16 @@ class AnthropicChatClient(BaseChatClient):
                     http_client=self.http_client,
                     backend_name=self.BACKEND_NAME,
                 ).create_completion(
-                    messages, model, False, temperature, max_tokens, _tools, _tool_choice, response_format, **kwargs
+                    messages=messages,
+                    model=model,
+                    stream=False,
+                    temperature=temperature,
+                    max_tokens=max_tokens,
+                    tools=_tools,
+                    tool_choice=_tool_choice,
+                    response_format=response_format,
+                    stream_options=stream_options,
+                    **kwargs,
                 )
 
         assert isinstance(self.raw_client, Anthropic | AnthropicVertex)
@@ -554,6 +568,7 @@ class AsyncAnthropicChatClient(BaseAsyncChatClient):
         tools: Iterable[ToolParam] | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None = None,
         **kwargs,
     ) -> ChatCompletionMessage:
         pass
@@ -569,6 +584,7 @@ class AsyncAnthropicChatClient(BaseAsyncChatClient):
         tools: Iterable[ToolParam] | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None = None,
         **kwargs,
     ) -> AsyncGenerator[ChatCompletionDeltaMessage, Any]:
         pass
@@ -584,6 +600,7 @@ class AsyncAnthropicChatClient(BaseAsyncChatClient):
         tools: Iterable[ToolParam] | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None = None,
         **kwargs,
     ) -> ChatCompletionMessage | AsyncGenerator[ChatCompletionDeltaMessage, Any]:
         pass
@@ -598,6 +615,7 @@ class AsyncAnthropicChatClient(BaseAsyncChatClient):
         tools: Iterable[ToolParam] | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None = None,
         **kwargs,
     ):
         if model is not None:
@@ -639,7 +657,16 @@ class AsyncAnthropicChatClient(BaseAsyncChatClient):
                         backend_name=self.BACKEND_NAME,
                     )
                     response = await client.create_completion(
-                        messages, model, True, temperature, max_tokens, _tools, _tool_choice, response_format, **kwargs
+                        messages=messages,
+                        model=model,
+                        stream=True,
+                        temperature=temperature,
+                        max_tokens=max_tokens,
+                        tools=_tools,
+                        tool_choice=_tool_choice,
+                        response_format=response_format,
+                        stream_options=stream_options,
+                        **kwargs,
                     )
                     async for chunk in response:
                         yield chunk

vectorvein/chat_clients/base_client.py

@@ -18,6 +18,7 @@ from ..types.llm_parameters import (
     ToolChoice,
     ChatCompletionMessage,
     ChatCompletionDeltaMessage,
+    ChatCompletionStreamOptionsParam,
 )
 
 
@@ -71,6 +72,7 @@ class BaseChatClient(ABC):
         tools: Iterable[ToolParam] | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None = None,
         **kwargs,
     ) -> ChatCompletionMessage:
         pass
@@ -87,6 +89,7 @@ class BaseChatClient(ABC):
         tools: Iterable[ToolParam] | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None = None,
         **kwargs,
     ) -> Generator[ChatCompletionDeltaMessage, Any, None]:
         pass
@@ -103,6 +106,7 @@ class BaseChatClient(ABC):
         tools: Iterable[ToolParam] | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None = None,
         **kwargs,
     ) -> ChatCompletionMessage | Generator[ChatCompletionDeltaMessage, Any, None]:
         pass
@@ -118,6 +122,7 @@ class BaseChatClient(ABC):
         tools: Iterable[ToolParam] | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None = None,
         **kwargs,
     ) -> ChatCompletionMessage | Generator[ChatCompletionDeltaMessage, Any, None]:
         pass
@@ -131,6 +136,7 @@ class BaseChatClient(ABC):
         tools: Iterable[ToolParam] | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None = None,
         **kwargs,
     ) -> Generator[ChatCompletionDeltaMessage, Any, None]:
         return self.create_completion(
@@ -142,6 +148,7 @@ class BaseChatClient(ABC):
             tools=tools,
             tool_choice=tool_choice,
             response_format=response_format,
+            stream_options=stream_options,
             **kwargs,
         )
 
@@ -198,6 +205,7 @@ class BaseAsyncChatClient(ABC):
         tools: list | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None = None,
         **kwargs,
     ) -> ChatCompletionMessage:
         pass
@@ -214,6 +222,7 @@ class BaseAsyncChatClient(ABC):
         tools: list | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None = None,
         **kwargs,
     ) -> AsyncGenerator[ChatCompletionDeltaMessage, None]:
         pass
@@ -230,6 +239,7 @@ class BaseAsyncChatClient(ABC):
         tools: list | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None = None,
         **kwargs,
     ) -> ChatCompletionMessage | AsyncGenerator[ChatCompletionDeltaMessage, None]:
         pass
@@ -245,6 +255,7 @@ class BaseAsyncChatClient(ABC):
         tools: list | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None = None,
         **kwargs,
     ) -> ChatCompletionMessage | AsyncGenerator[ChatCompletionDeltaMessage, None]:
         pass
@@ -258,6 +269,7 @@ class BaseAsyncChatClient(ABC):
         tools: list | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None = None,
         **kwargs,
     ) -> AsyncGenerator[ChatCompletionDeltaMessage, None]:
         return await self.create_completion(
@@ -269,5 +281,6 @@ class BaseAsyncChatClient(ABC):
             tools=tools,
             tool_choice=tool_choice,
             response_format=response_format,
+            stream_options=stream_options,
             **kwargs,
         )

vectorvein/chat_clients/gemini_client.py

@@ -19,6 +19,7 @@ from ..types.llm_parameters import (
     ToolChoice,
     ChatCompletionMessage,
     ChatCompletionDeltaMessage,
+    ChatCompletionStreamOptionsParam,
 )
 
 
@@ -63,6 +64,7 @@ class GeminiChatClient(BaseChatClient):
         tools: Iterable[ToolParam] | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None = None,
         **kwargs,
     ) -> ChatCompletionMessage:
         pass
@@ -78,6 +80,7 @@ class GeminiChatClient(BaseChatClient):
         tools: Iterable[ToolParam] | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None = None,
         **kwargs,
     ) -> Generator[ChatCompletionDeltaMessage, None, None]:
         pass
@@ -93,6 +96,7 @@ class GeminiChatClient(BaseChatClient):
         tools: Iterable[ToolParam] | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None = None,
         **kwargs,
     ) -> ChatCompletionMessage | Generator[ChatCompletionDeltaMessage, Any, None]:
         pass
@@ -107,6 +111,7 @@ class GeminiChatClient(BaseChatClient):
         tools: Iterable[ToolParam] | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None = None,
         **kwargs,
     ):
         if model is not None:
@@ -144,7 +149,12 @@ class GeminiChatClient(BaseChatClient):
 
         if self.random_endpoint:
             self.random_endpoint = True
-            self.endpoint_id = random.choice(self.backend_settings.models[self.model].endpoints)
+            endpoint_choice = random.choice(self.backend_settings.models[self.model].endpoints)
+            if isinstance(endpoint_choice, dict):
+                self.endpoint_id = endpoint_choice["endpoint_id"]
+                self.model_id = endpoint_choice["model_id"]
+            else:
+                self.endpoint_id = endpoint_choice
             self.endpoint = settings.get_endpoint(self.endpoint_id)
 
         request_body = {
@@ -291,6 +301,7 @@ class AsyncGeminiChatClient(BaseAsyncChatClient):
         tools: Iterable[ToolParam] | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None = None,
         **kwargs,
     ) -> ChatCompletionMessage:
         pass
@@ -306,6 +317,7 @@ class AsyncGeminiChatClient(BaseAsyncChatClient):
         tools: Iterable[ToolParam] | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None = None,
         **kwargs,
     ) -> AsyncGenerator[ChatCompletionDeltaMessage, Any]:
         pass
@@ -321,6 +333,7 @@ class AsyncGeminiChatClient(BaseAsyncChatClient):
         tools: Iterable[ToolParam] | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None = None,
         **kwargs,
     ) -> ChatCompletionMessage | AsyncGenerator[ChatCompletionDeltaMessage, Any]:
         pass
@@ -335,6 +348,7 @@ class AsyncGeminiChatClient(BaseAsyncChatClient):
         tools: Iterable[ToolParam] | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None = None,
         **kwargs,
     ):
         if model is not None:
@@ -372,7 +386,12 @@ class AsyncGeminiChatClient(BaseAsyncChatClient):
 
         if self.random_endpoint:
             self.random_endpoint = True
-            self.endpoint_id = random.choice(self.backend_settings.models[self.model].endpoints)
+            endpoint_choice = random.choice(self.backend_settings.models[self.model].endpoints)
+            if isinstance(endpoint_choice, dict):
+                self.endpoint_id = endpoint_choice["endpoint_id"]
+                self.model_id = endpoint_choice["model_id"]
+            else:
+                self.endpoint_id = endpoint_choice
             self.endpoint = settings.get_endpoint(self.endpoint_id)
 
         request_body = {

vectorvein/chat_clients/minimax_client.py

@@ -18,6 +18,7 @@ from ..types.llm_parameters import (
     ToolChoice,
     ChatCompletionMessage,
     ChatCompletionDeltaMessage,
+    ChatCompletionStreamOptionsParam,
 )
 
 
@@ -72,6 +73,7 @@ class MiniMaxChatClient(BaseChatClient):
             self.http_client = http_client
         else:
             self.http_client = httpx.Client()
+        self.model_id = None
 
     @cached_property
     def raw_client(self):
@@ -88,6 +90,7 @@ class MiniMaxChatClient(BaseChatClient):
         tools: Iterable[ToolParam] | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None = None,
         **kwargs,
     ) -> ChatCompletionMessage:
         pass
@@ -103,6 +106,7 @@ class MiniMaxChatClient(BaseChatClient):
         tools: Iterable[ToolParam] | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None = None,
         **kwargs,
     ) -> Generator[ChatCompletionDeltaMessage, None, None]:
         pass
@@ -118,6 +122,7 @@ class MiniMaxChatClient(BaseChatClient):
         tools: Iterable[ToolParam] | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None = None,
         **kwargs,
     ) -> ChatCompletionMessage | Generator[ChatCompletionDeltaMessage, Any, None]:
         pass
@@ -132,6 +137,7 @@ class MiniMaxChatClient(BaseChatClient):
         tools: Iterable[ToolParam] | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None = None,
         **kwargs,
     ):
         if model is not None:
@@ -144,9 +150,16 @@ class MiniMaxChatClient(BaseChatClient):
             tool_choice = "auto"
 
         self.model_setting = self.backend_settings.models[self.model]
+        if self.model_id is None:
+            self.model_id = self.model_setting.id
         if self.random_endpoint:
             self.random_endpoint = True
-            self.endpoint_id = random.choice(self.backend_settings.models[self.model].endpoints)
+            endpoint_choice = random.choice(self.backend_settings.models[self.model].endpoints)
+            if isinstance(endpoint_choice, dict):
+                self.endpoint_id = endpoint_choice["endpoint_id"]
+                self.model_id = endpoint_choice["model_id"]
+            else:
+                self.endpoint_id = endpoint_choice
             self.endpoint = settings.get_endpoint(self.endpoint_id)
 
         if self.context_length_control == ContextLengthControlType.Latest:
@@ -191,7 +204,7 @@ class MiniMaxChatClient(BaseChatClient):
         self.headers = {"Authorization": f"Bearer {self.endpoint.api_key}", "Content-Type": "application/json"}
 
         request_body = {
-            "model": self.model,
+            "model": self.model_id,
             "messages": messages,
             "max_tokens": max_tokens,
             "temperature": self.temperature,
@@ -287,6 +300,7 @@ class AsyncMiniMaxChatClient(BaseAsyncChatClient):
             self.http_client = http_client
         else:
             self.http_client = httpx.AsyncClient()
+        self.model_id = None
 
     @cached_property
     def raw_client(self):
@@ -303,6 +317,7 @@ class AsyncMiniMaxChatClient(BaseAsyncChatClient):
         tools: Iterable[ToolParam] | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None = None,
         **kwargs,
     ) -> ChatCompletionMessage:
         pass
@@ -318,6 +333,7 @@ class AsyncMiniMaxChatClient(BaseAsyncChatClient):
         tools: Iterable[ToolParam] | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None = None,
         **kwargs,
     ) -> AsyncGenerator[ChatCompletionDeltaMessage, Any]:
         pass
@@ -333,6 +349,7 @@ class AsyncMiniMaxChatClient(BaseAsyncChatClient):
         tools: Iterable[ToolParam] | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None = None,
         **kwargs,
     ) -> ChatCompletionMessage | AsyncGenerator[ChatCompletionDeltaMessage, Any]:
         pass
@@ -347,6 +364,7 @@ class AsyncMiniMaxChatClient(BaseAsyncChatClient):
         tools: Iterable[ToolParam] | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None = None,
         **kwargs,
     ):
         if model is not None:
@@ -359,9 +377,16 @@ class AsyncMiniMaxChatClient(BaseAsyncChatClient):
             tool_choice = "auto"
 
         self.model_setting = self.backend_settings.models[self.model]
+        if self.model_id is None:
+            self.model_id = self.model_setting.id
         if self.random_endpoint:
             self.random_endpoint = True
-            self.endpoint_id = random.choice(self.backend_settings.models[self.model].endpoints)
+            endpoint_choice = random.choice(self.backend_settings.models[self.model].endpoints)
+            if isinstance(endpoint_choice, dict):
+                self.endpoint_id = endpoint_choice["endpoint_id"]
+                self.model_id = endpoint_choice["model_id"]
+            else:
+                self.endpoint_id = endpoint_choice
             self.endpoint = settings.get_endpoint(self.endpoint_id)
 
         if self.context_length_control == ContextLengthControlType.Latest:
@@ -404,7 +429,7 @@ class AsyncMiniMaxChatClient(BaseAsyncChatClient):
         self.headers = {"Authorization": f"Bearer {self.endpoint.api_key}", "Content-Type": "application/json"}
 
         request_body = {
-            "model": self.model,
+            "model": self.model_id,
             "messages": messages,
             "max_tokens": max_tokens,
             "temperature": self.temperature,

vectorvein/chat_clients/openai_compatible_client.py

@@ -6,8 +6,6 @@ from functools import cached_property
 from typing import overload, Generator, AsyncGenerator, Any, Literal, Iterable
 
 import httpx
-from openai._streaming import Stream, AsyncStream
-from openai.types.chat import ChatCompletion, ChatCompletionChunk
 from openai import OpenAI, AsyncOpenAI, AzureOpenAI, AsyncAzureOpenAI
 
 from .base_client import BaseChatClient, BaseAsyncChatClient
@@ -25,8 +23,11 @@ from ..types.llm_parameters import (
     NOT_GIVEN,
     ToolParam,
     ToolChoice,
+    OpenAINotGiven,
+    Usage,
     ChatCompletionMessage,
     ChatCompletionDeltaMessage,
+    ChatCompletionStreamOptionsParam,
 )
 
 
@@ -74,7 +75,7 @@ class OpenAICompatibleChatClient(BaseChatClient):
             return AzureOpenAI(
                 azure_endpoint=self.endpoint.api_base,
                 api_key=self.endpoint.api_key,
-                api_version="2024-08-01-preview",
+                api_version="2024-10-01-preview",
                 http_client=self.http_client,
             )
         else:
@@ -95,6 +96,7 @@ class OpenAICompatibleChatClient(BaseChatClient):
         tools: Iterable[ToolParam] | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None | OpenAINotGiven = NOT_GIVEN,
         **kwargs,
     ) -> ChatCompletionMessage:
         pass
@@ -110,6 +112,7 @@ class OpenAICompatibleChatClient(BaseChatClient):
         tools: Iterable[ToolParam] | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None | OpenAINotGiven = NOT_GIVEN,
         **kwargs,
     ) -> Generator[ChatCompletionDeltaMessage, None, None]:
         pass
@@ -125,6 +128,7 @@ class OpenAICompatibleChatClient(BaseChatClient):
         tools: Iterable[ToolParam] | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None | OpenAINotGiven = NOT_GIVEN,
         **kwargs,
     ) -> ChatCompletionMessage | Generator[ChatCompletionDeltaMessage, Any, None]:
         pass
@@ -139,6 +143,7 @@ class OpenAICompatibleChatClient(BaseChatClient):
         tools: Iterable[ToolParam] | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None | OpenAINotGiven = NOT_GIVEN,
         **kwargs,
     ):
         if model is not None:
@@ -189,13 +194,19 @@ class OpenAICompatibleChatClient(BaseChatClient):
         else:
             self.response_format = {}
 
+        if stream_options:
+            _stream_options_params = {"stream_options": stream_options}
+        else:
+            _stream_options_params = {}
+
         if self.stream:
-            stream_response: Stream[ChatCompletionChunk] = raw_client.chat.completions.create(
+            stream_response = raw_client.chat.completions.create(
                 model=self.model_id,
                 messages=messages,
                 stream=True,
                 temperature=self.temperature,
                 max_tokens=max_tokens,
+                **_stream_options_params,
                 **self.response_format,
                 **tools_params,
                 **kwargs,
@@ -204,7 +215,16 @@ class OpenAICompatibleChatClient(BaseChatClient):
             def generator():
                 full_content = ""
                 result = {}
+                usage = None
                 for chunk in stream_response:
+                    if chunk.usage and chunk.usage.total_tokens:
+                        usage = Usage(
+                            completion_tokens=chunk.usage.completion_tokens or 0,
+                            prompt_tokens=chunk.usage.prompt_tokens or 0,
+                            total_tokens=chunk.usage.total_tokens or 0,
+                        )
+                    else:
+                        usage = None
                     if len(chunk.choices) == 0:
                         continue
                     if not chunk.choices[0].delta:
@@ -213,7 +233,7 @@ class OpenAICompatibleChatClient(BaseChatClient):
                         if chunk.choices[0].delta.tool_calls:
                             for index, tool_call in enumerate(chunk.choices[0].delta.tool_calls):
                                 tool_call.index = index
-                        yield ChatCompletionDeltaMessage(**chunk.choices[0].delta.model_dump())
+                        yield ChatCompletionDeltaMessage(**chunk.choices[0].delta.model_dump(), usage=usage)
                     else:
                         message = chunk.choices[0].delta.model_dump()
                         full_content += message["content"] if message["content"] else ""
@@ -225,13 +245,13 @@ class OpenAICompatibleChatClient(BaseChatClient):
                             message["content"] = ""
                             result = message
                             continue
-                        yield ChatCompletionDeltaMessage(**message)
+                        yield ChatCompletionDeltaMessage(**message, usage=usage)
                 if result:
-                    yield ChatCompletionDeltaMessage(**result)
+                    yield ChatCompletionDeltaMessage(**result, usage=usage)
 
             return generator()
         else:
-            response: ChatCompletion = raw_client.chat.completions.create(
+            response = raw_client.chat.completions.create(
                 model=self.model_id,
                 messages=messages,
                 stream=False,
@@ -306,7 +326,7 @@ class AsyncOpenAICompatibleChatClient(BaseAsyncChatClient):
             return AsyncAzureOpenAI(
                 azure_endpoint=self.endpoint.api_base,
                 api_key=self.endpoint.api_key,
-                api_version="2024-08-01-preview",
+                api_version="2024-10-01-preview",
                 http_client=self.http_client,
             )
         else:
@@ -327,6 +347,7 @@ class AsyncOpenAICompatibleChatClient(BaseAsyncChatClient):
         tools: Iterable[ToolParam] | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None | OpenAINotGiven = NOT_GIVEN,
         **kwargs,
     ) -> ChatCompletionMessage:
         pass
@@ -342,6 +363,7 @@ class AsyncOpenAICompatibleChatClient(BaseAsyncChatClient):
         tools: Iterable[ToolParam] | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None | OpenAINotGiven = NOT_GIVEN,
         **kwargs,
     ) -> AsyncGenerator[ChatCompletionDeltaMessage, Any]:
         pass
@@ -357,6 +379,7 @@ class AsyncOpenAICompatibleChatClient(BaseAsyncChatClient):
         tools: Iterable[ToolParam] | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None | OpenAINotGiven = NOT_GIVEN,
         **kwargs,
     ) -> ChatCompletionMessage | AsyncGenerator[ChatCompletionDeltaMessage, Any]:
         pass
@@ -371,6 +394,7 @@ class AsyncOpenAICompatibleChatClient(BaseAsyncChatClient):
         tools: Iterable[ToolParam] | NotGiven = NOT_GIVEN,
         tool_choice: ToolChoice | NotGiven = NOT_GIVEN,
         response_format: dict | None = None,
+        stream_options: ChatCompletionStreamOptionsParam | None | OpenAINotGiven = NOT_GIVEN,
         **kwargs,
     ):
         if model is not None:
@@ -412,6 +436,11 @@ class AsyncOpenAICompatibleChatClient(BaseAsyncChatClient):
         else:
             self.response_format = {}
 
+        if stream_options:
+            _stream_options_params = {"stream_options": stream_options}
+        else:
+            _stream_options_params = {}
+
         if max_tokens is None:
             max_output_tokens = self.model_setting.max_output_tokens
             token_counts = get_message_token_counts(messages=messages, tools=tools, model=self.model)
@@ -422,12 +451,13 @@ class AsyncOpenAICompatibleChatClient(BaseAsyncChatClient):
                 max_tokens = self.model_setting.context_length - token_counts - 64
 
         if self.stream:
-            stream_response: AsyncStream[ChatCompletionChunk] = await raw_client.chat.completions.create(
+            stream_response = await raw_client.chat.completions.create(
                 model=self.model_id,
                 messages=messages,
                 stream=self.stream,
                 temperature=self.temperature,
                 max_tokens=max_tokens,
+                **_stream_options_params,
                 **self.response_format,
                 **tools_params,
                 **kwargs,
@@ -436,7 +466,16 @@ class AsyncOpenAICompatibleChatClient(BaseAsyncChatClient):
             async def generator():
                 full_content = ""
                 result = {}
+                usage = None
                 async for chunk in stream_response:
+                    if chunk.usage and chunk.usage.total_tokens:
+                        usage = Usage(
+                            completion_tokens=chunk.usage.completion_tokens or 0,
+                            prompt_tokens=chunk.usage.prompt_tokens or 0,
+                            total_tokens=chunk.usage.total_tokens or 0,
+                        )
+                    else:
+                        usage = None
                     if len(chunk.choices) == 0:
                         continue
                     if not chunk.choices[0].delta:
@@ -445,7 +484,7 @@ class AsyncOpenAICompatibleChatClient(BaseAsyncChatClient):
                         if chunk.choices[0].delta.tool_calls:
                             for index, tool_call in enumerate(chunk.choices[0].delta.tool_calls):
                                 tool_call.index = index
-                        yield ChatCompletionDeltaMessage(**chunk.choices[0].delta.model_dump())
+                        yield ChatCompletionDeltaMessage(**chunk.choices[0].delta.model_dump(), usage=usage)
                     else:
                         message = chunk.choices[0].delta.model_dump()
                         full_content += message["content"] if message["content"] else ""
@@ -457,13 +496,13 @@ class AsyncOpenAICompatibleChatClient(BaseAsyncChatClient):
                             message["content"] = ""
                             result = message
                             continue
-                        yield ChatCompletionDeltaMessage(**message)
+                        yield ChatCompletionDeltaMessage(**message, usage=usage)
                 if result:
-                    yield ChatCompletionDeltaMessage(**result)
+                    yield ChatCompletionDeltaMessage(**result, usage=usage)
 
             return generator()
         else:
-            response: ChatCompletion = await raw_client.chat.completions.create(
+            response = await raw_client.chat.completions.create(
                 model=self.model_id,
                 messages=messages,
                 stream=self.stream,

vectorvein/types/defaults.py

@@ -526,7 +526,7 @@ OPENAI_MODELS: Final[Dict[str, Dict[str, Any]]] = {
 }
 
 # Anthropic models
-ANTHROPIC_DEFAULT_MODEL: Final[str] = "claude-3-5-sonnet-20240620"
+ANTHROPIC_DEFAULT_MODEL: Final[str] = "claude-3-5-sonnet-20241022"
 ANTHROPIC_MODELS: Final[Dict[str, Dict[str, Any]]] = {
     "claude-3-opus-20240229": {
         "id": "claude-3-opus-20240229",

vectorvein/types/llm_parameters.py

@@ -14,6 +14,7 @@ from openai._types import NOT_GIVEN as OPENAI_NOT_GIVEN
 from openai.types.chat.chat_completion_chunk import ChoiceDeltaToolCall
 from openai.types.chat.chat_completion_tool_param import ChatCompletionToolParam
 from openai.types.chat.chat_completion_message_tool_call import ChatCompletionMessageToolCall
+from openai.types.chat.chat_completion_stream_options_param import ChatCompletionStreamOptionsParam
 from openai.types.chat.chat_completion_tool_choice_option_param import ChatCompletionToolChoiceOptionParam
 
 from . import defaults as defs
@@ -122,6 +123,7 @@ __all__ = [
     "Usage",
     "ChatCompletionMessage",
     "ChatCompletionDeltaMessage",
+    "ChatCompletionStreamOptionsParam",
     "NotGiven",
     "NOT_GIVEN",
     "OpenAIToolParam",

{vectorvein-0.1.45.dist-info → vectorvein-0.1.47.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vectorvein
-Version: 0.1.45
+Version: 0.1.47
 Summary: Default template for PDM package
 Author-Email: Anderson <andersonby@163.com>
 License: MIT

{vectorvein-0.1.45.dist-info → vectorvein-0.1.47.dist-info}/RECORD

@@ -1,20 +1,20 @@
-vectorvein-0.1.45.dist-info/METADATA,sha256=Er7GHrx6wrxhlRAT_xJ2oKWVVHUyFT4BD4aqVBL8I7s,644
-vectorvein-0.1.45.dist-info/WHEEL,sha256=thaaA2w1JzcGC48WYufAs8nrYZjJm8LqNfnXFOFyCC4,90
-vectorvein-0.1.45.dist-info/entry_points.txt,sha256=6OYgBcLyFCUgeqLgnvMyOJxPCWzgy7se4rLPKtNonMs,34
+vectorvein-0.1.47.dist-info/METADATA,sha256=wHK12p2NAwLjy_gei3wiErFfYwWRSV1dVx7nEda3TQU,644
+vectorvein-0.1.47.dist-info/WHEEL,sha256=thaaA2w1JzcGC48WYufAs8nrYZjJm8LqNfnXFOFyCC4,90
+vectorvein-0.1.47.dist-info/entry_points.txt,sha256=6OYgBcLyFCUgeqLgnvMyOJxPCWzgy7se4rLPKtNonMs,34
 vectorvein/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vectorvein/chat_clients/__init__.py,sha256=dW169oK1n3v8Z0uD8itghzlCP72rxiaS-XYn6fvI2xM,16788
-vectorvein/chat_clients/anthropic_client.py,sha256=leEA9DqXe0cerwgRJ8UciENGl_7l7cyMa64htAkBeAc,32797
+vectorvein/chat_clients/anthropic_client.py,sha256=i1fMYSkUovd-Lc9B64bWMgzSTUFw4S3fj3AJ_pDokT4,34029
 vectorvein/chat_clients/baichuan_client.py,sha256=CVMvpgjdrZGv0BWnTOBD-f2ufZ3wq3496wqukumsAr4,526
-vectorvein/chat_clients/base_client.py,sha256=7i456Yn-tqY0oPeIj_wHWNGGzCKPAbX5Ufxy2wWGMNY,8653
+vectorvein/chat_clients/base_client.py,sha256=0Uj0e-JR0a68sRS_WfUMVd91Av7lzJh6-DukjutlaD0,9497
 vectorvein/chat_clients/deepseek_client.py,sha256=3qWu01NlJAP2N-Ff62d5-CZXZitlizE1fzb20LNetig,526
-vectorvein/chat_clients/gemini_client.py,sha256=6LKy03iVDXtj0Bn99o9M8rRURMzozX-kf0FoazxPQt0,18739
+vectorvein/chat_clients/gemini_client.py,sha256=TJZUP8dbufuDnqkRI_gxIDDCYHrIhMPCqKGI6sSwqZo,19860
 vectorvein/chat_clients/groq_client.py,sha256=Uow4pgdmFi93ZQSoOol2-0PhhqkW-S0XuSldvppz5U4,498
 vectorvein/chat_clients/local_client.py,sha256=55nOsxzqUf79q3Y14MKROA71zxhsT7p7FsDZ89rts2M,422
-vectorvein/chat_clients/minimax_client.py,sha256=0MVMb4g0K_VKnPGHYX81jHiBaQUGWFG5vV8wyClCT_8,17258
+vectorvein/chat_clients/minimax_client.py,sha256=cLpSJ5vIaDi0Kmrt7XGlbjnlFnByJiI0YWY3gw1y1Qs,18592
 vectorvein/chat_clients/mistral_client.py,sha256=1aKSylzBDaLYcFnaBIL4-sXSzWmXfBeON9Q0rq-ziWw,534
 vectorvein/chat_clients/moonshot_client.py,sha256=gbu-6nGxx8uM_U2WlI4Wus881rFRotzHtMSoYOcruGU,526
 vectorvein/chat_clients/openai_client.py,sha256=Nz6tV45pWcsOupxjnsRsGTicbQNJWIZyxuJoJ5DGMpg,527
-vectorvein/chat_clients/openai_compatible_client.py,sha256=FVm_ZYL9UP6t6hTUNxPyoxGYqXHJMw37UGOQ-t63gaw,19709
+vectorvein/chat_clients/openai_compatible_client.py,sha256=jsEYttMKNZmgKeOIk2MbNUz0Tri2jeQCR1iaCs_TIok,21728
 vectorvein/chat_clients/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vectorvein/chat_clients/qwen_client.py,sha256=-ryh-m9PgsO0fc4ulcCmPTy1155J8YUy15uPoJQOHA0,513
 vectorvein/chat_clients/stepfun_client.py,sha256=zsD2W5ahmR4DD9cqQTXmJr3txrGuvxbRWhFlRdwNijI,519
@@ -25,11 +25,11 @@ vectorvein/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vectorvein/server/token_server.py,sha256=36F9PKSNOX8ZtYBXY_l-76GQTpUSmQ2Y8EMy1H7wtdQ,1353
 vectorvein/settings/__init__.py,sha256=dyTCLhevXiKVJhOb1tjgZGMH38Indy4dkWVdDX543g0,3771
 vectorvein/settings/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-vectorvein/types/defaults.py,sha256=3sAC8NtNfPOG3Jz-d7Mjt-7JyTgfJingwpp9f76cbF0,22379
+vectorvein/types/defaults.py,sha256=L8ExeMttmxsCiJM0AUGqbvbJKC--HQhA883mayjNoz8,22379
 vectorvein/types/enums.py,sha256=x_S0IJiEWijOAEiMNdiGDGEWGtmt7TwMriJVDqrDmTo,1637
 vectorvein/types/exception.py,sha256=gnW4GnJ76jND6UGnodk9xmqkcbeS7Cz2rvncA2HpD5E,69
-vectorvein/types/llm_parameters.py,sha256=vhleSgCHzDl7EULYJ3dUYlu9KLbfs9y6dcPD0BkaRdg,5114
+vectorvein/types/llm_parameters.py,sha256=5o-C_yXxxQWZy_e8OWowB2107GTS-Eawx4Mvb1q55Co,5256
 vectorvein/types/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vectorvein/utilities/media_processing.py,sha256=cnzLrU1OaJvSv87IOnc36FrDXtmGMDStPbxtIJ33YN4,5880
 vectorvein/utilities/retry.py,sha256=6KFS9R2HdhqM3_9jkjD4F36ZSpEx2YNFGOVlpOsUetM,2208
-vectorvein-0.1.45.dist-info/RECORD,,
+vectorvein-0.1.47.dist-info/RECORD,,