vectorvein 0.2.9__py3-none-any.whl → 0.2.11__py3-none-any.whl

vectorvein/chat_clients/__init__.py

@@ -1,7 +1,7 @@
 # @Author: Bi Ying
 # @Date:   2024-07-26 14:48:55
 import httpx
-from typing import overload, Literal
+from typing import overload, Literal, TYPE_CHECKING
 
 from .base_client import BaseChatClient, BaseAsyncChatClient
 
@@ -26,6 +26,11 @@ from ..types.enums import BackendType, ContextLengthControlType
 from .anthropic_client import AnthropicChatClient, AsyncAnthropicChatClient
 from .utils import format_messages, get_token_counts, get_message_token_counts, ToolCallContentProcessor
 
+
+if TYPE_CHECKING:
+    from ..settings import Settings
+    from ..types.settings import SettingsDict
+
 # 后端映射
 BackendMap = {
     "sync": {
@@ -75,6 +80,7 @@ def create_chat_client(
     random_endpoint: bool = True,
     endpoint_id: str = "",
     http_client: httpx.Client | None = None,
+    settings: "Settings | SettingsDict | None" = None,  # Use default settings if not provided
     **kwargs,
 ) -> AnthropicChatClient: ...
 
@@ -89,6 +95,7 @@ def create_chat_client(
     random_endpoint: bool = True,
     endpoint_id: str = "",
     http_client: httpx.Client | None = None,
+    settings: "Settings | SettingsDict | None" = None,  # Use default settings if not provided
     **kwargs,
 ) -> DeepSeekChatClient: ...
 
@@ -103,6 +110,7 @@ def create_chat_client(
     random_endpoint: bool = True,
     endpoint_id: str = "",
     http_client: httpx.Client | None = None,
+    settings: "Settings | SettingsDict | None" = None,  # Use default settings if not provided
     **kwargs,
 ) -> GeminiChatClient: ...
 
@@ -117,6 +125,7 @@ def create_chat_client(
     random_endpoint: bool = True,
     endpoint_id: str = "",
     http_client: httpx.Client | None = None,
+    settings: "Settings | SettingsDict | None" = None,  # Use default settings if not provided
     **kwargs,
 ) -> GroqChatClient: ...
 
@@ -131,6 +140,7 @@ def create_chat_client(
     random_endpoint: bool = True,
     endpoint_id: str = "",
     http_client: httpx.Client | None = None,
+    settings: "Settings | SettingsDict | None" = None,  # Use default settings if not provided
     **kwargs,
 ) -> LocalChatClient: ...
 
@@ -145,6 +155,7 @@ def create_chat_client(
     random_endpoint: bool = True,
     endpoint_id: str = "",
     http_client: httpx.Client | None = None,
+    settings: "Settings | SettingsDict | None" = None,  # Use default settings if not provided
     **kwargs,
 ) -> MiniMaxChatClient: ...
 
@@ -159,6 +170,7 @@ def create_chat_client(
     random_endpoint: bool = True,
     endpoint_id: str = "",
     http_client: httpx.Client | None = None,
+    settings: "Settings | SettingsDict | None" = None,  # Use default settings if not provided
     **kwargs,
 ) -> MistralChatClient: ...
 
@@ -173,6 +185,7 @@ def create_chat_client(
     random_endpoint: bool = True,
     endpoint_id: str = "",
     http_client: httpx.Client | None = None,
+    settings: "Settings | SettingsDict | None" = None,  # Use default settings if not provided
     **kwargs,
 ) -> MoonshotChatClient: ...
 
@@ -187,6 +200,7 @@ def create_chat_client(
     random_endpoint: bool = True,
     endpoint_id: str = "",
     http_client: httpx.Client | None = None,
+    settings: "Settings | SettingsDict | None" = None,  # Use default settings if not provided
     **kwargs,
 ) -> OpenAIChatClient: ...
 
@@ -201,6 +215,7 @@ def create_chat_client(
     random_endpoint: bool = True,
     endpoint_id: str = "",
     http_client: httpx.Client | None = None,
+    settings: "Settings | SettingsDict | None" = None,  # Use default settings if not provided
     **kwargs,
 ) -> QwenChatClient: ...
 
@@ -215,6 +230,7 @@ def create_chat_client(
     random_endpoint: bool = True,
     endpoint_id: str = "",
     http_client: httpx.Client | None = None,
+    settings: "Settings | SettingsDict | None" = None,  # Use default settings if not provided
     **kwargs,
 ) -> YiChatClient: ...
 
@@ -229,6 +245,7 @@ def create_chat_client(
     random_endpoint: bool = True,
     endpoint_id: str = "",
     http_client: httpx.Client | None = None,
+    settings: "Settings | SettingsDict | None" = None,  # Use default settings if not provided
     **kwargs,
 ) -> ZhiPuAIChatClient: ...
 
@@ -243,6 +260,7 @@ def create_chat_client(
     random_endpoint: bool = True,
     endpoint_id: str = "",
     http_client: httpx.Client | None = None,
+    settings: "Settings | SettingsDict | None" = None,  # Use default settings if not provided
     **kwargs,
 ) -> BaichuanChatClient: ...
 
@@ -257,6 +275,7 @@ def create_chat_client(
     random_endpoint: bool = True,
     endpoint_id: str = "",
     http_client: httpx.Client | None = None,
+    settings: "Settings | SettingsDict | None" = None,  # Use default settings if not provided
     **kwargs,
 ) -> StepFunChatClient: ...
 
@@ -271,6 +290,7 @@ def create_chat_client(
     random_endpoint: bool = True,
     endpoint_id: str = "",
     http_client: httpx.Client | None = None,
+    settings: "Settings | SettingsDict | None" = None,  # Use default settings if not provided
     **kwargs,
 ) -> XAIChatClient: ...
 
@@ -285,6 +305,7 @@ def create_chat_client(
     random_endpoint: bool = True,
     endpoint_id: str = "",
     http_client: httpx.Client | None = None,
+    settings: "Settings | SettingsDict | None" = None,  # Use default settings if not provided
     **kwargs,
 ) -> BaseChatClient: ...
 
@@ -298,6 +319,7 @@ def create_chat_client(
     random_endpoint: bool = True,
     endpoint_id: str = "",
     http_client: httpx.Client | None = None,
+    settings: "Settings | SettingsDict | None" = None,  # Use default settings if not provided
     **kwargs,
 ) -> BaseChatClient:
     if backend not in BackendMap["sync"]:
@@ -314,6 +336,7 @@ def create_chat_client(
         random_endpoint=random_endpoint,
         endpoint_id=endpoint_id,
         http_client=http_client,
+        settings=settings,
         **kwargs,
     )
 
@@ -328,6 +351,7 @@ def create_async_chat_client(
     random_endpoint: bool = True,
     endpoint_id: str = "",
     http_client: httpx.AsyncClient | None = None,
+    settings: "Settings | SettingsDict | None" = None,  # Use default settings if not provided
     **kwargs,
 ) -> AsyncAnthropicChatClient: ...
 
@@ -342,6 +366,7 @@ def create_async_chat_client(
     random_endpoint: bool = True,
     endpoint_id: str = "",
     http_client: httpx.AsyncClient | None = None,
+    settings: "Settings | SettingsDict | None" = None,  # Use default settings if not provided
     **kwargs,
 ) -> AsyncDeepSeekChatClient: ...
 
@@ -356,6 +381,7 @@ def create_async_chat_client(
     random_endpoint: bool = True,
     endpoint_id: str = "",
     http_client: httpx.AsyncClient | None = None,
+    settings: "Settings | SettingsDict | None" = None,  # Use default settings if not provided
     **kwargs,
 ) -> AsyncGeminiChatClient: ...
 
@@ -370,6 +396,7 @@ def create_async_chat_client(
     random_endpoint: bool = True,
     endpoint_id: str = "",
     http_client: httpx.AsyncClient | None = None,
+    settings: "Settings | SettingsDict | None" = None,  # Use default settings if not provided
     **kwargs,
 ) -> AsyncGroqChatClient: ...
 
@@ -384,6 +411,7 @@ def create_async_chat_client(
     random_endpoint: bool = True,
     endpoint_id: str = "",
     http_client: httpx.AsyncClient | None = None,
+    settings: "Settings | SettingsDict | None" = None,  # Use default settings if not provided
     **kwargs,
 ) -> AsyncLocalChatClient: ...
 
@@ -398,6 +426,7 @@ def create_async_chat_client(
     random_endpoint: bool = True,
     endpoint_id: str = "",
     http_client: httpx.AsyncClient | None = None,
+    settings: "Settings | SettingsDict | None" = None,  # Use default settings if not provided
     **kwargs,
 ) -> AsyncMiniMaxChatClient: ...
 
@@ -412,6 +441,7 @@ def create_async_chat_client(
     random_endpoint: bool = True,
     endpoint_id: str = "",
     http_client: httpx.AsyncClient | None = None,
+    settings: "Settings | SettingsDict | None" = None,  # Use default settings if not provided
     **kwargs,
 ) -> AsyncMistralChatClient: ...
 
@@ -426,6 +456,7 @@ def create_async_chat_client(
     random_endpoint: bool = True,
     endpoint_id: str = "",
     http_client: httpx.AsyncClient | None = None,
+    settings: "Settings | SettingsDict | None" = None,  # Use default settings if not provided
     **kwargs,
 ) -> AsyncMoonshotChatClient: ...
 
@@ -440,6 +471,7 @@ def create_async_chat_client(
     random_endpoint: bool = True,
     endpoint_id: str = "",
     http_client: httpx.AsyncClient | None = None,
+    settings: "Settings | SettingsDict | None" = None,  # Use default settings if not provided
     **kwargs,
 ) -> AsyncOpenAIChatClient: ...
 
@@ -454,6 +486,7 @@ def create_async_chat_client(
     random_endpoint: bool = True,
     endpoint_id: str = "",
     http_client: httpx.AsyncClient | None = None,
+    settings: "Settings | SettingsDict | None" = None,  # Use default settings if not provided
     **kwargs,
 ) -> AsyncQwenChatClient: ...
 
@@ -468,6 +501,7 @@ def create_async_chat_client(
     random_endpoint: bool = True,
     endpoint_id: str = "",
     http_client: httpx.AsyncClient | None = None,
+    settings: "Settings | SettingsDict | None" = None,  # Use default settings if not provided
     **kwargs,
 ) -> AsyncYiChatClient: ...
 
@@ -482,6 +516,7 @@ def create_async_chat_client(
     random_endpoint: bool = True,
     endpoint_id: str = "",
     http_client: httpx.AsyncClient | None = None,
+    settings: "Settings | SettingsDict | None" = None,  # Use default settings if not provided
     **kwargs,
 ) -> AsyncZhiPuAIChatClient: ...
 
@@ -496,6 +531,7 @@ def create_async_chat_client(
     random_endpoint: bool = True,
     endpoint_id: str = "",
     http_client: httpx.AsyncClient | None = None,
+    settings: "Settings | SettingsDict | None" = None,  # Use default settings if not provided
     **kwargs,
 ) -> AsyncBaichuanChatClient: ...
 
@@ -510,6 +546,7 @@ def create_async_chat_client(
     random_endpoint: bool = True,
     endpoint_id: str = "",
     http_client: httpx.AsyncClient | None = None,
+    settings: "Settings | SettingsDict | None" = None,  # Use default settings if not provided
     **kwargs,
 ) -> AsyncStepFunChatClient: ...
 
@@ -524,6 +561,7 @@ def create_async_chat_client(
     random_endpoint: bool = True,
     endpoint_id: str = "",
     http_client: httpx.AsyncClient | None = None,
+    settings: "Settings | SettingsDict | None" = None,  # Use default settings if not provided
     **kwargs,
 ) -> AsyncXAIChatClient: ...
 
@@ -538,6 +576,7 @@ def create_async_chat_client(
     random_endpoint: bool = True,
     endpoint_id: str = "",
     http_client: httpx.AsyncClient | None = None,
+    settings: "Settings | SettingsDict | None" = None,  # Use default settings if not provided
     **kwargs,
 ) -> BaseAsyncChatClient: ...
 
@@ -551,6 +590,7 @@ def create_async_chat_client(
     random_endpoint: bool = True,
     endpoint_id: str = "",
     http_client: httpx.AsyncClient | None = None,
+    settings: "Settings | SettingsDict | None" = None,  # Use default settings if not provided
     **kwargs,
 ) -> BaseAsyncChatClient:
     if backend not in BackendMap["async"]:
@@ -567,6 +607,7 @@ def create_async_chat_client(
         random_endpoint=random_endpoint,
         endpoint_id=endpoint_id,
         http_client=http_client,
+        settings=settings,
         **kwargs,
     )
 

vectorvein/chat_clients/anthropic_client.py

@@ -5,13 +5,14 @@ from typing import (
     Any,
     Dict,
     List,
+    TYPE_CHECKING,
+    overload,
+    Generator,
+    AsyncGenerator,
     Union,
     Literal,
     Iterable,
-    overload,
     Optional,
-    Generator,
-    AsyncGenerator,
 )
 
 import httpx
@@ -67,6 +68,10 @@ from ..types.llm_parameters import (
     ChatCompletionDeltaMessage,
 )
 
+if TYPE_CHECKING:
+    from ..settings import Settings
+    from ..types.settings import SettingsDict
+
 
 def refactor_tool_use_params(tools: Iterable[ChatCompletionToolParam]) -> list[AnthropicToolParam]:
     return [
@@ -186,6 +191,7 @@ class AnthropicChatClient(BaseChatClient):
         endpoint_id: str = "",
         http_client: httpx.Client | None = None,
         backend_name: str | None = None,
+        settings: "Settings | SettingsDict | None" = None,  # Use default settings if not provided
     ):
         super().__init__(
             model,
@@ -196,6 +202,7 @@ class AnthropicChatClient(BaseChatClient):
             endpoint_id,
             http_client,
             backend_name,
+            settings,
         )
         self.model_id = None
         self.endpoint = None
@@ -717,6 +724,7 @@ class AsyncAnthropicChatClient(BaseAsyncChatClient):
         endpoint_id: str = "",
         http_client: httpx.AsyncClient | None = None,
         backend_name: str | None = None,
+        settings: "Settings | SettingsDict | None" = None,  # Use default settings if not provided
     ):
         super().__init__(
             model,
@@ -727,6 +735,7 @@ class AsyncAnthropicChatClient(BaseAsyncChatClient):
             endpoint_id,
             http_client,
             backend_name,
+            settings,
         )
         self.model_id = None
         self.endpoint = None

vectorvein/chat_clients/base_client.py

@@ -27,8 +27,10 @@ from anthropic import (
 )
 from anthropic.types.thinking_config_param import ThinkingConfigParam
 
-from ..settings import settings
+from ..settings import Settings
+from ..settings import settings as default_settings
 from ..types import defaults as defs
+from ..types.settings import SettingsDict
 from ..types.enums import ContextLengthControlType, BackendType
 from ..types.llm_parameters import (
     NotGiven,
@@ -58,6 +60,7 @@ class BaseChatClient(ABC):
         endpoint_id: str = "",
         http_client: httpx.Client | None = None,
         backend_name: str | None = None,
+        settings: Settings | SettingsDict | None = None,  # Use default settings if not provided
     ):
         self.model = model or self.DEFAULT_MODEL
         self.stream = stream
@@ -70,7 +73,14 @@ class BaseChatClient(ABC):
         if backend_name is not None:
             self.BACKEND_NAME = BackendType(backend_name)
 
-        self.backend_settings = settings.get_backend(self.BACKEND_NAME)
+        if settings is None:
+            self.settings = default_settings
+        elif isinstance(settings, dict):
+            self.settings = Settings(**settings)
+        else:
+            self.settings = settings
+
+        self.backend_settings = self.settings.get_backend(self.BACKEND_NAME)
 
         self.rate_limiter = self._init_rate_limiter()
         self.active_requests = defaultdict(int)
@@ -81,29 +91,29 @@ class BaseChatClient(ABC):
         if endpoint_id:
             self.endpoint_id = endpoint_id
             self.random_endpoint = False
-            self.endpoint = settings.get_endpoint(self.endpoint_id)
+            self.endpoint = self.settings.get_endpoint(self.endpoint_id)
 
     def _init_rate_limiter(self):
-        if not settings.rate_limit:
+        if not self.settings.rate_limit:
             return None
-        if not settings.rate_limit.enabled:
+        if not self.settings.rate_limit.enabled:
             return None
 
-        if settings.rate_limit.backend == "memory":
+        if self.settings.rate_limit.backend == "memory":
             return SyncMemoryRateLimiter()
-        elif settings.rate_limit.backend == "redis":
-            if not settings.rate_limit.redis:
+        elif self.settings.rate_limit.backend == "redis":
+            if not self.settings.rate_limit.redis:
                 raise ValueError("Redis settings must be provided if Redis backend is selected.")
             return SyncRedisRateLimiter(
-                host=settings.rate_limit.redis.host,
-                port=settings.rate_limit.redis.port,
-                db=settings.rate_limit.redis.db,
+                host=self.settings.rate_limit.redis.host,
+                port=self.settings.rate_limit.redis.port,
+                db=self.settings.rate_limit.redis.db,
             )
-        elif settings.rate_limit.backend == "diskcache":
-            if not settings.rate_limit.diskcache:
+        elif self.settings.rate_limit.backend == "diskcache":
+            if not self.settings.rate_limit.diskcache:
                 raise ValueError("Diskcache settings must be provided if Diskcache backend is selected.")
             return SyncDiskCacheRateLimiter(
-                cache_dir=settings.rate_limit.diskcache.cache_dir,
+                cache_dir=self.settings.rate_limit.diskcache.cache_dir,
             )
         return None
 
@@ -115,8 +125,10 @@ class BaseChatClient(ABC):
 
         # Get rate limit parameters
         # Priority: parameters in model.endpoints > parameters in endpoint > default parameters
-        rpm = self.rpm or endpoint.rpm or (settings.rate_limit.default_rpm if settings.rate_limit else 60)
-        tpm = self.tpm or endpoint.tpm or (settings.rate_limit.default_tpm if settings.rate_limit else 1000000)
+        rpm = self.rpm or endpoint.rpm or (self.settings.rate_limit.default_rpm if self.settings.rate_limit else 60)
+        tpm = (
+            self.tpm or endpoint.tpm or (self.settings.rate_limit.default_tpm if self.settings.rate_limit else 1000000)
+        )
 
         while self.rate_limiter:
             allowed, wait_time = self.rate_limiter.check_limit(key, rpm, tpm, self._estimate_request_tokens(messages))
@@ -151,9 +163,9 @@ class BaseChatClient(ABC):
                     self.concurrent_requests = endpoint.get("concurrent_requests", None)
                 else:
                     self.endpoint_id = endpoint
-                self.endpoint = settings.get_endpoint(self.endpoint_id)
+                self.endpoint = self.settings.get_endpoint(self.endpoint_id)
             else:
-                self.endpoint = settings.get_endpoint(self.endpoint_id)
+                self.endpoint = self.settings.get_endpoint(self.endpoint_id)
                 self.set_model_id_by_endpoint_id(self.endpoint_id)
         elif isinstance(self.endpoint, EndpointSetting):
             self.endpoint_id = self.endpoint.id
@@ -411,6 +423,15 @@ class BaseChatClient(ABC):
             timeout=timeout,
         )
 
+    def model_list(self):
+        _raw_client = self.raw_client
+        if isinstance(_raw_client, (OpenAI, AzureOpenAI)):
+            return _raw_client.models.list().model_dump()
+        elif isinstance(_raw_client, Anthropic):
+            return _raw_client.models.list(limit=1000).model_dump()
+        else:
+            raise ValueError(f"Unsupported client type: {type(_raw_client)}")
+
 
 class BaseAsyncChatClient(ABC):
     DEFAULT_MODEL: str
@@ -426,6 +447,7 @@ class BaseAsyncChatClient(ABC):
         endpoint_id: str = "",
         http_client: httpx.AsyncClient | None = None,
         backend_name: str | None = None,
+        settings: Settings | SettingsDict | None = None,  # Use default settings if not provided
     ):
         self.model = model or self.DEFAULT_MODEL
         self.stream = stream
@@ -438,7 +460,14 @@ class BaseAsyncChatClient(ABC):
         if backend_name is not None:
             self.BACKEND_NAME = BackendType(backend_name)
 
-        self.backend_settings = settings.get_backend(self.BACKEND_NAME)
+        if settings is None:
+            self.settings = default_settings
+        elif isinstance(settings, dict):
+            self.settings = Settings(**settings)
+        else:
+            self.settings = settings
+
+        self.backend_settings = self.settings.get_backend(self.BACKEND_NAME)
 
         self.rate_limiter = self._init_rate_limiter()
         self.active_requests = defaultdict(int)
@@ -449,29 +478,29 @@ class BaseAsyncChatClient(ABC):
         if endpoint_id:
             self.endpoint_id = endpoint_id
             self.random_endpoint = False
-            self.endpoint = settings.get_endpoint(self.endpoint_id)
+            self.endpoint = self.settings.get_endpoint(self.endpoint_id)
 
     def _init_rate_limiter(self):
-        if not settings.rate_limit:
+        if not self.settings.rate_limit:
             return None
-        if not settings.rate_limit.enabled:
+        if not self.settings.rate_limit.enabled:
             return None
 
-        if settings.rate_limit.backend == "memory":
+        if self.settings.rate_limit.backend == "memory":
             return AsyncMemoryRateLimiter()
-        elif settings.rate_limit.backend == "redis":
-            if not settings.rate_limit.redis:
+        elif self.settings.rate_limit.backend == "redis":
+            if not self.settings.rate_limit.redis:
                 raise ValueError("Redis settings must be provided if Redis backend is selected.")
             return AsyncRedisRateLimiter(
-                host=settings.rate_limit.redis.host,
-                port=settings.rate_limit.redis.port,
-                db=settings.rate_limit.redis.db,
+                host=self.settings.rate_limit.redis.host,
+                port=self.settings.rate_limit.redis.port,
+                db=self.settings.rate_limit.redis.db,
             )
-        elif settings.rate_limit.backend == "diskcache":
-            if not settings.rate_limit.diskcache:
+        elif self.settings.rate_limit.backend == "diskcache":
+            if not self.settings.rate_limit.diskcache:
                 raise ValueError("Diskcache settings must be provided if Diskcache backend is selected.")
             return AsyncDiskCacheRateLimiter(
-                cache_dir=settings.rate_limit.diskcache.cache_dir,
+                cache_dir=self.settings.rate_limit.diskcache.cache_dir,
             )
         return None
 
@@ -483,8 +512,10 @@ class BaseAsyncChatClient(ABC):
 
         # Get rate limit parameters
         # Priority: parameters in model.endpoints > parameters in endpoint > default parameters
-        rpm = self.rpm or endpoint.rpm or (settings.rate_limit.default_rpm if settings.rate_limit else 60)
-        tpm = self.tpm or endpoint.tpm or (settings.rate_limit.default_tpm if settings.rate_limit else 1000000)
+        rpm = self.rpm or endpoint.rpm or (self.settings.rate_limit.default_rpm if self.settings.rate_limit else 60)
+        tpm = (
+            self.tpm or endpoint.tpm or (self.settings.rate_limit.default_tpm if self.settings.rate_limit else 1000000)
+        )
 
         while self.rate_limiter:
             allowed, wait_time = await self.rate_limiter.check_limit(
@@ -521,9 +552,9 @@ class BaseAsyncChatClient(ABC):
                     self.concurrent_requests = endpoint.get("concurrent_requests", None)
                 else:
                     self.endpoint_id = endpoint
-                self.endpoint = settings.get_endpoint(self.endpoint_id)
+                self.endpoint = self.settings.get_endpoint(self.endpoint_id)
             else:
-                self.endpoint = settings.get_endpoint(self.endpoint_id)
+                self.endpoint = self.settings.get_endpoint(self.endpoint_id)
                 self.set_model_id_by_endpoint_id(self.endpoint_id)
         elif isinstance(self.endpoint, EndpointSetting):
             self.endpoint_id = self.endpoint.id
@@ -629,7 +660,7 @@ class BaseAsyncChatClient(ABC):
         extra_query: Query | None = None,
         extra_body: Body | None = None,
         timeout: float | httpx.Timeout | None | OpenAINotGiven = NOT_GIVEN,
-    ) -> AsyncGenerator[ChatCompletionDeltaMessage, None]:
+    ) -> AsyncGenerator[ChatCompletionDeltaMessage, Any]:
         pass
 
     @overload
@@ -671,7 +702,7 @@ class BaseAsyncChatClient(ABC):
         extra_query: Query | None = None,
         extra_body: Body | None = None,
         timeout: float | httpx.Timeout | None | OpenAINotGiven = NOT_GIVEN,
-    ) -> ChatCompletionMessage | AsyncGenerator[ChatCompletionDeltaMessage, None]:
+    ) -> ChatCompletionMessage | AsyncGenerator[ChatCompletionDeltaMessage, Any]:
         pass
 
     @abstractmethod
@@ -712,7 +743,7 @@ class BaseAsyncChatClient(ABC):
         extra_query: Query | None = None,
         extra_body: Body | None = None,
         timeout: float | httpx.Timeout | None | OpenAINotGiven = NOT_GIVEN,
-    ) -> ChatCompletionMessage | AsyncGenerator[ChatCompletionDeltaMessage, None]:
+    ) -> ChatCompletionMessage | AsyncGenerator[ChatCompletionDeltaMessage, Any]:
         pass
 
     async def create_stream(
@@ -751,7 +782,7 @@ class BaseAsyncChatClient(ABC):
         extra_query: Query | None = None,
         extra_body: Body | None = None,
         timeout: float | httpx.Timeout | None | OpenAINotGiven = NOT_GIVEN,
-    ) -> AsyncGenerator[ChatCompletionDeltaMessage, None]:
+    ) -> AsyncGenerator[ChatCompletionDeltaMessage, Any]:
         return await self.create_completion(
             messages=messages,
             model=model,
@@ -788,3 +819,12 @@ class BaseAsyncChatClient(ABC):
             extra_body=extra_body,
             timeout=timeout,
         )
+
+    async def model_list(self):
+        _raw_client = self.raw_client
+        if isinstance(_raw_client, (AsyncOpenAI, AsyncAzureOpenAI)):
+            return (await _raw_client.models.list()).model_dump()
+        elif isinstance(_raw_client, AsyncAnthropic):
+            return (await _raw_client.models.list(limit=1000)).model_dump()
+        else:
+            raise ValueError(f"Unsupported client type: {type(_raw_client)}")

vectorvein/chat_clients/openai_compatible_client.py

@@ -3,7 +3,19 @@
 import re
 import json
 from functools import cached_property
-from typing import overload, Generator, AsyncGenerator, Any, Literal, Iterable, Optional, Dict, List, Union
+from typing import (
+    Any,
+    Dict,
+    List,
+    TYPE_CHECKING,
+    overload,
+    Generator,
+    AsyncGenerator,
+    Union,
+    Literal,
+    Iterable,
+    Optional,
+)
 
 import httpx
 from openai import OpenAI, AsyncOpenAI, AzureOpenAI, AsyncAzureOpenAI
@@ -39,6 +51,10 @@ from ..types.llm_parameters import (
     ChatCompletionDeltaMessage,
 )
 
+if TYPE_CHECKING:
+    from ..settings import Settings
+    from ..types.settings import SettingsDict
+
 
 class OpenAICompatibleChatClient(BaseChatClient):
     DEFAULT_MODEL: str = ""
@@ -54,6 +70,7 @@ class OpenAICompatibleChatClient(BaseChatClient):
         endpoint_id: str = "",
         http_client: httpx.Client | None = None,
         backend_name: str | None = None,
+        settings: "Settings | SettingsDict | None" = None,  # Use default settings if not provided
     ):
         super().__init__(
             model,
@@ -64,6 +81,7 @@ class OpenAICompatibleChatClient(BaseChatClient):
             endpoint_id,
             http_client,
             backend_name,
+            settings,
         )
         self.model_id = None
         self.endpoint = None
@@ -527,6 +545,7 @@ class AsyncOpenAICompatibleChatClient(BaseAsyncChatClient):
         endpoint_id: str = "",
         http_client: httpx.AsyncClient | None = None,
         backend_name: str | None = None,
+        settings: "Settings | SettingsDict | None" = None,  # Use default settings if not provided
     ):
         super().__init__(
             model,
@@ -537,6 +556,7 @@ class AsyncOpenAICompatibleChatClient(BaseAsyncChatClient):
             endpoint_id,
             http_client,
             backend_name,
+            settings,
         )
         self.model_id = None
         self.endpoint = None

vectorvein/types/__init__.py

@@ -0,0 +1,129 @@
+from .defaults import (
+    CONTEXT_LENGTH_CONTROL,
+    ENDPOINT_CONCURRENT_REQUESTS,
+    ENDPOINT_RPM,
+    ENDPOINT_TPM,
+    MODEL_CONTEXT_LENGTH,
+    MOONSHOT_DEFAULT_MODEL,
+    MOONSHOT_MODELS,
+    DEEPSEEK_DEFAULT_MODEL,
+    DEEPSEEK_MODELS,
+    BAICHUAN_DEFAULT_MODEL,
+    BAICHUAN_MODELS,
+    GROQ_DEFAULT_MODEL,
+    GROQ_MODELS,
+    QWEN_DEFAULT_MODEL,
+    QWEN_MODELS,
+    YI_DEFAULT_MODEL,
+    YI_MODELS,
+    ZHIPUAI_DEFAULT_MODEL,
+    ZHIPUAI_MODELS,
+    MISTRAL_DEFAULT_MODEL,
+    MISTRAL_MODELS,
+    OPENAI_DEFAULT_MODEL,
+    OPENAI_MODELS,
+    ANTHROPIC_DEFAULT_MODEL,
+    ANTHROPIC_MODELS,
+    MINIMAX_DEFAULT_MODEL,
+    MINIMAX_MODELS,
+    GEMINI_DEFAULT_MODEL,
+    GEMINI_MODELS,
+    ERNIE_DEFAULT_MODEL,
+    ERNIE_MODELS,
+    STEPFUN_DEFAULT_MODEL,
+    STEPFUN_MODELS,
+    XAI_DEFAULT_MODEL,
+    XAI_MODELS,
+)
+from .enums import BackendType, LLMType, ContextLengthControlType
+from .exception import APIStatusError
+from .llm_parameters import (
+    EndpointSetting,
+    ModelSetting,
+    BackendSettings,
+    Usage,
+    ChatCompletionMessage,
+    ChatCompletionDeltaMessage,
+    NotGiven,
+    NOT_GIVEN,
+    OpenAIToolParam,
+    ToolParam,
+    Tools,
+    ToolChoice,
+)
+from .settings import (
+    RedisConfigDict,
+    DiskCacheConfigDict,
+    RateLimitConfigDict,
+    ServerDict,
+    EndpointOptionDict,
+    ModelConfigDict,
+    BackendSettingsDict,
+    EndpointSettingDict,
+    SettingsDict,
+)
+
+
+__all__ = [
+    "CONTEXT_LENGTH_CONTROL",
+    "ENDPOINT_CONCURRENT_REQUESTS",
+    "ENDPOINT_RPM",
+    "ENDPOINT_TPM",
+    "MODEL_CONTEXT_LENGTH",
+    "MOONSHOT_DEFAULT_MODEL",
+    "MOONSHOT_MODELS",
+    "DEEPSEEK_DEFAULT_MODEL",
+    "DEEPSEEK_MODELS",
+    "BAICHUAN_DEFAULT_MODEL",
+    "BAICHUAN_MODELS",
+    "GROQ_DEFAULT_MODEL",
+    "GROQ_MODELS",
+    "QWEN_DEFAULT_MODEL",
+    "QWEN_MODELS",
+    "YI_DEFAULT_MODEL",
+    "YI_MODELS",
+    "ZHIPUAI_DEFAULT_MODEL",
+    "ZHIPUAI_MODELS",
+    "MISTRAL_DEFAULT_MODEL",
+    "MISTRAL_MODELS",
+    "OPENAI_DEFAULT_MODEL",
+    "OPENAI_MODELS",
+    "ANTHROPIC_DEFAULT_MODEL",
+    "ANTHROPIC_MODELS",
+    "MINIMAX_DEFAULT_MODEL",
+    "MINIMAX_MODELS",
+    "GEMINI_DEFAULT_MODEL",
+    "GEMINI_MODELS",
+    "ERNIE_DEFAULT_MODEL",
+    "ERNIE_MODELS",
+    "STEPFUN_DEFAULT_MODEL",
+    "STEPFUN_MODELS",
+    "XAI_DEFAULT_MODEL",
+    "XAI_MODELS",
+    "BackendType",
+    "LLMType",
+    "ContextLengthControlType",
+    "APIStatusError",
+    "EndpointOptionDict",
+    "EndpointSetting",
+    "ModelSetting",
+    "BackendSettings",
+    "Usage",
+    "ChatCompletionMessage",
+    "ChatCompletionDeltaMessage",
+    "NotGiven",
+    "NOT_GIVEN",
+    "OpenAIToolParam",
+    "ToolParam",
+    "Tools",
+    "ToolChoice",
+    "RedisConfigDict",
+    "DiskCacheConfigDict",
+    "RateLimitConfigDict",
+    "ServerDict",
+    "EndpointOptionDict",
+    "ModelConfigDict",
+    "BackendSettingsDict",
+    "EndpointSettingDict",
+    "SettingsDict",
+]

vectorvein/types/llm_parameters.py

@@ -1,7 +1,6 @@
 # @Author: Bi Ying
 # @Date:   2024-07-26 23:48:04
 from typing import List, Dict, Optional, Union, Iterable
-from typing_extensions import TypedDict, NotRequired  # Required by pydantic under Python < 3.12
 
 from pydantic import BaseModel, Field
 
@@ -22,14 +21,7 @@ from openai.types.chat.chat_completion_stream_options_param import ChatCompletio
 from openai.types.chat.chat_completion_tool_choice_option_param import ChatCompletionToolChoiceOptionParam
 
 from . import defaults as defs
-
-
-class EndpointOptionDict(TypedDict):
-    endpoint_id: str
-    model_id: str
-    rpm: NotRequired[int]
-    tpm: NotRequired[int]
-    concurrent_requests: NotRequired[int]
+from .settings import EndpointOptionDict
 
 
 class EndpointSetting(BaseModel):

vectorvein/types/settings.py

@@ -0,0 +1,105 @@
+from typing import Dict, List, Optional, Union, Literal
+from typing_extensions import TypedDict, NotRequired  # Required by pydantic under Python < 3.12
+
+
+class RedisConfigDict(TypedDict):
+    """TypedDict representing the RedisConfig structure."""
+
+    host: str
+    port: int
+    db: int
+
+
+class DiskCacheConfigDict(TypedDict):
+    """TypedDict representing the DiskCacheConfig structure."""
+
+    cache_dir: str
+
+
+class RateLimitConfigDict(TypedDict):
+    """TypedDict representing the RateLimitConfig structure."""
+
+    enabled: bool
+    backend: Literal["memory", "redis", "diskcache"]
+    redis: Optional[RedisConfigDict]
+    diskcache: Optional[DiskCacheConfigDict]
+    default_rpm: int
+    default_tpm: int
+
+
+class ServerDict(TypedDict):
+    """TypedDict representing the Server structure."""
+
+    host: str
+    port: int
+    url: Optional[str]
+
+
+class EndpointOptionDict(TypedDict):
+    """TypedDict representing the model endpoint option structure."""
+
+    endpoint_id: str
+    model_id: str
+    rpm: NotRequired[int]
+    tpm: NotRequired[int]
+    concurrent_requests: NotRequired[int]
+
+
+class ModelConfigDict(TypedDict):
+    """TypedDict representing the model configuration structure."""
+
+    id: str
+    endpoints: List[Union[str, EndpointOptionDict]]
+    function_call_available: bool
+    response_format_available: bool
+    native_multimodal: bool
+    context_length: int
+    max_output_tokens: Optional[int]
+
+
+class BackendSettingsDict(TypedDict):
+    """TypedDict representing the BackendSettings structure."""
+
+    models: Dict[str, ModelConfigDict]
+
+
+class EndpointSettingDict(TypedDict):
+    """TypedDict representing the EndpointSetting structure."""
+
+    id: str
+    api_base: Optional[str]
+    api_key: str
+    region: Optional[str]
+    api_schema_type: Optional[str]
+    credentials: Optional[dict]
+    is_azure: Optional[bool]
+    is_vertex: Optional[bool]
+    is_bedrock: Optional[bool]
+    rpm: Optional[int]
+    tpm: Optional[int]
+    concurrent_requests: Optional[int]
+    proxy: Optional[str]
+
+
+class SettingsDict(TypedDict):
+    """TypedDict representing the expected structure of the settings dictionary."""
+
+    endpoints: List[EndpointSettingDict]
+    token_server: Optional[ServerDict]
+    rate_limit: Optional[RateLimitConfigDict]
+    # 各模型后端配置
+    anthropic: BackendSettingsDict
+    deepseek: BackendSettingsDict
+    gemini: BackendSettingsDict
+    groq: BackendSettingsDict
+    local: BackendSettingsDict
+    minimax: BackendSettingsDict
+    mistral: BackendSettingsDict
+    moonshot: BackendSettingsDict
+    openai: BackendSettingsDict
+    qwen: BackendSettingsDict
+    yi: BackendSettingsDict
+    zhipuai: BackendSettingsDict
+    baichuan: BackendSettingsDict
+    stepfun: BackendSettingsDict
+    xai: BackendSettingsDict

{vectorvein-0.2.9.dist-info → vectorvein-0.2.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vectorvein
-Version: 0.2.9
+Version: 0.2.11
 Summary: VectorVein python SDK
 Author-Email: Anderson <andersonby@163.com>
 License: MIT

{vectorvein-0.2.9.dist-info → vectorvein-0.2.11.dist-info}/RECORD

@@ -1,15 +1,15 @@
-vectorvein-0.2.9.dist-info/METADATA,sha256=0-oCeh7DA-fSPf3gKR0i79tcp_qLi5iOf1w4-mW9WaU,4413
-vectorvein-0.2.9.dist-info/WHEEL,sha256=thaaA2w1JzcGC48WYufAs8nrYZjJm8LqNfnXFOFyCC4,90
-vectorvein-0.2.9.dist-info/entry_points.txt,sha256=6OYgBcLyFCUgeqLgnvMyOJxPCWzgy7se4rLPKtNonMs,34
+vectorvein-0.2.11.dist-info/METADATA,sha256=gSQgVDuAhXDNgYkTNflojyYrSZudUpV76aDOSrNsb_g,4414
+vectorvein-0.2.11.dist-info/WHEEL,sha256=thaaA2w1JzcGC48WYufAs8nrYZjJm8LqNfnXFOFyCC4,90
+vectorvein-0.2.11.dist-info/entry_points.txt,sha256=6OYgBcLyFCUgeqLgnvMyOJxPCWzgy7se4rLPKtNonMs,34
 vectorvein/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vectorvein/api/__init__.py,sha256=lfY-XA46fgD2iIZTU0VYP8i07AwA03Egj4Qua0vUKrQ,738
 vectorvein/api/client.py,sha256=xF-leKDQzVyyy9FnIRaz0k4eElYW1XbbzeRLcpnyk90,33047
 vectorvein/api/exceptions.py,sha256=uS_PAdx0ksC0r3dgfSGWdbLMZm4qdLeWSSqCv1g3_Gc,772
 vectorvein/api/models.py,sha256=xtPWMsB0yIJI7i-gY4B6MtvXv0ZIXnoeKspmeInH6fU,1449
-vectorvein/chat_clients/__init__.py,sha256=1ElJPca32YqSxD4dWTHkTEWhiJE_N6J-e5ZGilpT7ks,19178
-vectorvein/chat_clients/anthropic_client.py,sha256=Yx0H2kW-XWOZId9Dnzke5eWWS8FqMf6OfgczZF7yeuM,58192
+vectorvein/chat_clients/__init__.py,sha256=d3qraTPO_J5K9FJIqsLVCfrjev6fkm7y48g_umPaqW0,22620
+vectorvein/chat_clients/anthropic_client.py,sha256=KFOyUR-Vg9Ux75KYtqgOJz2B3cSYXKQoqaaapxqZfRs,58554
 vectorvein/chat_clients/baichuan_client.py,sha256=CVMvpgjdrZGv0BWnTOBD-f2ufZ3wq3496wqukumsAr4,526
-vectorvein/chat_clients/base_client.py,sha256=RcAJwwJ83qC73t3W7GoPNvQXb-qPJ7d8U1J5nwzcYls,37044
+vectorvein/chat_clients/base_client.py,sha256=p7s-G4Wh9MSpDKEfG8wuFAeWy5DGvj5Go31hqrpQPhM,38817
 vectorvein/chat_clients/deepseek_client.py,sha256=3qWu01NlJAP2N-Ff62d5-CZXZitlizE1fzb20LNetig,526
 vectorvein/chat_clients/gemini_client.py,sha256=ufovIZrmAE3RLEe8h5WXombf7bARAZxnkj6ydNK2FQM,475
 vectorvein/chat_clients/groq_client.py,sha256=Uow4pgdmFi93ZQSoOol2-0PhhqkW-S0XuSldvppz5U4,498
@@ -18,7 +18,7 @@ vectorvein/chat_clients/minimax_client.py,sha256=YOILWcsHsN5tihLTMbKJIyJr9TJREMI
 vectorvein/chat_clients/mistral_client.py,sha256=1aKSylzBDaLYcFnaBIL4-sXSzWmXfBeON9Q0rq-ziWw,534
 vectorvein/chat_clients/moonshot_client.py,sha256=gbu-6nGxx8uM_U2WlI4Wus881rFRotzHtMSoYOcruGU,526
 vectorvein/chat_clients/openai_client.py,sha256=Nz6tV45pWcsOupxjnsRsGTicbQNJWIZyxuJoJ5DGMpg,527
-vectorvein/chat_clients/openai_compatible_client.py,sha256=IY3EKkkAespVeVV8sv7M1DN71BCdBRWm_dOtPLn2Rgo,48071
+vectorvein/chat_clients/openai_compatible_client.py,sha256=9ldl6TPinXLLH1kPtP-pVYO2D-VBy_s1Gf-XNNdd3CY,48498
 vectorvein/chat_clients/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vectorvein/chat_clients/qwen_client.py,sha256=-ryh-m9PgsO0fc4ulcCmPTy1155J8YUy15uPoJQOHA0,513
 vectorvein/chat_clients/stepfun_client.py,sha256=zsD2W5ahmR4DD9cqQTXmJr3txrGuvxbRWhFlRdwNijI,519
@@ -30,11 +30,13 @@ vectorvein/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 vectorvein/server/token_server.py,sha256=36F9PKSNOX8ZtYBXY_l-76GQTpUSmQ2Y8EMy1H7wtdQ,1353
 vectorvein/settings/__init__.py,sha256=ecGyrE_6YfX9z6Igb1rDCu1Q-qMTcVozWF3WEl_hiKA,4871
 vectorvein/settings/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+vectorvein/types/__init__.py,sha256=ie7H3rTMq_Fg836vOmy96m3wzjDkqfekQecPXXEDbcM,3005
 vectorvein/types/defaults.py,sha256=VrkQoyHqC_eK3g1b6egpPYLLo0ltwMHqxDscCX4y-N0,27417
 vectorvein/types/enums.py,sha256=7KTJSVtQueImmbr1fSwv3rQVtc0RyMWXJmoE2tDOaso,1667
 vectorvein/types/exception.py,sha256=gnW4GnJ76jND6UGnodk9xmqkcbeS7Cz2rvncA2HpD5E,69
-vectorvein/types/llm_parameters.py,sha256=uby71bteIHmZ3YtFBDbapOaQunBVqcI5qAfQmjWlniM,6339
+vectorvein/types/llm_parameters.py,sha256=8ot02N3PS794bJTc0jlPk_XoXDGWQDTxyyF-1NmqVn0,6103
 vectorvein/types/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+vectorvein/types/settings.py,sha256=hzLtE7ryyIIGxcExAsqh9sMNKP8-r4WczhYWIBNm-uM,2880
 vectorvein/utilities/media_processing.py,sha256=7KtbLFzOYIn1e9QTN9G6C76NH8CBlV9kfAgiRKEIeXY,6263
 vectorvein/utilities/rate_limiter.py,sha256=dwolIUVw2wP83Odqpx0AAaE77de1GzxkYDGH4tM_u_4,10300
 vectorvein/utilities/retry.py,sha256=6KFS9R2HdhqM3_9jkjD4F36ZSpEx2YNFGOVlpOsUetM,2208
@@ -59,4 +61,4 @@ vectorvein/workflow/nodes/vector_db.py,sha256=t6I17q6iR3yQreiDHpRrksMdWDPIvgqJs0
 vectorvein/workflow/nodes/video_generation.py,sha256=qmdg-t_idpxq1veukd-jv_ChICMOoInKxprV9Z4Vi2w,4118
 vectorvein/workflow/nodes/web_crawlers.py,sha256=LsqomfXfqrXfHJDO1cl0Ox48f4St7X_SL12DSbAMSOw,5415
 vectorvein/workflow/utils/json_to_code.py,sha256=F7dhDy8kGc8ndOeihGLRLGFGlquoxVlb02ENtxnQ0C8,5914
-vectorvein-0.2.9.dist-info/RECORD,,
+vectorvein-0.2.11.dist-info/RECORD,,