keycycle 0.3.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- keycycle/__init__.py +48 -0
- keycycle/adapters/__init__.py +22 -0
- keycycle/adapters/generic_adapter.py +663 -0
- keycycle/adapters/openai_adapter.py +381 -0
- keycycle/config/__init__.py +25 -0
- keycycle/config/constants.py +35 -0
- keycycle/config/dataclasses.py +225 -0
- keycycle/config/enums.py +5 -0
- keycycle/config/loader.py +54 -0
- keycycle/config/log_config.py +53 -0
- keycycle/config/models/cerebras.yaml +36 -0
- keycycle/config/models/cohere.yaml +14 -0
- keycycle/config/models/gemini.yaml +69 -0
- keycycle/config/models/groq.yaml +161 -0
- keycycle/config/models/mistral.yaml +13 -0
- keycycle/config/models/moonshot.yaml +59 -0
- keycycle/config/models/openrouter.yaml +4 -0
- keycycle/config/models/openrouter_models.yaml +97 -0
- keycycle/config/models.py +53 -0
- keycycle/core/__init__.py +40 -0
- keycycle/core/backoff.py +79 -0
- keycycle/core/exceptions.py +94 -0
- keycycle/core/utils.py +480 -0
- keycycle/key_rotation/__init__.py +7 -0
- keycycle/key_rotation/rotating_mixin.py +392 -0
- keycycle/key_rotation/rotation_manager.py +231 -0
- keycycle/legacy_multi_provider_wrapper.py +734 -0
- keycycle/multi_client_wrapper.py +372 -0
- keycycle/py.typed +0 -0
- keycycle/usage/__init__.py +6 -0
- keycycle/usage/db_logic.py +98 -0
- keycycle/usage/usage_logger.py +89 -0
- keycycle/utils.py +27 -0
- keycycle-0.3.2.dist-info/METADATA +44 -0
- keycycle-0.3.2.dist-info/RECORD +37 -0
- keycycle-0.3.2.dist-info/WHEEL +5 -0
- keycycle-0.3.2.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,734 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import time
|
|
3
|
+
import logging
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from threading import RLock
|
|
6
|
+
from typing import Any, Callable, Dict, List, Optional, Tuple, Type, TypeVar, Union
|
|
7
|
+
|
|
8
|
+
from dotenv import load_dotenv
|
|
9
|
+
from rich.console import Console
|
|
10
|
+
from rich.table import Table
|
|
11
|
+
from rich.panel import Panel
|
|
12
|
+
from rich import box
|
|
13
|
+
from rich.markup import escape
|
|
14
|
+
|
|
15
|
+
from .utils import get_agno_model_class
|
|
16
|
+
from .key_rotation.rotation_manager import RotatingKeyManager
|
|
17
|
+
from .key_rotation.rotating_mixin import RotatingCredentialsMixin
|
|
18
|
+
from .config.dataclasses import KeyUsage, RateLimits, UsageSnapshot, KeyLimitOverride
|
|
19
|
+
from .config.enums import RateLimitStrategy
|
|
20
|
+
from .config.models import MODEL_LIMITS, PROVIDER_STRATEGIES
|
|
21
|
+
from .config.constants import DEFAULT_COOLDOWN_SECONDS
|
|
22
|
+
from .core.utils import (
|
|
23
|
+
validate_api_key,
|
|
24
|
+
get_key_suffix,
|
|
25
|
+
KeyEntry,
|
|
26
|
+
load_api_keys as _load_api_keys,
|
|
27
|
+
normalize_key_limits as _normalize_key_limits,
|
|
28
|
+
resolve_limits as _resolve_limits,
|
|
29
|
+
)
|
|
30
|
+
from .core.exceptions import NoAvailableKeyError, KeyNotFoundError
|
|
31
|
+
from .core.backoff import ExponentialBackoff, BackoffConfig
|
|
32
|
+
from .usage.db_logic import UsageDatabase
|
|
33
|
+
from .config.log_config import default_logger
|
|
34
|
+
from .adapters.openai_adapter import RotatingOpenAIClient, RotatingAsyncOpenAIClient
|
|
35
|
+
from .adapters.generic_adapter import (
|
|
36
|
+
create_rotating_client,
|
|
37
|
+
SyncGenericRotatingClient,
|
|
38
|
+
AsyncGenericRotatingClient,
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
T = TypeVar("T")
|
|
42
|
+
|
|
43
|
+
class MultiProviderWrapper:
|
|
44
|
+
"""Wrapper for Agno models with rotating API keys"""
|
|
45
|
+
|
|
46
|
+
PROVIDER_STRATEGIES = PROVIDER_STRATEGIES
|
|
47
|
+
MODEL_LIMITS = MODEL_LIMITS
|
|
48
|
+
|
|
49
|
+
@staticmethod
|
|
50
|
+
def load_api_keys(
|
|
51
|
+
provider: str,
|
|
52
|
+
env_file: Optional[str] = None,
|
|
53
|
+
extra_params: Optional[List[str]] = None,
|
|
54
|
+
) -> List[KeyEntry]:
|
|
55
|
+
"""
|
|
56
|
+
Load API keys from environment variables.
|
|
57
|
+
|
|
58
|
+
Args:
|
|
59
|
+
provider: Provider name (e.g., 'twelvelabs', 'anthropic')
|
|
60
|
+
env_file: Path to .env file (optional)
|
|
61
|
+
extra_params: List of extra parameter names to load alongside API keys.
|
|
62
|
+
For each param, loads {PROVIDER}_{PARAM}_N environment variables.
|
|
63
|
+
|
|
64
|
+
Returns:
|
|
65
|
+
List of key entries. If extra_params is provided, returns list of dicts
|
|
66
|
+
with api_key and extra params. Otherwise returns list of strings.
|
|
67
|
+
|
|
68
|
+
Example:
|
|
69
|
+
>>> # With extra_params=["index_id"], loads:
|
|
70
|
+
>>> # TWELVELABS_API_KEY_1, TWELVELABS_INDEX_ID_1
|
|
71
|
+
>>> # TWELVELABS_API_KEY_2, TWELVELABS_INDEX_ID_2
|
|
72
|
+
>>> keys = MultiProviderWrapper.load_api_keys("twelvelabs", extra_params=["index_id"])
|
|
73
|
+
>>> # Returns: [{"api_key": "key1", "index_id": "idx1"}, ...]
|
|
74
|
+
"""
|
|
75
|
+
return _load_api_keys(provider, env_file, extra_params)
|
|
76
|
+
|
|
77
|
+
@classmethod
|
|
78
|
+
def from_env(
|
|
79
|
+
cls,
|
|
80
|
+
provider: str,
|
|
81
|
+
default_model_id: str,
|
|
82
|
+
env_file: Optional[str] = None,
|
|
83
|
+
db_url: Optional[str] = None,
|
|
84
|
+
db_env_var: Optional[str] = "TIDB_DB_URL",
|
|
85
|
+
debug: bool = False,
|
|
86
|
+
logger: Optional[logging.Logger] = None,
|
|
87
|
+
key_limits: Optional[Dict[Union[int, str], KeyLimitOverride]] = None,
|
|
88
|
+
key_tiers: Optional[Dict[int, str]] = None,
|
|
89
|
+
tier_limits: Optional[Dict[str, RateLimits]] = None,
|
|
90
|
+
extra_params: Optional[List[str]] = None,
|
|
91
|
+
**kwargs
|
|
92
|
+
):
|
|
93
|
+
"""
|
|
94
|
+
Create a MultiProviderWrapper from environment variables.
|
|
95
|
+
|
|
96
|
+
Args:
|
|
97
|
+
provider: Provider name (e.g., 'gemini', 'openai')
|
|
98
|
+
default_model_id: Default model identifier
|
|
99
|
+
env_file: Path to .env file (optional)
|
|
100
|
+
db_url: Database URL for usage persistence
|
|
101
|
+
db_env_var: Environment variable name for database URL
|
|
102
|
+
debug: Enable debug logging
|
|
103
|
+
logger: Custom logger instance
|
|
104
|
+
key_limits: Per-key rate limit overrides (index/suffix -> RateLimits or dict)
|
|
105
|
+
key_tiers: Map key indices to tier names (e.g., {0: 'free', 1: 'pro'})
|
|
106
|
+
tier_limits: Map tier names to rate limits (e.g., {'pro': RateLimits(...)})
|
|
107
|
+
extra_params: List of extra parameter names to load alongside API keys.
|
|
108
|
+
For each param, loads {PROVIDER}_{PARAM}_N environment variables.
|
|
109
|
+
**kwargs: Additional arguments passed to the wrapper
|
|
110
|
+
|
|
111
|
+
Example with tiers:
|
|
112
|
+
>>> wrapper = MultiProviderWrapper.from_env(
|
|
113
|
+
... provider='gemini',
|
|
114
|
+
... default_model_id='gemini-2.5-flash',
|
|
115
|
+
... key_tiers={0: 'free', 1: 'pro'},
|
|
116
|
+
... tier_limits={
|
|
117
|
+
... 'free': RateLimits(5, 300, 20, 250000),
|
|
118
|
+
... 'pro': RateLimits(100, 6000, 1000, 1000000),
|
|
119
|
+
... }
|
|
120
|
+
... )
|
|
121
|
+
|
|
122
|
+
Example with extra_params (TwelveLabs with index_id):
|
|
123
|
+
>>> wrapper = MultiProviderWrapper.from_env(
|
|
124
|
+
... provider='twelvelabs',
|
|
125
|
+
... default_model_id='pegasus-1',
|
|
126
|
+
... extra_params=['index_id'],
|
|
127
|
+
... )
|
|
128
|
+
"""
|
|
129
|
+
try:
|
|
130
|
+
model_class = get_agno_model_class(provider)
|
|
131
|
+
except (ValueError, ImportError):
|
|
132
|
+
# Agno might not be installed, or provider not supported by Agno utils.
|
|
133
|
+
# We proceed without a model class, allowing only OpenAI client usage.
|
|
134
|
+
model_class = None
|
|
135
|
+
|
|
136
|
+
api_keys = cls.load_api_keys(provider, env_file, extra_params)
|
|
137
|
+
db_url = db_url or os.getenv(db_env_var)
|
|
138
|
+
|
|
139
|
+
# Convert tier config to key_limits
|
|
140
|
+
if key_tiers and tier_limits:
|
|
141
|
+
key_limits = key_limits or {}
|
|
142
|
+
for key_index, tier_name in key_tiers.items():
|
|
143
|
+
if tier_name in tier_limits:
|
|
144
|
+
key_limits[key_index] = tier_limits[tier_name]
|
|
145
|
+
|
|
146
|
+
return cls(
|
|
147
|
+
provider, api_keys, default_model_id,
|
|
148
|
+
model_class, db_url, db_env_var, debug, logger,
|
|
149
|
+
key_limits=key_limits, **kwargs
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
def __init__(
|
|
153
|
+
self,
|
|
154
|
+
provider: str,
|
|
155
|
+
api_keys: List[KeyEntry],
|
|
156
|
+
default_model_id: str,
|
|
157
|
+
model_class: Optional[Any] = None,
|
|
158
|
+
db_url: Optional[str] = None,
|
|
159
|
+
db_env_var: Optional[str] = "TIDB_DB_URL",
|
|
160
|
+
debug: bool = False,
|
|
161
|
+
logger: Optional[logging.Logger] = None,
|
|
162
|
+
cooldown_seconds: int = DEFAULT_COOLDOWN_SECONDS,
|
|
163
|
+
key_limits: Optional[Dict[Union[int, str], KeyLimitOverride]] = None,
|
|
164
|
+
**kwargs
|
|
165
|
+
):
|
|
166
|
+
self.provider = provider.lower()
|
|
167
|
+
self.logger = logger or default_logger
|
|
168
|
+
self.model_class = model_class
|
|
169
|
+
self.default_model_id = default_model_id
|
|
170
|
+
self.model_kwargs = kwargs
|
|
171
|
+
self.cooldown_seconds = cooldown_seconds
|
|
172
|
+
self.toggle_debug(debug)
|
|
173
|
+
|
|
174
|
+
# Extract primary keys for validation (handle both str and dict entries)
|
|
175
|
+
primary_keys = [
|
|
176
|
+
entry if isinstance(entry, str) else entry.get("api_key", "")
|
|
177
|
+
for entry in api_keys
|
|
178
|
+
]
|
|
179
|
+
|
|
180
|
+
# Validate API keys
|
|
181
|
+
for i, key in enumerate(primary_keys):
|
|
182
|
+
if not validate_api_key(key):
|
|
183
|
+
self.logger.warning("API key #%d appears invalid or is a placeholder", i + 1)
|
|
184
|
+
|
|
185
|
+
# Normalize key_limits: convert key identifiers (index/suffix/full key) to suffixes
|
|
186
|
+
self._key_limits: Dict[str, KeyLimitOverride] = self._normalize_key_limits_internal(primary_keys, key_limits)
|
|
187
|
+
|
|
188
|
+
self.db = UsageDatabase(db_url, db_env_var)
|
|
189
|
+
self.strategy = self.PROVIDER_STRATEGIES.get(self.provider, RateLimitStrategy.PER_MODEL)
|
|
190
|
+
self.manager = RotatingKeyManager(
|
|
191
|
+
api_keys, self.provider, self.strategy, self.db,
|
|
192
|
+
cooldown_seconds=cooldown_seconds,
|
|
193
|
+
limit_resolver=self._resolve_limits_internal,
|
|
194
|
+
)
|
|
195
|
+
self._model_cache_lock = RLock() # Thread safety for RotatingClass creation
|
|
196
|
+
self._RotatingClass = None
|
|
197
|
+
self.console = Console()
|
|
198
|
+
|
|
199
|
+
def toggle_debug(self, enable: bool) -> None:
|
|
200
|
+
"""
|
|
201
|
+
Dynamically switches logging verbosity for this module.
|
|
202
|
+
enable=True -> Shows detailed rotation/reservation logs (DEBUG)
|
|
203
|
+
enable=False -> Shows only key info/warnings (INFO)
|
|
204
|
+
"""
|
|
205
|
+
level = logging.DEBUG if enable else logging.INFO
|
|
206
|
+
self.logger.setLevel(level)
|
|
207
|
+
|
|
208
|
+
status = "ENABLED" if enable else "DISABLED"
|
|
209
|
+
self.logger.info("Debug logging %s for %s", status, self.provider)
|
|
210
|
+
|
|
211
|
+
def _normalize_key_limits_internal(
|
|
212
|
+
self,
|
|
213
|
+
api_keys: List[str],
|
|
214
|
+
key_limits: Optional[Dict[Union[int, str], KeyLimitOverride]]
|
|
215
|
+
) -> Dict[str, KeyLimitOverride]:
|
|
216
|
+
"""
|
|
217
|
+
Normalize key_limits by converting all identifiers to key suffixes.
|
|
218
|
+
|
|
219
|
+
Accepts:
|
|
220
|
+
- Integer index (0-based): Maps to the key at that position
|
|
221
|
+
- String suffix: Matched against the last N characters of each key
|
|
222
|
+
- Full key string: Exact match
|
|
223
|
+
|
|
224
|
+
Returns:
|
|
225
|
+
Dict mapping key suffixes to their limit overrides
|
|
226
|
+
"""
|
|
227
|
+
return _normalize_key_limits(api_keys, key_limits, self.logger)
|
|
228
|
+
|
|
229
|
+
def _resolve_limits_internal(self, model_id: str, key_suffix: Optional[str] = None) -> RateLimits:
|
|
230
|
+
"""
|
|
231
|
+
Resolve rate limits for a model, optionally with per-key overrides.
|
|
232
|
+
|
|
233
|
+
Args:
|
|
234
|
+
model_id: The model identifier
|
|
235
|
+
key_suffix: Optional key suffix for per-key limit lookup
|
|
236
|
+
|
|
237
|
+
Returns:
|
|
238
|
+
RateLimits for the model/key combination
|
|
239
|
+
"""
|
|
240
|
+
return _resolve_limits(
|
|
241
|
+
model_id=model_id,
|
|
242
|
+
default_model_id=self.default_model_id,
|
|
243
|
+
key_suffix=key_suffix,
|
|
244
|
+
key_limits=self._key_limits,
|
|
245
|
+
model_limits=self.MODEL_LIMITS,
|
|
246
|
+
provider=self.provider,
|
|
247
|
+
default_limits_factory=lambda: RateLimits(10, 100, 1000),
|
|
248
|
+
)
|
|
249
|
+
|
|
250
|
+
def get_key_usage(
|
|
251
|
+
self,
|
|
252
|
+
model_id: str = None,
|
|
253
|
+
estimated_tokens: int = 1000,
|
|
254
|
+
wait: bool = True,
|
|
255
|
+
timeout: float = 10,
|
|
256
|
+
key_id: Union[int, str] = None
|
|
257
|
+
) -> KeyUsage:
|
|
258
|
+
"""
|
|
259
|
+
Finds the first valid key, or a specific key if key_id is provided.
|
|
260
|
+
|
|
261
|
+
Args:
|
|
262
|
+
model_id: Model identifier
|
|
263
|
+
estimated_tokens: Estimated token usage
|
|
264
|
+
wait: Whether to wait for a key if none available (ignored if key_id is set)
|
|
265
|
+
timeout: Max wait time
|
|
266
|
+
key_id: Optional index (int) or suffix/key (str) to force a specific key
|
|
267
|
+
|
|
268
|
+
Returns:
|
|
269
|
+
KeyUsage object for the selected key
|
|
270
|
+
|
|
271
|
+
Raises:
|
|
272
|
+
KeyNotFoundError: If key_id is specified but not found
|
|
273
|
+
NoAvailableKeyError: If no keys are available within timeout
|
|
274
|
+
"""
|
|
275
|
+
mid = model_id or self.default_model_id
|
|
276
|
+
|
|
277
|
+
# Specific Key Request (Bypass Rotation Logic)
|
|
278
|
+
if key_id is not None:
|
|
279
|
+
key_usage = self.manager.get_specific_key(key_id, mid, estimated_tokens)
|
|
280
|
+
if not key_usage:
|
|
281
|
+
raise KeyNotFoundError(key_id)
|
|
282
|
+
return key_usage
|
|
283
|
+
|
|
284
|
+
# Standard Rotation Logic with Exponential Backoff
|
|
285
|
+
limits = self._resolve_limits_internal(mid)
|
|
286
|
+
start = time.time()
|
|
287
|
+
backoff = ExponentialBackoff(BackoffConfig(
|
|
288
|
+
initial_interval=0.5,
|
|
289
|
+
max_interval=2.0, # Cap at 2 seconds for key polling
|
|
290
|
+
multiplier=1.5
|
|
291
|
+
))
|
|
292
|
+
|
|
293
|
+
while True:
|
|
294
|
+
key_usage = self.manager.get_key(mid, limits, estimated_tokens)
|
|
295
|
+
if key_usage:
|
|
296
|
+
return key_usage
|
|
297
|
+
if not wait:
|
|
298
|
+
raise NoAvailableKeyError(
|
|
299
|
+
self.provider, mid, wait=False, timeout=timeout,
|
|
300
|
+
total_keys=len(self.manager.keys)
|
|
301
|
+
)
|
|
302
|
+
if time.time() - start > timeout:
|
|
303
|
+
# Count cooling down keys for better error message
|
|
304
|
+
cooling_down = sum(1 for k in self.manager.keys if k.is_cooling_down(self.cooldown_seconds))
|
|
305
|
+
raise NoAvailableKeyError(
|
|
306
|
+
self.provider, mid, wait=True, timeout=timeout,
|
|
307
|
+
total_keys=len(self.manager.keys),
|
|
308
|
+
cooling_down=cooling_down
|
|
309
|
+
)
|
|
310
|
+
backoff.wait()
|
|
311
|
+
|
|
312
|
+
def get_openai_client(
|
|
313
|
+
self,
|
|
314
|
+
estimated_tokens: int = 1000,
|
|
315
|
+
max_retries: int = 5,
|
|
316
|
+
**kwargs
|
|
317
|
+
) -> RotatingOpenAIClient:
|
|
318
|
+
"""
|
|
319
|
+
Returns a rotating OpenAI client (Sync)
|
|
320
|
+
|
|
321
|
+
Args:
|
|
322
|
+
estimated_tokens: Estimated tokens per request for rate limiting
|
|
323
|
+
max_retries: Maximum retries on rate limit errors
|
|
324
|
+
**kwargs: Additional arguments passed to the OpenAI client
|
|
325
|
+
"""
|
|
326
|
+
return RotatingOpenAIClient(
|
|
327
|
+
manager=self.manager,
|
|
328
|
+
limit_resolver=self._resolve_limits_internal,
|
|
329
|
+
default_model=self.default_model_id,
|
|
330
|
+
estimated_tokens=estimated_tokens,
|
|
331
|
+
max_retries=max_retries,
|
|
332
|
+
provider=self.provider, # Pass provider so it can look up base_url
|
|
333
|
+
client_kwargs={**self.model_kwargs, **kwargs},
|
|
334
|
+
)
|
|
335
|
+
|
|
336
|
+
def get_async_openai_client(
|
|
337
|
+
self,
|
|
338
|
+
estimated_tokens: int = 1000,
|
|
339
|
+
max_retries: int = 5,
|
|
340
|
+
**kwargs
|
|
341
|
+
) -> RotatingAsyncOpenAIClient:
|
|
342
|
+
"""
|
|
343
|
+
Returns a rotating OpenAI client (Async)
|
|
344
|
+
|
|
345
|
+
Args:
|
|
346
|
+
estimated_tokens: Estimated tokens per request for rate limiting
|
|
347
|
+
max_retries: Maximum retries on rate limit errors
|
|
348
|
+
**kwargs: Additional arguments passed to the AsyncOpenAI client
|
|
349
|
+
"""
|
|
350
|
+
return RotatingAsyncOpenAIClient(
|
|
351
|
+
manager=self.manager,
|
|
352
|
+
limit_resolver=self._resolve_limits_internal,
|
|
353
|
+
default_model=self.default_model_id,
|
|
354
|
+
estimated_tokens=estimated_tokens,
|
|
355
|
+
max_retries=max_retries,
|
|
356
|
+
provider=self.provider, # Pass provider so it can look up base_url
|
|
357
|
+
client_kwargs={**self.model_kwargs, **kwargs}
|
|
358
|
+
)
|
|
359
|
+
|
|
360
|
+
def get_rotating_client(
|
|
361
|
+
self,
|
|
362
|
+
client_class: Type[T],
|
|
363
|
+
api_key_param: str = "api_key",
|
|
364
|
+
is_async: Optional[bool] = None,
|
|
365
|
+
usage_extractor: Optional[Callable[[Any], int]] = None,
|
|
366
|
+
estimated_tokens: int = 1000,
|
|
367
|
+
max_retries: int = 5,
|
|
368
|
+
model_param: str = "model",
|
|
369
|
+
excluded_kwargs: Optional[List[str]] = None,
|
|
370
|
+
**client_kwargs,
|
|
371
|
+
) -> Union[SyncGenericRotatingClient[T], AsyncGenericRotatingClient[T]]:
|
|
372
|
+
"""
|
|
373
|
+
Returns a rotating client that wraps any Python client class accepting api_key=.
|
|
374
|
+
|
|
375
|
+
This creates a drop-in replacement for the original client class that automatically
|
|
376
|
+
rotates API keys on rate limit errors.
|
|
377
|
+
|
|
378
|
+
Args:
|
|
379
|
+
client_class: The client class to wrap (e.g., Anthropic, TwelveLabs)
|
|
380
|
+
api_key_param: Name of the API key parameter in the client constructor
|
|
381
|
+
is_async: Whether the client is async. If None, auto-detected
|
|
382
|
+
usage_extractor: Custom function to extract token usage from responses.
|
|
383
|
+
If None, uses a default extractor that handles OpenAI, Anthropic,
|
|
384
|
+
and Cohere response patterns
|
|
385
|
+
estimated_tokens: Estimated tokens per request for rate limiting
|
|
386
|
+
max_retries: Maximum number of key rotations on rate limit errors
|
|
387
|
+
model_param: Name of the model parameter in API calls
|
|
388
|
+
excluded_kwargs: List of kwarg names to exclude from client constructor.
|
|
389
|
+
Useful for clients that don't accept certain params (e.g., 'model' for TwelveLabs).
|
|
390
|
+
**client_kwargs: Additional kwargs to pass to the client constructor
|
|
391
|
+
|
|
392
|
+
Returns:
|
|
393
|
+
A rotating client wrapper that can be used like the original client
|
|
394
|
+
|
|
395
|
+
Examples:
|
|
396
|
+
>>> # Anthropic
|
|
397
|
+
>>> from anthropic import Anthropic, AsyncAnthropic
|
|
398
|
+
>>> wrapper = MultiProviderWrapper.from_env(provider="anthropic", default_model_id="claude-3-sonnet")
|
|
399
|
+
>>> client = wrapper.get_rotating_client(Anthropic)
|
|
400
|
+
>>> response = client.messages.create(model="claude-3-sonnet", messages=[...], max_tokens=100)
|
|
401
|
+
|
|
402
|
+
>>> # Async variant
|
|
403
|
+
>>> async_client = wrapper.get_rotating_client(AsyncAnthropic)
|
|
404
|
+
>>> response = await async_client.messages.create(...)
|
|
405
|
+
|
|
406
|
+
>>> # TwelveLabs with excluded_kwargs (doesn't accept 'model')
|
|
407
|
+
>>> from twelvelabs import TwelveLabs
|
|
408
|
+
>>> wrapper = MultiProviderWrapper.from_env(provider="twelvelabs", default_model_id="marengo2.6")
|
|
409
|
+
>>> client = wrapper.get_rotating_client(TwelveLabs, excluded_kwargs=["model"])
|
|
410
|
+
>>> indexes = client.index.list()
|
|
411
|
+
|
|
412
|
+
>>> # Custom usage extractor for Cohere
|
|
413
|
+
>>> client = wrapper.get_rotating_client(
|
|
414
|
+
... cohere.Client,
|
|
415
|
+
... usage_extractor=lambda r: (
|
|
416
|
+
... (r.meta.billed_units.input_tokens or 0) +
|
|
417
|
+
... (r.meta.billed_units.output_tokens or 0)
|
|
418
|
+
... )
|
|
419
|
+
... )
|
|
420
|
+
"""
|
|
421
|
+
return create_rotating_client(
|
|
422
|
+
client_class=client_class,
|
|
423
|
+
manager=self.manager,
|
|
424
|
+
limit_resolver=self._resolve_limits_internal,
|
|
425
|
+
default_model=self.default_model_id,
|
|
426
|
+
api_key_param=api_key_param,
|
|
427
|
+
is_async=is_async,
|
|
428
|
+
usage_extractor=usage_extractor,
|
|
429
|
+
estimated_tokens=estimated_tokens,
|
|
430
|
+
max_retries=max_retries,
|
|
431
|
+
model_param=model_param,
|
|
432
|
+
excluded_kwargs=excluded_kwargs,
|
|
433
|
+
**{**self.model_kwargs, **client_kwargs},
|
|
434
|
+
)
|
|
435
|
+
|
|
436
|
+
def get_model(
|
|
437
|
+
self,
|
|
438
|
+
estimated_tokens: int = 1000,
|
|
439
|
+
wait: bool = True,
|
|
440
|
+
timeout: float = 10,
|
|
441
|
+
max_retries: int = 5,
|
|
442
|
+
key_id: Union[int, str] = None,
|
|
443
|
+
pin_key: bool = False,
|
|
444
|
+
**kwargs
|
|
445
|
+
) -> Any:
|
|
446
|
+
"""Dynamically creates a rotating model for ANY provider."""
|
|
447
|
+
if self.model_class is None:
|
|
448
|
+
raise RuntimeError(
|
|
449
|
+
"Agno model class is not available. "
|
|
450
|
+
"Ensure 'agno' is installed and that the provider is supported."
|
|
451
|
+
)
|
|
452
|
+
|
|
453
|
+
# Thread-safe creation of rotating class
|
|
454
|
+
with self._model_cache_lock:
|
|
455
|
+
if self._RotatingClass is None:
|
|
456
|
+
self._RotatingClass = type(
|
|
457
|
+
f"Rotating{self.model_class.__name__}",
|
|
458
|
+
(RotatingCredentialsMixin, self.model_class),
|
|
459
|
+
{}
|
|
460
|
+
)
|
|
461
|
+
RotatingProviderClass = self._RotatingClass
|
|
462
|
+
|
|
463
|
+
model_id = kwargs.get('id', self.default_model_id)
|
|
464
|
+
final_kwargs = {**self.model_kwargs, **kwargs}
|
|
465
|
+
if 'id' not in final_kwargs:
|
|
466
|
+
final_kwargs['id'] = model_id
|
|
467
|
+
|
|
468
|
+
initial_key_usage = self.get_key_usage(
|
|
469
|
+
model_id=model_id,
|
|
470
|
+
estimated_tokens=estimated_tokens,
|
|
471
|
+
wait=wait,
|
|
472
|
+
timeout=timeout,
|
|
473
|
+
key_id=key_id
|
|
474
|
+
)
|
|
475
|
+
|
|
476
|
+
fixed_key_id = key_id if pin_key else None
|
|
477
|
+
|
|
478
|
+
model_instance = RotatingProviderClass(
|
|
479
|
+
api_key=initial_key_usage.api_key,
|
|
480
|
+
model_id=model_id,
|
|
481
|
+
wrapper=self,
|
|
482
|
+
rotating_wait=wait,
|
|
483
|
+
rotating_timeout=timeout,
|
|
484
|
+
rotating_estimated_tokens=estimated_tokens,
|
|
485
|
+
rotating_max_retries=max_retries,
|
|
486
|
+
rotating_fixed_key_id=fixed_key_id,
|
|
487
|
+
**final_kwargs
|
|
488
|
+
)
|
|
489
|
+
|
|
490
|
+
return model_instance
|
|
491
|
+
|
|
492
|
+
def get_api_key(
|
|
493
|
+
self,
|
|
494
|
+
model_id: Optional[str] = None,
|
|
495
|
+
estimated_tokens: int = 1000,
|
|
496
|
+
wait: bool = True,
|
|
497
|
+
timeout: float = 10,
|
|
498
|
+
key_id: Union[int, str] = None
|
|
499
|
+
) -> str:
|
|
500
|
+
"""
|
|
501
|
+
Get a valid API key for direct use (e.g., embeddings, custom endpoints).
|
|
502
|
+
|
|
503
|
+
Args:
|
|
504
|
+
model_id: Model identifier for rate limiting (uses default if None)
|
|
505
|
+
estimated_tokens: Estimated tokens for this request
|
|
506
|
+
wait: Whether to wait for an available key
|
|
507
|
+
timeout: Maximum time to wait for a key
|
|
508
|
+
key_id: Optional index (int) or suffix/key (str) to force a specific key
|
|
509
|
+
|
|
510
|
+
Returns:
|
|
511
|
+
A valid API key string
|
|
512
|
+
|
|
513
|
+
Example:
|
|
514
|
+
>>> wrapper = MultiProviderWrapper.from_env('cohere', 'command-r-plus')
|
|
515
|
+
>>> api_key = wrapper.get_api_key()
|
|
516
|
+
>>> # Use with cohere SDK directly
|
|
517
|
+
>>> import cohere
|
|
518
|
+
>>> co = cohere.Client(api_key)
|
|
519
|
+
>>> response = co.embed(texts=["hello"], model="embed-english-v3.0")
|
|
520
|
+
"""
|
|
521
|
+
key_usage = self.get_key_usage(model_id, estimated_tokens, wait, timeout, key_id=key_id)
|
|
522
|
+
return key_usage.api_key
|
|
523
|
+
|
|
524
|
+
def get_api_key_with_context(
|
|
525
|
+
self,
|
|
526
|
+
model_id: Optional[str] = None,
|
|
527
|
+
estimated_tokens: int = 1000,
|
|
528
|
+
wait: bool = True,
|
|
529
|
+
timeout: float = 10,
|
|
530
|
+
key_id: Union[int, str] = None
|
|
531
|
+
) -> Tuple[str, KeyUsage]:
|
|
532
|
+
"""
|
|
533
|
+
Get an API key along with its usage context for manual tracking.
|
|
534
|
+
This gives you both the key and the key_usage object for more control.
|
|
535
|
+
|
|
536
|
+
Args:
|
|
537
|
+
model_id: Model identifier for rate limiting
|
|
538
|
+
estimated_tokens: Estimated tokens for this request
|
|
539
|
+
wait: Whether to wait for an available key
|
|
540
|
+
timeout: Maximum time to wait
|
|
541
|
+
key_id: Optional index (int) or suffix/key (str) to force a specific key
|
|
542
|
+
|
|
543
|
+
Returns:
|
|
544
|
+
Tuple of (api_key: str, key_usage_obj: KeyUsage)
|
|
545
|
+
"""
|
|
546
|
+
key_usage = self.get_key_usage(model_id, estimated_tokens, wait, timeout, key_id=key_id)
|
|
547
|
+
return key_usage.api_key, key_usage
|
|
548
|
+
|
|
549
|
+
def record_key_usage(
|
|
550
|
+
self,
|
|
551
|
+
api_key: str,
|
|
552
|
+
model_id: Optional[str] = None,
|
|
553
|
+
actual_tokens: int = 0,
|
|
554
|
+
estimated_tokens: int = 1000
|
|
555
|
+
) -> None:
|
|
556
|
+
"""
|
|
557
|
+
Record usage for a key obtained via get_api_key().
|
|
558
|
+
Call this after you're done using the key to update usage tracking.
|
|
559
|
+
|
|
560
|
+
Args:
|
|
561
|
+
api_key: The API key that was used
|
|
562
|
+
model_id: Model that was used (uses default if None)
|
|
563
|
+
actual_tokens: Actual tokens consumed (if known)
|
|
564
|
+
estimated_tokens: Estimated tokens (used if actual unknown)
|
|
565
|
+
|
|
566
|
+
Example:
|
|
567
|
+
>>> api_key = wrapper.get_api_key(estimated_tokens=500)
|
|
568
|
+
>>> # ... use api_key for embeddings ...
|
|
569
|
+
>>> wrapper.record_key_usage(api_key, model_id="embed-english-v3.0", actual_tokens=450)
|
|
570
|
+
"""
|
|
571
|
+
mid = model_id or self.default_model_id
|
|
572
|
+
|
|
573
|
+
# Find the key_usage object for this api_key
|
|
574
|
+
key_obj = next((k for k in self.manager.keys if k.api_key == api_key), None)
|
|
575
|
+
if key_obj:
|
|
576
|
+
self.manager.record_usage(key_obj, mid, actual_tokens, estimated_tokens)
|
|
577
|
+
else:
|
|
578
|
+
self.logger.warning("API key not found in manager for recording usage")
|
|
579
|
+
|
|
580
|
+
# --- PRINTING HELPERS ---
|
|
581
|
+
|
|
582
|
+
def _create_usage_table(self, title: str, data: List[Tuple[str, UsageSnapshot]]) -> Table:
|
|
583
|
+
"""
|
|
584
|
+
Generates a standardized table for usage stats.
|
|
585
|
+
data format: [(Label, Snapshot), ...]
|
|
586
|
+
"""
|
|
587
|
+
# Palette
|
|
588
|
+
c_title = "#bae1ff" # Pastel Rose
|
|
589
|
+
c_head = "#f2f2f2" # Pastel Cream
|
|
590
|
+
c_req = "#faa0a0" # Pastel Periwinkle
|
|
591
|
+
c_tok = "#e5baff" # Pastel Peach
|
|
592
|
+
c_border= "#B9B9B9" # Muted Grey
|
|
593
|
+
c_identifier = "#7cd292" # Soft Mauve
|
|
594
|
+
|
|
595
|
+
table = Table(
|
|
596
|
+
title=title,
|
|
597
|
+
box=box.ROUNDED,
|
|
598
|
+
expand=False,
|
|
599
|
+
title_style=f"bold {c_title}",
|
|
600
|
+
title_justify="left",
|
|
601
|
+
border_style=c_border,
|
|
602
|
+
header_style=f"{c_head}"
|
|
603
|
+
)
|
|
604
|
+
|
|
605
|
+
# Define Columns
|
|
606
|
+
table.add_column("Identifier", style=f"bold {c_identifier}", no_wrap=True)
|
|
607
|
+
table.add_column("Requests ([white]m / h / d[/])", justify="center", style=c_req, no_wrap=True)
|
|
608
|
+
table.add_column("Tokens ([white]m / h / d[/])", justify="center", style=c_tok, no_wrap=True)
|
|
609
|
+
table.add_column("Total Requests", justify="center", style=f"{c_req}", no_wrap=True)
|
|
610
|
+
table.add_column("Total Tokens", justify="center", style=f"bold {c_tok}", no_wrap=True)
|
|
611
|
+
|
|
612
|
+
for label, s in data:
|
|
613
|
+
req_str = f"{s.rpm} / {s.rph} / {s.rpd}"
|
|
614
|
+
tok_str = f"{s.tpm:,} / {s.tph:,} / {s.tpd:,}"
|
|
615
|
+
|
|
616
|
+
table.add_row(
|
|
617
|
+
label,
|
|
618
|
+
req_str,
|
|
619
|
+
tok_str,
|
|
620
|
+
f"{s.total_requests}",
|
|
621
|
+
f"{s.total_tokens:,}"
|
|
622
|
+
)
|
|
623
|
+
return table
|
|
624
|
+
|
|
625
|
+
def print_global_stats(self) -> None:
|
|
626
|
+
stats = self.manager.get_global_stats()
|
|
627
|
+
|
|
628
|
+
# 1. Prepare Data for the Table
|
|
629
|
+
rows = []
|
|
630
|
+
for k in stats.keys:
|
|
631
|
+
label = f"Key #{k.index+1} (..{k.suffix})"
|
|
632
|
+
rows.append((label, k.snapshot))
|
|
633
|
+
|
|
634
|
+
# 2. Create and Print Table
|
|
635
|
+
table = self._create_usage_table(
|
|
636
|
+
title=f"GLOBAL STATS: {escape(self.provider.upper())}",
|
|
637
|
+
data=rows
|
|
638
|
+
)
|
|
639
|
+
|
|
640
|
+
# 3. Add a Summary Footer (using a Panel for the Total)
|
|
641
|
+
total_s = stats.total
|
|
642
|
+
grid = Table.grid(padding=(0, 4))
|
|
643
|
+
grid.add_column(style="#e0e0e0") # Label Color
|
|
644
|
+
grid.add_column(style="bold", justify="left") # Value Color
|
|
645
|
+
|
|
646
|
+
grid.add_row("Total Requests:", f"[{'#faa0a0'}]{total_s.total_requests}[/]")
|
|
647
|
+
grid.add_row("Total Tokens:", f"[{'#e5baff'}]{total_s.total_tokens:,}[/]")
|
|
648
|
+
|
|
649
|
+
self.console.print()
|
|
650
|
+
self.console.print(Panel(
|
|
651
|
+
grid,
|
|
652
|
+
title="[bold #bae1ff] AGGREGATE TOTALS [/]",
|
|
653
|
+
border_style="#bae1ff",
|
|
654
|
+
expand=False
|
|
655
|
+
))
|
|
656
|
+
self.console.print(table)
|
|
657
|
+
|
|
658
|
+
def print_key_stats(self, identifier: Union[int, str]) -> None:
|
|
659
|
+
stats = self.manager.get_key_stats(identifier)
|
|
660
|
+
if not stats:
|
|
661
|
+
self.console.print(f"[bold red]Key not found:[/][white] {identifier}[/]")
|
|
662
|
+
return
|
|
663
|
+
|
|
664
|
+
self.console.print()
|
|
665
|
+
self.console.rule(f"[bold]Key Report: {stats.suffix}[/]")
|
|
666
|
+
|
|
667
|
+
# 1. Total Snapshot Panel
|
|
668
|
+
s = stats.total
|
|
669
|
+
grid = Table.grid(padding=(0, 4))
|
|
670
|
+
grid.add_column(style="#e0e0e0")
|
|
671
|
+
grid.add_column(justify="left")
|
|
672
|
+
|
|
673
|
+
grid.add_row("Total Requests:", f"[{'#faa0a0'}]{s.total_requests}[/]")
|
|
674
|
+
grid.add_row("Total Tokens:", f"[{'#e5baff'}]{s.total_tokens:,}[/]")
|
|
675
|
+
|
|
676
|
+
self.console.print(Panel(
|
|
677
|
+
grid,
|
|
678
|
+
title=f"[bold #97e3e9]Key #{stats.index+1} Overview[/]",
|
|
679
|
+
border_style="#bae1ff",
|
|
680
|
+
expand=False
|
|
681
|
+
))
|
|
682
|
+
|
|
683
|
+
# 2. Breakdown Table
|
|
684
|
+
if not stats.breakdown:
|
|
685
|
+
self.console.print("[italic dim]No usage recorded for this key yet.[/]")
|
|
686
|
+
else:
|
|
687
|
+
rows = [(model_id, snap) for model_id, snap in stats.breakdown.items()]
|
|
688
|
+
table = self._create_usage_table(title="Breakdown by Model", data=rows)
|
|
689
|
+
self.console.print(table)
|
|
690
|
+
|
|
691
|
+
def print_model_stats(self, model_id: str) -> None:
|
|
692
|
+
data = self.manager.get_model_stats(model_id)
|
|
693
|
+
|
|
694
|
+
self.console.print()
|
|
695
|
+
self.console.rule(f"[bold]Model Report: [blue]{model_id}[/][/]", style="#B9B9B9")
|
|
696
|
+
|
|
697
|
+
# 1. Total Summary
|
|
698
|
+
s = data.total
|
|
699
|
+
self.console.print(f"Total Tokens Consumed: [bold green]{s.total_tokens:,}[/]")
|
|
700
|
+
|
|
701
|
+
# 2. Contributing Keys Table
|
|
702
|
+
if not data.keys:
|
|
703
|
+
self.console.print("[italic dim]No keys have used this model.[/]")
|
|
704
|
+
else:
|
|
705
|
+
rows = []
|
|
706
|
+
for k in data.keys:
|
|
707
|
+
label = f"Key #{k.index+1} (..{k.suffix})"
|
|
708
|
+
rows.append((label, k.snapshot))
|
|
709
|
+
|
|
710
|
+
table = self._create_usage_table(title="Contributing Keys", data=rows)
|
|
711
|
+
self.console.print(table)
|
|
712
|
+
|
|
713
|
+
def print_granular_stats(self, identifier: Union[int, str], model_id: str) -> None:
|
|
714
|
+
data = self.manager.get_granular_stats(identifier, model_id)
|
|
715
|
+
|
|
716
|
+
if not data:
|
|
717
|
+
self.console.print(f"[bold red]Key '{identifier}' not found.[/]")
|
|
718
|
+
return
|
|
719
|
+
|
|
720
|
+
self.console.print()
|
|
721
|
+
if data.snapshot:
|
|
722
|
+
# Re-use the table builder for a single row just for consistency
|
|
723
|
+
label = f"Key #{data.index+1} (..{data.suffix})"
|
|
724
|
+
table = self._create_usage_table(
|
|
725
|
+
title=f"Granular: {model_id}",
|
|
726
|
+
data=[(label, data.snapshot)]
|
|
727
|
+
)
|
|
728
|
+
self.console.print(table)
|
|
729
|
+
else:
|
|
730
|
+
self.console.print(Panel(
|
|
731
|
+
f"No usage for model [bold]{model_id}[/] on key [bold]..{data.suffix}[/]",
|
|
732
|
+
style="#e5baff",
|
|
733
|
+
border_style="#B9B9B9"
|
|
734
|
+
))
|