langroid 0.56.10__tar.gz → 0.56.11__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {langroid-0.56.10 → langroid-0.56.11}/PKG-INFO +1 -1
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/chat_agent.py +12 -0
- langroid-0.56.11/langroid/language_models/client_cache.py +255 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/language_models/openai_gpt.py +57 -27
- {langroid-0.56.10 → langroid-0.56.11}/pyproject.toml +1 -1
- {langroid-0.56.10 → langroid-0.56.11}/.gitignore +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/LICENSE +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/README.md +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/__init__.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/__init__.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/base.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/batch.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/callbacks/__init__.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/callbacks/chainlit.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/chat_document.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/done_sequence_parser.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/openai_assistant.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/__init__.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/arangodb/__init__.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/arangodb/arangodb_agent.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/arangodb/system_messages.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/arangodb/tools.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/arangodb/utils.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/doc_chat_agent.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/doc_chat_task.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/lance_doc_chat_agent.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/lance_rag/__init__.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/lance_rag/critic_agent.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/lance_rag/lance_rag_task.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/lance_rag/query_planner_agent.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/lance_tools.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/neo4j/__init__.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/neo4j/csv_kg_chat.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/neo4j/neo4j_chat_agent.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/neo4j/system_messages.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/neo4j/tools.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/relevance_extractor_agent.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/retriever_agent.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/sql/__init__.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/sql/sql_chat_agent.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/sql/utils/__init__.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/sql/utils/description_extractors.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/sql/utils/populate_metadata.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/sql/utils/system_message.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/sql/utils/tools.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/table_chat_agent.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/task.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/tool_message.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/tools/__init__.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/tools/duckduckgo_search_tool.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/tools/exa_search_tool.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/tools/file_tools.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/tools/google_search_tool.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/tools/mcp/__init__.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/tools/mcp/decorators.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/tools/mcp/fastmcp_client.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/tools/metaphor_search_tool.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/tools/orchestration.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/tools/recipient_tool.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/tools/retrieval_tool.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/tools/rewind_tool.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/tools/segment_extract_tool.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/tools/task_tool.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/tools/tavily_search_tool.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/xml_tool_message.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/cachedb/__init__.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/cachedb/base.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/cachedb/redis_cachedb.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/embedding_models/__init__.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/embedding_models/base.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/embedding_models/models.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/embedding_models/protoc/__init__.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/embedding_models/protoc/embeddings.proto +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/embedding_models/protoc/embeddings_pb2.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/embedding_models/protoc/embeddings_pb2.pyi +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/embedding_models/protoc/embeddings_pb2_grpc.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/embedding_models/remote_embeds.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/exceptions.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/language_models/__init__.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/language_models/azure_openai.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/language_models/base.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/language_models/config.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/language_models/mock_lm.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/language_models/model_info.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/language_models/prompt_formatter/__init__.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/language_models/prompt_formatter/base.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/language_models/prompt_formatter/hf_formatter.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/language_models/prompt_formatter/llama2_formatter.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/language_models/provider_params.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/language_models/utils.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/mcp/__init__.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/mcp/server/__init__.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/mytypes.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/parsing/__init__.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/parsing/agent_chats.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/parsing/code_parser.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/parsing/document_parser.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/parsing/file_attachment.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/parsing/md_parser.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/parsing/para_sentence_split.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/parsing/parse_json.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/parsing/parser.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/parsing/pdf_utils.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/parsing/repo_loader.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/parsing/routing.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/parsing/search.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/parsing/spider.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/parsing/table_loader.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/parsing/url_loader.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/parsing/urls.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/parsing/utils.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/parsing/web_search.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/prompts/__init__.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/prompts/dialog.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/prompts/prompts_config.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/prompts/templates.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/py.typed +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/pydantic_v1/__init__.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/pydantic_v1/main.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/utils/__init__.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/utils/algorithms/__init__.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/utils/algorithms/graph.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/utils/configuration.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/utils/constants.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/utils/git_utils.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/utils/globals.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/utils/logging.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/utils/object_registry.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/utils/output/__init__.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/utils/output/citations.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/utils/output/printing.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/utils/output/status.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/utils/pandas_utils.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/utils/pydantic_utils.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/utils/system.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/utils/types.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/vector_store/__init__.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/vector_store/base.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/vector_store/chromadb.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/vector_store/lancedb.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/vector_store/meilisearch.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/vector_store/pineconedb.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/vector_store/postgres.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/vector_store/qdrantdb.py +0 -0
- {langroid-0.56.10 → langroid-0.56.11}/langroid/vector_store/weaviatedb.py +0 -0
@@ -2068,3 +2068,15 @@ class ChatAgent(Agent):
|
|
2068
2068
|
return str(self.message_history[i])
|
2069
2069
|
else:
|
2070
2070
|
return "\n".join([str(m) for m in self.message_history[i:]])
|
2071
|
+
|
2072
|
+
def __del__(self) -> None:
|
2073
|
+
"""
|
2074
|
+
Cleanup method called when the ChatAgent is garbage collected.
|
2075
|
+
Note: We don't close LLM clients here because they may be shared
|
2076
|
+
across multiple agents when client caching is enabled.
|
2077
|
+
The clients are managed centrally and cleaned up via atexit hooks.
|
2078
|
+
"""
|
2079
|
+
# Previously we closed clients here, but this caused issues when
|
2080
|
+
# multiple agents shared the same cached client instance.
|
2081
|
+
# Clients are now managed centrally in langroid.language_models.client_cache
|
2082
|
+
pass
|
@@ -0,0 +1,255 @@
|
|
1
|
+
"""
|
2
|
+
Client caching/singleton pattern for LLM clients to prevent connection pool exhaustion.
|
3
|
+
"""
|
4
|
+
|
5
|
+
import atexit
|
6
|
+
import hashlib
|
7
|
+
import weakref
|
8
|
+
from typing import Any, Dict, Optional, Union, cast
|
9
|
+
|
10
|
+
from cerebras.cloud.sdk import AsyncCerebras, Cerebras
|
11
|
+
from groq import AsyncGroq, Groq
|
12
|
+
from httpx import Timeout
|
13
|
+
from openai import AsyncOpenAI, OpenAI
|
14
|
+
|
15
|
+
# Cache for client instances, keyed by hashed configuration parameters
|
16
|
+
_client_cache: Dict[str, Any] = {}
|
17
|
+
|
18
|
+
# Keep track of clients for cleanup
|
19
|
+
_all_clients: weakref.WeakSet[Any] = weakref.WeakSet()
|
20
|
+
|
21
|
+
|
22
|
+
def _get_cache_key(client_type: str, **kwargs: Any) -> str:
|
23
|
+
"""
|
24
|
+
Generate a cache key from client type and configuration parameters.
|
25
|
+
Uses the same approach as OpenAIGPT._cache_lookup for consistency.
|
26
|
+
|
27
|
+
Args:
|
28
|
+
client_type: Type of client (e.g., "openai", "groq", "cerebras")
|
29
|
+
**kwargs: Configuration parameters (api_key, base_url, timeout, etc.)
|
30
|
+
|
31
|
+
Returns:
|
32
|
+
SHA256 hash of the configuration as a hex string
|
33
|
+
"""
|
34
|
+
# Convert kwargs to sorted string representation
|
35
|
+
sorted_kwargs_str = str(sorted(kwargs.items()))
|
36
|
+
|
37
|
+
# Create raw key combining client type and sorted kwargs
|
38
|
+
raw_key = f"{client_type}:{sorted_kwargs_str}"
|
39
|
+
|
40
|
+
# Hash the key for consistent length and to handle complex objects
|
41
|
+
hashed_key = hashlib.sha256(raw_key.encode()).hexdigest()
|
42
|
+
|
43
|
+
return hashed_key
|
44
|
+
|
45
|
+
|
46
|
+
def get_openai_client(
|
47
|
+
api_key: str,
|
48
|
+
base_url: Optional[str] = None,
|
49
|
+
organization: Optional[str] = None,
|
50
|
+
timeout: Union[float, Timeout] = 120.0,
|
51
|
+
default_headers: Optional[Dict[str, str]] = None,
|
52
|
+
) -> OpenAI:
|
53
|
+
"""
|
54
|
+
Get or create a singleton OpenAI client with the given configuration.
|
55
|
+
|
56
|
+
Args:
|
57
|
+
api_key: OpenAI API key
|
58
|
+
base_url: Optional base URL for API
|
59
|
+
organization: Optional organization ID
|
60
|
+
timeout: Request timeout
|
61
|
+
default_headers: Optional default headers
|
62
|
+
|
63
|
+
Returns:
|
64
|
+
OpenAI client instance
|
65
|
+
"""
|
66
|
+
if isinstance(timeout, (int, float)):
|
67
|
+
timeout = Timeout(timeout)
|
68
|
+
|
69
|
+
cache_key = _get_cache_key(
|
70
|
+
"openai",
|
71
|
+
api_key=api_key,
|
72
|
+
base_url=base_url,
|
73
|
+
organization=organization,
|
74
|
+
timeout=timeout,
|
75
|
+
default_headers=default_headers,
|
76
|
+
)
|
77
|
+
|
78
|
+
if cache_key in _client_cache:
|
79
|
+
return cast(OpenAI, _client_cache[cache_key])
|
80
|
+
|
81
|
+
client = OpenAI(
|
82
|
+
api_key=api_key,
|
83
|
+
base_url=base_url,
|
84
|
+
organization=organization,
|
85
|
+
timeout=timeout,
|
86
|
+
default_headers=default_headers,
|
87
|
+
)
|
88
|
+
|
89
|
+
_client_cache[cache_key] = client
|
90
|
+
_all_clients.add(client)
|
91
|
+
return client
|
92
|
+
|
93
|
+
|
94
|
+
def get_async_openai_client(
|
95
|
+
api_key: str,
|
96
|
+
base_url: Optional[str] = None,
|
97
|
+
organization: Optional[str] = None,
|
98
|
+
timeout: Union[float, Timeout] = 120.0,
|
99
|
+
default_headers: Optional[Dict[str, str]] = None,
|
100
|
+
) -> AsyncOpenAI:
|
101
|
+
"""
|
102
|
+
Get or create a singleton AsyncOpenAI client with the given configuration.
|
103
|
+
|
104
|
+
Args:
|
105
|
+
api_key: OpenAI API key
|
106
|
+
base_url: Optional base URL for API
|
107
|
+
organization: Optional organization ID
|
108
|
+
timeout: Request timeout
|
109
|
+
default_headers: Optional default headers
|
110
|
+
|
111
|
+
Returns:
|
112
|
+
AsyncOpenAI client instance
|
113
|
+
"""
|
114
|
+
if isinstance(timeout, (int, float)):
|
115
|
+
timeout = Timeout(timeout)
|
116
|
+
|
117
|
+
cache_key = _get_cache_key(
|
118
|
+
"async_openai",
|
119
|
+
api_key=api_key,
|
120
|
+
base_url=base_url,
|
121
|
+
organization=organization,
|
122
|
+
timeout=timeout,
|
123
|
+
default_headers=default_headers,
|
124
|
+
)
|
125
|
+
|
126
|
+
if cache_key in _client_cache:
|
127
|
+
return cast(AsyncOpenAI, _client_cache[cache_key])
|
128
|
+
|
129
|
+
client = AsyncOpenAI(
|
130
|
+
api_key=api_key,
|
131
|
+
base_url=base_url,
|
132
|
+
organization=organization,
|
133
|
+
timeout=timeout,
|
134
|
+
default_headers=default_headers,
|
135
|
+
)
|
136
|
+
|
137
|
+
_client_cache[cache_key] = client
|
138
|
+
_all_clients.add(client)
|
139
|
+
return client
|
140
|
+
|
141
|
+
|
142
|
+
def get_groq_client(api_key: str) -> Groq:
|
143
|
+
"""
|
144
|
+
Get or create a singleton Groq client with the given configuration.
|
145
|
+
|
146
|
+
Args:
|
147
|
+
api_key: Groq API key
|
148
|
+
|
149
|
+
Returns:
|
150
|
+
Groq client instance
|
151
|
+
"""
|
152
|
+
cache_key = _get_cache_key("groq", api_key=api_key)
|
153
|
+
|
154
|
+
if cache_key in _client_cache:
|
155
|
+
return cast(Groq, _client_cache[cache_key])
|
156
|
+
|
157
|
+
client = Groq(api_key=api_key)
|
158
|
+
_client_cache[cache_key] = client
|
159
|
+
_all_clients.add(client)
|
160
|
+
return client
|
161
|
+
|
162
|
+
|
163
|
+
def get_async_groq_client(api_key: str) -> AsyncGroq:
|
164
|
+
"""
|
165
|
+
Get or create a singleton AsyncGroq client with the given configuration.
|
166
|
+
|
167
|
+
Args:
|
168
|
+
api_key: Groq API key
|
169
|
+
|
170
|
+
Returns:
|
171
|
+
AsyncGroq client instance
|
172
|
+
"""
|
173
|
+
cache_key = _get_cache_key("async_groq", api_key=api_key)
|
174
|
+
|
175
|
+
if cache_key in _client_cache:
|
176
|
+
return cast(AsyncGroq, _client_cache[cache_key])
|
177
|
+
|
178
|
+
client = AsyncGroq(api_key=api_key)
|
179
|
+
_client_cache[cache_key] = client
|
180
|
+
_all_clients.add(client)
|
181
|
+
return client
|
182
|
+
|
183
|
+
|
184
|
+
def get_cerebras_client(api_key: str) -> Cerebras:
|
185
|
+
"""
|
186
|
+
Get or create a singleton Cerebras client with the given configuration.
|
187
|
+
|
188
|
+
Args:
|
189
|
+
api_key: Cerebras API key
|
190
|
+
|
191
|
+
Returns:
|
192
|
+
Cerebras client instance
|
193
|
+
"""
|
194
|
+
cache_key = _get_cache_key("cerebras", api_key=api_key)
|
195
|
+
|
196
|
+
if cache_key in _client_cache:
|
197
|
+
return cast(Cerebras, _client_cache[cache_key])
|
198
|
+
|
199
|
+
client = Cerebras(api_key=api_key)
|
200
|
+
_client_cache[cache_key] = client
|
201
|
+
_all_clients.add(client)
|
202
|
+
return client
|
203
|
+
|
204
|
+
|
205
|
+
def get_async_cerebras_client(api_key: str) -> AsyncCerebras:
|
206
|
+
"""
|
207
|
+
Get or create a singleton AsyncCerebras client with the given configuration.
|
208
|
+
|
209
|
+
Args:
|
210
|
+
api_key: Cerebras API key
|
211
|
+
|
212
|
+
Returns:
|
213
|
+
AsyncCerebras client instance
|
214
|
+
"""
|
215
|
+
cache_key = _get_cache_key("async_cerebras", api_key=api_key)
|
216
|
+
|
217
|
+
if cache_key in _client_cache:
|
218
|
+
return cast(AsyncCerebras, _client_cache[cache_key])
|
219
|
+
|
220
|
+
client = AsyncCerebras(api_key=api_key)
|
221
|
+
_client_cache[cache_key] = client
|
222
|
+
_all_clients.add(client)
|
223
|
+
return client
|
224
|
+
|
225
|
+
|
226
|
+
def _cleanup_clients() -> None:
|
227
|
+
"""
|
228
|
+
Cleanup function to close all cached clients on exit.
|
229
|
+
Called automatically via atexit.
|
230
|
+
"""
|
231
|
+
import inspect
|
232
|
+
|
233
|
+
for client in list(_all_clients):
|
234
|
+
if hasattr(client, "close") and callable(client.close):
|
235
|
+
try:
|
236
|
+
# Check if close is a coroutine function (async)
|
237
|
+
if inspect.iscoroutinefunction(client.close):
|
238
|
+
# For async clients, we can't await in atexit
|
239
|
+
# They will be cleaned up by the OS
|
240
|
+
pass
|
241
|
+
else:
|
242
|
+
# Sync clients can be closed directly
|
243
|
+
client.close()
|
244
|
+
except Exception:
|
245
|
+
pass # Ignore errors during cleanup
|
246
|
+
|
247
|
+
|
248
|
+
# Register cleanup function to run on exit
|
249
|
+
atexit.register(_cleanup_clients)
|
250
|
+
|
251
|
+
|
252
|
+
# For testing purposes
|
253
|
+
def _clear_cache() -> None:
|
254
|
+
"""Clear the client cache. Only for testing."""
|
255
|
+
_client_cache.clear()
|
@@ -45,6 +45,14 @@ from langroid.language_models.base import (
|
|
45
45
|
StreamEventType,
|
46
46
|
ToolChoiceTypes,
|
47
47
|
)
|
48
|
+
from langroid.language_models.client_cache import (
|
49
|
+
get_async_cerebras_client,
|
50
|
+
get_async_groq_client,
|
51
|
+
get_async_openai_client,
|
52
|
+
get_cerebras_client,
|
53
|
+
get_groq_client,
|
54
|
+
get_openai_client,
|
55
|
+
)
|
48
56
|
from langroid.language_models.config import HFPromptFormatterConfig
|
49
57
|
from langroid.language_models.model_info import (
|
50
58
|
DeepSeekModel,
|
@@ -256,6 +264,9 @@ class OpenAIGPTConfig(LLMConfig):
|
|
256
264
|
temperature: float = 0.2
|
257
265
|
seed: int | None = 42
|
258
266
|
params: OpenAICallParams | None = None
|
267
|
+
use_cached_client: bool = (
|
268
|
+
True # Whether to reuse cached clients (prevents resource exhaustion)
|
269
|
+
)
|
259
270
|
# these can be any model name that is served at an OpenAI-compatible API end point
|
260
271
|
chat_model: str = default_openai_chat_model
|
261
272
|
chat_model_orig: str = default_openai_chat_model
|
@@ -529,24 +540,26 @@ class OpenAIGPT(LanguageModel):
|
|
529
540
|
self.config.chat_model = self.config.chat_model.replace("groq/", "")
|
530
541
|
if self.api_key == OPENAI_API_KEY:
|
531
542
|
self.api_key = os.getenv("GROQ_API_KEY", DUMMY_API_KEY)
|
532
|
-
self.
|
533
|
-
api_key=self.api_key
|
534
|
-
|
535
|
-
|
536
|
-
|
537
|
-
|
543
|
+
if self.config.use_cached_client:
|
544
|
+
self.client = get_groq_client(api_key=self.api_key)
|
545
|
+
self.async_client = get_async_groq_client(api_key=self.api_key)
|
546
|
+
else:
|
547
|
+
# Create new clients without caching
|
548
|
+
self.client = Groq(api_key=self.api_key)
|
549
|
+
self.async_client = AsyncGroq(api_key=self.api_key)
|
538
550
|
elif self.is_cerebras:
|
539
551
|
# use cerebras-specific client
|
540
552
|
self.config.chat_model = self.config.chat_model.replace("cerebras/", "")
|
541
553
|
if self.api_key == OPENAI_API_KEY:
|
542
554
|
self.api_key = os.getenv("CEREBRAS_API_KEY", DUMMY_API_KEY)
|
543
|
-
self.
|
544
|
-
api_key=self.api_key
|
545
|
-
|
546
|
-
|
547
|
-
|
548
|
-
|
549
|
-
|
555
|
+
if self.config.use_cached_client:
|
556
|
+
self.client = get_cerebras_client(api_key=self.api_key)
|
557
|
+
# TODO there is not async client, so should we do anything here?
|
558
|
+
self.async_client = get_async_cerebras_client(api_key=self.api_key)
|
559
|
+
else:
|
560
|
+
# Create new clients without caching
|
561
|
+
self.client = Cerebras(api_key=self.api_key)
|
562
|
+
self.async_client = AsyncCerebras(api_key=self.api_key)
|
550
563
|
else:
|
551
564
|
# in these cases, there's no specific client: OpenAI python client suffices
|
552
565
|
if self.is_litellm_proxy:
|
@@ -618,20 +631,37 @@ class OpenAIGPT(LanguageModel):
|
|
618
631
|
# Add Portkey-specific headers
|
619
632
|
self.config.headers.update(self.config.portkey_params.get_headers())
|
620
633
|
|
621
|
-
self.
|
622
|
-
|
623
|
-
|
624
|
-
|
625
|
-
|
626
|
-
|
627
|
-
|
628
|
-
|
629
|
-
|
630
|
-
|
631
|
-
|
632
|
-
|
633
|
-
|
634
|
-
|
634
|
+
if self.config.use_cached_client:
|
635
|
+
self.client = get_openai_client(
|
636
|
+
api_key=self.api_key,
|
637
|
+
base_url=self.api_base,
|
638
|
+
organization=self.config.organization,
|
639
|
+
timeout=Timeout(self.config.timeout),
|
640
|
+
default_headers=self.config.headers,
|
641
|
+
)
|
642
|
+
self.async_client = get_async_openai_client(
|
643
|
+
api_key=self.api_key,
|
644
|
+
base_url=self.api_base,
|
645
|
+
organization=self.config.organization,
|
646
|
+
timeout=Timeout(self.config.timeout),
|
647
|
+
default_headers=self.config.headers,
|
648
|
+
)
|
649
|
+
else:
|
650
|
+
# Create new clients without caching
|
651
|
+
self.client = OpenAI(
|
652
|
+
api_key=self.api_key,
|
653
|
+
base_url=self.api_base,
|
654
|
+
organization=self.config.organization,
|
655
|
+
timeout=Timeout(self.config.timeout),
|
656
|
+
default_headers=self.config.headers,
|
657
|
+
)
|
658
|
+
self.async_client = AsyncOpenAI(
|
659
|
+
api_key=self.api_key,
|
660
|
+
base_url=self.api_base,
|
661
|
+
organization=self.config.organization,
|
662
|
+
timeout=Timeout(self.config.timeout),
|
663
|
+
default_headers=self.config.headers,
|
664
|
+
)
|
635
665
|
|
636
666
|
self.cache: CacheDB | None = None
|
637
667
|
use_cache = self.config.cache_config is not None
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
{langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/lance_rag/query_planner_agent.py
RENAMED
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
{langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/sql/utils/description_extractors.py
RENAMED
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
{langroid-0.56.10 → langroid-0.56.11}/langroid/embedding_models/protoc/embeddings_pb2_grpc.py
RENAMED
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
{langroid-0.56.10 → langroid-0.56.11}/langroid/language_models/prompt_formatter/hf_formatter.py
RENAMED
File without changes
|
{langroid-0.56.10 → langroid-0.56.11}/langroid/language_models/prompt_formatter/llama2_formatter.py
RENAMED
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|