langroid 0.56.10__tar.gz → 0.56.11__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. {langroid-0.56.10 → langroid-0.56.11}/PKG-INFO +1 -1
  2. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/chat_agent.py +12 -0
  3. langroid-0.56.11/langroid/language_models/client_cache.py +255 -0
  4. {langroid-0.56.10 → langroid-0.56.11}/langroid/language_models/openai_gpt.py +57 -27
  5. {langroid-0.56.10 → langroid-0.56.11}/pyproject.toml +1 -1
  6. {langroid-0.56.10 → langroid-0.56.11}/.gitignore +0 -0
  7. {langroid-0.56.10 → langroid-0.56.11}/LICENSE +0 -0
  8. {langroid-0.56.10 → langroid-0.56.11}/README.md +0 -0
  9. {langroid-0.56.10 → langroid-0.56.11}/langroid/__init__.py +0 -0
  10. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/__init__.py +0 -0
  11. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/base.py +0 -0
  12. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/batch.py +0 -0
  13. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/callbacks/__init__.py +0 -0
  14. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/callbacks/chainlit.py +0 -0
  15. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/chat_document.py +0 -0
  16. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/done_sequence_parser.py +0 -0
  17. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/openai_assistant.py +0 -0
  18. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/__init__.py +0 -0
  19. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/arangodb/__init__.py +0 -0
  20. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/arangodb/arangodb_agent.py +0 -0
  21. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/arangodb/system_messages.py +0 -0
  22. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/arangodb/tools.py +0 -0
  23. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/arangodb/utils.py +0 -0
  24. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/doc_chat_agent.py +0 -0
  25. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/doc_chat_task.py +0 -0
  26. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/lance_doc_chat_agent.py +0 -0
  27. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/lance_rag/__init__.py +0 -0
  28. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/lance_rag/critic_agent.py +0 -0
  29. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/lance_rag/lance_rag_task.py +0 -0
  30. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/lance_rag/query_planner_agent.py +0 -0
  31. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/lance_tools.py +0 -0
  32. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/neo4j/__init__.py +0 -0
  33. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/neo4j/csv_kg_chat.py +0 -0
  34. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/neo4j/neo4j_chat_agent.py +0 -0
  35. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/neo4j/system_messages.py +0 -0
  36. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/neo4j/tools.py +0 -0
  37. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/relevance_extractor_agent.py +0 -0
  38. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/retriever_agent.py +0 -0
  39. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/sql/__init__.py +0 -0
  40. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/sql/sql_chat_agent.py +0 -0
  41. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/sql/utils/__init__.py +0 -0
  42. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/sql/utils/description_extractors.py +0 -0
  43. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/sql/utils/populate_metadata.py +0 -0
  44. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/sql/utils/system_message.py +0 -0
  45. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/sql/utils/tools.py +0 -0
  46. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/special/table_chat_agent.py +0 -0
  47. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/task.py +0 -0
  48. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/tool_message.py +0 -0
  49. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/tools/__init__.py +0 -0
  50. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/tools/duckduckgo_search_tool.py +0 -0
  51. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/tools/exa_search_tool.py +0 -0
  52. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/tools/file_tools.py +0 -0
  53. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/tools/google_search_tool.py +0 -0
  54. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/tools/mcp/__init__.py +0 -0
  55. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/tools/mcp/decorators.py +0 -0
  56. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/tools/mcp/fastmcp_client.py +0 -0
  57. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/tools/metaphor_search_tool.py +0 -0
  58. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/tools/orchestration.py +0 -0
  59. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/tools/recipient_tool.py +0 -0
  60. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/tools/retrieval_tool.py +0 -0
  61. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/tools/rewind_tool.py +0 -0
  62. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/tools/segment_extract_tool.py +0 -0
  63. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/tools/task_tool.py +0 -0
  64. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/tools/tavily_search_tool.py +0 -0
  65. {langroid-0.56.10 → langroid-0.56.11}/langroid/agent/xml_tool_message.py +0 -0
  66. {langroid-0.56.10 → langroid-0.56.11}/langroid/cachedb/__init__.py +0 -0
  67. {langroid-0.56.10 → langroid-0.56.11}/langroid/cachedb/base.py +0 -0
  68. {langroid-0.56.10 → langroid-0.56.11}/langroid/cachedb/redis_cachedb.py +0 -0
  69. {langroid-0.56.10 → langroid-0.56.11}/langroid/embedding_models/__init__.py +0 -0
  70. {langroid-0.56.10 → langroid-0.56.11}/langroid/embedding_models/base.py +0 -0
  71. {langroid-0.56.10 → langroid-0.56.11}/langroid/embedding_models/models.py +0 -0
  72. {langroid-0.56.10 → langroid-0.56.11}/langroid/embedding_models/protoc/__init__.py +0 -0
  73. {langroid-0.56.10 → langroid-0.56.11}/langroid/embedding_models/protoc/embeddings.proto +0 -0
  74. {langroid-0.56.10 → langroid-0.56.11}/langroid/embedding_models/protoc/embeddings_pb2.py +0 -0
  75. {langroid-0.56.10 → langroid-0.56.11}/langroid/embedding_models/protoc/embeddings_pb2.pyi +0 -0
  76. {langroid-0.56.10 → langroid-0.56.11}/langroid/embedding_models/protoc/embeddings_pb2_grpc.py +0 -0
  77. {langroid-0.56.10 → langroid-0.56.11}/langroid/embedding_models/remote_embeds.py +0 -0
  78. {langroid-0.56.10 → langroid-0.56.11}/langroid/exceptions.py +0 -0
  79. {langroid-0.56.10 → langroid-0.56.11}/langroid/language_models/__init__.py +0 -0
  80. {langroid-0.56.10 → langroid-0.56.11}/langroid/language_models/azure_openai.py +0 -0
  81. {langroid-0.56.10 → langroid-0.56.11}/langroid/language_models/base.py +0 -0
  82. {langroid-0.56.10 → langroid-0.56.11}/langroid/language_models/config.py +0 -0
  83. {langroid-0.56.10 → langroid-0.56.11}/langroid/language_models/mock_lm.py +0 -0
  84. {langroid-0.56.10 → langroid-0.56.11}/langroid/language_models/model_info.py +0 -0
  85. {langroid-0.56.10 → langroid-0.56.11}/langroid/language_models/prompt_formatter/__init__.py +0 -0
  86. {langroid-0.56.10 → langroid-0.56.11}/langroid/language_models/prompt_formatter/base.py +0 -0
  87. {langroid-0.56.10 → langroid-0.56.11}/langroid/language_models/prompt_formatter/hf_formatter.py +0 -0
  88. {langroid-0.56.10 → langroid-0.56.11}/langroid/language_models/prompt_formatter/llama2_formatter.py +0 -0
  89. {langroid-0.56.10 → langroid-0.56.11}/langroid/language_models/provider_params.py +0 -0
  90. {langroid-0.56.10 → langroid-0.56.11}/langroid/language_models/utils.py +0 -0
  91. {langroid-0.56.10 → langroid-0.56.11}/langroid/mcp/__init__.py +0 -0
  92. {langroid-0.56.10 → langroid-0.56.11}/langroid/mcp/server/__init__.py +0 -0
  93. {langroid-0.56.10 → langroid-0.56.11}/langroid/mytypes.py +0 -0
  94. {langroid-0.56.10 → langroid-0.56.11}/langroid/parsing/__init__.py +0 -0
  95. {langroid-0.56.10 → langroid-0.56.11}/langroid/parsing/agent_chats.py +0 -0
  96. {langroid-0.56.10 → langroid-0.56.11}/langroid/parsing/code_parser.py +0 -0
  97. {langroid-0.56.10 → langroid-0.56.11}/langroid/parsing/document_parser.py +0 -0
  98. {langroid-0.56.10 → langroid-0.56.11}/langroid/parsing/file_attachment.py +0 -0
  99. {langroid-0.56.10 → langroid-0.56.11}/langroid/parsing/md_parser.py +0 -0
  100. {langroid-0.56.10 → langroid-0.56.11}/langroid/parsing/para_sentence_split.py +0 -0
  101. {langroid-0.56.10 → langroid-0.56.11}/langroid/parsing/parse_json.py +0 -0
  102. {langroid-0.56.10 → langroid-0.56.11}/langroid/parsing/parser.py +0 -0
  103. {langroid-0.56.10 → langroid-0.56.11}/langroid/parsing/pdf_utils.py +0 -0
  104. {langroid-0.56.10 → langroid-0.56.11}/langroid/parsing/repo_loader.py +0 -0
  105. {langroid-0.56.10 → langroid-0.56.11}/langroid/parsing/routing.py +0 -0
  106. {langroid-0.56.10 → langroid-0.56.11}/langroid/parsing/search.py +0 -0
  107. {langroid-0.56.10 → langroid-0.56.11}/langroid/parsing/spider.py +0 -0
  108. {langroid-0.56.10 → langroid-0.56.11}/langroid/parsing/table_loader.py +0 -0
  109. {langroid-0.56.10 → langroid-0.56.11}/langroid/parsing/url_loader.py +0 -0
  110. {langroid-0.56.10 → langroid-0.56.11}/langroid/parsing/urls.py +0 -0
  111. {langroid-0.56.10 → langroid-0.56.11}/langroid/parsing/utils.py +0 -0
  112. {langroid-0.56.10 → langroid-0.56.11}/langroid/parsing/web_search.py +0 -0
  113. {langroid-0.56.10 → langroid-0.56.11}/langroid/prompts/__init__.py +0 -0
  114. {langroid-0.56.10 → langroid-0.56.11}/langroid/prompts/dialog.py +0 -0
  115. {langroid-0.56.10 → langroid-0.56.11}/langroid/prompts/prompts_config.py +0 -0
  116. {langroid-0.56.10 → langroid-0.56.11}/langroid/prompts/templates.py +0 -0
  117. {langroid-0.56.10 → langroid-0.56.11}/langroid/py.typed +0 -0
  118. {langroid-0.56.10 → langroid-0.56.11}/langroid/pydantic_v1/__init__.py +0 -0
  119. {langroid-0.56.10 → langroid-0.56.11}/langroid/pydantic_v1/main.py +0 -0
  120. {langroid-0.56.10 → langroid-0.56.11}/langroid/utils/__init__.py +0 -0
  121. {langroid-0.56.10 → langroid-0.56.11}/langroid/utils/algorithms/__init__.py +0 -0
  122. {langroid-0.56.10 → langroid-0.56.11}/langroid/utils/algorithms/graph.py +0 -0
  123. {langroid-0.56.10 → langroid-0.56.11}/langroid/utils/configuration.py +0 -0
  124. {langroid-0.56.10 → langroid-0.56.11}/langroid/utils/constants.py +0 -0
  125. {langroid-0.56.10 → langroid-0.56.11}/langroid/utils/git_utils.py +0 -0
  126. {langroid-0.56.10 → langroid-0.56.11}/langroid/utils/globals.py +0 -0
  127. {langroid-0.56.10 → langroid-0.56.11}/langroid/utils/logging.py +0 -0
  128. {langroid-0.56.10 → langroid-0.56.11}/langroid/utils/object_registry.py +0 -0
  129. {langroid-0.56.10 → langroid-0.56.11}/langroid/utils/output/__init__.py +0 -0
  130. {langroid-0.56.10 → langroid-0.56.11}/langroid/utils/output/citations.py +0 -0
  131. {langroid-0.56.10 → langroid-0.56.11}/langroid/utils/output/printing.py +0 -0
  132. {langroid-0.56.10 → langroid-0.56.11}/langroid/utils/output/status.py +0 -0
  133. {langroid-0.56.10 → langroid-0.56.11}/langroid/utils/pandas_utils.py +0 -0
  134. {langroid-0.56.10 → langroid-0.56.11}/langroid/utils/pydantic_utils.py +0 -0
  135. {langroid-0.56.10 → langroid-0.56.11}/langroid/utils/system.py +0 -0
  136. {langroid-0.56.10 → langroid-0.56.11}/langroid/utils/types.py +0 -0
  137. {langroid-0.56.10 → langroid-0.56.11}/langroid/vector_store/__init__.py +0 -0
  138. {langroid-0.56.10 → langroid-0.56.11}/langroid/vector_store/base.py +0 -0
  139. {langroid-0.56.10 → langroid-0.56.11}/langroid/vector_store/chromadb.py +0 -0
  140. {langroid-0.56.10 → langroid-0.56.11}/langroid/vector_store/lancedb.py +0 -0
  141. {langroid-0.56.10 → langroid-0.56.11}/langroid/vector_store/meilisearch.py +0 -0
  142. {langroid-0.56.10 → langroid-0.56.11}/langroid/vector_store/pineconedb.py +0 -0
  143. {langroid-0.56.10 → langroid-0.56.11}/langroid/vector_store/postgres.py +0 -0
  144. {langroid-0.56.10 → langroid-0.56.11}/langroid/vector_store/qdrantdb.py +0 -0
  145. {langroid-0.56.10 → langroid-0.56.11}/langroid/vector_store/weaviatedb.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: langroid
3
- Version: 0.56.10
3
+ Version: 0.56.11
4
4
  Summary: Harness LLMs with Multi-Agent Programming
5
5
  Author-email: Prasad Chalasani <pchalasani@gmail.com>
6
6
  License: MIT
@@ -2068,3 +2068,15 @@ class ChatAgent(Agent):
2068
2068
  return str(self.message_history[i])
2069
2069
  else:
2070
2070
  return "\n".join([str(m) for m in self.message_history[i:]])
2071
+
2072
+ def __del__(self) -> None:
2073
+ """
2074
+ Cleanup method called when the ChatAgent is garbage collected.
2075
+ Note: We don't close LLM clients here because they may be shared
2076
+ across multiple agents when client caching is enabled.
2077
+ The clients are managed centrally and cleaned up via atexit hooks.
2078
+ """
2079
+ # Previously we closed clients here, but this caused issues when
2080
+ # multiple agents shared the same cached client instance.
2081
+ # Clients are now managed centrally in langroid.language_models.client_cache
2082
+ pass
@@ -0,0 +1,255 @@
1
+ """
2
+ Client caching/singleton pattern for LLM clients to prevent connection pool exhaustion.
3
+ """
4
+
5
+ import atexit
6
+ import hashlib
7
+ import weakref
8
+ from typing import Any, Dict, Optional, Union, cast
9
+
10
+ from cerebras.cloud.sdk import AsyncCerebras, Cerebras
11
+ from groq import AsyncGroq, Groq
12
+ from httpx import Timeout
13
+ from openai import AsyncOpenAI, OpenAI
14
+
15
+ # Cache for client instances, keyed by hashed configuration parameters
16
+ _client_cache: Dict[str, Any] = {}
17
+
18
+ # Keep track of clients for cleanup
19
+ _all_clients: weakref.WeakSet[Any] = weakref.WeakSet()
20
+
21
+
22
+ def _get_cache_key(client_type: str, **kwargs: Any) -> str:
23
+ """
24
+ Generate a cache key from client type and configuration parameters.
25
+ Uses the same approach as OpenAIGPT._cache_lookup for consistency.
26
+
27
+ Args:
28
+ client_type: Type of client (e.g., "openai", "groq", "cerebras")
29
+ **kwargs: Configuration parameters (api_key, base_url, timeout, etc.)
30
+
31
+ Returns:
32
+ SHA256 hash of the configuration as a hex string
33
+ """
34
+ # Convert kwargs to sorted string representation
35
+ sorted_kwargs_str = str(sorted(kwargs.items()))
36
+
37
+ # Create raw key combining client type and sorted kwargs
38
+ raw_key = f"{client_type}:{sorted_kwargs_str}"
39
+
40
+ # Hash the key for consistent length and to handle complex objects
41
+ hashed_key = hashlib.sha256(raw_key.encode()).hexdigest()
42
+
43
+ return hashed_key
44
+
45
+
46
+ def get_openai_client(
47
+ api_key: str,
48
+ base_url: Optional[str] = None,
49
+ organization: Optional[str] = None,
50
+ timeout: Union[float, Timeout] = 120.0,
51
+ default_headers: Optional[Dict[str, str]] = None,
52
+ ) -> OpenAI:
53
+ """
54
+ Get or create a singleton OpenAI client with the given configuration.
55
+
56
+ Args:
57
+ api_key: OpenAI API key
58
+ base_url: Optional base URL for API
59
+ organization: Optional organization ID
60
+ timeout: Request timeout
61
+ default_headers: Optional default headers
62
+
63
+ Returns:
64
+ OpenAI client instance
65
+ """
66
+ if isinstance(timeout, (int, float)):
67
+ timeout = Timeout(timeout)
68
+
69
+ cache_key = _get_cache_key(
70
+ "openai",
71
+ api_key=api_key,
72
+ base_url=base_url,
73
+ organization=organization,
74
+ timeout=timeout,
75
+ default_headers=default_headers,
76
+ )
77
+
78
+ if cache_key in _client_cache:
79
+ return cast(OpenAI, _client_cache[cache_key])
80
+
81
+ client = OpenAI(
82
+ api_key=api_key,
83
+ base_url=base_url,
84
+ organization=organization,
85
+ timeout=timeout,
86
+ default_headers=default_headers,
87
+ )
88
+
89
+ _client_cache[cache_key] = client
90
+ _all_clients.add(client)
91
+ return client
92
+
93
+
94
+ def get_async_openai_client(
95
+ api_key: str,
96
+ base_url: Optional[str] = None,
97
+ organization: Optional[str] = None,
98
+ timeout: Union[float, Timeout] = 120.0,
99
+ default_headers: Optional[Dict[str, str]] = None,
100
+ ) -> AsyncOpenAI:
101
+ """
102
+ Get or create a singleton AsyncOpenAI client with the given configuration.
103
+
104
+ Args:
105
+ api_key: OpenAI API key
106
+ base_url: Optional base URL for API
107
+ organization: Optional organization ID
108
+ timeout: Request timeout
109
+ default_headers: Optional default headers
110
+
111
+ Returns:
112
+ AsyncOpenAI client instance
113
+ """
114
+ if isinstance(timeout, (int, float)):
115
+ timeout = Timeout(timeout)
116
+
117
+ cache_key = _get_cache_key(
118
+ "async_openai",
119
+ api_key=api_key,
120
+ base_url=base_url,
121
+ organization=organization,
122
+ timeout=timeout,
123
+ default_headers=default_headers,
124
+ )
125
+
126
+ if cache_key in _client_cache:
127
+ return cast(AsyncOpenAI, _client_cache[cache_key])
128
+
129
+ client = AsyncOpenAI(
130
+ api_key=api_key,
131
+ base_url=base_url,
132
+ organization=organization,
133
+ timeout=timeout,
134
+ default_headers=default_headers,
135
+ )
136
+
137
+ _client_cache[cache_key] = client
138
+ _all_clients.add(client)
139
+ return client
140
+
141
+
142
+ def get_groq_client(api_key: str) -> Groq:
143
+ """
144
+ Get or create a singleton Groq client with the given configuration.
145
+
146
+ Args:
147
+ api_key: Groq API key
148
+
149
+ Returns:
150
+ Groq client instance
151
+ """
152
+ cache_key = _get_cache_key("groq", api_key=api_key)
153
+
154
+ if cache_key in _client_cache:
155
+ return cast(Groq, _client_cache[cache_key])
156
+
157
+ client = Groq(api_key=api_key)
158
+ _client_cache[cache_key] = client
159
+ _all_clients.add(client)
160
+ return client
161
+
162
+
163
+ def get_async_groq_client(api_key: str) -> AsyncGroq:
164
+ """
165
+ Get or create a singleton AsyncGroq client with the given configuration.
166
+
167
+ Args:
168
+ api_key: Groq API key
169
+
170
+ Returns:
171
+ AsyncGroq client instance
172
+ """
173
+ cache_key = _get_cache_key("async_groq", api_key=api_key)
174
+
175
+ if cache_key in _client_cache:
176
+ return cast(AsyncGroq, _client_cache[cache_key])
177
+
178
+ client = AsyncGroq(api_key=api_key)
179
+ _client_cache[cache_key] = client
180
+ _all_clients.add(client)
181
+ return client
182
+
183
+
184
+ def get_cerebras_client(api_key: str) -> Cerebras:
185
+ """
186
+ Get or create a singleton Cerebras client with the given configuration.
187
+
188
+ Args:
189
+ api_key: Cerebras API key
190
+
191
+ Returns:
192
+ Cerebras client instance
193
+ """
194
+ cache_key = _get_cache_key("cerebras", api_key=api_key)
195
+
196
+ if cache_key in _client_cache:
197
+ return cast(Cerebras, _client_cache[cache_key])
198
+
199
+ client = Cerebras(api_key=api_key)
200
+ _client_cache[cache_key] = client
201
+ _all_clients.add(client)
202
+ return client
203
+
204
+
205
+ def get_async_cerebras_client(api_key: str) -> AsyncCerebras:
206
+ """
207
+ Get or create a singleton AsyncCerebras client with the given configuration.
208
+
209
+ Args:
210
+ api_key: Cerebras API key
211
+
212
+ Returns:
213
+ AsyncCerebras client instance
214
+ """
215
+ cache_key = _get_cache_key("async_cerebras", api_key=api_key)
216
+
217
+ if cache_key in _client_cache:
218
+ return cast(AsyncCerebras, _client_cache[cache_key])
219
+
220
+ client = AsyncCerebras(api_key=api_key)
221
+ _client_cache[cache_key] = client
222
+ _all_clients.add(client)
223
+ return client
224
+
225
+
226
+ def _cleanup_clients() -> None:
227
+ """
228
+ Cleanup function to close all cached clients on exit.
229
+ Called automatically via atexit.
230
+ """
231
+ import inspect
232
+
233
+ for client in list(_all_clients):
234
+ if hasattr(client, "close") and callable(client.close):
235
+ try:
236
+ # Check if close is a coroutine function (async)
237
+ if inspect.iscoroutinefunction(client.close):
238
+ # For async clients, we can't await in atexit
239
+ # They will be cleaned up by the OS
240
+ pass
241
+ else:
242
+ # Sync clients can be closed directly
243
+ client.close()
244
+ except Exception:
245
+ pass # Ignore errors during cleanup
246
+
247
+
248
+ # Register cleanup function to run on exit
249
+ atexit.register(_cleanup_clients)
250
+
251
+
252
+ # For testing purposes
253
+ def _clear_cache() -> None:
254
+ """Clear the client cache. Only for testing."""
255
+ _client_cache.clear()
@@ -45,6 +45,14 @@ from langroid.language_models.base import (
45
45
  StreamEventType,
46
46
  ToolChoiceTypes,
47
47
  )
48
+ from langroid.language_models.client_cache import (
49
+ get_async_cerebras_client,
50
+ get_async_groq_client,
51
+ get_async_openai_client,
52
+ get_cerebras_client,
53
+ get_groq_client,
54
+ get_openai_client,
55
+ )
48
56
  from langroid.language_models.config import HFPromptFormatterConfig
49
57
  from langroid.language_models.model_info import (
50
58
  DeepSeekModel,
@@ -256,6 +264,9 @@ class OpenAIGPTConfig(LLMConfig):
256
264
  temperature: float = 0.2
257
265
  seed: int | None = 42
258
266
  params: OpenAICallParams | None = None
267
+ use_cached_client: bool = (
268
+ True # Whether to reuse cached clients (prevents resource exhaustion)
269
+ )
259
270
  # these can be any model name that is served at an OpenAI-compatible API end point
260
271
  chat_model: str = default_openai_chat_model
261
272
  chat_model_orig: str = default_openai_chat_model
@@ -529,24 +540,26 @@ class OpenAIGPT(LanguageModel):
529
540
  self.config.chat_model = self.config.chat_model.replace("groq/", "")
530
541
  if self.api_key == OPENAI_API_KEY:
531
542
  self.api_key = os.getenv("GROQ_API_KEY", DUMMY_API_KEY)
532
- self.client = Groq(
533
- api_key=self.api_key,
534
- )
535
- self.async_client = AsyncGroq(
536
- api_key=self.api_key,
537
- )
543
+ if self.config.use_cached_client:
544
+ self.client = get_groq_client(api_key=self.api_key)
545
+ self.async_client = get_async_groq_client(api_key=self.api_key)
546
+ else:
547
+ # Create new clients without caching
548
+ self.client = Groq(api_key=self.api_key)
549
+ self.async_client = AsyncGroq(api_key=self.api_key)
538
550
  elif self.is_cerebras:
539
551
  # use cerebras-specific client
540
552
  self.config.chat_model = self.config.chat_model.replace("cerebras/", "")
541
553
  if self.api_key == OPENAI_API_KEY:
542
554
  self.api_key = os.getenv("CEREBRAS_API_KEY", DUMMY_API_KEY)
543
- self.client = Cerebras(
544
- api_key=self.api_key,
545
- )
546
- # TODO there is not async client, so should we do anything here?
547
- self.async_client = AsyncCerebras(
548
- api_key=self.api_key,
549
- )
555
+ if self.config.use_cached_client:
556
+ self.client = get_cerebras_client(api_key=self.api_key)
557
+ # TODO there is not async client, so should we do anything here?
558
+ self.async_client = get_async_cerebras_client(api_key=self.api_key)
559
+ else:
560
+ # Create new clients without caching
561
+ self.client = Cerebras(api_key=self.api_key)
562
+ self.async_client = AsyncCerebras(api_key=self.api_key)
550
563
  else:
551
564
  # in these cases, there's no specific client: OpenAI python client suffices
552
565
  if self.is_litellm_proxy:
@@ -618,20 +631,37 @@ class OpenAIGPT(LanguageModel):
618
631
  # Add Portkey-specific headers
619
632
  self.config.headers.update(self.config.portkey_params.get_headers())
620
633
 
621
- self.client = OpenAI(
622
- api_key=self.api_key,
623
- base_url=self.api_base,
624
- organization=self.config.organization,
625
- timeout=Timeout(self.config.timeout),
626
- default_headers=self.config.headers,
627
- )
628
- self.async_client = AsyncOpenAI(
629
- api_key=self.api_key,
630
- organization=self.config.organization,
631
- base_url=self.api_base,
632
- timeout=Timeout(self.config.timeout),
633
- default_headers=self.config.headers,
634
- )
634
+ if self.config.use_cached_client:
635
+ self.client = get_openai_client(
636
+ api_key=self.api_key,
637
+ base_url=self.api_base,
638
+ organization=self.config.organization,
639
+ timeout=Timeout(self.config.timeout),
640
+ default_headers=self.config.headers,
641
+ )
642
+ self.async_client = get_async_openai_client(
643
+ api_key=self.api_key,
644
+ base_url=self.api_base,
645
+ organization=self.config.organization,
646
+ timeout=Timeout(self.config.timeout),
647
+ default_headers=self.config.headers,
648
+ )
649
+ else:
650
+ # Create new clients without caching
651
+ self.client = OpenAI(
652
+ api_key=self.api_key,
653
+ base_url=self.api_base,
654
+ organization=self.config.organization,
655
+ timeout=Timeout(self.config.timeout),
656
+ default_headers=self.config.headers,
657
+ )
658
+ self.async_client = AsyncOpenAI(
659
+ api_key=self.api_key,
660
+ base_url=self.api_base,
661
+ organization=self.config.organization,
662
+ timeout=Timeout(self.config.timeout),
663
+ default_headers=self.config.headers,
664
+ )
635
665
 
636
666
  self.cache: CacheDB | None = None
637
667
  use_cache = self.config.cache_config is not None
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "langroid"
3
- version = "0.56.10"
3
+ version = "0.56.11"
4
4
  authors = [
5
5
  {name = "Prasad Chalasani", email = "pchalasani@gmail.com"},
6
6
  ]
File without changes
File without changes
File without changes
File without changes