camel-ai 0.2.67__py3-none-any.whl → 0.2.80a2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (224) hide show
  1. camel/__init__.py +1 -1
  2. camel/agents/_types.py +6 -2
  3. camel/agents/_utils.py +38 -0
  4. camel/agents/chat_agent.py +4014 -410
  5. camel/agents/mcp_agent.py +30 -27
  6. camel/agents/repo_agent.py +2 -1
  7. camel/benchmarks/browsecomp.py +6 -6
  8. camel/configs/__init__.py +15 -0
  9. camel/configs/aihubmix_config.py +88 -0
  10. camel/configs/amd_config.py +70 -0
  11. camel/configs/cometapi_config.py +104 -0
  12. camel/configs/minimax_config.py +93 -0
  13. camel/configs/nebius_config.py +103 -0
  14. camel/configs/vllm_config.py +2 -0
  15. camel/data_collectors/alpaca_collector.py +15 -6
  16. camel/datagen/self_improving_cot.py +1 -1
  17. camel/datasets/base_generator.py +39 -10
  18. camel/environments/__init__.py +12 -0
  19. camel/environments/rlcards_env.py +860 -0
  20. camel/environments/single_step.py +28 -3
  21. camel/environments/tic_tac_toe.py +1 -1
  22. camel/interpreters/__init__.py +2 -0
  23. camel/interpreters/docker/Dockerfile +4 -16
  24. camel/interpreters/docker_interpreter.py +3 -2
  25. camel/interpreters/e2b_interpreter.py +34 -1
  26. camel/interpreters/internal_python_interpreter.py +51 -2
  27. camel/interpreters/microsandbox_interpreter.py +395 -0
  28. camel/loaders/__init__.py +11 -2
  29. camel/loaders/base_loader.py +85 -0
  30. camel/loaders/chunkr_reader.py +9 -0
  31. camel/loaders/firecrawl_reader.py +4 -4
  32. camel/logger.py +1 -1
  33. camel/memories/agent_memories.py +84 -1
  34. camel/memories/base.py +34 -0
  35. camel/memories/blocks/chat_history_block.py +122 -4
  36. camel/memories/blocks/vectordb_block.py +8 -1
  37. camel/memories/context_creators/score_based.py +29 -237
  38. camel/memories/records.py +88 -8
  39. camel/messages/base.py +166 -40
  40. camel/messages/func_message.py +32 -5
  41. camel/models/__init__.py +10 -0
  42. camel/models/aihubmix_model.py +83 -0
  43. camel/models/aiml_model.py +1 -16
  44. camel/models/amd_model.py +101 -0
  45. camel/models/anthropic_model.py +117 -18
  46. camel/models/aws_bedrock_model.py +2 -33
  47. camel/models/azure_openai_model.py +205 -91
  48. camel/models/base_audio_model.py +3 -1
  49. camel/models/base_model.py +189 -24
  50. camel/models/cohere_model.py +5 -17
  51. camel/models/cometapi_model.py +83 -0
  52. camel/models/crynux_model.py +1 -16
  53. camel/models/deepseek_model.py +6 -16
  54. camel/models/fish_audio_model.py +6 -0
  55. camel/models/gemini_model.py +71 -20
  56. camel/models/groq_model.py +1 -17
  57. camel/models/internlm_model.py +1 -16
  58. camel/models/litellm_model.py +49 -32
  59. camel/models/lmstudio_model.py +1 -17
  60. camel/models/minimax_model.py +83 -0
  61. camel/models/mistral_model.py +1 -16
  62. camel/models/model_factory.py +27 -1
  63. camel/models/model_manager.py +24 -6
  64. camel/models/modelscope_model.py +1 -16
  65. camel/models/moonshot_model.py +185 -19
  66. camel/models/nebius_model.py +83 -0
  67. camel/models/nemotron_model.py +0 -5
  68. camel/models/netmind_model.py +1 -16
  69. camel/models/novita_model.py +1 -16
  70. camel/models/nvidia_model.py +1 -16
  71. camel/models/ollama_model.py +4 -19
  72. camel/models/openai_compatible_model.py +171 -46
  73. camel/models/openai_model.py +205 -77
  74. camel/models/openrouter_model.py +1 -17
  75. camel/models/ppio_model.py +1 -16
  76. camel/models/qianfan_model.py +1 -16
  77. camel/models/qwen_model.py +1 -16
  78. camel/models/reka_model.py +1 -16
  79. camel/models/samba_model.py +34 -47
  80. camel/models/sglang_model.py +64 -31
  81. camel/models/siliconflow_model.py +1 -16
  82. camel/models/stub_model.py +0 -4
  83. camel/models/togetherai_model.py +1 -16
  84. camel/models/vllm_model.py +1 -16
  85. camel/models/volcano_model.py +0 -17
  86. camel/models/watsonx_model.py +1 -16
  87. camel/models/yi_model.py +1 -16
  88. camel/models/zhipuai_model.py +60 -16
  89. camel/parsers/__init__.py +18 -0
  90. camel/parsers/mcp_tool_call_parser.py +176 -0
  91. camel/retrievers/auto_retriever.py +1 -0
  92. camel/runtimes/configs.py +11 -11
  93. camel/runtimes/daytona_runtime.py +15 -16
  94. camel/runtimes/docker_runtime.py +6 -6
  95. camel/runtimes/remote_http_runtime.py +5 -5
  96. camel/services/agent_openapi_server.py +380 -0
  97. camel/societies/__init__.py +2 -0
  98. camel/societies/role_playing.py +26 -28
  99. camel/societies/workforce/__init__.py +2 -0
  100. camel/societies/workforce/events.py +122 -0
  101. camel/societies/workforce/prompts.py +249 -38
  102. camel/societies/workforce/role_playing_worker.py +82 -20
  103. camel/societies/workforce/single_agent_worker.py +634 -34
  104. camel/societies/workforce/structured_output_handler.py +512 -0
  105. camel/societies/workforce/task_channel.py +169 -23
  106. camel/societies/workforce/utils.py +176 -9
  107. camel/societies/workforce/worker.py +77 -23
  108. camel/societies/workforce/workflow_memory_manager.py +772 -0
  109. camel/societies/workforce/workforce.py +3168 -478
  110. camel/societies/workforce/workforce_callback.py +74 -0
  111. camel/societies/workforce/workforce_logger.py +203 -175
  112. camel/societies/workforce/workforce_metrics.py +33 -0
  113. camel/storages/__init__.py +4 -0
  114. camel/storages/key_value_storages/json.py +15 -2
  115. camel/storages/key_value_storages/mem0_cloud.py +48 -47
  116. camel/storages/object_storages/google_cloud.py +1 -1
  117. camel/storages/vectordb_storages/__init__.py +6 -0
  118. camel/storages/vectordb_storages/chroma.py +731 -0
  119. camel/storages/vectordb_storages/oceanbase.py +13 -13
  120. camel/storages/vectordb_storages/pgvector.py +349 -0
  121. camel/storages/vectordb_storages/qdrant.py +3 -3
  122. camel/storages/vectordb_storages/surreal.py +365 -0
  123. camel/storages/vectordb_storages/tidb.py +8 -6
  124. camel/tasks/task.py +244 -27
  125. camel/toolkits/__init__.py +46 -8
  126. camel/toolkits/aci_toolkit.py +64 -19
  127. camel/toolkits/arxiv_toolkit.py +6 -6
  128. camel/toolkits/base.py +63 -5
  129. camel/toolkits/code_execution.py +28 -1
  130. camel/toolkits/context_summarizer_toolkit.py +684 -0
  131. camel/toolkits/craw4ai_toolkit.py +93 -0
  132. camel/toolkits/dappier_toolkit.py +10 -6
  133. camel/toolkits/dingtalk.py +1135 -0
  134. camel/toolkits/edgeone_pages_mcp_toolkit.py +49 -0
  135. camel/toolkits/excel_toolkit.py +901 -67
  136. camel/toolkits/file_toolkit.py +1402 -0
  137. camel/toolkits/function_tool.py +30 -6
  138. camel/toolkits/github_toolkit.py +107 -20
  139. camel/toolkits/gmail_toolkit.py +1839 -0
  140. camel/toolkits/google_calendar_toolkit.py +38 -4
  141. camel/toolkits/google_drive_mcp_toolkit.py +54 -0
  142. camel/toolkits/human_toolkit.py +34 -10
  143. camel/toolkits/hybrid_browser_toolkit/__init__.py +18 -0
  144. camel/toolkits/hybrid_browser_toolkit/config_loader.py +185 -0
  145. camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit.py +246 -0
  146. camel/toolkits/hybrid_browser_toolkit/hybrid_browser_toolkit_ts.py +1973 -0
  147. camel/toolkits/hybrid_browser_toolkit/installer.py +203 -0
  148. camel/toolkits/hybrid_browser_toolkit/ts/package-lock.json +3749 -0
  149. camel/toolkits/hybrid_browser_toolkit/ts/package.json +32 -0
  150. camel/toolkits/hybrid_browser_toolkit/ts/src/browser-scripts.js +125 -0
  151. camel/toolkits/hybrid_browser_toolkit/ts/src/browser-session.ts +1815 -0
  152. camel/toolkits/hybrid_browser_toolkit/ts/src/config-loader.ts +233 -0
  153. camel/toolkits/hybrid_browser_toolkit/ts/src/hybrid-browser-toolkit.ts +590 -0
  154. camel/toolkits/hybrid_browser_toolkit/ts/src/index.ts +7 -0
  155. camel/toolkits/hybrid_browser_toolkit/ts/src/parent-child-filter.ts +226 -0
  156. camel/toolkits/hybrid_browser_toolkit/ts/src/snapshot-parser.ts +219 -0
  157. camel/toolkits/hybrid_browser_toolkit/ts/src/som-screenshot-injected.ts +543 -0
  158. camel/toolkits/hybrid_browser_toolkit/ts/src/types.ts +130 -0
  159. camel/toolkits/hybrid_browser_toolkit/ts/tsconfig.json +26 -0
  160. camel/toolkits/hybrid_browser_toolkit/ts/websocket-server.js +319 -0
  161. camel/toolkits/hybrid_browser_toolkit/ws_wrapper.py +1032 -0
  162. camel/toolkits/hybrid_browser_toolkit_py/__init__.py +17 -0
  163. camel/toolkits/hybrid_browser_toolkit_py/actions.py +575 -0
  164. camel/toolkits/hybrid_browser_toolkit_py/agent.py +311 -0
  165. camel/toolkits/hybrid_browser_toolkit_py/browser_session.py +787 -0
  166. camel/toolkits/hybrid_browser_toolkit_py/config_loader.py +490 -0
  167. camel/toolkits/hybrid_browser_toolkit_py/hybrid_browser_toolkit.py +2390 -0
  168. camel/toolkits/hybrid_browser_toolkit_py/snapshot.py +233 -0
  169. camel/toolkits/hybrid_browser_toolkit_py/stealth_script.js +0 -0
  170. camel/toolkits/hybrid_browser_toolkit_py/unified_analyzer.js +1043 -0
  171. camel/toolkits/image_generation_toolkit.py +390 -0
  172. camel/toolkits/jina_reranker_toolkit.py +3 -4
  173. camel/toolkits/klavis_toolkit.py +5 -1
  174. camel/toolkits/markitdown_toolkit.py +104 -0
  175. camel/toolkits/math_toolkit.py +64 -10
  176. camel/toolkits/mcp_toolkit.py +370 -45
  177. camel/toolkits/memory_toolkit.py +5 -1
  178. camel/toolkits/message_agent_toolkit.py +608 -0
  179. camel/toolkits/message_integration.py +724 -0
  180. camel/toolkits/minimax_mcp_toolkit.py +195 -0
  181. camel/toolkits/note_taking_toolkit.py +277 -0
  182. camel/toolkits/notion_mcp_toolkit.py +224 -0
  183. camel/toolkits/openbb_toolkit.py +5 -1
  184. camel/toolkits/origene_mcp_toolkit.py +56 -0
  185. camel/toolkits/playwright_mcp_toolkit.py +12 -31
  186. camel/toolkits/pptx_toolkit.py +25 -12
  187. camel/toolkits/resend_toolkit.py +168 -0
  188. camel/toolkits/screenshot_toolkit.py +213 -0
  189. camel/toolkits/search_toolkit.py +437 -142
  190. camel/toolkits/slack_toolkit.py +104 -50
  191. camel/toolkits/sympy_toolkit.py +1 -1
  192. camel/toolkits/task_planning_toolkit.py +3 -3
  193. camel/toolkits/terminal_toolkit/__init__.py +18 -0
  194. camel/toolkits/terminal_toolkit/terminal_toolkit.py +957 -0
  195. camel/toolkits/terminal_toolkit/utils.py +532 -0
  196. camel/toolkits/thinking_toolkit.py +1 -1
  197. camel/toolkits/vertex_ai_veo_toolkit.py +590 -0
  198. camel/toolkits/video_analysis_toolkit.py +106 -26
  199. camel/toolkits/video_download_toolkit.py +17 -14
  200. camel/toolkits/web_deploy_toolkit.py +1219 -0
  201. camel/toolkits/wechat_official_toolkit.py +483 -0
  202. camel/toolkits/zapier_toolkit.py +5 -1
  203. camel/types/__init__.py +2 -2
  204. camel/types/agents/tool_calling_record.py +4 -1
  205. camel/types/enums.py +316 -40
  206. camel/types/openai_types.py +2 -2
  207. camel/types/unified_model_type.py +31 -4
  208. camel/utils/commons.py +36 -5
  209. camel/utils/constants.py +3 -0
  210. camel/utils/context_utils.py +1003 -0
  211. camel/utils/mcp.py +138 -4
  212. camel/utils/mcp_client.py +45 -1
  213. camel/utils/message_summarizer.py +148 -0
  214. camel/utils/token_counting.py +43 -20
  215. camel/utils/tool_result.py +44 -0
  216. {camel_ai-0.2.67.dist-info → camel_ai-0.2.80a2.dist-info}/METADATA +296 -85
  217. {camel_ai-0.2.67.dist-info → camel_ai-0.2.80a2.dist-info}/RECORD +219 -146
  218. camel/loaders/pandas_reader.py +0 -368
  219. camel/toolkits/dalle_toolkit.py +0 -175
  220. camel/toolkits/file_write_toolkit.py +0 -444
  221. camel/toolkits/openai_agent_toolkit.py +0 -135
  222. camel/toolkits/terminal_toolkit.py +0 -1037
  223. {camel_ai-0.2.67.dist-info → camel_ai-0.2.80a2.dist-info}/WHEEL +0 -0
  224. {camel_ai-0.2.67.dist-info → camel_ai-0.2.80a2.dist-info}/licenses/LICENSE +0 -0
@@ -0,0 +1,957 @@
1
+ # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
2
+ # Licensed under the Apache License, Version 2.0 (the "License");
3
+ # you may not use this file except in compliance with the License.
4
+ # You may obtain a copy of the License at
5
+ #
6
+ # http://www.apache.org/licenses/LICENSE-2.0
7
+ #
8
+ # Unless required by applicable law or agreed to in writing, software
9
+ # distributed under the License is distributed on an "AS IS" BASIS,
10
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11
+ # See the License for the specific language governing permissions and
12
+ # limitations under the License.
13
+ # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
14
+ import os
15
+ import platform
16
+ import select
17
+ import shlex
18
+ import subprocess
19
+ import sys
20
+ import threading
21
+ import time
22
+ from queue import Empty, Queue
23
+ from typing import Any, Dict, List, Optional
24
+
25
+ from camel.logger import get_logger
26
+ from camel.toolkits.base import BaseToolkit
27
+ from camel.toolkits.function_tool import FunctionTool
28
+ from camel.toolkits.terminal_toolkit.utils import (
29
+ check_nodejs_availability,
30
+ clone_current_environment,
31
+ ensure_uv_available,
32
+ sanitize_command,
33
+ setup_initial_env_with_uv,
34
+ setup_initial_env_with_venv,
35
+ )
36
+ from camel.utils import MCPServer
37
+
38
+ logger = get_logger(__name__)
39
+
40
+ # Try to import docker, but don't make it a hard requirement
41
+ try:
42
+ import docker
43
+ from docker.errors import APIError, NotFound
44
+ except ImportError:
45
+ docker = None
46
+ NotFound = None
47
+ APIError = None
48
+
49
+
50
+ def _to_plain(text: str) -> str:
51
+ r"""Convert ANSI text to plain text using rich if available."""
52
+ try:
53
+ from rich.text import Text as _RichText
54
+
55
+ return _RichText.from_ansi(text).plain
56
+ except Exception:
57
+ return text
58
+
59
+
60
+ @MCPServer()
61
+ class TerminalToolkit(BaseToolkit):
62
+ r"""A toolkit for LLM agents to execute and interact with terminal commands
63
+ in either a local or a sandboxed Docker environment.
64
+
65
+ Args:
66
+ timeout (Optional[float]): The default timeout in seconds for blocking
67
+ commands. Defaults to 20.0.
68
+ working_directory (Optional[str]): The base directory for operations.
69
+ For the local backend, this acts as a security sandbox.
70
+ For the Docker backend, this sets the working directory inside
71
+ the container.
72
+ If not specified, defaults to "./workspace" for local and
73
+ "/workspace" for Docker.
74
+ use_docker_backend (bool): If True, all commands are executed in a
75
+ Docker container. Defaults to False.
76
+ docker_container_name (Optional[str]): The name of the Docker
77
+ container to use. Required if use_docker_backend is True.
78
+ session_logs_dir (Optional[str]): The directory to store session
79
+ logs. Defaults to a 'terminal_logs' subfolder in the
80
+ working directory.
81
+ safe_mode (bool): Whether to apply security checks to commands.
82
+ Defaults to True.
83
+ allowed_commands (Optional[List[str]]): List of allowed commands
84
+ when safe_mode is True. If None, uses default safety rules.
85
+ clone_current_env (bool): Whether to clone the current Python
86
+ environment for local execution. Defaults to False.
87
+ """
88
+
89
+ def __init__(
90
+ self,
91
+ timeout: Optional[float] = 20.0,
92
+ working_directory: Optional[str] = None,
93
+ use_docker_backend: bool = False,
94
+ docker_container_name: Optional[str] = None,
95
+ session_logs_dir: Optional[str] = None,
96
+ safe_mode: bool = True,
97
+ allowed_commands: Optional[List[str]] = None,
98
+ clone_current_env: bool = False,
99
+ ):
100
+ self.use_docker_backend = use_docker_backend
101
+ self.timeout = timeout
102
+ self.shell_sessions: Dict[str, Dict[str, Any]] = {}
103
+ # Thread-safe guard for concurrent access to
104
+ # shell_sessions and session state
105
+ self._session_lock = threading.RLock()
106
+
107
+ # Initialize docker_workdir with proper type
108
+ self.docker_workdir: Optional[str] = None
109
+
110
+ if self.use_docker_backend:
111
+ # For Docker backend, working_directory is path inside container
112
+ if working_directory:
113
+ self.docker_workdir = working_directory
114
+ else:
115
+ self.docker_workdir = "/workspace"
116
+ # For logs and local file operations, use a local workspace
117
+ camel_workdir = os.environ.get("CAMEL_WORKDIR")
118
+ if camel_workdir:
119
+ self.working_dir = os.path.abspath(camel_workdir)
120
+ else:
121
+ self.working_dir = os.path.abspath("./workspace")
122
+ else:
123
+ # For local backend, working_directory is the local path
124
+ if working_directory:
125
+ self.working_dir = os.path.abspath(working_directory)
126
+ else:
127
+ camel_workdir = os.environ.get("CAMEL_WORKDIR")
128
+ if camel_workdir:
129
+ self.working_dir = os.path.abspath(camel_workdir)
130
+ else:
131
+ self.working_dir = os.path.abspath("./workspace")
132
+
133
+ # Only create local directory for logs and local backend operations
134
+ if not os.path.exists(self.working_dir):
135
+ os.makedirs(self.working_dir, exist_ok=True)
136
+ self.safe_mode = safe_mode
137
+
138
+ # Initialize whitelist of allowed commands if provided
139
+ self.allowed_commands = (
140
+ set(allowed_commands) if allowed_commands else None
141
+ )
142
+
143
+ # Environment management attributes
144
+ self.clone_current_env = clone_current_env
145
+ self.cloned_env_path: Optional[str] = None
146
+ self.initial_env_path: Optional[str] = None
147
+ self.python_executable = sys.executable
148
+
149
+ self.log_dir = os.path.abspath(
150
+ session_logs_dir or os.path.join(self.working_dir, "terminal_logs")
151
+ )
152
+ self.blocking_log_file = os.path.join(
153
+ self.log_dir, "blocking_commands.log"
154
+ )
155
+ self.os_type = platform.system()
156
+
157
+ os.makedirs(self.log_dir, exist_ok=True)
158
+
159
+ # Clean the file in terminal_logs folder
160
+ for file in os.listdir(self.log_dir):
161
+ if file.endswith(".log"):
162
+ os.remove(os.path.join(self.log_dir, file))
163
+
164
+ if self.use_docker_backend:
165
+ if docker is None:
166
+ raise ImportError(
167
+ "The 'docker' library is required to use the "
168
+ "Docker backend. Please install it with "
169
+ "'pip install docker'."
170
+ )
171
+ if not docker_container_name:
172
+ raise ValueError(
173
+ "docker_container_name must be "
174
+ "provided when using Docker backend."
175
+ )
176
+ try:
177
+ # APIClient is used for operations that need a timeout,
178
+ # like exec_start
179
+ self.docker_api_client = docker.APIClient(
180
+ base_url='unix://var/run/docker.sock', timeout=self.timeout
181
+ )
182
+ self.docker_client = docker.from_env()
183
+ self.container = self.docker_client.containers.get(
184
+ docker_container_name
185
+ )
186
+ logger.info(
187
+ f"Successfully attached to Docker container "
188
+ f"'{docker_container_name}'."
189
+ )
190
+ # Ensure the working directory exists inside the container
191
+ if self.docker_workdir:
192
+ try:
193
+ quoted_dir = shlex.quote(self.docker_workdir)
194
+ mkdir_cmd = f'sh -lc "mkdir -p -- {quoted_dir}"'
195
+ _init = self.docker_api_client.exec_create(
196
+ self.container.id, mkdir_cmd
197
+ )
198
+ self.docker_api_client.exec_start(_init['Id'])
199
+ except Exception as e:
200
+ logger.warning(
201
+ f"[Docker] Failed to ensure workdir "
202
+ f"'{self.docker_workdir}': {e}"
203
+ )
204
+ except NotFound:
205
+ raise RuntimeError(
206
+ f"Docker container '{docker_container_name}' not found."
207
+ )
208
+ except APIError as e:
209
+ raise RuntimeError(f"Failed to connect to Docker daemon: {e}")
210
+
211
+ # Set up environments (only for local backend)
212
+ if not self.use_docker_backend:
213
+ if self.clone_current_env:
214
+ self._setup_cloned_environment()
215
+ else:
216
+ # Default: set up initial environment with Python 3.10
217
+ self._setup_initial_environment()
218
+ elif self.clone_current_env:
219
+ logger.info(
220
+ "[ENV CLONE] Skipping environment setup for Docker backend "
221
+ "- container is already isolated"
222
+ )
223
+
224
+ def _setup_cloned_environment(self):
225
+ r"""Set up a cloned Python environment."""
226
+ self.cloned_env_path = os.path.join(self.working_dir, ".venv")
227
+
228
+ def update_callback(msg: str):
229
+ logger.info(f"[ENV CLONE] {msg.strip()}")
230
+
231
+ success = clone_current_environment(
232
+ self.cloned_env_path, self.working_dir, update_callback
233
+ )
234
+
235
+ if success:
236
+ # Update python executable to use the cloned environment
237
+ if self.os_type == 'Windows':
238
+ self.python_executable = os.path.join(
239
+ self.cloned_env_path, "Scripts", "python.exe"
240
+ )
241
+ else:
242
+ self.python_executable = os.path.join(
243
+ self.cloned_env_path, "bin", "python"
244
+ )
245
+ else:
246
+ logger.info(
247
+ "[ENV CLONE] Failed to create cloned environment, "
248
+ "using system Python"
249
+ )
250
+
251
+ def _setup_initial_environment(self):
252
+ r"""Set up an initial environment with Python 3.10."""
253
+ self.initial_env_path = os.path.join(self.working_dir, ".initial_env")
254
+
255
+ def update_callback(msg: str):
256
+ logger.info(f"[ENV INIT] {msg.strip()}")
257
+
258
+ # Try to ensure uv is available first
259
+ success, uv_path = ensure_uv_available(update_callback)
260
+
261
+ if success and uv_path:
262
+ success = setup_initial_env_with_uv(
263
+ self.initial_env_path,
264
+ uv_path,
265
+ self.working_dir,
266
+ update_callback,
267
+ )
268
+ else:
269
+ update_callback(
270
+ "Falling back to standard venv for environment setup\n"
271
+ )
272
+ success = setup_initial_env_with_venv(
273
+ self.initial_env_path, self.working_dir, update_callback
274
+ )
275
+
276
+ if success:
277
+ # Update python executable to use the initial environment
278
+ if self.os_type == 'Windows':
279
+ self.python_executable = os.path.join(
280
+ self.initial_env_path, "Scripts", "python.exe"
281
+ )
282
+ else:
283
+ self.python_executable = os.path.join(
284
+ self.initial_env_path, "bin", "python"
285
+ )
286
+
287
+ # Check Node.js availability
288
+ check_nodejs_availability(update_callback)
289
+ else:
290
+ logger.info(
291
+ "[ENV INIT] Failed to create initial environment, "
292
+ "using system Python"
293
+ )
294
+
295
+ def _adapt_command_for_environment(self, command: str) -> str:
296
+ r"""Adapt command to use virtual environment if available."""
297
+ # Only adapt for local backend
298
+ if self.use_docker_backend:
299
+ return command
300
+
301
+ # Check if we have any virtual environment (cloned or initial)
302
+ env_path = None
303
+ if self.cloned_env_path and os.path.exists(self.cloned_env_path):
304
+ env_path = self.cloned_env_path
305
+ elif self.initial_env_path and os.path.exists(self.initial_env_path):
306
+ env_path = self.initial_env_path
307
+
308
+ if not env_path:
309
+ return command
310
+
311
+ # Check if command starts with python or pip
312
+ command_lower = command.strip().lower()
313
+ if command_lower.startswith('python'):
314
+ # Replace 'python' with the virtual environment python
315
+ return command.replace('python', f'"{self.python_executable}"', 1)
316
+ elif command_lower.startswith('pip'):
317
+ # Replace 'pip' with python -m pip from virtual environment
318
+ return command.replace(
319
+ 'pip', f'"{self.python_executable}" -m pip', 1
320
+ )
321
+
322
+ return command
323
+
324
+ def _write_to_log(self, log_file: str, content: str) -> None:
325
+ r"""Write content to log file with optional ANSI stripping.
326
+
327
+ Args:
328
+ log_file (str): Path to the log file
329
+ content (str): Content to write
330
+ """
331
+ # Convert ANSI escape sequences to plain text
332
+ with open(log_file, "a", encoding="utf-8") as f:
333
+ f.write(_to_plain(content) + "\n")
334
+
335
+ def _sanitize_command(self, command: str) -> tuple[bool, str]:
336
+ r"""A comprehensive command sanitizer for both local and
337
+ Docker backends."""
338
+ return sanitize_command(
339
+ command=command,
340
+ use_docker_backend=self.use_docker_backend,
341
+ safe_mode=self.safe_mode,
342
+ working_dir=self.working_dir,
343
+ allowed_commands=self.allowed_commands,
344
+ )
345
+
346
+ def _start_output_reader_thread(self, session_id: str):
347
+ r"""Starts a thread to read stdout from a non-blocking process."""
348
+ with self._session_lock:
349
+ session = self.shell_sessions[session_id]
350
+
351
+ def reader():
352
+ try:
353
+ if session["backend"] == "local":
354
+ # For local processes, read line by line from stdout
355
+ try:
356
+ for line in iter(
357
+ session["process"].stdout.readline, ''
358
+ ):
359
+ session["output_stream"].put(line)
360
+ self._write_to_log(session["log_file"], line)
361
+ finally:
362
+ session["process"].stdout.close()
363
+ elif session["backend"] == "docker":
364
+ # For Docker, read from the raw socket
365
+ socket = session["process"]._sock
366
+ while True:
367
+ # Check if the socket is still open before reading
368
+ if socket.fileno() == -1:
369
+ break
370
+ try:
371
+ ready, _, _ = select.select([socket], [], [], 0.1)
372
+ except (ValueError, OSError):
373
+ # Socket may have been closed by another thread
374
+ break
375
+ if ready:
376
+ data = socket.recv(4096)
377
+ if not data:
378
+ break
379
+ decoded_data = data.decode(
380
+ 'utf-8', errors='ignore'
381
+ )
382
+ session["output_stream"].put(decoded_data)
383
+ self._write_to_log(
384
+ session["log_file"], decoded_data
385
+ )
386
+ # Check if the process is still running
387
+ if not self.docker_api_client.exec_inspect(
388
+ session["exec_id"]
389
+ )['Running']:
390
+ break
391
+ except Exception as e:
392
+ # Log the exception for diagnosis and store it on the session
393
+ logger.exception(f"[SESSION {session_id}] Reader thread error")
394
+ try:
395
+ with self._session_lock:
396
+ if session_id in self.shell_sessions:
397
+ self.shell_sessions[session_id]["error"] = str(e)
398
+ except Exception as cleanup_error:
399
+ logger.warning(
400
+ f"[SESSION {session_id}] Failed to store error state: "
401
+ f"{cleanup_error}"
402
+ )
403
+ finally:
404
+ try:
405
+ with self._session_lock:
406
+ if session_id in self.shell_sessions:
407
+ self.shell_sessions[session_id]["running"] = False
408
+ except Exception:
409
+ pass
410
+
411
+ thread = threading.Thread(target=reader, daemon=True)
412
+ thread.start()
413
+
414
+ def _collect_output_until_idle(
415
+ self,
416
+ id: str,
417
+ idle_duration: float = 0.5,
418
+ check_interval: float = 0.1,
419
+ max_wait: float = 5.0,
420
+ ) -> str:
421
+ r"""Collects output from a session until it's idle or a max wait time
422
+ is reached.
423
+
424
+ Args:
425
+ id (str): The session ID.
426
+ idle_duration (float): How long the stream must be empty to be
427
+ considered idle.(default: 0.5)
428
+ check_interval (float): The time to sleep between checks.
429
+ (default: 0.1)
430
+ max_wait (float): The maximum total time to wait for the process
431
+ to go idle. (default: 5.0)
432
+
433
+ Returns:
434
+ str: The collected output. If max_wait is reached while
435
+ the process is still outputting, a warning is appended.
436
+ """
437
+ with self._session_lock:
438
+ if id not in self.shell_sessions:
439
+ return f"Error: No session found with ID '{id}'."
440
+
441
+ output_parts = []
442
+ idle_time = 0.0
443
+ start_time = time.time()
444
+
445
+ while time.time() - start_time < max_wait:
446
+ new_output = self.shell_view(id)
447
+
448
+ # Check for terminal state messages from shell_view
449
+ if "--- SESSION TERMINATED ---" in new_output:
450
+ # Append the final output before the termination message
451
+ final_part = new_output.replace(
452
+ "--- SESSION TERMINATED ---", ""
453
+ ).strip()
454
+ if final_part:
455
+ output_parts.append(final_part)
456
+ # Session is dead, return what we have plus the message
457
+ return "".join(output_parts) + "\n--- SESSION TERMINATED ---"
458
+
459
+ if new_output.startswith("Error: No session found"):
460
+ return new_output
461
+
462
+ if new_output:
463
+ output_parts.append(new_output)
464
+ idle_time = 0.0 # Reset idle timer
465
+ else:
466
+ idle_time += check_interval
467
+ if idle_time >= idle_duration:
468
+ # Process is idle, success
469
+ return "".join(output_parts)
470
+ time.sleep(check_interval)
471
+
472
+ # If we exit the loop, it means max_wait was reached.
473
+ # Check one last time for any final output.
474
+ final_output = self.shell_view(id)
475
+ if final_output:
476
+ output_parts.append(final_output)
477
+
478
+ warning_message = (
479
+ "\n--- WARNING: Process is still actively outputting "
480
+ "after max wait time. Consider waiting before "
481
+ "sending the next command. ---"
482
+ )
483
+ return "".join(output_parts) + warning_message
484
+
485
+ def shell_exec(self, id: str, command: str, block: bool = True) -> str:
486
+ r"""Executes a shell command in blocking or non-blocking mode.
487
+
488
+ Args:
489
+ id (str): A unique identifier for the command's session. This ID is
490
+ used to interact with non-blocking processes.
491
+ command (str): The shell command to execute.
492
+ block (bool, optional): Determines the execution mode. Defaults to
493
+ True. If `True` (blocking mode), the function waits for the
494
+ command to complete and returns the full output. Use this for
495
+ most commands . If `False` (non-blocking mode), the function
496
+ starts the command in the background. Use this only for
497
+ interactive sessions or long-running tasks, or servers.
498
+
499
+ Returns:
500
+ str: The output of the command execution, which varies by mode.
501
+ In blocking mode, returns the complete standard output and
502
+ standard error from the command.
503
+ In non-blocking mode, returns a confirmation message with the
504
+ session `id`. To interact with the background process, use
505
+ other functions: `shell_view(id)` to see output,
506
+ `shell_write_to_process(id, "input")` to send input, and
507
+ `shell_kill_process(id)` to terminate.
508
+ """
509
+ if self.safe_mode:
510
+ is_safe, message = self._sanitize_command(command)
511
+ if not is_safe:
512
+ return f"Error: {message}"
513
+ command = message
514
+
515
+ if self.use_docker_backend:
516
+ # For Docker, we always run commands in a shell
517
+ # to support complex commands
518
+ command = f'bash -c "{command}"'
519
+ else:
520
+ # For local execution, check if we need to use cloned environment
521
+ command = self._adapt_command_for_environment(command)
522
+
523
+ session_id = id
524
+
525
+ if block:
526
+ # --- BLOCKING EXECUTION ---
527
+ log_entry = (
528
+ f"--- Executing blocking command at "
529
+ f"{time.ctime()} ---\n> {command}\n"
530
+ )
531
+ output = ""
532
+ try:
533
+ if not self.use_docker_backend:
534
+ # LOCAL BLOCKING
535
+ result = subprocess.run(
536
+ command,
537
+ capture_output=True,
538
+ text=True,
539
+ shell=True,
540
+ timeout=self.timeout,
541
+ cwd=self.working_dir,
542
+ encoding="utf-8",
543
+ )
544
+ stdout = result.stdout or ""
545
+ stderr = result.stderr or ""
546
+ output = stdout + (
547
+ f"\nSTDERR:\n{stderr}" if stderr else ""
548
+ )
549
+ else:
550
+ # DOCKER BLOCKING
551
+ assert (
552
+ self.docker_workdir is not None
553
+ ) # Docker backend always has workdir
554
+ exec_instance = self.docker_api_client.exec_create(
555
+ self.container.id, command, workdir=self.docker_workdir
556
+ )
557
+ exec_output = self.docker_api_client.exec_start(
558
+ exec_instance['Id']
559
+ )
560
+ output = exec_output.decode('utf-8', errors='ignore')
561
+
562
+ log_entry += f"--- Output ---\n{output}\n"
563
+ return _to_plain(output)
564
+ except subprocess.TimeoutExpired:
565
+ error_msg = (
566
+ f"Error: Command timed out after {self.timeout} seconds."
567
+ )
568
+ log_entry += f"--- Error ---\n{error_msg}\n"
569
+ return error_msg
570
+ except Exception as e:
571
+ if (
572
+ isinstance(e, (subprocess.TimeoutExpired, TimeoutError))
573
+ or "timed out" in str(e).lower()
574
+ ):
575
+ error_msg = (
576
+ f"Error: Command timed out after "
577
+ f"{self.timeout} seconds."
578
+ )
579
+ else:
580
+ error_msg = f"Error executing command: {e}"
581
+ log_entry += f"--- Error ---\n{error_msg}\n"
582
+ return error_msg
583
+ finally:
584
+ self._write_to_log(self.blocking_log_file, log_entry + "\n")
585
+ else:
586
+ # --- NON-BLOCKING EXECUTION ---
587
+ session_log_file = os.path.join(
588
+ self.log_dir, f"session_{session_id}.log"
589
+ )
590
+
591
+ self._write_to_log(
592
+ session_log_file,
593
+ f"--- Starting non-blocking session at {time.ctime()} ---\n"
594
+ f"> {command}\n",
595
+ )
596
+
597
+ # PYTHONUNBUFFERED=1 for real-time output
598
+ # Without this, Python subprocesses buffer output (4KB buffer)
599
+ # and shell_view() won't see output until buffer fills or process
600
+ # exits
601
+ env_vars = os.environ.copy()
602
+ env_vars["PYTHONUNBUFFERED"] = "1"
603
+ docker_env = {"PYTHONUNBUFFERED": "1"}
604
+
605
+ with self._session_lock:
606
+ self.shell_sessions[session_id] = {
607
+ "id": session_id,
608
+ "process": None,
609
+ "output_stream": Queue(),
610
+ "command_history": [command],
611
+ "running": True,
612
+ "log_file": session_log_file,
613
+ "backend": "docker"
614
+ if self.use_docker_backend
615
+ else "local",
616
+ }
617
+
618
+ process = None
619
+ exec_socket = None
620
+ try:
621
+ if not self.use_docker_backend:
622
+ process = subprocess.Popen(
623
+ command,
624
+ stdin=subprocess.PIPE,
625
+ stdout=subprocess.PIPE,
626
+ stderr=subprocess.STDOUT,
627
+ shell=True,
628
+ text=True,
629
+ cwd=self.working_dir,
630
+ encoding="utf-8",
631
+ env=env_vars,
632
+ )
633
+ with self._session_lock:
634
+ self.shell_sessions[session_id]["process"] = process
635
+ else:
636
+ assert (
637
+ self.docker_workdir is not None
638
+ ) # Docker backend always has workdir
639
+ exec_instance = self.docker_api_client.exec_create(
640
+ self.container.id,
641
+ command,
642
+ stdin=True,
643
+ tty=True,
644
+ workdir=self.docker_workdir,
645
+ environment=docker_env,
646
+ )
647
+ exec_id = exec_instance['Id']
648
+ exec_socket = self.docker_api_client.exec_start(
649
+ exec_id, tty=True, stream=True, socket=True
650
+ )
651
+ with self._session_lock:
652
+ self.shell_sessions[session_id]["process"] = (
653
+ exec_socket
654
+ )
655
+ self.shell_sessions[session_id]["exec_id"] = exec_id
656
+
657
+ self._start_output_reader_thread(session_id)
658
+
659
+ # Return immediately with session ID and instructions
660
+ return (
661
+ f"Session '{session_id}' started.\n\n"
662
+ f"You could use:\n"
663
+ f" - shell_view('{session_id}') - get output\n"
664
+ f" - shell_write_to_process('{session_id}', '<input>')"
665
+ f" - send input\n"
666
+ f" - shell_kill_process('{session_id}') - terminate"
667
+ )
668
+
669
+ except Exception as e:
670
+ # Clean up resources on failure
671
+ if process is not None:
672
+ try:
673
+ process.terminate()
674
+ except Exception:
675
+ pass
676
+ if exec_socket is not None:
677
+ try:
678
+ exec_socket.close()
679
+ except Exception:
680
+ pass
681
+
682
+ with self._session_lock:
683
+ if session_id in self.shell_sessions:
684
+ self.shell_sessions[session_id]["running"] = False
685
+ error_msg = f"Error starting non-blocking command: {e}"
686
+ self._write_to_log(
687
+ session_log_file, f"--- Error ---\n{error_msg}\n"
688
+ )
689
+ return error_msg
690
+
691
+ def shell_write_to_process(self, id: str, command: str) -> str:
692
+ r"""This function sends command to a running non-blocking
693
+ process and returns the resulting output after the process
694
+ becomes idle again. A newline \n is automatically appended
695
+ to the input command.
696
+
697
+ Args:
698
+ id (str): The unique session ID of the non-blocking process.
699
+ command (str): The text to write to the process's standard input.
700
+
701
+ Returns:
702
+ str: The output from the process after the command is sent.
703
+ """
704
+ with self._session_lock:
705
+ if (
706
+ id not in self.shell_sessions
707
+ or not self.shell_sessions[id]["running"]
708
+ ):
709
+ return (
710
+ f"Error: No active non-blocking "
711
+ f"session found with ID '{id}'."
712
+ )
713
+ session = self.shell_sessions[id]
714
+
715
+ # Flush any lingering output from previous commands.
716
+ self._collect_output_until_idle(id, idle_duration=0.3, max_wait=2.0)
717
+
718
+ with self._session_lock:
719
+ session["command_history"].append(command)
720
+ log_file = session["log_file"]
721
+ backend = session["backend"]
722
+ process = session["process"]
723
+
724
+ # Log command to the raw log file
725
+ self._write_to_log(log_file, f"> {command}\n")
726
+
727
+ try:
728
+ if backend == "local":
729
+ process.stdin.write(command + '\n')
730
+ process.stdin.flush()
731
+ else: # docker
732
+ socket = process._sock
733
+ socket.sendall((command + '\n').encode('utf-8'))
734
+
735
+ # Wait for and collect the new output
736
+ output = self._collect_output_until_idle(id)
737
+
738
+ return output
739
+
740
+ except Exception as e:
741
+ return f"Error writing to session '{id}': {e}"
742
+
743
+ def shell_view(self, id: str) -> str:
744
+ r"""Retrieves new output from a non-blocking session.
745
+
746
+ This function returns only NEW output since the last call. It does NOT
747
+ wait or block - it returns immediately with whatever is available.
748
+
749
+ Args:
750
+ id (str): The unique session ID of the non-blocking process.
751
+
752
+ Returns:
753
+ str: New output if available, or a status message.
754
+ """
755
+ with self._session_lock:
756
+ if id not in self.shell_sessions:
757
+ return f"Error: No session found with ID '{id}'."
758
+ session = self.shell_sessions[id]
759
+ is_running = session["running"]
760
+
761
+ # If session is terminated, drain the queue and return
762
+ if not is_running:
763
+ final_output = []
764
+ try:
765
+ while True:
766
+ final_output.append(session["output_stream"].get_nowait())
767
+ except Empty:
768
+ pass
769
+
770
+ if final_output:
771
+ return "".join(final_output) + "\n\n--- SESSION TERMINATED ---"
772
+ else:
773
+ return "--- SESSION TERMINATED (no new output) ---"
774
+
775
+ # For running session, check for new output
776
+ output = []
777
+ try:
778
+ while True:
779
+ output.append(session["output_stream"].get_nowait())
780
+ except Empty:
781
+ pass
782
+
783
+ if output:
784
+ return "".join(output)
785
+ else:
786
+ # No new output - guide the agent
787
+ return (
788
+ "[No new output]\n"
789
+ "Session is running but idle. Actions could take:\n"
790
+ " - For interactive sessions: Send input "
791
+ "with shell_write_to_process()\n"
792
+ " - For long tasks: Check again later (don't poll "
793
+ "too frequently)"
794
+ )
795
+
796
+ def shell_kill_process(self, id: str) -> str:
797
+ r"""This function forcibly terminates a running non-blocking process.
798
+
799
+ Args:
800
+ id (str): The unique session ID of the process to kill.
801
+
802
+ Returns:
803
+ str: A confirmation message indicating the process was terminated.
804
+ """
805
+ with self._session_lock:
806
+ if (
807
+ id not in self.shell_sessions
808
+ or not self.shell_sessions[id]["running"]
809
+ ):
810
+ return f"Error: No active session found with ID '{id}'."
811
+ session = self.shell_sessions[id]
812
+ try:
813
+ if session["backend"] == "local":
814
+ session["process"].terminate()
815
+ time.sleep(0.5)
816
+ if session["process"].poll() is None:
817
+ session["process"].kill()
818
+ # Ensure stdio streams are closed to unblock reader thread
819
+ try:
820
+ if getattr(session["process"], "stdin", None):
821
+ session["process"].stdin.close()
822
+ except Exception:
823
+ pass
824
+ try:
825
+ if getattr(session["process"], "stdout", None):
826
+ session["process"].stdout.close()
827
+ except Exception:
828
+ pass
829
+ else: # docker
830
+ # Docker exec processes stop when the socket is closed.
831
+ session["process"].close()
832
+ with self._session_lock:
833
+ if id in self.shell_sessions:
834
+ self.shell_sessions[id]["running"] = False
835
+ return f"Process in session '{id}' has been terminated."
836
+ except Exception as e:
837
+ return f"Error killing process in session '{id}': {e}"
838
+
839
+ def shell_ask_user_for_help(self, id: str, prompt: str) -> str:
840
+ r"""This function pauses execution and asks a human for help
841
+ with an interactive session.
842
+
843
+ This method can handle different scenarios:
844
+ 1. If session exists: Shows session output and allows interaction
845
+ 2. If session doesn't exist: Creates a temporary session for help
846
+
847
+ Args:
848
+ id (str): The session ID of the interactive process needing help.
849
+ Can be empty string for general help without session context.
850
+ prompt (str): The question or instruction from the LLM to show the
851
+ human user (e.g., "The program is asking for a filename. Please
852
+ enter 'config.json'.").
853
+
854
+ Returns:
855
+ str: The output from the shell session after the user's command has
856
+ been executed, or help information for general queries.
857
+ """
858
+ logger.info("\n" + "=" * 60)
859
+ logger.info("🤖 LLM Agent needs your help!")
860
+ logger.info(f"PROMPT: {prompt}")
861
+
862
+ # Case 1: Session doesn't exist - offer to create one
863
+ if id not in self.shell_sessions:
864
+ try:
865
+ user_input = input("Your response: ").strip()
866
+ if not user_input:
867
+ return "No user response."
868
+ else:
869
+ logger.info(
870
+ f"Creating session '{id}' and executing command..."
871
+ )
872
+ result = self.shell_exec(id, user_input, block=True)
873
+ return (
874
+ f"Session '{id}' created and "
875
+ f"executed command:\n{result}"
876
+ )
877
+ except EOFError:
878
+ return f"User input interrupted for session '{id}' creation."
879
+
880
+ # Case 2: Session exists - show context and interact
881
+ else:
882
+ # Get the latest output to show the user the current state
883
+ last_output = self._collect_output_until_idle(id)
884
+
885
+ logger.info(f"SESSION: '{id}' (active)")
886
+ logger.info("=" * 60)
887
+ logger.info("--- LAST OUTPUT ---")
888
+ logger.info(
889
+ last_output.strip()
890
+ if last_output.strip()
891
+ else "(no recent output)"
892
+ )
893
+ logger.info("-------------------")
894
+
895
+ try:
896
+ user_input = input("Your input: ").strip()
897
+ if not user_input:
898
+ return f"User provided no input for session '{id}'."
899
+ else:
900
+ # Send input to the existing session
901
+ return self.shell_write_to_process(id, user_input)
902
+ except EOFError:
903
+ return f"User input interrupted for session '{id}'."
904
+
905
+ def __enter__(self):
906
+ r"""Context manager entry."""
907
+ return self
908
+
909
+ def __exit__(self, exc_type, exc_val, exc_tb):
910
+ r"""Context manager exit - clean up all sessions."""
911
+ self.cleanup()
912
+ return False
913
+
914
+ def cleanup(self):
915
+ r"""Clean up all active sessions."""
916
+ with self._session_lock:
917
+ session_ids = list(self.shell_sessions.keys())
918
+ for session_id in session_ids:
919
+ with self._session_lock:
920
+ is_running = self.shell_sessions.get(session_id, {}).get(
921
+ "running", False
922
+ )
923
+ if is_running:
924
+ try:
925
+ self.shell_kill_process(session_id)
926
+ except Exception as e:
927
+ logger.warning(
928
+ f"Failed to kill session '{session_id}' "
929
+ f"during cleanup: {e}"
930
+ )
931
+
932
+ cleanup._manual_timeout = True # type: ignore[attr-defined]
933
+
934
+ def __del__(self):
935
+ r"""Fallback cleanup in destructor."""
936
+ try:
937
+ self.cleanup()
938
+ except Exception:
939
+ pass
940
+
941
+ __del__._manual_timeout = True # type: ignore[attr-defined]
942
+
943
+ def get_tools(self) -> List[FunctionTool]:
944
+ r"""Returns a list of FunctionTool objects representing the functions
945
+ in the toolkit.
946
+
947
+ Returns:
948
+ List[FunctionTool]: A list of FunctionTool objects representing the
949
+ functions in the toolkit.
950
+ """
951
+ return [
952
+ FunctionTool(self.shell_exec),
953
+ FunctionTool(self.shell_view),
954
+ FunctionTool(self.shell_write_to_process),
955
+ FunctionTool(self.shell_kill_process),
956
+ FunctionTool(self.shell_ask_user_for_help),
957
+ ]