camel-ai 0.2.76a4__py3-none-any.whl → 0.2.76a6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of camel-ai might be problematic. Click here for more details.
- camel/__init__.py +1 -1
- camel/agents/chat_agent.py +276 -21
- camel/configs/__init__.py +3 -0
- camel/configs/cometapi_config.py +104 -0
- camel/interpreters/docker/Dockerfile +3 -12
- camel/memories/blocks/chat_history_block.py +4 -1
- camel/memories/records.py +52 -8
- camel/messages/base.py +1 -1
- camel/models/__init__.py +2 -0
- camel/models/azure_openai_model.py +0 -6
- camel/models/cometapi_model.py +83 -0
- camel/models/model_factory.py +2 -0
- camel/models/openai_compatible_model.py +0 -6
- camel/models/zhipuai_model.py +61 -2
- camel/retrievers/auto_retriever.py +1 -0
- camel/societies/workforce/workforce.py +9 -7
- camel/storages/key_value_storages/json.py +15 -2
- camel/storages/vectordb_storages/tidb.py +8 -6
- camel/toolkits/__init__.py +4 -0
- camel/toolkits/dingtalk.py +1135 -0
- camel/toolkits/edgeone_pages_mcp_toolkit.py +11 -31
- camel/toolkits/google_drive_mcp_toolkit.py +12 -31
- camel/toolkits/message_integration.py +3 -0
- camel/toolkits/notion_mcp_toolkit.py +16 -26
- camel/toolkits/origene_mcp_toolkit.py +8 -49
- camel/toolkits/playwright_mcp_toolkit.py +12 -31
- camel/toolkits/resend_toolkit.py +168 -0
- camel/toolkits/terminal_toolkit/__init__.py +18 -0
- camel/toolkits/terminal_toolkit/terminal_toolkit.py +924 -0
- camel/toolkits/terminal_toolkit/utils.py +580 -0
- camel/types/enums.py +109 -0
- camel/types/unified_model_type.py +5 -0
- camel/utils/commons.py +2 -0
- {camel_ai-0.2.76a4.dist-info → camel_ai-0.2.76a6.dist-info}/METADATA +25 -6
- {camel_ai-0.2.76a4.dist-info → camel_ai-0.2.76a6.dist-info}/RECORD +37 -31
- camel/toolkits/terminal_toolkit.py +0 -1798
- {camel_ai-0.2.76a4.dist-info → camel_ai-0.2.76a6.dist-info}/WHEEL +0 -0
- {camel_ai-0.2.76a4.dist-info → camel_ai-0.2.76a6.dist-info}/licenses/LICENSE +0 -0
|
@@ -0,0 +1,924 @@
|
|
|
1
|
+
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
|
|
2
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
3
|
+
# you may not use this file except in compliance with the License.
|
|
4
|
+
# You may obtain a copy of the License at
|
|
5
|
+
#
|
|
6
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
7
|
+
#
|
|
8
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
9
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
10
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
11
|
+
# See the License for the specific language governing permissions and
|
|
12
|
+
# limitations under the License.
|
|
13
|
+
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
|
|
14
|
+
import atexit
|
|
15
|
+
import os
|
|
16
|
+
import platform
|
|
17
|
+
import select
|
|
18
|
+
import shlex
|
|
19
|
+
import subprocess
|
|
20
|
+
import sys
|
|
21
|
+
import threading
|
|
22
|
+
import time
|
|
23
|
+
from queue import Empty, Queue
|
|
24
|
+
from typing import Any, Dict, List, Optional
|
|
25
|
+
|
|
26
|
+
from camel.logger import get_logger
|
|
27
|
+
from camel.toolkits.base import BaseToolkit
|
|
28
|
+
from camel.toolkits.function_tool import FunctionTool
|
|
29
|
+
from camel.toolkits.terminal_toolkit.utils import (
|
|
30
|
+
check_nodejs_availability,
|
|
31
|
+
clone_current_environment,
|
|
32
|
+
ensure_uv_available,
|
|
33
|
+
sanitize_command,
|
|
34
|
+
setup_initial_env_with_uv,
|
|
35
|
+
setup_initial_env_with_venv,
|
|
36
|
+
)
|
|
37
|
+
from camel.utils import MCPServer
|
|
38
|
+
|
|
39
|
+
logger = get_logger(__name__)
|
|
40
|
+
|
|
41
|
+
# Try to import docker, but don't make it a hard requirement
|
|
42
|
+
try:
|
|
43
|
+
import docker
|
|
44
|
+
from docker.errors import APIError, NotFound
|
|
45
|
+
from docker.models.containers import Container
|
|
46
|
+
except ImportError:
|
|
47
|
+
docker = None
|
|
48
|
+
NotFound = None
|
|
49
|
+
APIError = None
|
|
50
|
+
Container = None
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _to_plain(text: str) -> str:
|
|
54
|
+
r"""Convert ANSI text to plain text using rich if available."""
|
|
55
|
+
try:
|
|
56
|
+
from rich.text import Text as _RichText
|
|
57
|
+
|
|
58
|
+
return _RichText.from_ansi(text).plain
|
|
59
|
+
except Exception:
|
|
60
|
+
return text
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@MCPServer()
|
|
64
|
+
class TerminalToolkit(BaseToolkit):
|
|
65
|
+
r"""A toolkit for LLM agents to execute and interact with terminal commands
|
|
66
|
+
in either a local or a sandboxed Docker environment.
|
|
67
|
+
|
|
68
|
+
Args:
|
|
69
|
+
timeout (Optional[float]): The default timeout in seconds for blocking
|
|
70
|
+
commands. Defaults to 20.0.
|
|
71
|
+
working_directory (Optional[str]): The base directory for operations.
|
|
72
|
+
For the local backend, this acts as a security sandbox.
|
|
73
|
+
For the Docker backend, this sets the working directory inside
|
|
74
|
+
the container.
|
|
75
|
+
If not specified, defaults to "./workspace" for local and
|
|
76
|
+
"/workspace" for Docker.
|
|
77
|
+
use_docker_backend (bool): If True, all commands are executed in a
|
|
78
|
+
Docker container. Defaults to False.
|
|
79
|
+
docker_container_name (Optional[str]): The name of the Docker
|
|
80
|
+
container to use. Required if use_docker_backend is True.
|
|
81
|
+
session_logs_dir (Optional[str]): The directory to store session
|
|
82
|
+
logs. Defaults to a 'terminal_logs' subfolder in the
|
|
83
|
+
working directory.
|
|
84
|
+
safe_mode (bool): Whether to apply security checks to commands.
|
|
85
|
+
Defaults to True.
|
|
86
|
+
allowed_commands (Optional[List[str]]): List of allowed commands
|
|
87
|
+
when safe_mode is True. If None, uses default safety rules.
|
|
88
|
+
clone_current_env (bool): Whether to clone the current Python
|
|
89
|
+
environment for local execution. Defaults to False.
|
|
90
|
+
"""
|
|
91
|
+
|
|
92
|
+
def __init__(
|
|
93
|
+
self,
|
|
94
|
+
timeout: Optional[float] = 20.0,
|
|
95
|
+
working_directory: Optional[str] = None,
|
|
96
|
+
use_docker_backend: bool = False,
|
|
97
|
+
docker_container_name: Optional[str] = None,
|
|
98
|
+
session_logs_dir: Optional[str] = None,
|
|
99
|
+
safe_mode: bool = True,
|
|
100
|
+
allowed_commands: Optional[List[str]] = None,
|
|
101
|
+
clone_current_env: bool = False,
|
|
102
|
+
):
|
|
103
|
+
self.use_docker_backend = use_docker_backend
|
|
104
|
+
self.timeout = timeout
|
|
105
|
+
self.shell_sessions: Dict[str, Dict[str, Any]] = {}
|
|
106
|
+
# Thread-safe guard for concurrent access to
|
|
107
|
+
# shell_sessions and session state
|
|
108
|
+
self._session_lock = threading.RLock()
|
|
109
|
+
|
|
110
|
+
# Initialize docker_workdir with proper type
|
|
111
|
+
self.docker_workdir: Optional[str] = None
|
|
112
|
+
|
|
113
|
+
if self.use_docker_backend:
|
|
114
|
+
# For Docker backend, working_directory is path inside container
|
|
115
|
+
if working_directory:
|
|
116
|
+
self.docker_workdir = working_directory
|
|
117
|
+
else:
|
|
118
|
+
self.docker_workdir = "/workspace"
|
|
119
|
+
# For logs and local file operations, use a local workspace
|
|
120
|
+
camel_workdir = os.environ.get("CAMEL_WORKDIR")
|
|
121
|
+
if camel_workdir:
|
|
122
|
+
self.working_dir = os.path.abspath(camel_workdir)
|
|
123
|
+
else:
|
|
124
|
+
self.working_dir = os.path.abspath("./workspace")
|
|
125
|
+
else:
|
|
126
|
+
# For local backend, working_directory is the local path
|
|
127
|
+
if working_directory:
|
|
128
|
+
self.working_dir = os.path.abspath(working_directory)
|
|
129
|
+
else:
|
|
130
|
+
camel_workdir = os.environ.get("CAMEL_WORKDIR")
|
|
131
|
+
if camel_workdir:
|
|
132
|
+
self.working_dir = os.path.abspath(camel_workdir)
|
|
133
|
+
else:
|
|
134
|
+
self.working_dir = os.path.abspath("./workspace")
|
|
135
|
+
|
|
136
|
+
# Only create local directory for logs and local backend operations
|
|
137
|
+
if not os.path.exists(self.working_dir):
|
|
138
|
+
os.makedirs(self.working_dir, exist_ok=True)
|
|
139
|
+
self.safe_mode = safe_mode
|
|
140
|
+
|
|
141
|
+
# Initialize whitelist of allowed commands if provided
|
|
142
|
+
self.allowed_commands = (
|
|
143
|
+
set(allowed_commands) if allowed_commands else None
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
# Environment management attributes
|
|
147
|
+
self.clone_current_env = clone_current_env
|
|
148
|
+
self.cloned_env_path: Optional[str] = None
|
|
149
|
+
self.initial_env_path: Optional[str] = None
|
|
150
|
+
self.python_executable = sys.executable
|
|
151
|
+
|
|
152
|
+
atexit.register(self.__del__)
|
|
153
|
+
|
|
154
|
+
self.log_dir = os.path.abspath(
|
|
155
|
+
session_logs_dir or os.path.join(self.working_dir, "terminal_logs")
|
|
156
|
+
)
|
|
157
|
+
self.blocking_log_file = os.path.join(
|
|
158
|
+
self.log_dir, "blocking_commands.log"
|
|
159
|
+
)
|
|
160
|
+
self.os_type = platform.system()
|
|
161
|
+
|
|
162
|
+
os.makedirs(self.log_dir, exist_ok=True)
|
|
163
|
+
|
|
164
|
+
# Clean the file in terminal_logs folder
|
|
165
|
+
for file in os.listdir(self.log_dir):
|
|
166
|
+
if file.endswith(".log"):
|
|
167
|
+
os.remove(os.path.join(self.log_dir, file))
|
|
168
|
+
|
|
169
|
+
if self.use_docker_backend:
|
|
170
|
+
if docker is None:
|
|
171
|
+
raise ImportError(
|
|
172
|
+
"The 'docker' library is required to use the "
|
|
173
|
+
"Docker backend. Please install it with "
|
|
174
|
+
"'pip install docker'."
|
|
175
|
+
)
|
|
176
|
+
if not docker_container_name:
|
|
177
|
+
raise ValueError(
|
|
178
|
+
"docker_container_name must be "
|
|
179
|
+
"provided when using Docker backend."
|
|
180
|
+
)
|
|
181
|
+
try:
|
|
182
|
+
# APIClient is used for operations that need a timeout,
|
|
183
|
+
# like exec_start
|
|
184
|
+
self.docker_api_client = docker.APIClient(
|
|
185
|
+
base_url='unix://var/run/docker.sock', timeout=self.timeout
|
|
186
|
+
)
|
|
187
|
+
self.docker_client = docker.from_env()
|
|
188
|
+
self.container = self.docker_client.containers.get(
|
|
189
|
+
docker_container_name
|
|
190
|
+
)
|
|
191
|
+
logger.info(
|
|
192
|
+
f"Successfully attached to Docker container "
|
|
193
|
+
f"'{docker_container_name}'."
|
|
194
|
+
)
|
|
195
|
+
# Ensure the working directory exists inside the container
|
|
196
|
+
if self.docker_workdir:
|
|
197
|
+
try:
|
|
198
|
+
quoted_dir = shlex.quote(self.docker_workdir)
|
|
199
|
+
mkdir_cmd = f'sh -lc "mkdir -p -- {quoted_dir}"'
|
|
200
|
+
_init = self.docker_api_client.exec_create(
|
|
201
|
+
self.container.id, mkdir_cmd
|
|
202
|
+
)
|
|
203
|
+
self.docker_api_client.exec_start(_init['Id'])
|
|
204
|
+
except Exception as e:
|
|
205
|
+
logger.warning(
|
|
206
|
+
f"[Docker] Failed to ensure workdir "
|
|
207
|
+
f"'{self.docker_workdir}': {e}"
|
|
208
|
+
)
|
|
209
|
+
except NotFound:
|
|
210
|
+
raise RuntimeError(
|
|
211
|
+
f"Docker container '{docker_container_name}' not found."
|
|
212
|
+
)
|
|
213
|
+
except APIError as e:
|
|
214
|
+
raise RuntimeError(f"Failed to connect to Docker daemon: {e}")
|
|
215
|
+
|
|
216
|
+
# Set up environments (only for local backend)
|
|
217
|
+
if not self.use_docker_backend:
|
|
218
|
+
if self.clone_current_env:
|
|
219
|
+
self._setup_cloned_environment()
|
|
220
|
+
else:
|
|
221
|
+
# Default: set up initial environment with Python 3.10
|
|
222
|
+
self._setup_initial_environment()
|
|
223
|
+
elif self.clone_current_env:
|
|
224
|
+
logger.info(
|
|
225
|
+
"[ENV CLONE] Skipping environment setup for Docker backend "
|
|
226
|
+
"- container is already isolated"
|
|
227
|
+
)
|
|
228
|
+
|
|
229
|
+
def _setup_cloned_environment(self):
|
|
230
|
+
r"""Set up a cloned Python environment."""
|
|
231
|
+
self.cloned_env_path = os.path.join(self.working_dir, ".venv")
|
|
232
|
+
|
|
233
|
+
def update_callback(msg: str):
|
|
234
|
+
logger.info(f"[ENV CLONE] {msg.strip()}")
|
|
235
|
+
|
|
236
|
+
success = clone_current_environment(
|
|
237
|
+
self.cloned_env_path, self.working_dir, update_callback
|
|
238
|
+
)
|
|
239
|
+
|
|
240
|
+
if success:
|
|
241
|
+
# Update python executable to use the cloned environment
|
|
242
|
+
if self.os_type == 'Windows':
|
|
243
|
+
self.python_executable = os.path.join(
|
|
244
|
+
self.cloned_env_path, "Scripts", "python.exe"
|
|
245
|
+
)
|
|
246
|
+
else:
|
|
247
|
+
self.python_executable = os.path.join(
|
|
248
|
+
self.cloned_env_path, "bin", "python"
|
|
249
|
+
)
|
|
250
|
+
else:
|
|
251
|
+
logger.info(
|
|
252
|
+
"[ENV CLONE] Failed to create cloned environment, "
|
|
253
|
+
"using system Python"
|
|
254
|
+
)
|
|
255
|
+
|
|
256
|
+
def _setup_initial_environment(self):
|
|
257
|
+
r"""Set up an initial environment with Python 3.10."""
|
|
258
|
+
self.initial_env_path = os.path.join(self.working_dir, ".initial_env")
|
|
259
|
+
|
|
260
|
+
def update_callback(msg: str):
|
|
261
|
+
logger.info(f"[ENV INIT] {msg.strip()}")
|
|
262
|
+
|
|
263
|
+
# Try to ensure uv is available first
|
|
264
|
+
success, uv_path = ensure_uv_available(update_callback)
|
|
265
|
+
|
|
266
|
+
if success and uv_path:
|
|
267
|
+
success = setup_initial_env_with_uv(
|
|
268
|
+
self.initial_env_path,
|
|
269
|
+
uv_path,
|
|
270
|
+
self.working_dir,
|
|
271
|
+
update_callback,
|
|
272
|
+
)
|
|
273
|
+
else:
|
|
274
|
+
update_callback(
|
|
275
|
+
"Falling back to standard venv for environment setup\n"
|
|
276
|
+
)
|
|
277
|
+
success = setup_initial_env_with_venv(
|
|
278
|
+
self.initial_env_path, self.working_dir, update_callback
|
|
279
|
+
)
|
|
280
|
+
|
|
281
|
+
if success:
|
|
282
|
+
# Update python executable to use the initial environment
|
|
283
|
+
if self.os_type == 'Windows':
|
|
284
|
+
self.python_executable = os.path.join(
|
|
285
|
+
self.initial_env_path, "Scripts", "python.exe"
|
|
286
|
+
)
|
|
287
|
+
else:
|
|
288
|
+
self.python_executable = os.path.join(
|
|
289
|
+
self.initial_env_path, "bin", "python"
|
|
290
|
+
)
|
|
291
|
+
|
|
292
|
+
# Check Node.js availability
|
|
293
|
+
check_nodejs_availability(update_callback)
|
|
294
|
+
else:
|
|
295
|
+
logger.info(
|
|
296
|
+
"[ENV INIT] Failed to create initial environment, "
|
|
297
|
+
"using system Python"
|
|
298
|
+
)
|
|
299
|
+
|
|
300
|
+
def _adapt_command_for_environment(self, command: str) -> str:
|
|
301
|
+
r"""Adapt command to use virtual environment if available."""
|
|
302
|
+
# Only adapt for local backend
|
|
303
|
+
if self.use_docker_backend:
|
|
304
|
+
return command
|
|
305
|
+
|
|
306
|
+
# Check if we have any virtual environment (cloned or initial)
|
|
307
|
+
env_path = None
|
|
308
|
+
if self.cloned_env_path and os.path.exists(self.cloned_env_path):
|
|
309
|
+
env_path = self.cloned_env_path
|
|
310
|
+
elif self.initial_env_path and os.path.exists(self.initial_env_path):
|
|
311
|
+
env_path = self.initial_env_path
|
|
312
|
+
|
|
313
|
+
if not env_path:
|
|
314
|
+
return command
|
|
315
|
+
|
|
316
|
+
# Check if command starts with python or pip
|
|
317
|
+
command_lower = command.strip().lower()
|
|
318
|
+
if command_lower.startswith('python'):
|
|
319
|
+
# Replace 'python' with the virtual environment python
|
|
320
|
+
return command.replace('python', f'"{self.python_executable}"', 1)
|
|
321
|
+
elif command_lower.startswith('pip'):
|
|
322
|
+
# Replace 'pip' with python -m pip from virtual environment
|
|
323
|
+
return command.replace(
|
|
324
|
+
'pip', f'"{self.python_executable}" -m pip', 1
|
|
325
|
+
)
|
|
326
|
+
|
|
327
|
+
return command
|
|
328
|
+
|
|
329
|
+
def _write_to_log(self, log_file: str, content: str) -> None:
|
|
330
|
+
r"""Write content to log file with optional ANSI stripping.
|
|
331
|
+
|
|
332
|
+
Args:
|
|
333
|
+
log_file (str): Path to the log file
|
|
334
|
+
content (str): Content to write
|
|
335
|
+
"""
|
|
336
|
+
# Convert ANSI escape sequences to plain text
|
|
337
|
+
with open(log_file, "a", encoding="utf-8") as f:
|
|
338
|
+
f.write(_to_plain(content) + "\n")
|
|
339
|
+
|
|
340
|
+
def _sanitize_command(self, command: str) -> tuple[bool, str]:
|
|
341
|
+
r"""A comprehensive command sanitizer for both local and
|
|
342
|
+
Docker backends."""
|
|
343
|
+
return sanitize_command(
|
|
344
|
+
command=command,
|
|
345
|
+
use_docker_backend=self.use_docker_backend,
|
|
346
|
+
safe_mode=self.safe_mode,
|
|
347
|
+
working_dir=self.working_dir,
|
|
348
|
+
allowed_commands=self.allowed_commands,
|
|
349
|
+
)
|
|
350
|
+
|
|
351
|
+
def _start_output_reader_thread(self, session_id: str):
|
|
352
|
+
r"""Starts a thread to read stdout from a non-blocking process."""
|
|
353
|
+
with self._session_lock:
|
|
354
|
+
session = self.shell_sessions[session_id]
|
|
355
|
+
|
|
356
|
+
def reader():
|
|
357
|
+
try:
|
|
358
|
+
if session["backend"] == "local":
|
|
359
|
+
# For local processes, read line by line from stdout
|
|
360
|
+
try:
|
|
361
|
+
for line in iter(
|
|
362
|
+
session["process"].stdout.readline, ''
|
|
363
|
+
):
|
|
364
|
+
session["output_stream"].put(line)
|
|
365
|
+
self._write_to_log(session["log_file"], line)
|
|
366
|
+
finally:
|
|
367
|
+
session["process"].stdout.close()
|
|
368
|
+
elif session["backend"] == "docker":
|
|
369
|
+
# For Docker, read from the raw socket
|
|
370
|
+
socket = session["process"]._sock
|
|
371
|
+
while True:
|
|
372
|
+
# Check if the socket is still open before reading
|
|
373
|
+
if socket.fileno() == -1:
|
|
374
|
+
break
|
|
375
|
+
try:
|
|
376
|
+
ready, _, _ = select.select([socket], [], [], 0.1)
|
|
377
|
+
except (ValueError, OSError):
|
|
378
|
+
# Socket may have been closed by another thread
|
|
379
|
+
break
|
|
380
|
+
if ready:
|
|
381
|
+
data = socket.recv(4096)
|
|
382
|
+
if not data:
|
|
383
|
+
break
|
|
384
|
+
decoded_data = data.decode(
|
|
385
|
+
'utf-8', errors='ignore'
|
|
386
|
+
)
|
|
387
|
+
session["output_stream"].put(decoded_data)
|
|
388
|
+
self._write_to_log(
|
|
389
|
+
session["log_file"], decoded_data
|
|
390
|
+
)
|
|
391
|
+
# Check if the process is still running
|
|
392
|
+
if not self.docker_api_client.exec_inspect(
|
|
393
|
+
session["exec_id"]
|
|
394
|
+
)['Running']:
|
|
395
|
+
break
|
|
396
|
+
except Exception as e:
|
|
397
|
+
# Log the exception for diagnosis and store it on the session
|
|
398
|
+
logger.exception(f"[SESSION {session_id}] Reader thread error")
|
|
399
|
+
try:
|
|
400
|
+
with self._session_lock:
|
|
401
|
+
if session_id in self.shell_sessions:
|
|
402
|
+
self.shell_sessions[session_id]["error"] = str(e)
|
|
403
|
+
except Exception:
|
|
404
|
+
# Swallow any secondary errors during cleanup
|
|
405
|
+
pass
|
|
406
|
+
finally:
|
|
407
|
+
try:
|
|
408
|
+
with self._session_lock:
|
|
409
|
+
if session_id in self.shell_sessions:
|
|
410
|
+
self.shell_sessions[session_id]["running"] = False
|
|
411
|
+
except Exception:
|
|
412
|
+
pass
|
|
413
|
+
|
|
414
|
+
thread = threading.Thread(target=reader, daemon=True)
|
|
415
|
+
thread.start()
|
|
416
|
+
|
|
417
|
+
def _collect_output_until_idle(
|
|
418
|
+
self,
|
|
419
|
+
id: str,
|
|
420
|
+
idle_duration: float = 0.5,
|
|
421
|
+
check_interval: float = 0.1,
|
|
422
|
+
max_wait: float = 5.0,
|
|
423
|
+
) -> str:
|
|
424
|
+
r"""Collects output from a session until it's idle or a max wait time
|
|
425
|
+
is reached.
|
|
426
|
+
|
|
427
|
+
Args:
|
|
428
|
+
id (str): The session ID.
|
|
429
|
+
idle_duration (float): How long the stream must be empty to be
|
|
430
|
+
considered idle.(default: 0.5)
|
|
431
|
+
check_interval (float): The time to sleep between checks.
|
|
432
|
+
(default: 0.1)
|
|
433
|
+
max_wait (float): The maximum total time to wait for the process
|
|
434
|
+
to go idle. (default: 5.0)
|
|
435
|
+
|
|
436
|
+
Returns:
|
|
437
|
+
str: The collected output. If max_wait is reached while
|
|
438
|
+
the process is still outputting, a warning is appended.
|
|
439
|
+
"""
|
|
440
|
+
with self._session_lock:
|
|
441
|
+
if id not in self.shell_sessions:
|
|
442
|
+
return f"Error: No session found with ID '{id}'."
|
|
443
|
+
|
|
444
|
+
output_parts = []
|
|
445
|
+
idle_time = 0.0
|
|
446
|
+
start_time = time.time()
|
|
447
|
+
|
|
448
|
+
while time.time() - start_time < max_wait:
|
|
449
|
+
new_output = self.shell_view(id)
|
|
450
|
+
|
|
451
|
+
# Check for terminal state messages from shell_view
|
|
452
|
+
if "--- SESSION TERMINATED ---" in new_output:
|
|
453
|
+
# Append the final output before the termination message
|
|
454
|
+
final_part = new_output.replace(
|
|
455
|
+
"--- SESSION TERMINATED ---", ""
|
|
456
|
+
).strip()
|
|
457
|
+
if final_part:
|
|
458
|
+
output_parts.append(final_part)
|
|
459
|
+
# Session is dead, return what we have plus the message
|
|
460
|
+
return "".join(output_parts) + "\n--- SESSION TERMINATED ---"
|
|
461
|
+
|
|
462
|
+
if new_output.startswith("Error: No session found"):
|
|
463
|
+
return new_output
|
|
464
|
+
|
|
465
|
+
if new_output:
|
|
466
|
+
output_parts.append(new_output)
|
|
467
|
+
idle_time = 0.0 # Reset idle timer
|
|
468
|
+
else:
|
|
469
|
+
idle_time += check_interval
|
|
470
|
+
if idle_time >= idle_duration:
|
|
471
|
+
# Process is idle, success
|
|
472
|
+
return "".join(output_parts)
|
|
473
|
+
time.sleep(check_interval)
|
|
474
|
+
|
|
475
|
+
# If we exit the loop, it means max_wait was reached.
|
|
476
|
+
# Check one last time for any final output.
|
|
477
|
+
final_output = self.shell_view(id)
|
|
478
|
+
if final_output:
|
|
479
|
+
output_parts.append(final_output)
|
|
480
|
+
|
|
481
|
+
warning_message = (
|
|
482
|
+
"\n--- WARNING: Process is still actively outputting "
|
|
483
|
+
"after max wait time. Consider using shell_wait() "
|
|
484
|
+
"before sending the next command. ---"
|
|
485
|
+
)
|
|
486
|
+
return "".join(output_parts) + warning_message
|
|
487
|
+
|
|
488
|
+
def shell_exec(self, id: str, command: str, block: bool = True) -> str:
|
|
489
|
+
r"""This function executes a shell command. The command can run in
|
|
490
|
+
blocking mode (waits for completion) or non-blocking mode
|
|
491
|
+
(runs in the background). A unique session ID is created for
|
|
492
|
+
each session.
|
|
493
|
+
|
|
494
|
+
Args:
|
|
495
|
+
command (str): The command to execute.
|
|
496
|
+
block (bool): If True, the command runs synchronously,
|
|
497
|
+
waiting for it to complete or time out, and returns
|
|
498
|
+
its full output. If False, the command runs
|
|
499
|
+
asynchronously in the background.
|
|
500
|
+
id (Optional[str]): A specific ID for the session. If not provided,
|
|
501
|
+
a unique ID is generated for non-blocking sessions.
|
|
502
|
+
|
|
503
|
+
Returns:
|
|
504
|
+
str: If block is True, returns the complete stdout and stderr.
|
|
505
|
+
If block is False, returns a message containing the new
|
|
506
|
+
session ID and the initial output from the command after
|
|
507
|
+
it goes idle.
|
|
508
|
+
"""
|
|
509
|
+
if self.safe_mode:
|
|
510
|
+
is_safe, message = self._sanitize_command(command)
|
|
511
|
+
if not is_safe:
|
|
512
|
+
return f"Error: {message}"
|
|
513
|
+
command = message
|
|
514
|
+
else:
|
|
515
|
+
command = command
|
|
516
|
+
|
|
517
|
+
if self.use_docker_backend:
|
|
518
|
+
# For Docker, we always run commands in a shell
|
|
519
|
+
# to support complex commands
|
|
520
|
+
command = f'bash -c "{command}"'
|
|
521
|
+
else:
|
|
522
|
+
# For local execution, check if we need to use cloned environment
|
|
523
|
+
command = self._adapt_command_for_environment(command)
|
|
524
|
+
|
|
525
|
+
session_id = id
|
|
526
|
+
|
|
527
|
+
if block:
|
|
528
|
+
# --- BLOCKING EXECUTION ---
|
|
529
|
+
log_entry = (
|
|
530
|
+
f"--- Executing blocking command at "
|
|
531
|
+
f"{time.ctime()} ---\n> {command}\n"
|
|
532
|
+
)
|
|
533
|
+
output = ""
|
|
534
|
+
try:
|
|
535
|
+
if not self.use_docker_backend:
|
|
536
|
+
# LOCAL BLOCKING
|
|
537
|
+
result = subprocess.run(
|
|
538
|
+
command,
|
|
539
|
+
capture_output=True,
|
|
540
|
+
text=True,
|
|
541
|
+
shell=True,
|
|
542
|
+
timeout=self.timeout,
|
|
543
|
+
cwd=self.working_dir,
|
|
544
|
+
encoding="utf-8",
|
|
545
|
+
)
|
|
546
|
+
stdout = result.stdout or ""
|
|
547
|
+
stderr = result.stderr or ""
|
|
548
|
+
output = stdout + (
|
|
549
|
+
f"\nSTDERR:\n{stderr}" if stderr else ""
|
|
550
|
+
)
|
|
551
|
+
else:
|
|
552
|
+
# DOCKER BLOCKING
|
|
553
|
+
assert (
|
|
554
|
+
self.docker_workdir is not None
|
|
555
|
+
) # Docker backend always has workdir
|
|
556
|
+
exec_instance = self.docker_api_client.exec_create(
|
|
557
|
+
self.container.id, command, workdir=self.docker_workdir
|
|
558
|
+
)
|
|
559
|
+
exec_output = self.docker_api_client.exec_start(
|
|
560
|
+
exec_instance['Id']
|
|
561
|
+
)
|
|
562
|
+
output = exec_output.decode('utf-8', errors='ignore')
|
|
563
|
+
|
|
564
|
+
log_entry += f"--- Output ---\n{output}\n"
|
|
565
|
+
return _to_plain(output)
|
|
566
|
+
except subprocess.TimeoutExpired:
|
|
567
|
+
error_msg = (
|
|
568
|
+
f"Error: Command timed out after {self.timeout} seconds."
|
|
569
|
+
)
|
|
570
|
+
log_entry += f"--- Error ---\n{error_msg}\n"
|
|
571
|
+
return error_msg
|
|
572
|
+
except Exception as e:
|
|
573
|
+
if "Read timed out" in str(e):
|
|
574
|
+
error_msg = (
|
|
575
|
+
f"Error: Command timed out after "
|
|
576
|
+
f"{self.timeout} seconds."
|
|
577
|
+
)
|
|
578
|
+
else:
|
|
579
|
+
error_msg = f"Error executing command: {e}"
|
|
580
|
+
log_entry += f"--- Error ---\n{error_msg}\n"
|
|
581
|
+
return error_msg
|
|
582
|
+
finally:
|
|
583
|
+
self._write_to_log(self.blocking_log_file, log_entry + "\n")
|
|
584
|
+
else:
|
|
585
|
+
# --- NON-BLOCKING EXECUTION ---
|
|
586
|
+
session_log_file = os.path.join(
|
|
587
|
+
self.log_dir, f"session_{session_id}.log"
|
|
588
|
+
)
|
|
589
|
+
|
|
590
|
+
self._write_to_log(
|
|
591
|
+
session_log_file,
|
|
592
|
+
f"--- Starting non-blocking session at {time.ctime()} ---\n"
|
|
593
|
+
f"> {command}\n",
|
|
594
|
+
)
|
|
595
|
+
|
|
596
|
+
with self._session_lock:
|
|
597
|
+
self.shell_sessions[session_id] = {
|
|
598
|
+
"id": session_id,
|
|
599
|
+
"process": None,
|
|
600
|
+
"output_stream": Queue(),
|
|
601
|
+
"command_history": [command],
|
|
602
|
+
"running": True,
|
|
603
|
+
"log_file": session_log_file,
|
|
604
|
+
"backend": "docker"
|
|
605
|
+
if self.use_docker_backend
|
|
606
|
+
else "local",
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
try:
|
|
610
|
+
if not self.use_docker_backend:
|
|
611
|
+
process = subprocess.Popen(
|
|
612
|
+
command,
|
|
613
|
+
stdin=subprocess.PIPE,
|
|
614
|
+
stdout=subprocess.PIPE,
|
|
615
|
+
stderr=subprocess.STDOUT,
|
|
616
|
+
shell=True,
|
|
617
|
+
text=True,
|
|
618
|
+
cwd=self.working_dir,
|
|
619
|
+
encoding="utf-8",
|
|
620
|
+
)
|
|
621
|
+
with self._session_lock:
|
|
622
|
+
self.shell_sessions[session_id]["process"] = process
|
|
623
|
+
else:
|
|
624
|
+
assert (
|
|
625
|
+
self.docker_workdir is not None
|
|
626
|
+
) # Docker backend always has workdir
|
|
627
|
+
exec_instance = self.docker_api_client.exec_create(
|
|
628
|
+
self.container.id,
|
|
629
|
+
command,
|
|
630
|
+
stdin=True,
|
|
631
|
+
tty=True,
|
|
632
|
+
workdir=self.docker_workdir,
|
|
633
|
+
)
|
|
634
|
+
exec_id = exec_instance['Id']
|
|
635
|
+
exec_socket = self.docker_api_client.exec_start(
|
|
636
|
+
exec_id, tty=True, stream=True, socket=True
|
|
637
|
+
)
|
|
638
|
+
with self._session_lock:
|
|
639
|
+
self.shell_sessions[session_id]["process"] = (
|
|
640
|
+
exec_socket
|
|
641
|
+
)
|
|
642
|
+
self.shell_sessions[session_id]["exec_id"] = exec_id
|
|
643
|
+
|
|
644
|
+
self._start_output_reader_thread(session_id)
|
|
645
|
+
|
|
646
|
+
# time.sleep(0.1)
|
|
647
|
+
initial_output = self._collect_output_until_idle(session_id)
|
|
648
|
+
|
|
649
|
+
return (
|
|
650
|
+
f"Session started with ID: {session_id}\n\n"
|
|
651
|
+
f"[Initial Output]:\n{initial_output}"
|
|
652
|
+
)
|
|
653
|
+
|
|
654
|
+
except Exception as e:
|
|
655
|
+
with self._session_lock:
|
|
656
|
+
if session_id in self.shell_sessions:
|
|
657
|
+
self.shell_sessions[session_id]["running"] = False
|
|
658
|
+
error_msg = f"Error starting non-blocking command: {e}"
|
|
659
|
+
self._write_to_log(
|
|
660
|
+
session_log_file, f"--- Error ---\n{error_msg}\n"
|
|
661
|
+
)
|
|
662
|
+
return error_msg
|
|
663
|
+
|
|
664
|
+
def shell_write_to_process(self, id: str, command: str) -> str:
|
|
665
|
+
r"""This function sends command to a running non-blocking
|
|
666
|
+
process and returns the resulting output after the process
|
|
667
|
+
becomes idle again. A newline \n is automatically appended
|
|
668
|
+
to the input command.
|
|
669
|
+
|
|
670
|
+
Args:
|
|
671
|
+
id (str): The unique session ID of the non-blocking process.
|
|
672
|
+
command (str): The text to write to the process's standard input.
|
|
673
|
+
|
|
674
|
+
Returns:
|
|
675
|
+
str: The output from the process after the command is sent.
|
|
676
|
+
"""
|
|
677
|
+
with self._session_lock:
|
|
678
|
+
if (
|
|
679
|
+
id not in self.shell_sessions
|
|
680
|
+
or not self.shell_sessions[id]["running"]
|
|
681
|
+
):
|
|
682
|
+
return (
|
|
683
|
+
f"Error: No active non-blocking "
|
|
684
|
+
f"session found with ID '{id}'."
|
|
685
|
+
)
|
|
686
|
+
session = self.shell_sessions[id]
|
|
687
|
+
|
|
688
|
+
# Flush any lingering output from previous commands.
|
|
689
|
+
self._collect_output_until_idle(id, idle_duration=0.3, max_wait=2.0)
|
|
690
|
+
|
|
691
|
+
with self._session_lock:
|
|
692
|
+
session["command_history"].append(command)
|
|
693
|
+
log_file = session["log_file"]
|
|
694
|
+
backend = session["backend"]
|
|
695
|
+
process = session["process"]
|
|
696
|
+
|
|
697
|
+
# Log command to the raw log file
|
|
698
|
+
self._write_to_log(log_file, f"> {command}\n")
|
|
699
|
+
|
|
700
|
+
try:
|
|
701
|
+
if backend == "local":
|
|
702
|
+
process.stdin.write(command + '\n')
|
|
703
|
+
process.stdin.flush()
|
|
704
|
+
else: # docker
|
|
705
|
+
socket = process._sock
|
|
706
|
+
socket.sendall((command + '\n').encode('utf-8'))
|
|
707
|
+
|
|
708
|
+
# Wait for and collect the new output
|
|
709
|
+
output = self._collect_output_until_idle(id)
|
|
710
|
+
|
|
711
|
+
return output
|
|
712
|
+
|
|
713
|
+
except Exception as e:
|
|
714
|
+
return f"Error writing to session '{id}': {e}"
|
|
715
|
+
|
|
716
|
+
def shell_view(self, id: str) -> str:
|
|
717
|
+
r"""This function retrieves any new output from a non-blocking session
|
|
718
|
+
since the last time this function was called. If the process has
|
|
719
|
+
terminated, it drains the output queue and appends a termination
|
|
720
|
+
message. If the process is still running, it simply returns any
|
|
721
|
+
new output.
|
|
722
|
+
|
|
723
|
+
Args:
|
|
724
|
+
id (str): The unique session ID of the non-blocking process.
|
|
725
|
+
|
|
726
|
+
Returns:
|
|
727
|
+
str: The new output from the process's stdout and stderr. Returns
|
|
728
|
+
an empty string if there is no new output.
|
|
729
|
+
"""
|
|
730
|
+
with self._session_lock:
|
|
731
|
+
if id not in self.shell_sessions:
|
|
732
|
+
return f"Error: No session found with ID '{id}'."
|
|
733
|
+
session = self.shell_sessions[id]
|
|
734
|
+
is_running = session["running"]
|
|
735
|
+
|
|
736
|
+
# If session is terminated, drain the queue and return
|
|
737
|
+
# with a status message.
|
|
738
|
+
if not is_running:
|
|
739
|
+
final_output = []
|
|
740
|
+
try:
|
|
741
|
+
while True:
|
|
742
|
+
final_output.append(session["output_stream"].get_nowait())
|
|
743
|
+
except Empty:
|
|
744
|
+
pass
|
|
745
|
+
return "".join(final_output) + "\n--- SESSION TERMINATED ---"
|
|
746
|
+
|
|
747
|
+
# Otherwise, just drain the queue for a live session.
|
|
748
|
+
output = []
|
|
749
|
+
try:
|
|
750
|
+
while True:
|
|
751
|
+
output.append(session["output_stream"].get_nowait())
|
|
752
|
+
except Empty:
|
|
753
|
+
pass
|
|
754
|
+
|
|
755
|
+
return "".join(output)
|
|
756
|
+
|
|
757
|
+
def shell_wait(self, id: str, wait_seconds: float = 5.0) -> str:
|
|
758
|
+
r"""This function waits for a specified duration for a
|
|
759
|
+
non-blocking process to produce more output or terminate.
|
|
760
|
+
|
|
761
|
+
Args:
|
|
762
|
+
id (str): The unique session ID of the non-blocking process.
|
|
763
|
+
wait_seconds (float): The maximum number of seconds to wait.
|
|
764
|
+
|
|
765
|
+
Returns:
|
|
766
|
+
str: All output collected during the wait period.
|
|
767
|
+
"""
|
|
768
|
+
with self._session_lock:
|
|
769
|
+
if id not in self.shell_sessions:
|
|
770
|
+
return f"Error: No session found with ID '{id}'."
|
|
771
|
+
session = self.shell_sessions[id]
|
|
772
|
+
if not session["running"]:
|
|
773
|
+
return (
|
|
774
|
+
"Session is no longer running. "
|
|
775
|
+
"Use shell_view to get final output."
|
|
776
|
+
)
|
|
777
|
+
|
|
778
|
+
output_collected = []
|
|
779
|
+
end_time = time.time() + wait_seconds
|
|
780
|
+
while time.time() < end_time and session["running"]:
|
|
781
|
+
new_output = self.shell_view(id)
|
|
782
|
+
if new_output:
|
|
783
|
+
output_collected.append(new_output)
|
|
784
|
+
time.sleep(0.2)
|
|
785
|
+
|
|
786
|
+
return "".join(output_collected)
|
|
787
|
+
|
|
788
|
+
def shell_kill_process(self, id: str) -> str:
|
|
789
|
+
r"""This function forcibly terminates a running non-blocking process.
|
|
790
|
+
|
|
791
|
+
Args:
|
|
792
|
+
id (str): The unique session ID of the process to kill.
|
|
793
|
+
|
|
794
|
+
Returns:
|
|
795
|
+
str: A confirmation message indicating the process was terminated.
|
|
796
|
+
"""
|
|
797
|
+
with self._session_lock:
|
|
798
|
+
if (
|
|
799
|
+
id not in self.shell_sessions
|
|
800
|
+
or not self.shell_sessions[id]["running"]
|
|
801
|
+
):
|
|
802
|
+
return f"Error: No active session found with ID '{id}'."
|
|
803
|
+
session = self.shell_sessions[id]
|
|
804
|
+
try:
|
|
805
|
+
if session["backend"] == "local":
|
|
806
|
+
session["process"].terminate()
|
|
807
|
+
time.sleep(0.5)
|
|
808
|
+
if session["process"].poll() is None:
|
|
809
|
+
session["process"].kill()
|
|
810
|
+
# Ensure stdio streams are closed to unblock reader thread
|
|
811
|
+
try:
|
|
812
|
+
if getattr(session["process"], "stdin", None):
|
|
813
|
+
session["process"].stdin.close()
|
|
814
|
+
except Exception:
|
|
815
|
+
pass
|
|
816
|
+
try:
|
|
817
|
+
if getattr(session["process"], "stdout", None):
|
|
818
|
+
session["process"].stdout.close()
|
|
819
|
+
except Exception:
|
|
820
|
+
pass
|
|
821
|
+
else: # docker
|
|
822
|
+
# Docker exec processes stop when the socket is closed.
|
|
823
|
+
session["process"].close()
|
|
824
|
+
with self._session_lock:
|
|
825
|
+
if id in self.shell_sessions:
|
|
826
|
+
self.shell_sessions[id]["running"] = False
|
|
827
|
+
return f"Process in session '{id}' has been terminated."
|
|
828
|
+
except Exception as e:
|
|
829
|
+
return f"Error killing process in session '{id}': {e}"
|
|
830
|
+
|
|
831
|
+
def shell_ask_user_for_help(self, id: str, prompt: str) -> str:
|
|
832
|
+
r"""This function pauses execution and asks a human for help
|
|
833
|
+
with an interactive session.
|
|
834
|
+
|
|
835
|
+
This method can handle different scenarios:
|
|
836
|
+
1. If session exists: Shows session output and allows interaction
|
|
837
|
+
2. If session doesn't exist: Creates a temporary session for help
|
|
838
|
+
|
|
839
|
+
Args:
|
|
840
|
+
id (str): The session ID of the interactive process needing help.
|
|
841
|
+
Can be empty string for general help without session context.
|
|
842
|
+
prompt (str): The question or instruction from the LLM to show the
|
|
843
|
+
human user (e.g., "The program is asking for a filename. Please
|
|
844
|
+
enter 'config.json'.").
|
|
845
|
+
|
|
846
|
+
Returns:
|
|
847
|
+
str: The output from the shell session after the user's command has
|
|
848
|
+
been executed, or help information for general queries.
|
|
849
|
+
"""
|
|
850
|
+
logger.info("\n" + "=" * 60)
|
|
851
|
+
logger.info("🤖 LLM Agent needs your help!")
|
|
852
|
+
logger.info(f"PROMPT: {prompt}")
|
|
853
|
+
|
|
854
|
+
# Case 1: Session doesn't exist - offer to create one
|
|
855
|
+
if id not in self.shell_sessions:
|
|
856
|
+
try:
|
|
857
|
+
user_input = input("Your response: ").strip()
|
|
858
|
+
if not user_input:
|
|
859
|
+
return "No user response."
|
|
860
|
+
else:
|
|
861
|
+
logger.info(
|
|
862
|
+
f"Creating session '{id}' and executing command..."
|
|
863
|
+
)
|
|
864
|
+
result = self.shell_exec(id, user_input, block=True)
|
|
865
|
+
return (
|
|
866
|
+
f"Session '{id}' created and "
|
|
867
|
+
f"executed command:\n{result}"
|
|
868
|
+
)
|
|
869
|
+
except EOFError:
|
|
870
|
+
return f"User input interrupted for session '{id}' creation."
|
|
871
|
+
|
|
872
|
+
# Case 2: Session exists - show context and interact
|
|
873
|
+
else:
|
|
874
|
+
# Get the latest output to show the user the current state
|
|
875
|
+
last_output = self._collect_output_until_idle(id)
|
|
876
|
+
|
|
877
|
+
logger.info(f"SESSION: '{id}' (active)")
|
|
878
|
+
logger.info("=" * 60)
|
|
879
|
+
logger.info("--- LAST OUTPUT ---")
|
|
880
|
+
logger.info(
|
|
881
|
+
last_output.strip()
|
|
882
|
+
if last_output.strip()
|
|
883
|
+
else "(no recent output)"
|
|
884
|
+
)
|
|
885
|
+
logger.info("-------------------")
|
|
886
|
+
|
|
887
|
+
try:
|
|
888
|
+
user_input = input("Your input: ").strip()
|
|
889
|
+
if not user_input:
|
|
890
|
+
return f"User provided no input for session '{id}'."
|
|
891
|
+
else:
|
|
892
|
+
# Send input to the existing session
|
|
893
|
+
return self.shell_write_to_process(id, user_input)
|
|
894
|
+
except EOFError:
|
|
895
|
+
return f"User input interrupted for session '{id}'."
|
|
896
|
+
|
|
897
|
+
def __del__(self):
|
|
898
|
+
# Clean up any sessions
|
|
899
|
+
with self._session_lock:
|
|
900
|
+
session_ids = list(self.shell_sessions.keys())
|
|
901
|
+
for session_id in session_ids:
|
|
902
|
+
with self._session_lock:
|
|
903
|
+
is_running = self.shell_sessions.get(session_id, {}).get(
|
|
904
|
+
"running", False
|
|
905
|
+
)
|
|
906
|
+
if is_running:
|
|
907
|
+
self.shell_kill_process(session_id)
|
|
908
|
+
|
|
909
|
+
def get_tools(self) -> List[FunctionTool]:
|
|
910
|
+
r"""Returns a list of FunctionTool objects representing the functions
|
|
911
|
+
in the toolkit.
|
|
912
|
+
|
|
913
|
+
Returns:
|
|
914
|
+
List[FunctionTool]: A list of FunctionTool objects representing the
|
|
915
|
+
functions in the toolkit.
|
|
916
|
+
"""
|
|
917
|
+
return [
|
|
918
|
+
FunctionTool(self.shell_exec),
|
|
919
|
+
FunctionTool(self.shell_view),
|
|
920
|
+
FunctionTool(self.shell_wait),
|
|
921
|
+
FunctionTool(self.shell_write_to_process),
|
|
922
|
+
FunctionTool(self.shell_kill_process),
|
|
923
|
+
FunctionTool(self.shell_ask_user_for_help),
|
|
924
|
+
]
|