open-data-sci 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- open_data_sci-0.1.0.dist-info/METADATA +629 -0
- open_data_sci-0.1.0.dist-info/RECORD +85 -0
- open_data_sci-0.1.0.dist-info/WHEEL +4 -0
- open_data_sci-0.1.0.dist-info/entry_points.txt +2 -0
- open_data_sci-0.1.0.dist-info/licenses/LICENSE +201 -0
- opendatasci/__init__.py +47 -0
- opendatasci/_tui/__init__.py +1 -0
- opendatasci/_tui/adapter.py +102 -0
- opendatasci/_tui/app.py +429 -0
- opendatasci/_tui/commands.py +95 -0
- opendatasci/_tui/completion.py +139 -0
- opendatasci/_tui/controller.py +644 -0
- opendatasci/_tui/file_refs.py +153 -0
- opendatasci/_tui/models.py +4 -0
- opendatasci/_tui/presenter.py +259 -0
- opendatasci/_tui/service.py +78 -0
- opendatasci/_tui/session.py +53 -0
- opendatasci/_tui/styles.tcss +248 -0
- opendatasci/_tui/styles_visible.tcss +245 -0
- opendatasci/_tui/theme.py +113 -0
- opendatasci/_tui/tools_display.py +86 -0
- opendatasci/_tui/widgets.py +1001 -0
- opendatasci/_utils/__init__.py +0 -0
- opendatasci/_utils/async_utils.py +11 -0
- opendatasci/_utils/data_formats.py +135 -0
- opendatasci/_utils/hash_utils.py +52 -0
- opendatasci/_utils/langchain_utils.py +155 -0
- opendatasci/_utils/streaming_utils.py +23 -0
- opendatasci/agents/__init__.py +12 -0
- opendatasci/agents/agents.py +515 -0
- opendatasci/agents/agents_factory.py +71 -0
- opendatasci/agents/chat_memory.py +397 -0
- opendatasci/agents/graphs.py +84 -0
- opendatasci/agents/nodes.py +74 -0
- opendatasci/agents/states.py +36 -0
- opendatasci/agents/turn_memory.py +124 -0
- opendatasci/configs.py +275 -0
- opendatasci/context/__init__.py +7 -0
- opendatasci/context/base.py +56 -0
- opendatasci/context/local.py +236 -0
- opendatasci/models/__init__.py +7 -0
- opendatasci/models/anthropic.py +40 -0
- opendatasci/models/aws.py +86 -0
- opendatasci/models/factory.py +179 -0
- opendatasci/models/google.py +79 -0
- opendatasci/models/local.py +79 -0
- opendatasci/models/microsoft.py +62 -0
- opendatasci/models/openai.py +49 -0
- opendatasci/models/providers.py +12 -0
- opendatasci/prompts/__init__.py +5 -0
- opendatasci/prompts/builders.py +85 -0
- opendatasci/prompts/caching.py +42 -0
- opendatasci/prompts/message_templates.py +7 -0
- opendatasci/prompts/prompt_templates.py +227 -0
- opendatasci/resources/skills/competitive_data_science.md +241 -0
- opendatasci/resources/skills/data_science.md +55 -0
- opendatasci/resources/skills/data_science_education.md +42 -0
- opendatasci/resources/skills/deep_learning.md +205 -0
- opendatasci/resources/skills/machine_learning.md +68 -0
- opendatasci/resources/skills/quantitative_analysis.md +45 -0
- opendatasci/sandbox/__init__.py +14 -0
- opendatasci/sandbox/_runner.py +114 -0
- opendatasci/sandbox/base.py +170 -0
- opendatasci/sandbox/srt.py +490 -0
- opendatasci/skills/__init__.py +9 -0
- opendatasci/skills/base.py +28 -0
- opendatasci/skills/local.py +131 -0
- opendatasci/streaming/__init__.py +37 -0
- opendatasci/streaming/events.py +159 -0
- opendatasci/streaming/processors.py +387 -0
- opendatasci/tools/__init__.py +58 -0
- opendatasci/tools/coding.py +261 -0
- opendatasci/tools/critic.py +136 -0
- opendatasci/tools/dataset_info.py +391 -0
- opendatasci/tools/factory.py +172 -0
- opendatasci/tools/mcp.py +179 -0
- opendatasci/tools/planning.py +88 -0
- opendatasci/tools/skills.py +90 -0
- opendatasci/tools/user_interaction.py +54 -0
- opendatasci/tools/web.py +236 -0
- opendatasci/tools/workers.py +237 -0
- opendatasci/tools/workspace.py +55 -0
- opendatasci/workspace/__init__.py +9 -0
- opendatasci/workspace/base.py +20 -0
- opendatasci/workspace/local.py +25 -0
|
@@ -0,0 +1,490 @@
|
|
|
1
|
+
"""SRT-backed sandbox for OpenDataSci sessions.
|
|
2
|
+
|
|
3
|
+
Platform support: the underlying Sandbox Runtime only sandboxes on macOS and
|
|
4
|
+
Linux. Windows is unsupported: ``sandbox_runtime`` imports the Unix-only
|
|
5
|
+
``resource`` module at import time, so this module fails to import at all on
|
|
6
|
+
Windows; this class is therefore exercised only under mocks on such hosts.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import asyncio
|
|
10
|
+
import base64
|
|
11
|
+
import json
|
|
12
|
+
import logging
|
|
13
|
+
import os
|
|
14
|
+
import shlex
|
|
15
|
+
import shutil
|
|
16
|
+
import signal
|
|
17
|
+
import subprocess
|
|
18
|
+
import sys
|
|
19
|
+
import tempfile
|
|
20
|
+
import traceback
|
|
21
|
+
import warnings
|
|
22
|
+
from contextlib import asynccontextmanager
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
from typing import Any, AsyncIterator
|
|
25
|
+
|
|
26
|
+
from sandbox_runtime import SandboxManager, SandboxRuntimeConfig
|
|
27
|
+
from sandbox_runtime.utils.platform import get_platform
|
|
28
|
+
|
|
29
|
+
from opendatasci.sandbox.base import (
|
|
30
|
+
BaseSandbox,
|
|
31
|
+
BaseSandboxFactory,
|
|
32
|
+
SandboxExecResult,
|
|
33
|
+
validate_cli_command,
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
logger = logging.getLogger(__name__)
|
|
37
|
+
|
|
38
|
+
# Install commands per platform, used to build an actionable error message when
|
|
39
|
+
# the native sandbox binaries (bwrap/socat/ripgrep) are missing. ``pip install``
|
|
40
|
+
# cannot provide these — they must come from the OS package manager.
|
|
41
|
+
_INSTALL_HINTS: dict[str, str] = {
|
|
42
|
+
"macos": "brew install ripgrep",
|
|
43
|
+
"linux": (
|
|
44
|
+
"sudo apt-get install -y bubblewrap socat ripgrep # Debian/Ubuntu\n"
|
|
45
|
+
" sudo dnf install -y bubblewrap socat ripgrep # Fedora\n"
|
|
46
|
+
" sudo pacman -S --noconfirm bubblewrap socat ripgrep # Arch"
|
|
47
|
+
),
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def check_sandbox_dependencies() -> None:
|
|
52
|
+
"""Raise ``RuntimeError`` with actionable guidance if the sandbox cannot run.
|
|
53
|
+
|
|
54
|
+
Call this as early as possible (e.g. at agent construction) so a missing
|
|
55
|
+
system dependency surfaces immediately, rather than on the first sandboxed
|
|
56
|
+
code execution deep into a session.
|
|
57
|
+
"""
|
|
58
|
+
if SandboxManager.check_dependencies():
|
|
59
|
+
return
|
|
60
|
+
|
|
61
|
+
platform = get_platform()
|
|
62
|
+
if not SandboxManager.is_supported_platform(platform):
|
|
63
|
+
raise RuntimeError(
|
|
64
|
+
f"OpenDataSci's sandbox is not supported on platform '{platform}'. "
|
|
65
|
+
"Only macOS and Linux are supported."
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
install_hint = _INSTALL_HINTS.get(platform)
|
|
69
|
+
required = (
|
|
70
|
+
"ripgrep (rg)" if platform == "macos" else "ripgrep (rg), bubblewrap (bwrap), and socat"
|
|
71
|
+
)
|
|
72
|
+
message = f"Missing required sandbox dependencies for {platform}: {required}."
|
|
73
|
+
if install_hint:
|
|
74
|
+
message += f" Install with:\n {install_hint}"
|
|
75
|
+
raise RuntimeError(message)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
_RUNNER_SRC = Path(__file__).parent / "_runner.py"
|
|
79
|
+
|
|
80
|
+
# Sensitive host locations the sandbox must never expose to model-generated
|
|
81
|
+
# code. These are expanded to absolute, symlink-resolved paths before being
|
|
82
|
+
# handed to SRT, which resolves deny rules relative to the workspace cwd and
|
|
83
|
+
# does *not* expand ``~`` itself.
|
|
84
|
+
_SENSITIVE_READ_PATHS: tuple[str, ...] = (
|
|
85
|
+
"~/.ssh",
|
|
86
|
+
"~/.aws",
|
|
87
|
+
"~/.gnupg",
|
|
88
|
+
"~/.config/gcloud",
|
|
89
|
+
"~/.kube",
|
|
90
|
+
"~/.docker",
|
|
91
|
+
"~/.netrc",
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
# Allowlist of host environment variables propagated into the sandboxed
|
|
95
|
+
# subprocess. The host env carries secrets (API keys, DB URLs, cloud creds);
|
|
96
|
+
# model-generated code runs via ``exec`` inside the runner and could otherwise
|
|
97
|
+
# read and exfiltrate them through the workspace. Only variables needed for the
|
|
98
|
+
# interpreter, the sandbox wrapper (bwrap/sandbox-exec), and locale/temp
|
|
99
|
+
# resolution are forwarded.
|
|
100
|
+
_ENV_PASSTHROUGH: tuple[str, ...] = (
|
|
101
|
+
# POSIX essentials
|
|
102
|
+
"PATH",
|
|
103
|
+
"HOME",
|
|
104
|
+
"USER",
|
|
105
|
+
"LOGNAME",
|
|
106
|
+
"SHELL",
|
|
107
|
+
"LANG",
|
|
108
|
+
"LANGUAGE",
|
|
109
|
+
"LC_ALL",
|
|
110
|
+
"LC_CTYPE",
|
|
111
|
+
"TZ",
|
|
112
|
+
"TERM",
|
|
113
|
+
"TMPDIR",
|
|
114
|
+
"TEMP",
|
|
115
|
+
"TMP",
|
|
116
|
+
# Windows essentials (mock/dev hosts only)
|
|
117
|
+
"SYSTEMROOT",
|
|
118
|
+
"SYSTEMDRIVE",
|
|
119
|
+
"WINDIR",
|
|
120
|
+
"PATHEXT",
|
|
121
|
+
"COMSPEC",
|
|
122
|
+
"USERPROFILE",
|
|
123
|
+
"PROCESSOR_ARCHITECTURE",
|
|
124
|
+
"NUMBER_OF_PROCESSORS",
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
# ---------------------------------------------------------------------------
|
|
129
|
+
# Process-global manager lifecycle
|
|
130
|
+
# ---------------------------------------------------------------------------
|
|
131
|
+
#
|
|
132
|
+
# ``SandboxManager`` is module-level singleton state inside ``sandbox_runtime``:
|
|
133
|
+
# the network proxies and (on Linux) the network bridge are shared by every
|
|
134
|
+
# sandbox in the process. It must therefore be initialized exactly once per
|
|
135
|
+
# process and torn down only at process exit — a job the library already
|
|
136
|
+
# performs via its own ``atexit``/signal handlers. An individual session-scoped
|
|
137
|
+
# sandbox must never call ``SandboxManager.reset()``, or it would rip the shared
|
|
138
|
+
# infrastructure out from under its concurrently running siblings.
|
|
139
|
+
_manager_lock = asyncio.Lock()
|
|
140
|
+
_manager_initialized = False
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
async def _ensure_manager_initialized(config: SandboxRuntimeConfig) -> None:
|
|
144
|
+
"""Initialize the global ``SandboxManager`` once per process (idempotent)."""
|
|
145
|
+
global _manager_initialized
|
|
146
|
+
if _manager_initialized:
|
|
147
|
+
return
|
|
148
|
+
async with _manager_lock:
|
|
149
|
+
if _manager_initialized:
|
|
150
|
+
return
|
|
151
|
+
await SandboxManager.initialize(config)
|
|
152
|
+
_manager_initialized = True
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _base_sandbox_env() -> dict[str, str]:
|
|
156
|
+
"""Return a minimal, allowlisted copy of the host environment."""
|
|
157
|
+
return {key: os.environ[key] for key in _ENV_PASSTHROUGH if key in os.environ}
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
class SRTSandbox(BaseSandbox):
|
|
161
|
+
"""Session-scoped sandbox powered by Anthropic's Sandbox Runtime (SRT).
|
|
162
|
+
|
|
163
|
+
Executes Python snippets and allowlisted TUI commands in an OS-level
|
|
164
|
+
sandbox — no Docker or remote container is required. Python state
|
|
165
|
+
(variables, results) is preserved across calls within the same instance.
|
|
166
|
+
|
|
167
|
+
Each :meth:`execute`/:meth:`execute_cli`/:meth:`reset` call is serialized by
|
|
168
|
+
a per-instance lock so overlapping calls cannot interleave their
|
|
169
|
+
read-modify-write of the on-disk ``state.pkl``.
|
|
170
|
+
|
|
171
|
+
Note: every :meth:`execute` spins up a fresh interpreter that unpickles the
|
|
172
|
+
entire namespace, runs, and re-pickles it. This keeps executions hermetic
|
|
173
|
+
but makes cost O(state) per call; a persistent-kernel runner would remove
|
|
174
|
+
that overhead and is the natural next step for long interactive sessions.
|
|
175
|
+
"""
|
|
176
|
+
|
|
177
|
+
def __init__(
|
|
178
|
+
self,
|
|
179
|
+
workspace_path: Path | None = None,
|
|
180
|
+
*,
|
|
181
|
+
command_timeout: int | None = None,
|
|
182
|
+
) -> None:
|
|
183
|
+
self._workspace_path = workspace_path
|
|
184
|
+
self._command_timeout = command_timeout if command_timeout is not None else 1800
|
|
185
|
+
|
|
186
|
+
self._session_dir = Path(tempfile.mkdtemp(prefix="opendatasci_srt_"))
|
|
187
|
+
self._state_path = self._session_dir / "state.pkl"
|
|
188
|
+
self._runner_path = self._session_dir / "runner.py"
|
|
189
|
+
|
|
190
|
+
self._history: list[SandboxExecResult] = []
|
|
191
|
+
self._results: dict[str, str] = {}
|
|
192
|
+
self._var_info: dict[str, str] = {}
|
|
193
|
+
self._sandbox_config: SandboxRuntimeConfig | None = None
|
|
194
|
+
self._initialized = False
|
|
195
|
+
# Set by reset(); consumed under _lock at the start of the next execute
|
|
196
|
+
# so the on-disk wipe happens inside the serialized critical section.
|
|
197
|
+
self._reset_pending = False
|
|
198
|
+
self._lock = asyncio.Lock()
|
|
199
|
+
|
|
200
|
+
# ------------------------------------------------------------------
|
|
201
|
+
# Sandbox protocol
|
|
202
|
+
# ------------------------------------------------------------------
|
|
203
|
+
|
|
204
|
+
async def execute(self, code: str) -> SandboxExecResult:
|
|
205
|
+
async with self._lock:
|
|
206
|
+
try:
|
|
207
|
+
await self._ensure_initialized()
|
|
208
|
+
|
|
209
|
+
if self._reset_pending:
|
|
210
|
+
# Deleting the state file clears both variables and saved
|
|
211
|
+
# results (the latter live inside the pickle under
|
|
212
|
+
# RESULTS_KEY), so a single unlink is a complete wipe.
|
|
213
|
+
self._state_path.unlink(missing_ok=True)
|
|
214
|
+
self._reset_pending = False
|
|
215
|
+
|
|
216
|
+
workspace = str(self._workspace_path or self._session_dir)
|
|
217
|
+
env = {
|
|
218
|
+
**_base_sandbox_env(),
|
|
219
|
+
"OPENDATASCI_CODE_B64": base64.b64encode(code.encode("utf-8")).decode("ascii"),
|
|
220
|
+
"OPENDATASCI_STATE_PATH": str(self._state_path),
|
|
221
|
+
"OPENDATASCI_WORKSPACE": workspace,
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
command = f"{shlex.quote(sys.executable)} {shlex.quote(str(self._runner_path))}"
|
|
225
|
+
wrapped = await SandboxManager.wrap_with_sandbox(
|
|
226
|
+
command, custom_config=self._make_config()
|
|
227
|
+
)
|
|
228
|
+
stdout_str, stderr_str, _ = await self._run_subprocess(
|
|
229
|
+
wrapped, env=env, cwd=workspace
|
|
230
|
+
)
|
|
231
|
+
|
|
232
|
+
payload = self._parse_runner_payload(stdout_str, stderr_str)
|
|
233
|
+
self._var_info.update(payload.get("var_info", {}))
|
|
234
|
+
self._results.update(payload.get("saved_results", {}))
|
|
235
|
+
|
|
236
|
+
stdout = payload.get("stdout", "")
|
|
237
|
+
dropped_vars = payload.get("dropped_vars", [])
|
|
238
|
+
if dropped_vars:
|
|
239
|
+
warning = f"Warning: variable(s) not persisted (not picklable): {', '.join(dropped_vars)}"
|
|
240
|
+
stdout = f"{stdout}\n{warning}" if stdout else warning
|
|
241
|
+
|
|
242
|
+
if payload.get("success"):
|
|
243
|
+
result = SandboxExecResult(
|
|
244
|
+
success=True,
|
|
245
|
+
output=payload.get("result"),
|
|
246
|
+
stdout=stdout,
|
|
247
|
+
code=code,
|
|
248
|
+
)
|
|
249
|
+
else:
|
|
250
|
+
result = SandboxExecResult(
|
|
251
|
+
success=False,
|
|
252
|
+
error=payload.get("error", "Unknown execution error"),
|
|
253
|
+
stdout=stdout,
|
|
254
|
+
code=code,
|
|
255
|
+
)
|
|
256
|
+
except TimeoutError:
|
|
257
|
+
result = self._fail(
|
|
258
|
+
code,
|
|
259
|
+
f"TimeoutError: execution timed out after {self._command_timeout}s",
|
|
260
|
+
)
|
|
261
|
+
except Exception as exc:
|
|
262
|
+
result = self._fail(code, f"SRTError: {exc}\n{traceback.format_exc()}")
|
|
263
|
+
|
|
264
|
+
self._history.append(result)
|
|
265
|
+
return result
|
|
266
|
+
|
|
267
|
+
async def execute_cli(self, command: str) -> SandboxExecResult:
|
|
268
|
+
error = validate_cli_command(command)
|
|
269
|
+
if error:
|
|
270
|
+
result = self._fail(command, f"Error: {error}")
|
|
271
|
+
self._history.append(result)
|
|
272
|
+
return result
|
|
273
|
+
|
|
274
|
+
async with self._lock:
|
|
275
|
+
try:
|
|
276
|
+
await self._ensure_initialized()
|
|
277
|
+
|
|
278
|
+
workspace = str(self._workspace_path or self._session_dir)
|
|
279
|
+
wrapped = await SandboxManager.wrap_with_sandbox(
|
|
280
|
+
command, custom_config=self._make_config()
|
|
281
|
+
)
|
|
282
|
+
stdout_str, stderr_str, exit_code = await self._run_subprocess(
|
|
283
|
+
wrapped, env=_base_sandbox_env(), cwd=workspace
|
|
284
|
+
)
|
|
285
|
+
|
|
286
|
+
combined = "\n".join(filter(None, [stdout_str, stderr_str]))
|
|
287
|
+
success = exit_code == 0
|
|
288
|
+
result = SandboxExecResult(
|
|
289
|
+
success=success,
|
|
290
|
+
stdout=combined,
|
|
291
|
+
error=None if success else f"Command failed (exit {exit_code})",
|
|
292
|
+
code=command,
|
|
293
|
+
)
|
|
294
|
+
except TimeoutError:
|
|
295
|
+
result = self._fail(
|
|
296
|
+
command,
|
|
297
|
+
f"TimeoutError: command timed out after {self._command_timeout}s",
|
|
298
|
+
)
|
|
299
|
+
except Exception as exc:
|
|
300
|
+
result = self._fail(command, f"SRTCLIError: {exc}")
|
|
301
|
+
|
|
302
|
+
self._history.append(result)
|
|
303
|
+
return result
|
|
304
|
+
|
|
305
|
+
def get_history(self) -> list[SandboxExecResult]:
|
|
306
|
+
return list(self._history)
|
|
307
|
+
|
|
308
|
+
def reset(self) -> None:
|
|
309
|
+
# Clear the in-memory views eagerly; defer the on-disk state wipe to the
|
|
310
|
+
# next execute so it runs inside the serialized critical section (and
|
|
311
|
+
# cannot clobber an in-flight execution's pickle write). The two views
|
|
312
|
+
# only diverge in the window before the next execute, which itself
|
|
313
|
+
# reconciles them — no public read path observes the difference.
|
|
314
|
+
self._history.clear()
|
|
315
|
+
self._var_info.clear()
|
|
316
|
+
self._results.clear()
|
|
317
|
+
self._reset_pending = True
|
|
318
|
+
|
|
319
|
+
async def close(self) -> None:
|
|
320
|
+
# The SandboxManager is a process-global singleton shared with every
|
|
321
|
+
# concurrent sibling sandbox; tearing it down here would break them.
|
|
322
|
+
# Its own atexit/signal handlers perform the single process-level
|
|
323
|
+
# teardown. We own only our session directory.
|
|
324
|
+
self._initialized = False
|
|
325
|
+
shutil.rmtree(self._session_dir, ignore_errors=True)
|
|
326
|
+
|
|
327
|
+
# ------------------------------------------------------------------
|
|
328
|
+
# Internal helpers
|
|
329
|
+
# ------------------------------------------------------------------
|
|
330
|
+
|
|
331
|
+
async def _ensure_initialized(self) -> None:
|
|
332
|
+
# Caller holds ``self._lock``, so the per-instance bookkeeping below
|
|
333
|
+
# (runner copy + flag) cannot race a concurrent first call.
|
|
334
|
+
if self._initialized:
|
|
335
|
+
return
|
|
336
|
+
await _ensure_manager_initialized(self._make_config())
|
|
337
|
+
shutil.copy2(_RUNNER_SRC, self._runner_path)
|
|
338
|
+
self._initialized = True
|
|
339
|
+
|
|
340
|
+
def _make_config(self) -> SandboxRuntimeConfig:
|
|
341
|
+
if self._sandbox_config is None:
|
|
342
|
+
workspace = str(self._workspace_path or self._session_dir)
|
|
343
|
+
deny_read = [
|
|
344
|
+
os.path.realpath(os.path.expanduser(path)) for path in _SENSITIVE_READ_PATHS
|
|
345
|
+
]
|
|
346
|
+
self._sandbox_config = SandboxRuntimeConfig(
|
|
347
|
+
network={"allowed_domains": [], "denied_domains": []},
|
|
348
|
+
filesystem={
|
|
349
|
+
"deny_read": deny_read,
|
|
350
|
+
"allow_write": [workspace, str(self._session_dir)],
|
|
351
|
+
"deny_write": [],
|
|
352
|
+
},
|
|
353
|
+
)
|
|
354
|
+
return self._sandbox_config
|
|
355
|
+
|
|
356
|
+
async def _run_subprocess(
|
|
357
|
+
self, command: str, env: dict[str, str], cwd: str
|
|
358
|
+
) -> tuple[str, str, int]:
|
|
359
|
+
# Launch the wrapped command in its own process group/session so a
|
|
360
|
+
# timeout can signal the *entire* tree (shell → bwrap/sandbox-exec →
|
|
361
|
+
# python), not just the top-level shell, which would otherwise leak the
|
|
362
|
+
# sandbox and its python child as orphans.
|
|
363
|
+
spawn_kwargs: dict[str, Any]
|
|
364
|
+
if sys.platform != "win32":
|
|
365
|
+
spawn_kwargs = {"start_new_session": True}
|
|
366
|
+
else:
|
|
367
|
+
spawn_kwargs = {"creationflags": subprocess.CREATE_NEW_PROCESS_GROUP}
|
|
368
|
+
|
|
369
|
+
proc: asyncio.subprocess.Process | None = None
|
|
370
|
+
try:
|
|
371
|
+
proc = await asyncio.create_subprocess_shell(
|
|
372
|
+
command,
|
|
373
|
+
stdout=asyncio.subprocess.PIPE,
|
|
374
|
+
stderr=asyncio.subprocess.PIPE,
|
|
375
|
+
env=env,
|
|
376
|
+
cwd=cwd,
|
|
377
|
+
**spawn_kwargs,
|
|
378
|
+
)
|
|
379
|
+
stdout_bytes, stderr_bytes = await asyncio.wait_for(
|
|
380
|
+
proc.communicate(),
|
|
381
|
+
timeout=self._command_timeout,
|
|
382
|
+
)
|
|
383
|
+
returncode = proc.returncode
|
|
384
|
+
if returncode is None:
|
|
385
|
+
# Should not happen after communicate(); surface it as a failure
|
|
386
|
+
# rather than masking it as success (exit 0).
|
|
387
|
+
logger.warning(
|
|
388
|
+
"Subprocess returncode is None after communicate(); treating as failure"
|
|
389
|
+
)
|
|
390
|
+
returncode = -1
|
|
391
|
+
return (
|
|
392
|
+
stdout_bytes.decode("utf-8", errors="replace").strip(),
|
|
393
|
+
stderr_bytes.decode("utf-8", errors="replace").strip(),
|
|
394
|
+
returncode,
|
|
395
|
+
)
|
|
396
|
+
except asyncio.TimeoutError:
|
|
397
|
+
if proc is not None:
|
|
398
|
+
await self._terminate_process_tree(proc)
|
|
399
|
+
raise TimeoutError(f"Command timed out after {self._command_timeout}s: {command!r}")
|
|
400
|
+
|
|
401
|
+
async def _terminate_process_tree(self, proc: asyncio.subprocess.Process) -> None:
|
|
402
|
+
"""Kill the subprocess's whole group and reap it, so no orphans or
|
|
403
|
+
unreaped transports remain."""
|
|
404
|
+
try:
|
|
405
|
+
if sys.platform != "win32":
|
|
406
|
+
os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
|
|
407
|
+
else:
|
|
408
|
+
proc.kill()
|
|
409
|
+
except (ProcessLookupError, PermissionError):
|
|
410
|
+
pass
|
|
411
|
+
try:
|
|
412
|
+
await proc.wait()
|
|
413
|
+
except Exception:
|
|
414
|
+
logger.exception("Failed to reap timed-out subprocess")
|
|
415
|
+
|
|
416
|
+
def _parse_runner_payload(self, raw_stdout: str, raw_stderr: str = "") -> dict[str, Any]:
|
|
417
|
+
# The runner emits its result as a single trailing JSON line, so we scan
|
|
418
|
+
# bottom-up: this survives arbitrary user ``print()`` output captured
|
|
419
|
+
# above it. (A user subprocess writing raw bytes directly to fd 1 *after*
|
|
420
|
+
# the payload line could still corrupt parsing — an accepted edge case.)
|
|
421
|
+
if not raw_stdout:
|
|
422
|
+
detail = f"stderr: {raw_stderr}" if raw_stderr else "no output"
|
|
423
|
+
raise ValueError(f"SRT runner returned no stdout payload ({detail}).")
|
|
424
|
+
|
|
425
|
+
for line in reversed(raw_stdout.splitlines()):
|
|
426
|
+
line = line.strip()
|
|
427
|
+
if not line:
|
|
428
|
+
continue
|
|
429
|
+
try:
|
|
430
|
+
parsed = json.loads(line)
|
|
431
|
+
except json.JSONDecodeError:
|
|
432
|
+
continue
|
|
433
|
+
if isinstance(parsed, dict):
|
|
434
|
+
return parsed
|
|
435
|
+
|
|
436
|
+
raise ValueError(f"SRT runner output was not JSON: {raw_stdout}")
|
|
437
|
+
|
|
438
|
+
def _fail(self, code: str, error: str) -> SandboxExecResult:
|
|
439
|
+
return SandboxExecResult(success=False, error=error, stdout="", code=code)
|
|
440
|
+
|
|
441
|
+
def __del__(self) -> None:
|
|
442
|
+
# Warn on un-closed sandboxes, but always reclaim the temp dir — it is
|
|
443
|
+
# allocated unconditionally in __init__, so even a sandbox that was never
|
|
444
|
+
# run (or never closed) must not leak it.
|
|
445
|
+
if getattr(self, "_initialized", False):
|
|
446
|
+
warnings.warn(
|
|
447
|
+
f"{self.__class__.__name__} was not properly closed; always use SRTSandboxFactory as a context manager",
|
|
448
|
+
ResourceWarning,
|
|
449
|
+
source=self,
|
|
450
|
+
)
|
|
451
|
+
session_dir = getattr(self, "_session_dir", None)
|
|
452
|
+
if session_dir is not None:
|
|
453
|
+
shutil.rmtree(session_dir, ignore_errors=True)
|
|
454
|
+
|
|
455
|
+
|
|
456
|
+
class SRTSandboxFactory(BaseSandboxFactory):
|
|
457
|
+
"""Factory that creates :class:`SRTSandbox` instances as async context managers.
|
|
458
|
+
|
|
459
|
+
Usage::
|
|
460
|
+
|
|
461
|
+
factory = SRTSandboxFactory()
|
|
462
|
+
async with factory.create(workspace_path=path) as sandbox:
|
|
463
|
+
result = await sandbox.execute(code)
|
|
464
|
+
# sandbox is closed here
|
|
465
|
+
|
|
466
|
+
Args:
|
|
467
|
+
command_timeout: Maximum seconds a single sandbox command may run
|
|
468
|
+
before being killed. Forwarded verbatim to every
|
|
469
|
+
:class:`SRTSandbox` created by :meth:`create`.
|
|
470
|
+
|
|
471
|
+
Raises:
|
|
472
|
+
RuntimeError: From :meth:`create`, if the host is missing a required
|
|
473
|
+
native sandbox dependency (e.g. bubblewrap/socat on Linux,
|
|
474
|
+
ripgrep on macOS) or the platform is unsupported (e.g. Windows).
|
|
475
|
+
"""
|
|
476
|
+
|
|
477
|
+
def __init__(self, *, command_timeout: int | None = None) -> None:
|
|
478
|
+
self._command_timeout = command_timeout
|
|
479
|
+
|
|
480
|
+
@asynccontextmanager
|
|
481
|
+
async def create(self, workspace_path: Path | None = None) -> AsyncIterator[SRTSandbox]:
|
|
482
|
+
check_sandbox_dependencies()
|
|
483
|
+
sandbox = SRTSandbox(
|
|
484
|
+
workspace_path=workspace_path,
|
|
485
|
+
command_timeout=self._command_timeout,
|
|
486
|
+
)
|
|
487
|
+
try:
|
|
488
|
+
yield sandbox
|
|
489
|
+
finally:
|
|
490
|
+
await sandbox.close()
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
from abc import ABC, abstractmethod
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
@dataclass(frozen=True)
|
|
6
|
+
class Skill:
|
|
7
|
+
"""A named prompt extension that specialises the agent for a domain.
|
|
8
|
+
|
|
9
|
+
Attributes:
|
|
10
|
+
name: Unique skill identifier (e.g. ``"machine_learning"``).
|
|
11
|
+
content: The prompt text injected into the agent's system prompt when
|
|
12
|
+
this skill is active.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
name: str
|
|
16
|
+
content: str
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class BaseSkillStore(ABC):
|
|
20
|
+
"""Registry of named skills available to the agent."""
|
|
21
|
+
|
|
22
|
+
@abstractmethod
|
|
23
|
+
def load(self, name: str) -> Skill | None:
|
|
24
|
+
"""Return the :class:`Skill` for *name*, or ``None`` if not found."""
|
|
25
|
+
|
|
26
|
+
@abstractmethod
|
|
27
|
+
def list(self) -> dict[str, Skill]:
|
|
28
|
+
"""Return all available skills keyed by name."""
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import logging
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
from opendatasci.skills.base import BaseSkillStore, Skill
|
|
6
|
+
|
|
7
|
+
logger = logging.getLogger(__name__)
|
|
8
|
+
|
|
9
|
+
_BUILTIN_SKILLS_DIRECTORY = Path(__file__).resolve().parents[1] / "resources" / "skills"
|
|
10
|
+
|
|
11
|
+
_BUILTIN_NAMES = [
|
|
12
|
+
"data_science",
|
|
13
|
+
"competitive_data_science",
|
|
14
|
+
"machine_learning",
|
|
15
|
+
"deep_learning",
|
|
16
|
+
"quantitative_analysis",
|
|
17
|
+
"data_science_education",
|
|
18
|
+
]
|
|
19
|
+
|
|
20
|
+
SKILL_LABELS: dict[str, str] = {
|
|
21
|
+
"data_science": "Data Scientist",
|
|
22
|
+
"competitive_data_science": "Competitive Data Scientist",
|
|
23
|
+
"machine_learning": "Machine Learning Eng.",
|
|
24
|
+
"quantitative_analysis": "Quantitative Analyst",
|
|
25
|
+
"data_science_education": "Data Science Educator",
|
|
26
|
+
"deep_learning": "Deep Learning Eng.",
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class LocalSkillStore(BaseSkillStore):
|
|
31
|
+
"""Loads skills from one or more local filesystem directories.
|
|
32
|
+
|
|
33
|
+
Directories are scanned in order; later directories override earlier ones when
|
|
34
|
+
skill names clash. Each directory may contain:
|
|
35
|
+
|
|
36
|
+
- ``.md`` files — loaded directly (filename stem used as skill name).
|
|
37
|
+
- ``.yaml`` / ``.yml`` / ``.json`` files — parsed for ``name`` and ``prompt`` keys.
|
|
38
|
+
|
|
39
|
+
When *paths* is ``None``, only the built-in skills directory is scanned.
|
|
40
|
+
|
|
41
|
+
Args:
|
|
42
|
+
paths: Ordered list of directories to scan. ``None`` loads only the
|
|
43
|
+
built-in skills bundled with the package.
|
|
44
|
+
strict: When ``True``, raise ``ValueError``
|
|
45
|
+
instead of warning if any structured skill file cannot be parsed.
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
def __init__(
|
|
49
|
+
self,
|
|
50
|
+
paths: list[Path] | None = None,
|
|
51
|
+
*,
|
|
52
|
+
strict: bool = True,
|
|
53
|
+
) -> None:
|
|
54
|
+
self._paths: list[Path] = paths if paths is not None else [_BUILTIN_SKILLS_DIRECTORY]
|
|
55
|
+
self._strict = strict
|
|
56
|
+
|
|
57
|
+
def load(self, name: str) -> Skill | None:
|
|
58
|
+
return self.list().get(name)
|
|
59
|
+
|
|
60
|
+
def list(self) -> dict[str, Skill]:
|
|
61
|
+
result: dict[str, Skill] = {}
|
|
62
|
+
for d in self._paths:
|
|
63
|
+
result.update(self._load_from_dir(d))
|
|
64
|
+
return result
|
|
65
|
+
|
|
66
|
+
def load_user_defined(self) -> dict[str, Skill]:
|
|
67
|
+
"""Return skills from all directories except the built-in skills directory."""
|
|
68
|
+
result: dict[str, Skill] = {}
|
|
69
|
+
for d in self._paths:
|
|
70
|
+
if d != _BUILTIN_SKILLS_DIRECTORY:
|
|
71
|
+
result.update(self._load_from_dir(d))
|
|
72
|
+
return result
|
|
73
|
+
|
|
74
|
+
def _load_from_dir(self, path: Path) -> dict[str, Skill]:
|
|
75
|
+
if not path.is_dir():
|
|
76
|
+
return {}
|
|
77
|
+
|
|
78
|
+
result: dict[str, Skill] = {}
|
|
79
|
+
failures: list[tuple[Path, str]] = []
|
|
80
|
+
|
|
81
|
+
for file in sorted(path.iterdir()):
|
|
82
|
+
if file.suffix == ".md":
|
|
83
|
+
result[file.stem] = Skill(name=file.stem, content=file.read_text(encoding="utf-8"))
|
|
84
|
+
continue
|
|
85
|
+
|
|
86
|
+
if file.suffix not in {".yaml", ".yml", ".json"}:
|
|
87
|
+
continue
|
|
88
|
+
|
|
89
|
+
try:
|
|
90
|
+
if file.suffix == ".json":
|
|
91
|
+
data = json.loads(file.read_text(encoding="utf-8"))
|
|
92
|
+
else:
|
|
93
|
+
try:
|
|
94
|
+
import yaml # type: ignore[import-untyped]
|
|
95
|
+
|
|
96
|
+
data = yaml.safe_load(file.read_text(encoding="utf-8"))
|
|
97
|
+
except ImportError:
|
|
98
|
+
data = json.loads(file.read_text(encoding="utf-8"))
|
|
99
|
+
except Exception as exc: # noqa: BLE001
|
|
100
|
+
failures.append((file, f"parse error: {exc}"))
|
|
101
|
+
continue
|
|
102
|
+
|
|
103
|
+
if not isinstance(data, dict):
|
|
104
|
+
failures.append(
|
|
105
|
+
(file, f"expected a mapping at the top level, got {type(data).__name__}")
|
|
106
|
+
)
|
|
107
|
+
continue
|
|
108
|
+
|
|
109
|
+
name = data.get("name")
|
|
110
|
+
prompt = data.get("prompt")
|
|
111
|
+
missing = [k for k, v in (("name", name), ("prompt", prompt)) if not v]
|
|
112
|
+
if missing:
|
|
113
|
+
failures.append((file, f"missing required key(s): {', '.join(missing)}"))
|
|
114
|
+
continue
|
|
115
|
+
|
|
116
|
+
result[str(name)] = Skill(name=str(name), content=str(prompt))
|
|
117
|
+
|
|
118
|
+
if failures:
|
|
119
|
+
summary = "; ".join(f"{f.name}: {reason}" for f, reason in failures)
|
|
120
|
+
if self._strict:
|
|
121
|
+
raise ValueError(
|
|
122
|
+
f"{len(failures)} skill file(s) in '{path}' could not be loaded: {summary}"
|
|
123
|
+
)
|
|
124
|
+
logger.warning(
|
|
125
|
+
"%d skill file(s) in '%s' could not be loaded: %s",
|
|
126
|
+
len(failures),
|
|
127
|
+
path,
|
|
128
|
+
summary,
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
return result
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
from opendatasci._utils.streaming_utils import format_stream_error
|
|
2
|
+
from opendatasci.streaming.events import (
|
|
3
|
+
AgentStreamEvent,
|
|
4
|
+
BaseAgentStreamEvent,
|
|
5
|
+
ErrorEvent,
|
|
6
|
+
InputRequiredEvent,
|
|
7
|
+
MessageEvent,
|
|
8
|
+
ReasoningEvent,
|
|
9
|
+
ResponseEvent,
|
|
10
|
+
SubagentEvent,
|
|
11
|
+
TokenEvent,
|
|
12
|
+
ToolCallEvent,
|
|
13
|
+
ToolCommunicationEvent,
|
|
14
|
+
ToolResultEvent,
|
|
15
|
+
UsageEvent,
|
|
16
|
+
WorkerDoneEvent,
|
|
17
|
+
)
|
|
18
|
+
from opendatasci.streaming.processors import AgentTurnStreamProcessor
|
|
19
|
+
|
|
20
|
+
__all__ = [
|
|
21
|
+
"AgentStreamEvent",
|
|
22
|
+
"AgentTurnStreamProcessor",
|
|
23
|
+
"format_stream_error",
|
|
24
|
+
"BaseAgentStreamEvent",
|
|
25
|
+
"ErrorEvent",
|
|
26
|
+
"InputRequiredEvent",
|
|
27
|
+
"MessageEvent",
|
|
28
|
+
"ReasoningEvent",
|
|
29
|
+
"ResponseEvent",
|
|
30
|
+
"SubagentEvent",
|
|
31
|
+
"TokenEvent",
|
|
32
|
+
"ToolCallEvent",
|
|
33
|
+
"ToolCommunicationEvent",
|
|
34
|
+
"ToolResultEvent",
|
|
35
|
+
"UsageEvent",
|
|
36
|
+
"WorkerDoneEvent",
|
|
37
|
+
]
|