snowflake-sandbox-python 0.2.1a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- snowflake/cli_sandbox/__init__.py +13 -0
- snowflake/cli_sandbox/_adapter.py +170 -0
- snowflake/cli_sandbox/_common.py +77 -0
- snowflake/cli_sandbox/_egress_flags.py +121 -0
- snowflake/cli_sandbox/_get_command.py +109 -0
- snowflake/cli_sandbox/_run_command.py +1091 -0
- snowflake/cli_sandbox/_shell_command.py +666 -0
- snowflake/cli_sandbox/_upload_plan.py +187 -0
- snowflake/cli_sandbox/commands.py +556 -0
- snowflake/cli_sandbox/plugin_spec.py +28 -0
- snowflake/cli_sandbox/py.typed +0 -0
- snowflake/sandbox/__init__.py +317 -0
- snowflake/sandbox/__main__.py +225 -0
- snowflake/sandbox/_ansi.py +206 -0
- snowflake/sandbox/_args.py +208 -0
- snowflake/sandbox/_assemble.py +256 -0
- snowflake/sandbox/_bundle.py +240 -0
- snowflake/sandbox/_connection_resolve.py +328 -0
- snowflake/sandbox/_deploy_spec.py +56 -0
- snowflake/sandbox/_diagnostics.py +501 -0
- snowflake/sandbox/_env.py +143 -0
- snowflake/sandbox/_files_mixin.py +280 -0
- snowflake/sandbox/_fs_ops.py +304 -0
- snowflake/sandbox/_globs.py +176 -0
- snowflake/sandbox/_hosts.py +110 -0
- snowflake/sandbox/_mcp_discovery.py +288 -0
- snowflake/sandbox/_mcp_status.py +183 -0
- snowflake/sandbox/_retry.py +94 -0
- snowflake/sandbox/_runtime/__init__.py +42 -0
- snowflake/sandbox/_runtime/_fs_helper.py +93 -0
- snowflake/sandbox/_runtime/_job_runner.py +111 -0
- snowflake/sandbox/_runtime/_protocol.py +53 -0
- snowflake/sandbox/_runtime/_shims.py +267 -0
- snowflake/sandbox/_sandbox_state.py +303 -0
- snowflake/sandbox/_session_registry.py +222 -0
- snowflake/sandbox/_sse.py +160 -0
- snowflake/sandbox/_stage.py +270 -0
- snowflake/sandbox/_sync_files_mixin.py +272 -0
- snowflake/sandbox/_sync_fs_ops.py +185 -0
- snowflake/sandbox/_sync_transport.py +737 -0
- snowflake/sandbox/_sync_watch.py +99 -0
- snowflake/sandbox/_transport.py +1366 -0
- snowflake/sandbox/_transport_errors.py +270 -0
- snowflake/sandbox/_upload_plan.py +497 -0
- snowflake/sandbox/_version.py +37 -0
- snowflake/sandbox/_watch.py +164 -0
- snowflake/sandbox/_wire.py +348 -0
- snowflake/sandbox/app.py +256 -0
- snowflake/sandbox/client.py +2356 -0
- snowflake/sandbox/config.py +1133 -0
- snowflake/sandbox/connect.py +288 -0
- snowflake/sandbox/deploy.py +499 -0
- snowflake/sandbox/egress.py +388 -0
- snowflake/sandbox/exceptions.py +253 -0
- snowflake/sandbox/exec_stream.py +264 -0
- snowflake/sandbox/files.py +547 -0
- snowflake/sandbox/function.py +567 -0
- snowflake/sandbox/image.py +46 -0
- snowflake/sandbox/jobs.py +649 -0
- snowflake/sandbox/lifecycle.py +67 -0
- snowflake/sandbox/log_stream.py +219 -0
- snowflake/sandbox/mcp.py +480 -0
- snowflake/sandbox/mount.py +161 -0
- snowflake/sandbox/py.typed +0 -0
- snowflake/sandbox/secret.py +244 -0
- snowflake/sandbox/session_app.py +244 -0
- snowflake/sandbox/shell.py +556 -0
- snowflake/sandbox/sync_client.py +2245 -0
- snowflake/sandbox/sync_exec_stream.py +238 -0
- snowflake/sandbox/sync_files.py +377 -0
- snowflake/sandbox/sync_log_stream.py +142 -0
- snowflake/sandbox/sync_shell.py +413 -0
- snowflake/sandbox/types.py +193 -0
- snowflake/sandbox/warm_session.py +700 -0
- snowflake_sandbox_python-0.2.1a1.dist-info/METADATA +339 -0
- snowflake_sandbox_python-0.2.1a1.dist-info/RECORD +80 -0
- snowflake_sandbox_python-0.2.1a1.dist-info/WHEEL +5 -0
- snowflake_sandbox_python-0.2.1a1.dist-info/entry_points.txt +2 -0
- snowflake_sandbox_python-0.2.1a1.dist-info/licenses/LICENSE +202 -0
- snowflake_sandbox_python-0.2.1a1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
"""Terminal escape-sequence filtering.
|
|
2
|
+
|
|
3
|
+
Generic byte-stream filtering for ANSI/OSC control sequences. Nothing here knows
|
|
4
|
+
what a sandbox is: it takes pty bytes in and gives pty bytes out.
|
|
5
|
+
|
|
6
|
+
It lives apart from `shell.py` because it outlives it. `shell.py` says in its own
|
|
7
|
+
docstring that it is a stopgap to be deleted once a duplex stream reaches the
|
|
8
|
+
sandbox -- but the reason to strip OSC 52 before writing remote bytes to a local
|
|
9
|
+
terminal has nothing to do with which transport carried them, so a duplex
|
|
10
|
+
replacement needs this filter unchanged. Leaving it inside the module marked for
|
|
11
|
+
deletion invited it to be deleted along with it.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
__all__ = ["_ClipboardStripper"]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class _ClipboardStripper:
|
|
20
|
+
"""Removes OSC 52 (clipboard-write) sequences from a byte stream.
|
|
21
|
+
|
|
22
|
+
Why this one sequence and not a general "sanitizer": a sandbox runs untrusted
|
|
23
|
+
and often agent-generated code, and `attach()` hands its raw output to the
|
|
24
|
+
operator's *local* terminal emulator. That is the same trust boundary `ssh`
|
|
25
|
+
and `docker attach` carry, but our remote end is less trusted than a host you
|
|
26
|
+
chose to log into, so the sequences that act on the operator's machine rather
|
|
27
|
+
than on the screen deserve removing.
|
|
28
|
+
|
|
29
|
+
OSC 52 qualifies unambiguously: it writes the local clipboard, has no display
|
|
30
|
+
purpose, and a sandbox silently replacing what the operator is about to paste
|
|
31
|
+
-- then having them paste it into a trusted shell -- is a real consequence.
|
|
32
|
+
|
|
33
|
+
Deliberately NOT filtered, because filtering them would break more than it
|
|
34
|
+
protects:
|
|
35
|
+
|
|
36
|
+
* **Display sequences** (CSI colours, cursor movement, alt-screen). Removing
|
|
37
|
+
these is removing the terminal, and `vim`/`htop` stop working.
|
|
38
|
+
* **Capability and cursor-position queries** (DA, DSR). A malicious sandbox
|
|
39
|
+
can use these to make the local terminal *reply*, and the reply arrives as
|
|
40
|
+
typed input on the next prompt -- a genuine residual risk. But legitimate
|
|
41
|
+
TUIs query these routinely, and an app whose query is swallowed hangs
|
|
42
|
+
waiting for an answer that never comes. Silently breaking working programs
|
|
43
|
+
to narrow an already-narrow window is the wrong trade; the honest answer is
|
|
44
|
+
to document it (see `attach`).
|
|
45
|
+
* **Window-title writes** (OSC 0/2). Annoying, not dangerous.
|
|
46
|
+
|
|
47
|
+
**This decides from the first bytes of the sequence, not from finding its
|
|
48
|
+
end**, and that shape is load-bearing rather than incidental. The first
|
|
49
|
+
version searched for the terminator and only then classified the body, which
|
|
50
|
+
forced it to buffer an unterminated sequence -- and to have a bound on that
|
|
51
|
+
buffer, past which it emitted the held bytes *verbatim*. A sandbox could
|
|
52
|
+
therefore defeat the filter completely by sending `ESC ] 52 ; c ;` followed by
|
|
53
|
+
more than the bound in base64 and the terminator in a later chunk: the
|
|
54
|
+
introducer and payload were passed through and the terminal reassembled them.
|
|
55
|
+
Classifying up front means an OSC 52 is discarded as it streams, so its length
|
|
56
|
+
is irrelevant and nothing needs holding.
|
|
57
|
+
|
|
58
|
+
It also treats a bare `ESC` inside a sequence as ENDING it, which is what
|
|
59
|
+
xterm-family terminals do. Otherwise `ESC ] 0 ; x ESC ] 52 ; c ; <payload>
|
|
60
|
+
BEL` reads as one sequence whose parameter is `0` -- passed through intact --
|
|
61
|
+
while the terminal aborts the outer one at the inner `ESC` and honours the
|
|
62
|
+
clipboard write inside it.
|
|
63
|
+
|
|
64
|
+
The 8-bit C1 forms (`0x9d` introducer, `0x9c` terminator) are handled too, for
|
|
65
|
+
terminals that decode them.
|
|
66
|
+
"""
|
|
67
|
+
|
|
68
|
+
# An OSC parameter is a short number followed by ';'. Anything longer than
|
|
69
|
+
# this without resolving is not a parameter, so stop waiting and pass it on.
|
|
70
|
+
_MAX_PARAM = 16
|
|
71
|
+
|
|
72
|
+
def __init__(self) -> None:
|
|
73
|
+
# "pass" outside a sequence; "esc" saw ESC awaiting ']'; "param" inside a
|
|
74
|
+
# sequence still reading its parameter; "keep"/"drop" inside a classified
|
|
75
|
+
# sequence.
|
|
76
|
+
self._state = "pass"
|
|
77
|
+
self._param = bytearray()
|
|
78
|
+
# What we were doing when an ESC arrived. ESC is ambiguous: it may start a
|
|
79
|
+
# new sequence, or be the first half of the 2-byte ST terminator, and the
|
|
80
|
+
# right handling of ST depends on whether we were keeping or dropping.
|
|
81
|
+
self._pre_esc = "pass"
|
|
82
|
+
|
|
83
|
+
def feed(self, data: bytes) -> bytes:
|
|
84
|
+
"""Filter one chunk of the stream, returning the bytes safe to write.
|
|
85
|
+
|
|
86
|
+
Stateful across calls: a sequence split over two chunks is classified
|
|
87
|
+
from its first bytes, so an OSC 52 is dropped however it is fragmented.
|
|
88
|
+
Call `flush` at end of stream to release anything still withheld.
|
|
89
|
+
"""
|
|
90
|
+
out = bytearray()
|
|
91
|
+
for byte in data:
|
|
92
|
+
self._step(byte, out)
|
|
93
|
+
return bytes(out)
|
|
94
|
+
|
|
95
|
+
def _enter_param(self) -> None:
|
|
96
|
+
"""Begin a sequence, with its parameter not yet read."""
|
|
97
|
+
self._state = "param"
|
|
98
|
+
self._param.clear()
|
|
99
|
+
|
|
100
|
+
def _step(self, byte: int, out: bytearray) -> None:
|
|
101
|
+
# One handler per state (see __init__ for what each means); the states
|
|
102
|
+
# are few and fixed, so a dispatch reads more honestly than one long body.
|
|
103
|
+
state = self._state
|
|
104
|
+
if state == "pass":
|
|
105
|
+
self._step_pass(byte, out)
|
|
106
|
+
elif state == "esc":
|
|
107
|
+
self._step_esc(byte, out)
|
|
108
|
+
elif state == "param":
|
|
109
|
+
self._step_param(byte, out)
|
|
110
|
+
elif state == "keep":
|
|
111
|
+
self._step_keep(byte, out)
|
|
112
|
+
else: # "drop"
|
|
113
|
+
self._step_drop(byte, out)
|
|
114
|
+
|
|
115
|
+
def _step_pass(self, byte: int, out: bytearray) -> None:
|
|
116
|
+
if byte == 0x1B: # ESC -- may begin an OSC
|
|
117
|
+
self._pre_esc = "pass"
|
|
118
|
+
self._state = "esc"
|
|
119
|
+
elif byte == 0x9D: # 8-bit OSC introducer
|
|
120
|
+
self._enter_param()
|
|
121
|
+
else:
|
|
122
|
+
out.append(byte)
|
|
123
|
+
|
|
124
|
+
def _step_esc(self, byte: int, out: bytearray) -> None:
|
|
125
|
+
if byte == 0x5C: # '\\' -> this ESC was the 2-byte ST terminator
|
|
126
|
+
if self._pre_esc in ("keep", "pass"):
|
|
127
|
+
out += bytes((0x1B, 0x5C))
|
|
128
|
+
self._state = "pass"
|
|
129
|
+
self._param.clear()
|
|
130
|
+
elif byte == 0x5D: # ']' -> a new OSC begins
|
|
131
|
+
self._enter_param()
|
|
132
|
+
elif byte == 0x1B: # ESC ESC -- emit one, stay pending
|
|
133
|
+
if self._pre_esc in ("keep", "pass"):
|
|
134
|
+
out.append(0x1B)
|
|
135
|
+
else:
|
|
136
|
+
# Not a sequence we care about: replay the withheld ESC unless we
|
|
137
|
+
# were mid-drop, where everything is discarded anyway.
|
|
138
|
+
if self._pre_esc in ("keep", "pass"):
|
|
139
|
+
out += bytes((0x1B, byte))
|
|
140
|
+
self._state = "pass"
|
|
141
|
+
|
|
142
|
+
def _step_param(self, byte: int, out: bytearray) -> None:
|
|
143
|
+
if self._is_end(byte, out, keep=True):
|
|
144
|
+
return
|
|
145
|
+
if byte == 0x1B:
|
|
146
|
+
# A sequence aborted by a new ESC. Emit what we withheld (it was
|
|
147
|
+
# not OSC 52) and reprocess the ESC.
|
|
148
|
+
out += bytes((0x1B, 0x5D)) + bytes(self._param)
|
|
149
|
+
self._param.clear()
|
|
150
|
+
# Committed to passing this one through, so an ST that follows is
|
|
151
|
+
# echoed too.
|
|
152
|
+
self._pre_esc = "keep"
|
|
153
|
+
self._state = "esc"
|
|
154
|
+
return
|
|
155
|
+
self._param.append(byte)
|
|
156
|
+
if byte == 0x3B or len(self._param) > self._MAX_PARAM: # ';'
|
|
157
|
+
param = bytes(self._param).rstrip(b";")
|
|
158
|
+
if param == b"52":
|
|
159
|
+
self._state = "drop"
|
|
160
|
+
else:
|
|
161
|
+
out += bytes((0x1B, 0x5D)) + bytes(self._param)
|
|
162
|
+
self._state = "keep"
|
|
163
|
+
self._param.clear()
|
|
164
|
+
|
|
165
|
+
def _step_keep(self, byte: int, out: bytearray) -> None:
|
|
166
|
+
if self._is_end(byte, out, keep=True):
|
|
167
|
+
return
|
|
168
|
+
if byte == 0x1B:
|
|
169
|
+
self._pre_esc = "keep"
|
|
170
|
+
self._state = "esc"
|
|
171
|
+
return
|
|
172
|
+
out.append(byte)
|
|
173
|
+
|
|
174
|
+
def _step_drop(self, byte: int, out: bytearray) -> None:
|
|
175
|
+
# Discard everything until the sequence ends. Length is irrelevant here,
|
|
176
|
+
# which is the whole point.
|
|
177
|
+
if self._is_end(byte, out, keep=False):
|
|
178
|
+
return
|
|
179
|
+
if byte == 0x1B:
|
|
180
|
+
self._pre_esc = "drop"
|
|
181
|
+
self._state = "esc"
|
|
182
|
+
|
|
183
|
+
def _is_end(self, byte: int, out: bytearray, *, keep: bool) -> bool:
|
|
184
|
+
"""Handle BEL / 8-bit ST. Returns True if the byte ended the sequence."""
|
|
185
|
+
if byte in (0x07, 0x9C):
|
|
186
|
+
if keep:
|
|
187
|
+
out.append(byte)
|
|
188
|
+
self._state = "pass"
|
|
189
|
+
self._param.clear()
|
|
190
|
+
return True
|
|
191
|
+
return False
|
|
192
|
+
|
|
193
|
+
def flush(self) -> bytes:
|
|
194
|
+
"""Emit anything withheld, at end of stream.
|
|
195
|
+
|
|
196
|
+
A partially-read parameter is emitted (it was never classified as OSC 52),
|
|
197
|
+
a pending ESC is emitted, and an in-progress OSC 52 stays dropped.
|
|
198
|
+
"""
|
|
199
|
+
out = bytearray()
|
|
200
|
+
if self._state == "esc":
|
|
201
|
+
out.append(0x1B)
|
|
202
|
+
elif self._state == "param":
|
|
203
|
+
out += bytes((0x1B, 0x5D)) + bytes(self._param)
|
|
204
|
+
self._param.clear()
|
|
205
|
+
self._state = "pass"
|
|
206
|
+
return bytes(out)
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
"""Validating and coercing what the caller passed.
|
|
2
|
+
|
|
3
|
+
Author-time checks on `Sandbox.create` arguments, and the translation of a
|
|
4
|
+
caller's command into what the wire actually accepts. Failing here rather than at
|
|
5
|
+
the server turns a confusing remote response into a message that names the
|
|
6
|
+
parameter and the fix.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from collections.abc import Mapping, Sequence
|
|
12
|
+
|
|
13
|
+
from snowflake.sandbox.exceptions import SandboxError
|
|
14
|
+
|
|
15
|
+
__all__ = [
|
|
16
|
+
"_tags_post_create_message",
|
|
17
|
+
"_validate_tags",
|
|
18
|
+
"_dur_seconds",
|
|
19
|
+
"_require_nonempty_cmd",
|
|
20
|
+
"_require_str_cmd",
|
|
21
|
+
"_cmd_to_code",
|
|
22
|
+
"_wrap_detached",
|
|
23
|
+
]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _tags_post_create_message(create_call: str) -> str:
|
|
27
|
+
"""Why ``set_tags()`` cannot relabel a running sandbox, and what to do instead.
|
|
28
|
+
|
|
29
|
+
The rule is the platform's and is identical for both clients, but the remedy
|
|
30
|
+
has to be spelled in the caller's own idiom — ``await AsyncSandbox.create(...)``
|
|
31
|
+
on the async client, ``Sandbox.create(...)`` on the sync one. Sharing the text
|
|
32
|
+
with the async spelling baked in told sync callers to await a class they are
|
|
33
|
+
not using, so the differing half is a parameter rather than a copy.
|
|
34
|
+
"""
|
|
35
|
+
return (
|
|
36
|
+
"set_tags() cannot relabel a running sandbox: the backend stores tags (as "
|
|
37
|
+
"`labels`) only at create and exposes no update endpoint, so there is nowhere "
|
|
38
|
+
"to write them. Pass tags at create instead: "
|
|
39
|
+
f'{create_call}(..., tags={{"team": "sandbox"}}). Call set_tags() only '
|
|
40
|
+
"before the sandbox is created (it is then buffered into the create request)."
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _validate_tags(tags: Mapping[str, str] | None, *, context: str) -> dict[str, str]:
|
|
45
|
+
"""Coerce/validate a caller tag map into a plain ``dict[str, str]``.
|
|
46
|
+
|
|
47
|
+
Tags map to the server's ``labels`` field, which is a bare ``map[string]string``
|
|
48
|
+
with no server-side validation, so the only rules enforced are: keys and values
|
|
49
|
+
must be strings, and keys must be non-empty. An empty *value* is allowed (a
|
|
50
|
+
presence-style tag). An obvious mistake (a non-string value, an empty key) fails
|
|
51
|
+
at the call site rather than as a confusing server response or a
|
|
52
|
+
silently-stringified label. ``None`` -> ``{}``.
|
|
53
|
+
"""
|
|
54
|
+
if tags is None:
|
|
55
|
+
return {}
|
|
56
|
+
if not isinstance(tags, Mapping):
|
|
57
|
+
raise SandboxError(
|
|
58
|
+
f"{context}: tags must be a mapping of str->str, got {type(tags).__name__}"
|
|
59
|
+
)
|
|
60
|
+
out: dict[str, str] = {}
|
|
61
|
+
for k, v in tags.items():
|
|
62
|
+
if not isinstance(k, str) or not isinstance(v, str):
|
|
63
|
+
raise SandboxError(
|
|
64
|
+
f"{context}: every tag key and value must be a str, got "
|
|
65
|
+
f"{type(k).__name__}->{type(v).__name__}"
|
|
66
|
+
)
|
|
67
|
+
if not k:
|
|
68
|
+
raise SandboxError(f"{context}: tag keys must be non-empty")
|
|
69
|
+
out[k] = v
|
|
70
|
+
return out
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _dur_seconds(value: str | float) -> float:
|
|
74
|
+
"""Parse a duration (``"30s"`` / ``"10m"`` / ``"1h"`` or a number) to seconds."""
|
|
75
|
+
if isinstance(value, (int, float)):
|
|
76
|
+
return float(value)
|
|
77
|
+
s = str(value).strip().lower()
|
|
78
|
+
try:
|
|
79
|
+
if s.endswith("ms"):
|
|
80
|
+
return float(s[:-2]) / 1000.0
|
|
81
|
+
if s.endswith("s"):
|
|
82
|
+
return float(s[:-1])
|
|
83
|
+
if s.endswith("m"):
|
|
84
|
+
return float(s[:-1]) * 60.0
|
|
85
|
+
if s.endswith("h"):
|
|
86
|
+
return float(s[:-1]) * 3600.0
|
|
87
|
+
return float(s)
|
|
88
|
+
except ValueError as exc:
|
|
89
|
+
raise SandboxError(f"invalid duration: {value!r}") from exc
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _require_nonempty_cmd(cmd: Sequence[str], *, caller: str) -> None:
|
|
93
|
+
"""Reject an empty argv before it becomes a silent no-op.
|
|
94
|
+
|
|
95
|
+
An empty ``cmd`` used to run and "succeed": `_cmd_to_code([])` produces
|
|
96
|
+
``subprocess.run('', shell=True, ...)``, and a shell handed an empty string
|
|
97
|
+
exits 0 with no output. The caller got an `ExecResult` with ``exit_code == 0``
|
|
98
|
+
and empty streams -- indistinguishable from a command that ran and printed
|
|
99
|
+
nothing, so a bug that dropped the argv on the floor looked like a pass.
|
|
100
|
+
|
|
101
|
+
*caller* names the public method in the message so the steer is actionable
|
|
102
|
+
from either ``exec`` or ``exec_stream``.
|
|
103
|
+
"""
|
|
104
|
+
if not cmd:
|
|
105
|
+
raise SandboxError(
|
|
106
|
+
f"{caller}() requires at least one command element; got an empty "
|
|
107
|
+
f"sequence. An empty argv would have run an empty shell command and "
|
|
108
|
+
f"returned exit_code 0 with no output, which is indistinguishable "
|
|
109
|
+
f"from success. Pass the program and its arguments, e.g. "
|
|
110
|
+
f'{caller}(["python", "-c", "print(42)"]).'
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _require_str_cmd(cmd: Sequence[str], *, caller: str) -> None:
|
|
115
|
+
"""Reject a non-string argv element before shlex silently reinterprets it.
|
|
116
|
+
|
|
117
|
+
`_cmd_to_code` quotes with ``shlex.quote``, which starts ``if not s: return "''"``
|
|
118
|
+
-- so ``None`` is FALSY and comes back as an empty shell argument rather than
|
|
119
|
+
raising: ``["ok", None]`` ran as ``ok \'\'``, a different command than the caller
|
|
120
|
+
passed, with nothing said. An ``int`` is the milder half: shlex raises, but a bare
|
|
121
|
+
``TypeError`` that ``except SandboxError`` does not catch, unlike every other
|
|
122
|
+
input-validation failure here.
|
|
123
|
+
"""
|
|
124
|
+
if isinstance(cmd, str):
|
|
125
|
+
# A bare string is iterated character by character below, so every element
|
|
126
|
+
# is trivially a str and the loop never fires -- then `_cmd_to_code` quotes
|
|
127
|
+
# each character and runs a mangled command. Reject it here: argv is a
|
|
128
|
+
# sequence of program-plus-arguments, not a shell line.
|
|
129
|
+
raise SandboxError(
|
|
130
|
+
f"{caller}() takes an argv sequence, not a string: {cmd!r} would be "
|
|
131
|
+
f"iterated character by character. Pass a list, e.g. "
|
|
132
|
+
f'{caller}(["ls", "-la"]) or {caller}(["sh", "-c", {cmd!r}]).'
|
|
133
|
+
)
|
|
134
|
+
for i, arg in enumerate(cmd):
|
|
135
|
+
if not isinstance(arg, str):
|
|
136
|
+
raise SandboxError(
|
|
137
|
+
f"{caller}() cmd[{i}] must be str, got {type(arg).__name__}. Every "
|
|
138
|
+
f"element of the argv must be a string; pass str(x) if you meant to "
|
|
139
|
+
f"interpolate a value."
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _cmd_to_code(cmd: Sequence[str]) -> str:
|
|
144
|
+
"""Turn an argv into the single Python ``code`` string /execute accepts.
|
|
145
|
+
|
|
146
|
+
A ``python -c CODE`` invocation passes CODE through; any other argv is
|
|
147
|
+
wrapped in a subprocess call so arbitrary commands run.
|
|
148
|
+
|
|
149
|
+
Trailing arguments after ``-c CODE`` are delivered as ``sys.argv``. They used
|
|
150
|
+
to be dropped silently: ``exec(["python3", "-c", src, "AAA"])`` sent only
|
|
151
|
+
``src``, and the script saw ``sys.argv == ['-']`` (the /execute path feeds the
|
|
152
|
+
source to the interpreter on stdin), so callers had no way to parameterize a
|
|
153
|
+
run per exec and no error telling them so.
|
|
154
|
+
"""
|
|
155
|
+
if len(cmd) >= 3 and cmd[0] in ("python", "python3") and cmd[1] == "-c":
|
|
156
|
+
code, extra = cmd[2], list(cmd[3:])
|
|
157
|
+
if not extra:
|
|
158
|
+
return code
|
|
159
|
+
# Compile the caller's source as its own unit rather than prepending the
|
|
160
|
+
# assignment to it: prepending breaks `from __future__ import ...`, which
|
|
161
|
+
# must be the first statement. Passing __name__ == "__main__" keeps
|
|
162
|
+
# `if __name__ == "__main__":` entry points working.
|
|
163
|
+
return (
|
|
164
|
+
"import sys\n"
|
|
165
|
+
f"sys.argv = {['-c', *extra]!r}\n"
|
|
166
|
+
f"exec(compile({code!r}, '<sandbox-exec>', 'exec'), "
|
|
167
|
+
"{'__name__': '__main__', '__builtins__': __builtins__})\n"
|
|
168
|
+
)
|
|
169
|
+
import shlex
|
|
170
|
+
|
|
171
|
+
joined = " ".join(shlex.quote(a) for a in cmd)
|
|
172
|
+
# Inherited, not captured: capture_output=True is communicate(), which waits for
|
|
173
|
+
# pipe EOF rather than for the command to exit -- so a descendant left holding the
|
|
174
|
+
# pipe burned the whole server deadline (exit 124, or 137 on guest-init) and the
|
|
175
|
+
# buffered output was discarded with it. sandbox-server's drain loop already breaks
|
|
176
|
+
# on the child's exit regardless of pipe EOF, and reads the two streams separately,
|
|
177
|
+
# so inheriting hands the deadline back to it and keeps stdout/stderr apart. #266
|
|
178
|
+
return (
|
|
179
|
+
"import subprocess, sys; "
|
|
180
|
+
f"r = subprocess.run({joined!r}, shell=True); "
|
|
181
|
+
"sys.exit(r.returncode)"
|
|
182
|
+
)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _wrap_detached(cmd: Sequence[str]) -> tuple[list[str], str]:
|
|
186
|
+
"""Wrap *cmd* so a detached run prints a result sentinel on exit.
|
|
187
|
+
|
|
188
|
+
Returns ``(argv, nonce)``. Sets ``$SANDBOX_RESULT`` (creating its dir), runs
|
|
189
|
+
*cmd* as a subprocess, then emits
|
|
190
|
+
``__SANDBOX_RESULT__<base64(json)>`` — what `Sandbox.wait()` scans for —
|
|
191
|
+
carrying ``{status, exit_code, result, nonce}``. ``result`` is the parsed
|
|
192
|
+
``$SANDBOX_RESULT`` artifact the inner command wrote (matching the
|
|
193
|
+
``deploy_async`` job-runner), or ``{"exit_code": N}`` if it wrote none.
|
|
194
|
+
|
|
195
|
+
The ``nonce`` is minted here, per run, and kept in the wrapper only: it is
|
|
196
|
+
never exported to the inner command's environment. `wait()` requires it, so
|
|
197
|
+
an arbitrary ``__SANDBOX_RESULT__`` line printed by the workload itself is no
|
|
198
|
+
longer read as the run's authoritative outcome.
|
|
199
|
+
"""
|
|
200
|
+
import json as _json
|
|
201
|
+
import uuid as _uuid
|
|
202
|
+
|
|
203
|
+
from snowflake.sandbox._runtime._shims import _detached_wrapper_source
|
|
204
|
+
|
|
205
|
+
nonce = _uuid.uuid4().hex
|
|
206
|
+
payload = _json.dumps(list(cmd))
|
|
207
|
+
script = _detached_wrapper_source(payload, nonce)
|
|
208
|
+
return ["python3", "-c", script], nonce
|
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
"""Assemble the local bundle that gets shipped to a sandbox.
|
|
2
|
+
|
|
3
|
+
``Bundle`` is the declaration — which local tree, which include/exclude globs —
|
|
4
|
+
and the helpers below are what realize it: shallow-clone a git ref, copy a tree
|
|
5
|
+
while filtering excludes / symlinks / credential-shaped files, expand ``${VAR}``
|
|
6
|
+
references, and layer include-glob matches on top. Everything here is local and
|
|
7
|
+
filesystem-only: no Snowflake calls, no network beyond ``git clone``.
|
|
8
|
+
|
|
9
|
+
``deploy_spec`` (``deploy``), ``deploy_async`` (``jobs``), ``Function.remote``
|
|
10
|
+
(``function``) and ``agent_session`` (``warm_session``) all package their code
|
|
11
|
+
through this module, which is why it is a module and not four copies.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import fnmatch
|
|
17
|
+
import os
|
|
18
|
+
import shutil
|
|
19
|
+
import subprocess
|
|
20
|
+
import tempfile
|
|
21
|
+
from collections.abc import Mapping, Sequence
|
|
22
|
+
from dataclasses import dataclass, field
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
from typing import Any
|
|
25
|
+
|
|
26
|
+
from snowflake.sandbox.exceptions import SandboxError
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True)
|
|
30
|
+
class Bundle:
|
|
31
|
+
"""Source-tree slice bundled and uploaded to the sandbox.
|
|
32
|
+
|
|
33
|
+
Example:
|
|
34
|
+
bundle = Bundle.from_dir(
|
|
35
|
+
".",
|
|
36
|
+
include=["src/**", "config.toml"],
|
|
37
|
+
exclude=["*.pyc", ".DS_Store"],
|
|
38
|
+
)
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
root: str = "."
|
|
42
|
+
include: tuple[str, ...] = field(default_factory=tuple)
|
|
43
|
+
exclude: tuple[str, ...] = field(default_factory=tuple)
|
|
44
|
+
|
|
45
|
+
@staticmethod
|
|
46
|
+
def from_dir(
|
|
47
|
+
root: str,
|
|
48
|
+
*,
|
|
49
|
+
include: Sequence[str] = (),
|
|
50
|
+
exclude: Sequence[str] = (),
|
|
51
|
+
) -> Bundle:
|
|
52
|
+
"""Construct a Bundle from a root directory with optional glob patterns."""
|
|
53
|
+
return Bundle(root=root, include=tuple(include), exclude=tuple(exclude))
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _bundle_files(source_dir: Path, limit: int = 200) -> list[str]:
|
|
57
|
+
out = []
|
|
58
|
+
for p in sorted(source_dir.rglob("*")):
|
|
59
|
+
if p.is_file() and "__pycache__" not in p.parts and ".git" not in p.parts:
|
|
60
|
+
out.append(str(p.relative_to(source_dir)))
|
|
61
|
+
if len(out) >= limit:
|
|
62
|
+
out.append("... (truncated)")
|
|
63
|
+
break
|
|
64
|
+
return out
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
_DEFAULT_EXCLUDES = ("*.pyc", ".DS_Store")
|
|
68
|
+
_SKIP_DIR_NAMES = {"__pycache__", ".git", ".venv", "node_modules"}
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _clone_git_ref(repo: str, ref: str | None, subdir: str | None) -> tuple[Path, Path, str]:
|
|
72
|
+
"""Shallow-clone ``repo`` at ``ref`` into a temp dir.
|
|
73
|
+
|
|
74
|
+
Returns ``(clone_root, effective_source_dir, resolved_sha)``. The caller
|
|
75
|
+
is responsible for ``shutil.rmtree``-ing ``clone_root``.
|
|
76
|
+
|
|
77
|
+
Strategy: ``git clone --depth 1 <repo> <tmp>`` then
|
|
78
|
+
``git -C <tmp> fetch --depth 1 origin <ref> && git -C <tmp> checkout FETCH_HEAD``.
|
|
79
|
+
This handles both branch/tag names and full/abbreviated SHAs with a single
|
|
80
|
+
code path (fetch by refspec always works for branches; for SHAs it works on
|
|
81
|
+
GitHub and most servers that enable ``uploadpack.allowReachableSHA1InWant``).
|
|
82
|
+
``ref=None`` skips the second fetch and uses the default branch HEAD.
|
|
83
|
+
"""
|
|
84
|
+
tmp_parent = Path(tempfile.mkdtemp(prefix="sbx-git-"))
|
|
85
|
+
tmp = tmp_parent / "repo"
|
|
86
|
+
try:
|
|
87
|
+
# Initial shallow clone (gets default branch HEAD; also establishes the remote).
|
|
88
|
+
subprocess.run(
|
|
89
|
+
["git", "clone", "--depth", "1", repo, str(tmp)],
|
|
90
|
+
check=True,
|
|
91
|
+
capture_output=True,
|
|
92
|
+
text=True,
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
if ref is not None:
|
|
96
|
+
# Fetch the requested ref and check it out.
|
|
97
|
+
subprocess.run(
|
|
98
|
+
["git", "-C", str(tmp), "fetch", "--depth", "1", "origin", ref],
|
|
99
|
+
check=True,
|
|
100
|
+
capture_output=True,
|
|
101
|
+
text=True,
|
|
102
|
+
)
|
|
103
|
+
subprocess.run(
|
|
104
|
+
["git", "-C", str(tmp), "checkout", "FETCH_HEAD"],
|
|
105
|
+
check=True,
|
|
106
|
+
capture_output=True,
|
|
107
|
+
text=True,
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
# Resolve the HEAD SHA.
|
|
111
|
+
proc = subprocess.run(
|
|
112
|
+
["git", "-C", str(tmp), "rev-parse", "HEAD"],
|
|
113
|
+
check=True,
|
|
114
|
+
capture_output=True,
|
|
115
|
+
text=True,
|
|
116
|
+
)
|
|
117
|
+
sha = proc.stdout.strip()
|
|
118
|
+
|
|
119
|
+
effective = tmp / subdir if subdir else tmp
|
|
120
|
+
if not effective.is_dir():
|
|
121
|
+
raise SandboxError(f"git subdir not found in clone: {subdir}")
|
|
122
|
+
|
|
123
|
+
print(f"[deploy] resolved {repo}@{ref or 'HEAD'} -> {sha}")
|
|
124
|
+
return tmp_parent, effective, sha
|
|
125
|
+
except subprocess.CalledProcessError as exc:
|
|
126
|
+
shutil.rmtree(tmp_parent, ignore_errors=True)
|
|
127
|
+
stderr_msg = (exc.stderr or "").strip()
|
|
128
|
+
raise SandboxError(f"git clone/fetch failed: {stderr_msg or exc}") from exc
|
|
129
|
+
except Exception:
|
|
130
|
+
shutil.rmtree(tmp_parent, ignore_errors=True)
|
|
131
|
+
raise
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _excluded(rel: str, patterns: Sequence[str]) -> bool:
|
|
135
|
+
if _SKIP_DIR_NAMES & set(Path(rel).parts):
|
|
136
|
+
return True
|
|
137
|
+
return any(
|
|
138
|
+
fnmatch.fnmatch(rel, pat) or fnmatch.fnmatch(Path(rel).name, pat) for pat in patterns
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _copy_tree_into(src_root: Path, dest: Path, excludes: Sequence[str]) -> None:
|
|
143
|
+
from snowflake.sandbox._bundle import _is_credential_path
|
|
144
|
+
|
|
145
|
+
for src in sorted(src_root.rglob("*")):
|
|
146
|
+
# A symlink is followed by is_file()/copy2, so one pointing outside the
|
|
147
|
+
# project would put its target in the bundle (and then on a stage). The
|
|
148
|
+
# bundle is the project tree, nothing else.
|
|
149
|
+
if src.is_symlink():
|
|
150
|
+
continue
|
|
151
|
+
if not src.is_file():
|
|
152
|
+
continue
|
|
153
|
+
rel = src.relative_to(src_root)
|
|
154
|
+
if _excluded(str(rel), excludes):
|
|
155
|
+
continue
|
|
156
|
+
# `.env` / `id_rsa` / `*.pem` beside the entry point is the normal layout;
|
|
157
|
+
# the default excludes were only *.pyc + .DS_Store, so they shipped.
|
|
158
|
+
if _is_credential_path(rel):
|
|
159
|
+
continue
|
|
160
|
+
(dest / rel).parent.mkdir(parents=True, exist_ok=True)
|
|
161
|
+
shutil.copy2(src, dest / rel, follow_symlinks=False)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _subst_map(
|
|
165
|
+
var_overrides: Mapping[str, str] | None,
|
|
166
|
+
env_overrides: Mapping[str, str] | None,
|
|
167
|
+
) -> dict[str, str]:
|
|
168
|
+
"""Build the manifest ``${VAR}`` substitution map for ``[bundle]``/``[build]``.
|
|
169
|
+
|
|
170
|
+
Precedence (highest wins): ``var_overrides`` (build-time ``--var``) >
|
|
171
|
+
``env_overrides`` (runtime ``--env``, also usable for substitution since some
|
|
172
|
+
values like ``TRIAGE_TEAM`` are both) > ``os.environ`` (resolved lazily by
|
|
173
|
+
``_expand_vars`` for names not in the explicit map)."""
|
|
174
|
+
merged: dict[str, str] = {}
|
|
175
|
+
if env_overrides:
|
|
176
|
+
merged.update(env_overrides)
|
|
177
|
+
if var_overrides:
|
|
178
|
+
merged.update(var_overrides)
|
|
179
|
+
return merged
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def _expand_vars(text: str, subst: Mapping[str, str] | None) -> str:
|
|
183
|
+
"""Expand ``$VAR`` / ``${VAR}`` in *text* against *subst*.
|
|
184
|
+
|
|
185
|
+
*subst* is the build-time substitution map (see ``_subst_map``). Names absent
|
|
186
|
+
from it fall back to ``os.environ`` (so plain ``os.path.expandvars`` behaviour
|
|
187
|
+
is preserved for back-compat). Passing ``subst=None`` is exactly the old
|
|
188
|
+
``os.path.expandvars``."""
|
|
189
|
+
if subst is None:
|
|
190
|
+
return os.path.expandvars(text)
|
|
191
|
+
# Layer subst over os.environ for the duration of the expansion: explicit
|
|
192
|
+
# overrides win, anything else still resolves from the process environment.
|
|
193
|
+
saved = {k: os.environ.get(k) for k in subst}
|
|
194
|
+
try:
|
|
195
|
+
os.environ.update(subst)
|
|
196
|
+
return os.path.expandvars(text)
|
|
197
|
+
finally:
|
|
198
|
+
for k, old in saved.items():
|
|
199
|
+
if old is None:
|
|
200
|
+
os.environ.pop(k, None)
|
|
201
|
+
else:
|
|
202
|
+
os.environ[k] = old
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _assemble_bundle(
|
|
206
|
+
manifest_dir: Path, spec: dict[str, Any], subst: Mapping[str, str] | None = None
|
|
207
|
+
) -> Path:
|
|
208
|
+
"""Assemble a bundle from the project dir + ``[bundle].include`` globs.
|
|
209
|
+
|
|
210
|
+
The manifest's own directory is the base (entry, setup.sh, requirements.txt
|
|
211
|
+
ship as in a plain deploy). ``[bundle].include`` globs are resolved against
|
|
212
|
+
``[bundle].root`` (default: the manifest dir; ``$VARS`` and ``~`` expanded)
|
|
213
|
+
and layered under ``[bundle].mount`` (default: bundle root), preserving each
|
|
214
|
+
match's path relative to root. This lets a workload whose files are scattered
|
|
215
|
+
across a monorepo declare its slice instead of shipping a bespoke packer.
|
|
216
|
+
|
|
217
|
+
``$VAR`` expansion in ``root`` / ``include`` resolves from *subst* (the merged
|
|
218
|
+
build-time map: ``var_overrides`` > ``env_overrides`` > ``os.environ``) so a
|
|
219
|
+
relative ``root = ".."`` plus ``--var TRIAGE_TEAM=platform`` needs no shell export.
|
|
220
|
+
|
|
221
|
+
Note: glob ``**`` does not descend into symlinked directories — include such
|
|
222
|
+
targets explicitly (e.g. the real dir behind a symlink)."""
|
|
223
|
+
excludes = list(_DEFAULT_EXCLUDES) + list(spec.get("exclude") or [])
|
|
224
|
+
dest = Path(tempfile.mkdtemp(prefix="sbx-bundle-"))
|
|
225
|
+
_copy_tree_into(manifest_dir, dest, excludes)
|
|
226
|
+
|
|
227
|
+
root = Path(os.path.expanduser(_expand_vars(spec.get("root") or str(manifest_dir), subst)))
|
|
228
|
+
if not root.is_absolute():
|
|
229
|
+
root = (manifest_dir / root).resolve()
|
|
230
|
+
if not root.is_dir():
|
|
231
|
+
raise SandboxError(f"[bundle].root not found: {root}")
|
|
232
|
+
from snowflake.sandbox._bundle import _is_credential_path
|
|
233
|
+
|
|
234
|
+
mount = dest / (spec.get("mount") or ".")
|
|
235
|
+
copied: set[str] = set()
|
|
236
|
+
for pattern in spec.get("include") or []:
|
|
237
|
+
pattern = _expand_vars(pattern, subst) # e.g. teams/${TRIAGE_TEAM}/**
|
|
238
|
+
for match in sorted(root.glob(pattern)):
|
|
239
|
+
# A glob match is either a file (copy it) or a directory (copy the
|
|
240
|
+
# files beneath it) — `**` matches dirs, not the files under them.
|
|
241
|
+
files = [match] if match.is_file() else [p for p in match.rglob("*") if p.is_file()]
|
|
242
|
+
for src_file in files:
|
|
243
|
+
# The same credential + symlink filters the base copy runs. Without
|
|
244
|
+
# them here, `include=["**"]` / `root=".."` would upload `.env` /
|
|
245
|
+
# `id_rsa` / `~/.aws` to the stage: skip symlinks (a symlink out of
|
|
246
|
+
# the tree would drag its target in) and credential-shaped paths.
|
|
247
|
+
if src_file.is_symlink():
|
|
248
|
+
continue
|
|
249
|
+
rel = src_file.relative_to(root)
|
|
250
|
+
key = str(rel)
|
|
251
|
+
if key in copied or _excluded(key, excludes) or _is_credential_path(rel):
|
|
252
|
+
continue
|
|
253
|
+
(mount / rel).parent.mkdir(parents=True, exist_ok=True)
|
|
254
|
+
shutil.copy2(src_file, mount / rel, follow_symlinks=False)
|
|
255
|
+
copied.add(key)
|
|
256
|
+
return dest
|