snowflake-sandbox-python 0.2.1a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- snowflake/cli_sandbox/__init__.py +13 -0
- snowflake/cli_sandbox/_adapter.py +170 -0
- snowflake/cli_sandbox/_common.py +77 -0
- snowflake/cli_sandbox/_egress_flags.py +121 -0
- snowflake/cli_sandbox/_get_command.py +109 -0
- snowflake/cli_sandbox/_run_command.py +1091 -0
- snowflake/cli_sandbox/_shell_command.py +666 -0
- snowflake/cli_sandbox/_upload_plan.py +187 -0
- snowflake/cli_sandbox/commands.py +556 -0
- snowflake/cli_sandbox/plugin_spec.py +28 -0
- snowflake/cli_sandbox/py.typed +0 -0
- snowflake/sandbox/__init__.py +317 -0
- snowflake/sandbox/__main__.py +225 -0
- snowflake/sandbox/_ansi.py +206 -0
- snowflake/sandbox/_args.py +208 -0
- snowflake/sandbox/_assemble.py +256 -0
- snowflake/sandbox/_bundle.py +240 -0
- snowflake/sandbox/_connection_resolve.py +328 -0
- snowflake/sandbox/_deploy_spec.py +56 -0
- snowflake/sandbox/_diagnostics.py +501 -0
- snowflake/sandbox/_env.py +143 -0
- snowflake/sandbox/_files_mixin.py +280 -0
- snowflake/sandbox/_fs_ops.py +304 -0
- snowflake/sandbox/_globs.py +176 -0
- snowflake/sandbox/_hosts.py +110 -0
- snowflake/sandbox/_mcp_discovery.py +288 -0
- snowflake/sandbox/_mcp_status.py +183 -0
- snowflake/sandbox/_retry.py +94 -0
- snowflake/sandbox/_runtime/__init__.py +42 -0
- snowflake/sandbox/_runtime/_fs_helper.py +93 -0
- snowflake/sandbox/_runtime/_job_runner.py +111 -0
- snowflake/sandbox/_runtime/_protocol.py +53 -0
- snowflake/sandbox/_runtime/_shims.py +267 -0
- snowflake/sandbox/_sandbox_state.py +303 -0
- snowflake/sandbox/_session_registry.py +222 -0
- snowflake/sandbox/_sse.py +160 -0
- snowflake/sandbox/_stage.py +270 -0
- snowflake/sandbox/_sync_files_mixin.py +272 -0
- snowflake/sandbox/_sync_fs_ops.py +185 -0
- snowflake/sandbox/_sync_transport.py +737 -0
- snowflake/sandbox/_sync_watch.py +99 -0
- snowflake/sandbox/_transport.py +1366 -0
- snowflake/sandbox/_transport_errors.py +270 -0
- snowflake/sandbox/_upload_plan.py +497 -0
- snowflake/sandbox/_version.py +37 -0
- snowflake/sandbox/_watch.py +164 -0
- snowflake/sandbox/_wire.py +348 -0
- snowflake/sandbox/app.py +256 -0
- snowflake/sandbox/client.py +2356 -0
- snowflake/sandbox/config.py +1133 -0
- snowflake/sandbox/connect.py +288 -0
- snowflake/sandbox/deploy.py +499 -0
- snowflake/sandbox/egress.py +388 -0
- snowflake/sandbox/exceptions.py +253 -0
- snowflake/sandbox/exec_stream.py +264 -0
- snowflake/sandbox/files.py +547 -0
- snowflake/sandbox/function.py +567 -0
- snowflake/sandbox/image.py +46 -0
- snowflake/sandbox/jobs.py +649 -0
- snowflake/sandbox/lifecycle.py +67 -0
- snowflake/sandbox/log_stream.py +219 -0
- snowflake/sandbox/mcp.py +480 -0
- snowflake/sandbox/mount.py +161 -0
- snowflake/sandbox/py.typed +0 -0
- snowflake/sandbox/secret.py +244 -0
- snowflake/sandbox/session_app.py +244 -0
- snowflake/sandbox/shell.py +556 -0
- snowflake/sandbox/sync_client.py +2245 -0
- snowflake/sandbox/sync_exec_stream.py +238 -0
- snowflake/sandbox/sync_files.py +377 -0
- snowflake/sandbox/sync_log_stream.py +142 -0
- snowflake/sandbox/sync_shell.py +413 -0
- snowflake/sandbox/types.py +193 -0
- snowflake/sandbox/warm_session.py +700 -0
- snowflake_sandbox_python-0.2.1a1.dist-info/METADATA +339 -0
- snowflake_sandbox_python-0.2.1a1.dist-info/RECORD +80 -0
- snowflake_sandbox_python-0.2.1a1.dist-info/WHEEL +5 -0
- snowflake_sandbox_python-0.2.1a1.dist-info/entry_points.txt +2 -0
- snowflake_sandbox_python-0.2.1a1.dist-info/licenses/LICENSE +202 -0
- snowflake_sandbox_python-0.2.1a1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,497 @@
|
|
|
1
|
+
"""Deciding *what* a directory upload sends, and being able to say why.
|
|
2
|
+
|
|
3
|
+
This is the selection engine behind `Sandbox.upload_dir` and the `from_local`
|
|
4
|
+
bundle, and behind `snow sandbox run`'s TARGET handling. It lives here, in the SDK,
|
|
5
|
+
because the SDK is where the primitives already were (`_bundle._is_credential_path`,
|
|
6
|
+
`_globs`) and because two different walks had drifted apart: `_bundle._collect_tree`
|
|
7
|
+
used `rglob` and reported nothing, while this one used `os.scandir` and recorded a
|
|
8
|
+
reason per drop. One engine, so they cannot disagree again.
|
|
9
|
+
|
|
10
|
+
The CLI keeps the half that is genuinely about argv — `resolve_targets` and its
|
|
11
|
+
`TargetError`, whose messages name CLI flags — in `cli_sandbox._upload_plan`, and
|
|
12
|
+
imports the engine from here.
|
|
13
|
+
|
|
14
|
+
The module's job is to make every drop explainable. `UploadPlan.skipped` carries a
|
|
15
|
+
reason per path, so a caller can report "3 credential-shaped, 91 vcs/build, 2
|
|
16
|
+
symlinked directories" instead of silently sending fewer files than the user
|
|
17
|
+
expected. Silent under-upload is the failure mode users cannot debug.
|
|
18
|
+
|
|
19
|
+
Three deliberate choices worth knowing:
|
|
20
|
+
|
|
21
|
+
* **`os.scandir` recursion, not `rglob`.** `rglob` swallows per-entry errors, so an
|
|
22
|
+
unreadable directory is indistinguishable from an empty one, and it never
|
|
23
|
+
recurses into a symlinked directory — which means a pnpm/bazel workspace whose
|
|
24
|
+
packages are symlinks uploads *nothing* from those trees, without a word. Explicit
|
|
25
|
+
recursion lets both be counted and reported.
|
|
26
|
+
* **The credential test is fed the target-relative path, not the destination rel
|
|
27
|
+
and not a fixed-length tail.** `_is_credential_path` keys its directory rules on
|
|
28
|
+
everything but the last component, so the path it judges has to name every
|
|
29
|
+
directory between the tree root and the file — no more (a project that merely
|
|
30
|
+
lives under `~/.snowflake/` is not a credential) and no less (a truncated tail
|
|
31
|
+
hides `fixtures/.aws/sso/cache/token.json`). A single-file target has no tree to
|
|
32
|
+
be relative to, so there the absolute source path is judged instead.
|
|
33
|
+
* **`include` is compiled as a trailing negation.** gitignore's last-match-wins
|
|
34
|
+
then gives "include beats exclude" for free, and the reason a file survived can
|
|
35
|
+
still name the rule that saved it.
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
from __future__ import annotations
|
|
39
|
+
|
|
40
|
+
import os
|
|
41
|
+
from dataclasses import dataclass, field
|
|
42
|
+
from pathlib import Path
|
|
43
|
+
from typing import Literal
|
|
44
|
+
|
|
45
|
+
from snowflake.sandbox._bundle import _is_credential_path
|
|
46
|
+
from snowflake.sandbox._globs import GlobRule, anchored_head, compile_rules, match_rules
|
|
47
|
+
from snowflake.sandbox.exceptions import SandboxValidationError
|
|
48
|
+
|
|
49
|
+
__all__ = [
|
|
50
|
+
"SKIP_DIRS",
|
|
51
|
+
"ResolvedTarget",
|
|
52
|
+
"Selected",
|
|
53
|
+
"Skipped",
|
|
54
|
+
"UploadPlan",
|
|
55
|
+
"build_plan",
|
|
56
|
+
"plan_directory",
|
|
57
|
+
]
|
|
58
|
+
|
|
59
|
+
# Build output / VCS / editor state, never worth uploading. Carried over from
|
|
60
|
+
# `_run_command`; a `--include` can re-admit anything here, which is why the check
|
|
61
|
+
# runs before user rules rather than short-circuiting them.
|
|
62
|
+
SKIP_DIRS = frozenset(
|
|
63
|
+
{
|
|
64
|
+
".git",
|
|
65
|
+
".hg",
|
|
66
|
+
".svn",
|
|
67
|
+
"__pycache__",
|
|
68
|
+
".venv",
|
|
69
|
+
"venv",
|
|
70
|
+
"env",
|
|
71
|
+
"node_modules",
|
|
72
|
+
"target",
|
|
73
|
+
"dist",
|
|
74
|
+
"build",
|
|
75
|
+
".mypy_cache",
|
|
76
|
+
".pytest_cache",
|
|
77
|
+
".ruff_cache",
|
|
78
|
+
".tox",
|
|
79
|
+
".idea",
|
|
80
|
+
".vscode",
|
|
81
|
+
".eggs",
|
|
82
|
+
}
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
# Why a path was not uploaded. 'default' and 'rule' are ordinary filtering; the rest
|
|
86
|
+
# are conditions the user probably wants to know about.
|
|
87
|
+
SkipReason = Literal[
|
|
88
|
+
"credential",
|
|
89
|
+
"default",
|
|
90
|
+
"empty-dir",
|
|
91
|
+
"rule",
|
|
92
|
+
"symlink-file",
|
|
93
|
+
"symlink-dir",
|
|
94
|
+
"not-regular",
|
|
95
|
+
"unreadable",
|
|
96
|
+
"bad-name",
|
|
97
|
+
]
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@dataclass(frozen=True)
|
|
101
|
+
class ResolvedTarget:
|
|
102
|
+
"""One validated target: what the user typed, and where it lands."""
|
|
103
|
+
|
|
104
|
+
given: str
|
|
105
|
+
path: Path # absolute, symlinks resolved
|
|
106
|
+
is_dir: bool
|
|
107
|
+
basename: str # the single path component it occupies in the sandbox
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
@dataclass(frozen=True)
|
|
111
|
+
class Selected:
|
|
112
|
+
source: Path
|
|
113
|
+
rel: str # posix, relative to the sandbox destination root
|
|
114
|
+
size: int
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
@dataclass(frozen=True)
|
|
118
|
+
class Skipped:
|
|
119
|
+
rel: str
|
|
120
|
+
reason: SkipReason
|
|
121
|
+
detail: str = "" # the matching rule, or the OS error
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
@dataclass(frozen=True)
|
|
125
|
+
class UploadPlan:
|
|
126
|
+
targets: tuple[ResolvedTarget, ...]
|
|
127
|
+
selected: tuple[Selected, ...]
|
|
128
|
+
skipped: tuple[Skipped, ...]
|
|
129
|
+
total_bytes: int
|
|
130
|
+
cwd: str
|
|
131
|
+
oversized: tuple[Selected, ...] = field(default=())
|
|
132
|
+
|
|
133
|
+
@property
|
|
134
|
+
def file_count(self) -> int:
|
|
135
|
+
return len(self.selected)
|
|
136
|
+
|
|
137
|
+
def skipped_by_reason(self) -> dict[SkipReason, int]:
|
|
138
|
+
"""Counts per reason, for the manifest's "why" line."""
|
|
139
|
+
out: dict[SkipReason, int] = {}
|
|
140
|
+
for item in self.skipped:
|
|
141
|
+
out[item.reason] = out.get(item.reason, 0) + 1
|
|
142
|
+
return out
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _reject_unusable_name(name: str) -> str | None:
|
|
146
|
+
"""Return a reason string when *name* cannot ride the upload path.
|
|
147
|
+
|
|
148
|
+
`make_directory` used to reject NUL and newline client-side, but `upload_file`
|
|
149
|
+
creates parents itself so those calls are gone — taking the only guard with
|
|
150
|
+
them. The check moves here, where it can also be reported rather than raised.
|
|
151
|
+
"""
|
|
152
|
+
if "\x00" in name:
|
|
153
|
+
return "contains a NUL byte"
|
|
154
|
+
if "\n" in name or "\r" in name:
|
|
155
|
+
return "contains a newline"
|
|
156
|
+
return None
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _compile(exclude: list[str] | None, include: list[str] | None) -> tuple[GlobRule, ...]:
|
|
160
|
+
"""Excludes in order, then includes as trailing negations so include wins.
|
|
161
|
+
|
|
162
|
+
Two separate lists cannot express an interleaved order, so rather than pretend
|
|
163
|
+
otherwise the rule is stated plainly: every `include` is applied after every
|
|
164
|
+
`exclude`.
|
|
165
|
+
"""
|
|
166
|
+
for pattern in (*(exclude or []), *(include or [])):
|
|
167
|
+
if pattern.startswith("!"):
|
|
168
|
+
raise SandboxValidationError(
|
|
169
|
+
f"{pattern!r}: '!' is not accepted here — pass the pattern to "
|
|
170
|
+
f"include= to re-admit paths instead of negating an exclude=."
|
|
171
|
+
)
|
|
172
|
+
# Validated in the caller's own spelling, before the `!` below is prepended:
|
|
173
|
+
# `compile_rules` quotes the pattern it was handed, and quoting `'![abc'`
|
|
174
|
+
# names a pattern the caller never typed. Prepending `!` cannot change
|
|
175
|
+
# validity -- `_translate` strips it first -- so this is the same check.
|
|
176
|
+
try:
|
|
177
|
+
compile_rules([pattern])
|
|
178
|
+
except ValueError as exc:
|
|
179
|
+
raise SandboxValidationError(str(exc)) from exc
|
|
180
|
+
return compile_rules([*(exclude or []), *(f"!{p}" for p in (include or []))])
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def _credential_rel(source: Path, mrel: str, *, explicit: bool) -> Path:
|
|
184
|
+
"""The path `_is_credential_path` should judge.
|
|
185
|
+
|
|
186
|
+
Inside a walked tree that is *mrel*, the target-relative path. The denylist
|
|
187
|
+
inspects every component *but* the last, so it needs every directory between the
|
|
188
|
+
tree root and the file and nothing above it. A fixed-length tail gets this wrong
|
|
189
|
+
in both directions: `fixtures/.aws/sso/cache/token.json` hides `.aws` from a
|
|
190
|
+
three-component tail and ships a credential, while a tree that merely happens to
|
|
191
|
+
sit under `~/.snowflake/` has components *above* its own root judged and drops
|
|
192
|
+
its own source files as credentials — silently, since `credential` is not a
|
|
193
|
+
notable bundle skip.
|
|
194
|
+
|
|
195
|
+
A single named file is the one case with no tree to be relative to: *mrel* is a
|
|
196
|
+
bare basename, which hides the signal (`.ssh/config` tests True, `config`
|
|
197
|
+
False), so the absolute source path is judged. That direction only ever refuses
|
|
198
|
+
more, and `allow_credential_files=` is the escape hatch.
|
|
199
|
+
"""
|
|
200
|
+
return source if explicit else Path(mrel)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def build_plan(
|
|
204
|
+
targets: tuple[ResolvedTarget, ...],
|
|
205
|
+
*,
|
|
206
|
+
dest_root: str,
|
|
207
|
+
exclude: list[str] | None = None,
|
|
208
|
+
include: list[str] | None = None,
|
|
209
|
+
allow_credential_files: list[str] | None = None,
|
|
210
|
+
max_file_bytes: int | None = None,
|
|
211
|
+
) -> UploadPlan:
|
|
212
|
+
"""Walk *targets* and decide every file's fate, recording a reason for each."""
|
|
213
|
+
rules = _compile(exclude, include)
|
|
214
|
+
allowed = {Path(p).expanduser().resolve() for p in (allow_credential_files or [])}
|
|
215
|
+
|
|
216
|
+
selected: list[Selected] = []
|
|
217
|
+
skipped: list[Skipped] = []
|
|
218
|
+
|
|
219
|
+
for target in targets:
|
|
220
|
+
prefix = target.basename
|
|
221
|
+
if target.is_dir:
|
|
222
|
+
_walk_dir(target.path, prefix, rules, allowed, selected, skipped)
|
|
223
|
+
else:
|
|
224
|
+
# A lone file matches on its own name: `run proj/boot.py --exclude boot.py`.
|
|
225
|
+
_classify_file(
|
|
226
|
+
target.path,
|
|
227
|
+
prefix,
|
|
228
|
+
target.basename,
|
|
229
|
+
rules,
|
|
230
|
+
allowed,
|
|
231
|
+
selected,
|
|
232
|
+
skipped,
|
|
233
|
+
explicit=True,
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
oversized = tuple(s for s in selected if s.size > max_file_bytes) if max_file_bytes else ()
|
|
237
|
+
return UploadPlan(
|
|
238
|
+
targets=targets,
|
|
239
|
+
selected=tuple(selected),
|
|
240
|
+
skipped=tuple(skipped),
|
|
241
|
+
total_bytes=sum(s.size for s in selected),
|
|
242
|
+
cwd=_resolve_cwd(targets, dest_root),
|
|
243
|
+
oversized=oversized,
|
|
244
|
+
)
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def _resolve_cwd(targets: tuple[ResolvedTarget, ...], dest_root: str) -> str:
|
|
248
|
+
"""Where the command runs.
|
|
249
|
+
|
|
250
|
+
A single directory target runs *inside* that directory, which is what makes
|
|
251
|
+
`run ./project --command "python -m app"` behave as typed. Everything else runs
|
|
252
|
+
at the root, so a relative command can still reach every target by basename —
|
|
253
|
+
with one dir target among several, a cwd inside it would hide the siblings.
|
|
254
|
+
"""
|
|
255
|
+
if len(targets) == 1 and targets[0].is_dir:
|
|
256
|
+
return f"{dest_root}/{targets[0].basename}"
|
|
257
|
+
return dest_root
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def _walk_dir(
|
|
261
|
+
root: Path,
|
|
262
|
+
prefix: str,
|
|
263
|
+
rules: tuple[GlobRule, ...],
|
|
264
|
+
allowed: set[Path],
|
|
265
|
+
selected: list[Selected],
|
|
266
|
+
skipped: list[Skipped],
|
|
267
|
+
) -> None:
|
|
268
|
+
"""Recurse *root* with `os.scandir`, recording every exclusion with its reason.
|
|
269
|
+
|
|
270
|
+
Two paths are tracked per entry, and conflating them is a bug: *rel* is where the
|
|
271
|
+
file lands in the sandbox (prefixed with the target's basename) and is what the
|
|
272
|
+
manifest shows, while *mrel* is the path **relative to the target the user named**
|
|
273
|
+
and is what patterns match against. Users write `--exclude "src/**"` thinking in
|
|
274
|
+
terms of the directory they passed, exactly as `.gitignore` is relative to its own
|
|
275
|
+
directory; matching the prefixed path instead would make every *anchored* pattern
|
|
276
|
+
silently never match. It also keeps a target that happens to be named `build` or
|
|
277
|
+
`dist` from being mistaken for a default-skipped directory.
|
|
278
|
+
"""
|
|
279
|
+
stack: list[tuple[Path, str, str]] = [(root, prefix, "")]
|
|
280
|
+
while stack:
|
|
281
|
+
current, rel_prefix, match_prefix = stack.pop()
|
|
282
|
+
try:
|
|
283
|
+
entries = sorted(os.scandir(current), key=lambda e: e.name)
|
|
284
|
+
except OSError as exc:
|
|
285
|
+
skipped.append(Skipped(rel_prefix, "unreadable", exc.strerror or str(exc)))
|
|
286
|
+
continue
|
|
287
|
+
|
|
288
|
+
if not entries:
|
|
289
|
+
# Reported rather than dropped: the destination gets no such directory
|
|
290
|
+
# (there is no file to create it under), and an app that expects
|
|
291
|
+
# `logs/` to exist otherwise gets neither the directory nor a word
|
|
292
|
+
# about it -- against this module's "every drop is explainable" premise.
|
|
293
|
+
skipped.append(Skipped(f"{rel_prefix}/", "empty-dir"))
|
|
294
|
+
continue
|
|
295
|
+
|
|
296
|
+
for entry in entries:
|
|
297
|
+
rel = f"{rel_prefix}/{entry.name}"
|
|
298
|
+
mrel = f"{match_prefix}/{entry.name}" if match_prefix else entry.name
|
|
299
|
+
if bad := _reject_unusable_name(entry.name):
|
|
300
|
+
skipped.append(Skipped(rel, "bad-name", bad))
|
|
301
|
+
continue
|
|
302
|
+
try:
|
|
303
|
+
is_symlink = entry.is_symlink()
|
|
304
|
+
is_dir = entry.is_dir(follow_symlinks=False)
|
|
305
|
+
is_file = entry.is_file(follow_symlinks=False)
|
|
306
|
+
except OSError as exc:
|
|
307
|
+
skipped.append(Skipped(rel, "unreadable", exc.strerror or str(exc)))
|
|
308
|
+
continue
|
|
309
|
+
|
|
310
|
+
if is_symlink:
|
|
311
|
+
# Never followed, in either direction. Reported separately because a
|
|
312
|
+
# symlinked *directory* can silently omit an entire vendored tree,
|
|
313
|
+
# while a symlinked file omits one file.
|
|
314
|
+
points_at_dir = False
|
|
315
|
+
try:
|
|
316
|
+
points_at_dir = os.path.isdir(entry.path)
|
|
317
|
+
except OSError:
|
|
318
|
+
pass
|
|
319
|
+
skipped.append(Skipped(rel, "symlink-dir" if points_at_dir else "symlink-file"))
|
|
320
|
+
continue
|
|
321
|
+
|
|
322
|
+
if is_dir:
|
|
323
|
+
# Pruning a default-skipped directory is cheap and lets the manifest
|
|
324
|
+
# report one line instead of thousands. But `--include` means "re-admit
|
|
325
|
+
# what matches", and a pattern like `dist/**` cannot match the bare
|
|
326
|
+
# directory path -- so a skipped tree an include could still reach into
|
|
327
|
+
# has to be descended, and per-file evaluation decides.
|
|
328
|
+
#
|
|
329
|
+
# Asked per directory rather than once for the whole walk: any include
|
|
330
|
+
# at all used to disable pruning everywhere, so `include=["dist/**"]`
|
|
331
|
+
# walked `.git`, `node_modules` and `.venv` in full and allocated a
|
|
332
|
+
# `Skipped` per file -- hundreds of thousands on a large repo, plus a
|
|
333
|
+
# spurious "not-regular" warning off sockets under `.venv`.
|
|
334
|
+
if entry.name in SKIP_DIRS and not _include_may_reach(mrel, rules):
|
|
335
|
+
skipped.append(Skipped(rel + "/", "default", entry.name))
|
|
336
|
+
continue
|
|
337
|
+
stack.append((Path(entry.path), rel, mrel))
|
|
338
|
+
continue
|
|
339
|
+
|
|
340
|
+
if not is_file:
|
|
341
|
+
skipped.append(Skipped(rel, "not-regular"))
|
|
342
|
+
continue
|
|
343
|
+
|
|
344
|
+
_classify_file(
|
|
345
|
+
Path(entry.path), rel, mrel, rules, allowed, selected, skipped, explicit=False
|
|
346
|
+
)
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def _classify_file(
|
|
350
|
+
source: Path,
|
|
351
|
+
rel: str,
|
|
352
|
+
mrel: str,
|
|
353
|
+
rules: tuple[GlobRule, ...],
|
|
354
|
+
allowed: set[Path],
|
|
355
|
+
selected: list[Selected],
|
|
356
|
+
skipped: list[Skipped],
|
|
357
|
+
*,
|
|
358
|
+
explicit: bool,
|
|
359
|
+
) -> None:
|
|
360
|
+
"""Decide one file. *explicit* marks a path the user named as a TARGET.
|
|
361
|
+
|
|
362
|
+
*rel* is the destination path (reported); *mrel* is target-relative (matched).
|
|
363
|
+
"""
|
|
364
|
+
if bad := _reject_unusable_name(Path(rel).name):
|
|
365
|
+
skipped.append(Skipped(rel, "bad-name", bad))
|
|
366
|
+
return
|
|
367
|
+
|
|
368
|
+
hit = match_rules(mrel, rules)
|
|
369
|
+
if hit is not None and not hit.negated:
|
|
370
|
+
skipped.append(Skipped(rel, "rule", hit.source))
|
|
371
|
+
return
|
|
372
|
+
if hit is None and not explicit and _in_skipped_dir(mrel):
|
|
373
|
+
skipped.append(Skipped(rel, "default"))
|
|
374
|
+
return
|
|
375
|
+
|
|
376
|
+
# The credential filter runs last and is not overridable by a pattern: with
|
|
377
|
+
# negation available, a stray `--include "*"` would otherwise be one flag away
|
|
378
|
+
# from shipping ~/.aws/credentials. Only an exact --allow-credential-file PATH
|
|
379
|
+
# overrides, and naming a credential as a TARGET does not.
|
|
380
|
+
if (
|
|
381
|
+
_is_credential_path(_credential_rel(source, mrel, explicit=explicit))
|
|
382
|
+
and source not in allowed
|
|
383
|
+
):
|
|
384
|
+
skipped.append(Skipped(rel, "credential", source.name))
|
|
385
|
+
return
|
|
386
|
+
|
|
387
|
+
try:
|
|
388
|
+
size = source.stat().st_size
|
|
389
|
+
except OSError as exc:
|
|
390
|
+
skipped.append(Skipped(rel, "unreadable", exc.strerror or str(exc)))
|
|
391
|
+
return
|
|
392
|
+
selected.append(Selected(source, rel, size))
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
def _include_may_reach(mrel: str, rules: tuple[GlobRule, ...]) -> bool:
|
|
396
|
+
"""True when some `include=` could still match a path inside skipped dir *mrel*.
|
|
397
|
+
|
|
398
|
+
An unanchored include (`*.pyc`) matches at any depth, so it could; an include
|
|
399
|
+
anchored to a literal first segment (`dist/**`) can only reach into a tree whose
|
|
400
|
+
own first component is that segment. Anything this cannot decide counts as "it
|
|
401
|
+
could", so the answer is never wrong in the direction that loses files.
|
|
402
|
+
"""
|
|
403
|
+
first = mrel.split("/", 1)[0]
|
|
404
|
+
for rule in rules:
|
|
405
|
+
if not rule.negated:
|
|
406
|
+
continue
|
|
407
|
+
head = anchored_head(rule)
|
|
408
|
+
if head is None or head == first:
|
|
409
|
+
return True
|
|
410
|
+
return False
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
def _in_skipped_dir(rel: str) -> bool:
|
|
414
|
+
"""True when any parent component of *rel* is a default-skipped directory.
|
|
415
|
+
|
|
416
|
+
`_walk_dir` prunes these before descending, so this only catches the case where
|
|
417
|
+
a file is reached by another route.
|
|
418
|
+
"""
|
|
419
|
+
return bool(SKIP_DIRS & set(Path(rel).parts[:-1]))
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
def plan_directory(
|
|
423
|
+
local_dir: str | Path,
|
|
424
|
+
*,
|
|
425
|
+
dest_root: str,
|
|
426
|
+
exclude: list[str] | None = None,
|
|
427
|
+
include: list[str] | None = None,
|
|
428
|
+
allow_credential_files: list[str] | None = None,
|
|
429
|
+
max_file_bytes: int | None = None,
|
|
430
|
+
strip_root: bool = False,
|
|
431
|
+
) -> UploadPlan:
|
|
432
|
+
"""Plan the upload of one local directory, the shape both SDK callers need.
|
|
433
|
+
|
|
434
|
+
`build_plan` takes resolved *targets* because the CLI can be handed several at
|
|
435
|
+
once; the SDK's two callers each have exactly one directory, so this wraps the
|
|
436
|
+
single-target case and owns the two things that differ between them.
|
|
437
|
+
|
|
438
|
+
``strip_root`` chooses whether the directory's own name becomes a path component
|
|
439
|
+
in the destination. `Sandbox.upload_dir` strips it -- ``upload_dir("./site",
|
|
440
|
+
"/app")`` puts ``site/index.html`` at ``/app/index.html``, which is what "upload
|
|
441
|
+
the contents of this folder there" means, and mirrors ``cp -r src/. dst``. The
|
|
442
|
+
`from_local` bundle also strips it, because the zip is extracted *as* the project
|
|
443
|
+
root. `snow sandbox run` keeps it, so several targets can sit side by side.
|
|
444
|
+
|
|
445
|
+
Raises `SandboxValidationError` if *local_dir* is not a directory, so a caller
|
|
446
|
+
that passes a file gets told which of the two verbs it wanted.
|
|
447
|
+
"""
|
|
448
|
+
root = Path(local_dir).expanduser().resolve()
|
|
449
|
+
if not root.exists():
|
|
450
|
+
# Distinguished from the is-a-file case below: sending someone to
|
|
451
|
+
# `upload_file()` for a path that is not there is the same misdirection this
|
|
452
|
+
# message was introduced to remove from `upload_file`.
|
|
453
|
+
raise SandboxValidationError(f"{root} does not exist")
|
|
454
|
+
if not root.is_dir():
|
|
455
|
+
raise SandboxValidationError(
|
|
456
|
+
f"{root} is not a directory; use upload_file() for a single file"
|
|
457
|
+
)
|
|
458
|
+
target = ResolvedTarget(str(local_dir), root, is_dir=True, basename=root.name)
|
|
459
|
+
plan = build_plan(
|
|
460
|
+
(target,),
|
|
461
|
+
dest_root=dest_root,
|
|
462
|
+
exclude=exclude,
|
|
463
|
+
include=include,
|
|
464
|
+
allow_credential_files=allow_credential_files,
|
|
465
|
+
max_file_bytes=max_file_bytes,
|
|
466
|
+
)
|
|
467
|
+
if not strip_root:
|
|
468
|
+
return plan
|
|
469
|
+
# Re-root every path at dest_root by dropping the leading "<basename>/". Done
|
|
470
|
+
# here rather than by teaching `_walk_dir` a second prefix convention, so the
|
|
471
|
+
# walk keeps one rule and the rewrite stays visible in one place.
|
|
472
|
+
cut = len(target.basename) + 1
|
|
473
|
+
return UploadPlan(
|
|
474
|
+
targets=plan.targets,
|
|
475
|
+
selected=tuple(
|
|
476
|
+
Selected(s.source, s.rel[cut:], s.size)
|
|
477
|
+
if s.rel.startswith(target.basename + "/")
|
|
478
|
+
else s
|
|
479
|
+
for s in plan.selected
|
|
480
|
+
),
|
|
481
|
+
skipped=tuple(
|
|
482
|
+
# `or "."`: the root itself is reported when it is empty, and stripping
|
|
483
|
+
# its own name off leaves nothing to name it by.
|
|
484
|
+
Skipped(k.rel[cut:] or ".", k.reason, k.detail)
|
|
485
|
+
if k.rel.startswith(target.basename + "/")
|
|
486
|
+
else k
|
|
487
|
+
for k in plan.skipped
|
|
488
|
+
),
|
|
489
|
+
total_bytes=plan.total_bytes,
|
|
490
|
+
cwd=dest_root,
|
|
491
|
+
oversized=tuple(
|
|
492
|
+
Selected(s.source, s.rel[cut:], s.size)
|
|
493
|
+
if s.rel.startswith(target.basename + "/")
|
|
494
|
+
else s
|
|
495
|
+
for s in plan.oversized
|
|
496
|
+
),
|
|
497
|
+
)
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"""Single source of truth for the installed package version.
|
|
2
|
+
|
|
3
|
+
Resolves via ``importlib.metadata`` against the installed distribution
|
|
4
|
+
(the pinned wheel/sdist a caller ``pip install``s, or the dist-info an
|
|
5
|
+
editable install registers) so a runtime import always reflects the actual
|
|
6
|
+
installed build -- catching the "stale git checkout" class of incident this
|
|
7
|
+
was written for, where a long-stale SDK checkout silently routed to a dead
|
|
8
|
+
endpoint.
|
|
9
|
+
|
|
10
|
+
There is no ``setuptools-scm`` wiring: shallow CI checkouts can't derive a
|
|
11
|
+
version from git anyway. Instead this is a documented manual semver track:
|
|
12
|
+
``_FALLBACK_VERSION`` below must be bumped in lockstep with
|
|
13
|
+
``[project].version`` in ``pyproject.toml`` on every release, and the
|
|
14
|
+
release tag (``vX.Y.Z``) must match -- ``Jenkinsfile.release`` fails the
|
|
15
|
+
publish if the tag and ``pyproject.toml`` disagree. The fallback only
|
|
16
|
+
matters when the package metadata can't be found at all (e.g. running from
|
|
17
|
+
a raw source checkout with nothing installed).
|
|
18
|
+
|
|
19
|
+
Kept import-cheap: only stdlib ``importlib.metadata``.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
from importlib.metadata import PackageNotFoundError
|
|
25
|
+
from importlib.metadata import version as _pkg_version
|
|
26
|
+
|
|
27
|
+
__all__ = ["__version__"]
|
|
28
|
+
|
|
29
|
+
_DIST_NAME = "snowflake-sandbox-python"
|
|
30
|
+
|
|
31
|
+
# Keep in sync with [project].version in pyproject.toml.
|
|
32
|
+
_FALLBACK_VERSION = "0.2.1a1"
|
|
33
|
+
|
|
34
|
+
try:
|
|
35
|
+
__version__: str = _pkg_version(_DIST_NAME)
|
|
36
|
+
except PackageNotFoundError: # pragma: no cover - raw/uninstalled source checkout
|
|
37
|
+
__version__ = _FALLBACK_VERSION
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
"""Filesystem change notifications for ``AsyncSandbox``, over ``inotify(7)``.
|
|
2
|
+
|
|
3
|
+
One of the three separate mechanisms that reach a sandbox filesystem, split out of
|
|
4
|
+
``files.py``. `files` keeps the ``/files`` byte route; `_fs_ops` the directory
|
|
5
|
+
operations. Paths here follow the byte route rather than `_fs_ops`: ``watch``
|
|
6
|
+
requires an absolute path, and calls `files._require_absolute` to say so.
|
|
7
|
+
|
|
8
|
+
``watch`` streams filesystem-change events
|
|
9
|
+
(`FileWatchEvent`) as they happen. It needs no server route: it runs a tiny
|
|
10
|
+
``ctypes``-driven ``inotify(7)`` monitor in the container over the resumable
|
|
11
|
+
``exec_stream`` (the same channel the directory ops use), which reattaches to the
|
|
12
|
+
same exec session when the ingress drops a long-lived connection. Zero container
|
|
13
|
+
dependency — inotify is a kernel facility reached through ``libc``, so it does
|
|
14
|
+
not rely on an ``inotifywait`` binary (absent from the image) or any pip
|
|
15
|
+
install. See `watch` for the recursion/indefinite-timeout caveats.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import json
|
|
21
|
+
from typing import TYPE_CHECKING
|
|
22
|
+
|
|
23
|
+
from snowflake.sandbox._runtime._fs_helper import _watch_program
|
|
24
|
+
from snowflake.sandbox._runtime._protocol import (
|
|
25
|
+
_IN_ACCESS,
|
|
26
|
+
_IN_CLOSE_WRITE,
|
|
27
|
+
_IN_CREATE,
|
|
28
|
+
_IN_DELETE,
|
|
29
|
+
_IN_MODIFY,
|
|
30
|
+
_IN_MOVED_FROM,
|
|
31
|
+
_IN_MOVED_TO,
|
|
32
|
+
_IN_OPEN,
|
|
33
|
+
_WATCH_SENTINEL,
|
|
34
|
+
)
|
|
35
|
+
from snowflake.sandbox._transport import Transport
|
|
36
|
+
from snowflake.sandbox.files import _require_absolute
|
|
37
|
+
from snowflake.sandbox.types import FileWatchEvent, FileWatchEventType
|
|
38
|
+
|
|
39
|
+
if TYPE_CHECKING:
|
|
40
|
+
from collections.abc import AsyncIterator, Sequence
|
|
41
|
+
|
|
42
|
+
from snowflake.sandbox.client import AsyncSandbox
|
|
43
|
+
|
|
44
|
+
__all__ = ["watch"]
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
# ---- filesystem change notifications (Modal Sandbox.filesystem.watch) ------
|
|
48
|
+
#
|
|
49
|
+
# The monitor emits one sentinel-prefixed JSON line per raw inotify event; watch()
|
|
50
|
+
# parses those, maps the mask onto FileWatchEventType, applies the caller's filter,
|
|
51
|
+
# and yields FileWatchEvent. argv (path/recursive/timeout) is passed as separate
|
|
52
|
+
# argv entries -- never shell-interpolated -- so a path with spaces/quotes is safe.
|
|
53
|
+
# `inotify_init1` is always present on a Linux kernel, so the monitor needs no
|
|
54
|
+
# container package; see the module docstring for the rest of the mechanism.
|
|
55
|
+
|
|
56
|
+
# The single server-side exec budget a watch stream gets when no timeout is asked
|
|
57
|
+
# for. exec_stream caps an untimed exec at _DEFAULT_EXEC_BUDGET_S (600s) server
|
|
58
|
+
# side, so an "indefinite" watch is realised as a relaunch loop of bounded
|
|
59
|
+
# windows (see watch()); this is that window.
|
|
60
|
+
_WATCH_WINDOW_S = 3600.0
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _mask_to_event_type(mask: int) -> FileWatchEventType:
|
|
64
|
+
"""Collapse a raw inotify mask onto the coarse FileWatchEventType enum.
|
|
65
|
+
|
|
66
|
+
Order matters: a create/move-in is reported as Create even though the same
|
|
67
|
+
write often also carries MODIFY bits; delete/move-out wins similarly. This
|
|
68
|
+
matches Modal's granularity (it wraps the Rust ``notify`` crate, which makes
|
|
69
|
+
the same coarse distinctions).
|
|
70
|
+
"""
|
|
71
|
+
if mask & (_IN_CREATE | _IN_MOVED_TO):
|
|
72
|
+
return FileWatchEventType.Create
|
|
73
|
+
if mask & (_IN_DELETE | _IN_MOVED_FROM):
|
|
74
|
+
return FileWatchEventType.Delete
|
|
75
|
+
if mask & (_IN_MODIFY | _IN_CLOSE_WRITE):
|
|
76
|
+
return FileWatchEventType.Modify
|
|
77
|
+
if mask & (_IN_ACCESS | _IN_OPEN):
|
|
78
|
+
return FileWatchEventType.Access
|
|
79
|
+
return FileWatchEventType.Unknown
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
async def watch(
|
|
83
|
+
sandbox: AsyncSandbox,
|
|
84
|
+
path: str,
|
|
85
|
+
*,
|
|
86
|
+
filter: Sequence[FileWatchEventType] | None = None,
|
|
87
|
+
recursive: bool = False,
|
|
88
|
+
timeout: float | None = None,
|
|
89
|
+
transport: Transport | None = None,
|
|
90
|
+
) -> AsyncIterator[FileWatchEvent]:
|
|
91
|
+
"""Stream filesystem-change events under ``path`` (Modal
|
|
92
|
+
``Sandbox.filesystem.watch``). Async generator — ``async for ev in
|
|
93
|
+
sb.watch(...)``.
|
|
94
|
+
|
|
95
|
+
``path`` must be absolute. ``filter`` restricts the yielded
|
|
96
|
+
`FileWatchEventType`\\ s (``None`` = all). ``recursive`` also watches nested
|
|
97
|
+
subdirectories (and directories created later). ``timeout`` bounds the watch
|
|
98
|
+
in seconds; ``None`` watches until the caller stops iterating (or the sandbox
|
|
99
|
+
dies).
|
|
100
|
+
|
|
101
|
+
**How.** Runs a ctypes ``inotify(7)`` monitor in the container over the
|
|
102
|
+
resumable `exec_stream` — no server route, no container package. See the
|
|
103
|
+
module docstring.
|
|
104
|
+
|
|
105
|
+
**Caveats.**
|
|
106
|
+
* A single logical file save commonly yields a `Modify` (write) event; atomic
|
|
107
|
+
editors that save-by-rename surface as `Create`/`Delete` of a temp plus a
|
|
108
|
+
move — ordinary inotify granularity, matching Modal.
|
|
109
|
+
* A large ``recursive`` watch consumes one kernel watch descriptor per
|
|
110
|
+
directory (``fs.inotify.max_user_watches``); a very deep tree can exhaust it.
|
|
111
|
+
* ``timeout=None`` is realised as a relaunch loop of bounded exec windows
|
|
112
|
+
(the server caps an untimed exec at ~600s). Events that occur during the
|
|
113
|
+
sub-second relaunch gap between windows can be missed — a gap-free
|
|
114
|
+
indefinite watch would need a server-side ``/files/watch`` endpoint, which
|
|
115
|
+
does not exist yet. A bounded ``timeout`` runs as a single stream and has no
|
|
116
|
+
such gap.
|
|
117
|
+
"""
|
|
118
|
+
p = _require_absolute(path)
|
|
119
|
+
allowed = frozenset(filter) if filter else None
|
|
120
|
+
prog = _watch_program()
|
|
121
|
+
|
|
122
|
+
async def _one_window(window_timeout: float | None) -> AsyncIterator[FileWatchEvent]:
|
|
123
|
+
argv = [
|
|
124
|
+
"python3",
|
|
125
|
+
"-c",
|
|
126
|
+
prog,
|
|
127
|
+
p,
|
|
128
|
+
"1" if recursive else "0",
|
|
129
|
+
"" if window_timeout is None else repr(window_timeout),
|
|
130
|
+
]
|
|
131
|
+
# Client read budget must outlast the server exec budget so a live watch
|
|
132
|
+
# is never abandoned client-side; exec_stream adds its own grace on top.
|
|
133
|
+
stream = sandbox.exec_stream(argv, timeout=window_timeout)
|
|
134
|
+
async for line in stream:
|
|
135
|
+
if line.stream != "stdout":
|
|
136
|
+
continue
|
|
137
|
+
idx = line.data.find(_WATCH_SENTINEL)
|
|
138
|
+
if idx == -1:
|
|
139
|
+
continue
|
|
140
|
+
payload = line.data[idx + len(_WATCH_SENTINEL) :]
|
|
141
|
+
if payload.startswith("READY"):
|
|
142
|
+
continue
|
|
143
|
+
try:
|
|
144
|
+
obj = json.loads(payload)
|
|
145
|
+
except ValueError:
|
|
146
|
+
continue
|
|
147
|
+
etype = _mask_to_event_type(int(obj.get("mask", 0)))
|
|
148
|
+
if allowed is not None and etype not in allowed:
|
|
149
|
+
continue
|
|
150
|
+
yield FileWatchEvent(paths=[str(obj.get("path", p))], type=etype)
|
|
151
|
+
|
|
152
|
+
if timeout is not None:
|
|
153
|
+
# Bounded: a single stream (the resumable exec_stream keeps it alive
|
|
154
|
+
# across GOAWAYs up to the requested duration).
|
|
155
|
+
async for ev in _one_window(timeout):
|
|
156
|
+
yield ev
|
|
157
|
+
return
|
|
158
|
+
|
|
159
|
+
# Indefinite: relaunch bounded windows until the caller stops iterating.
|
|
160
|
+
# Breaking the async-for closes the generator, which closes the stream and
|
|
161
|
+
# kills the in-container monitor.
|
|
162
|
+
while True:
|
|
163
|
+
async for ev in _one_window(_WATCH_WINDOW_S):
|
|
164
|
+
yield ev
|