snowflake-sandbox-python 0.2.1a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- snowflake/cli_sandbox/__init__.py +13 -0
- snowflake/cli_sandbox/_adapter.py +170 -0
- snowflake/cli_sandbox/_common.py +77 -0
- snowflake/cli_sandbox/_egress_flags.py +121 -0
- snowflake/cli_sandbox/_get_command.py +109 -0
- snowflake/cli_sandbox/_run_command.py +1091 -0
- snowflake/cli_sandbox/_shell_command.py +666 -0
- snowflake/cli_sandbox/_upload_plan.py +187 -0
- snowflake/cli_sandbox/commands.py +556 -0
- snowflake/cli_sandbox/plugin_spec.py +28 -0
- snowflake/cli_sandbox/py.typed +0 -0
- snowflake/sandbox/__init__.py +317 -0
- snowflake/sandbox/__main__.py +225 -0
- snowflake/sandbox/_ansi.py +206 -0
- snowflake/sandbox/_args.py +208 -0
- snowflake/sandbox/_assemble.py +256 -0
- snowflake/sandbox/_bundle.py +240 -0
- snowflake/sandbox/_connection_resolve.py +328 -0
- snowflake/sandbox/_deploy_spec.py +56 -0
- snowflake/sandbox/_diagnostics.py +501 -0
- snowflake/sandbox/_env.py +143 -0
- snowflake/sandbox/_files_mixin.py +280 -0
- snowflake/sandbox/_fs_ops.py +304 -0
- snowflake/sandbox/_globs.py +176 -0
- snowflake/sandbox/_hosts.py +110 -0
- snowflake/sandbox/_mcp_discovery.py +288 -0
- snowflake/sandbox/_mcp_status.py +183 -0
- snowflake/sandbox/_retry.py +94 -0
- snowflake/sandbox/_runtime/__init__.py +42 -0
- snowflake/sandbox/_runtime/_fs_helper.py +93 -0
- snowflake/sandbox/_runtime/_job_runner.py +111 -0
- snowflake/sandbox/_runtime/_protocol.py +53 -0
- snowflake/sandbox/_runtime/_shims.py +267 -0
- snowflake/sandbox/_sandbox_state.py +303 -0
- snowflake/sandbox/_session_registry.py +222 -0
- snowflake/sandbox/_sse.py +160 -0
- snowflake/sandbox/_stage.py +270 -0
- snowflake/sandbox/_sync_files_mixin.py +272 -0
- snowflake/sandbox/_sync_fs_ops.py +185 -0
- snowflake/sandbox/_sync_transport.py +737 -0
- snowflake/sandbox/_sync_watch.py +99 -0
- snowflake/sandbox/_transport.py +1366 -0
- snowflake/sandbox/_transport_errors.py +270 -0
- snowflake/sandbox/_upload_plan.py +497 -0
- snowflake/sandbox/_version.py +37 -0
- snowflake/sandbox/_watch.py +164 -0
- snowflake/sandbox/_wire.py +348 -0
- snowflake/sandbox/app.py +256 -0
- snowflake/sandbox/client.py +2356 -0
- snowflake/sandbox/config.py +1133 -0
- snowflake/sandbox/connect.py +288 -0
- snowflake/sandbox/deploy.py +499 -0
- snowflake/sandbox/egress.py +388 -0
- snowflake/sandbox/exceptions.py +253 -0
- snowflake/sandbox/exec_stream.py +264 -0
- snowflake/sandbox/files.py +547 -0
- snowflake/sandbox/function.py +567 -0
- snowflake/sandbox/image.py +46 -0
- snowflake/sandbox/jobs.py +649 -0
- snowflake/sandbox/lifecycle.py +67 -0
- snowflake/sandbox/log_stream.py +219 -0
- snowflake/sandbox/mcp.py +480 -0
- snowflake/sandbox/mount.py +161 -0
- snowflake/sandbox/py.typed +0 -0
- snowflake/sandbox/secret.py +244 -0
- snowflake/sandbox/session_app.py +244 -0
- snowflake/sandbox/shell.py +556 -0
- snowflake/sandbox/sync_client.py +2245 -0
- snowflake/sandbox/sync_exec_stream.py +238 -0
- snowflake/sandbox/sync_files.py +377 -0
- snowflake/sandbox/sync_log_stream.py +142 -0
- snowflake/sandbox/sync_shell.py +413 -0
- snowflake/sandbox/types.py +193 -0
- snowflake/sandbox/warm_session.py +700 -0
- snowflake_sandbox_python-0.2.1a1.dist-info/METADATA +339 -0
- snowflake_sandbox_python-0.2.1a1.dist-info/RECORD +80 -0
- snowflake_sandbox_python-0.2.1a1.dist-info/WHEEL +5 -0
- snowflake_sandbox_python-0.2.1a1.dist-info/entry_points.txt +2 -0
- snowflake_sandbox_python-0.2.1a1.dist-info/licenses/LICENSE +202 -0
- snowflake_sandbox_python-0.2.1a1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
"""Building a code bundle from a local tree.
|
|
2
|
+
|
|
3
|
+
Collect, filter, hash and zip a directory into the archive that goes to a
|
|
4
|
+
Snowflake stage. Pure functions over paths and bytes — no transport, no client.
|
|
5
|
+
|
|
6
|
+
The credential exclusion is load-bearing, not hygiene: a project with a ``.env``
|
|
7
|
+
beside ``main.py`` is the normal layout, and without this the bundle would upload
|
|
8
|
+
it to the stage.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from collections.abc import Sequence
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import TYPE_CHECKING
|
|
16
|
+
|
|
17
|
+
from snowflake.sandbox.exceptions import SandboxValidationError
|
|
18
|
+
|
|
19
|
+
if TYPE_CHECKING:
|
|
20
|
+
from snowflake.sandbox._upload_plan import SkipReason, UploadPlan
|
|
21
|
+
|
|
22
|
+
__all__ = [
|
|
23
|
+
"_collect_tree",
|
|
24
|
+
"_hash_files",
|
|
25
|
+
"_zip_files",
|
|
26
|
+
"_is_credential_path",
|
|
27
|
+
"_warn_notable_bundle_skips",
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
# Filenames a code bundle must never carry to a Snowflake stage (see the module
|
|
32
|
+
# docstring for why this is load-bearing). Matched on the file name and on
|
|
33
|
+
# directory components, case-insensitively.
|
|
34
|
+
_CREDENTIAL_NAMES: frozenset[str] = frozenset(
|
|
35
|
+
{
|
|
36
|
+
".env",
|
|
37
|
+
".envrc",
|
|
38
|
+
".netrc",
|
|
39
|
+
".npmrc",
|
|
40
|
+
".pypirc",
|
|
41
|
+
".git-credentials",
|
|
42
|
+
"credentials",
|
|
43
|
+
"credentials.json",
|
|
44
|
+
"id_rsa",
|
|
45
|
+
"id_dsa",
|
|
46
|
+
"id_ecdsa",
|
|
47
|
+
"id_ed25519",
|
|
48
|
+
"service-account.json",
|
|
49
|
+
"connections.toml",
|
|
50
|
+
}
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
_CREDENTIAL_SUFFIXES: tuple[str, ...] = (".pem", ".key", ".p12", ".pfx", ".jks", ".keystore")
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
_CREDENTIAL_DIRS: frozenset[str] = frozenset({".ssh", ".aws", ".gnupg", ".snowflake"})
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _is_credential_path(rel: Path) -> bool:
|
|
61
|
+
"""True when *rel* (relative to the bundle root) looks like a credential."""
|
|
62
|
+
parts = [p.lower() for p in rel.parts]
|
|
63
|
+
if any(p in _CREDENTIAL_DIRS for p in parts[:-1]):
|
|
64
|
+
return True
|
|
65
|
+
name = parts[-1]
|
|
66
|
+
if name in _CREDENTIAL_NAMES or name.startswith(".env."):
|
|
67
|
+
return True
|
|
68
|
+
return name.endswith(_CREDENTIAL_SUFFIXES)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _collect_tree(
|
|
72
|
+
src: Path,
|
|
73
|
+
*,
|
|
74
|
+
exclude: list[str] | None = None,
|
|
75
|
+
include: list[str] | None = None,
|
|
76
|
+
) -> tuple[list[tuple[str, bytes]], UploadPlan]:
|
|
77
|
+
"""Sorted ``(relpath, content)`` for every file under *src*, plus the plan that
|
|
78
|
+
says what was left out and why. Used by ``from_local``.
|
|
79
|
+
|
|
80
|
+
Selection is delegated to `_upload_plan`, which is the one walk the CLI and
|
|
81
|
+
`Sandbox.upload_dir` also use. This function used to own a second, weaker walk:
|
|
82
|
+
``rglob`` with a four-entry skip set, which (a) reported nothing at all, so a
|
|
83
|
+
bundle that quietly lost a tree looked identical to a complete one, (b) never
|
|
84
|
+
recursed into a symlinked directory -- a pnpm/bazel workspace shipped without its
|
|
85
|
+
packages and without a word -- and (c) skipped no build output, so ``node_modules``
|
|
86
|
+
and ``dist`` rode along inside the zip. Sharing the engine fixes all three and
|
|
87
|
+
means the two walks cannot drift apart again.
|
|
88
|
+
|
|
89
|
+
The bytes are read here rather than in the engine: the engine plans an upload of
|
|
90
|
+
paths, and only this caller needs the contents in memory to zip them.
|
|
91
|
+
"""
|
|
92
|
+
# Imported in-function: `_upload_plan` imports `_is_credential_path` from this
|
|
93
|
+
# module, so a module-scope import here would close that into a cycle. Same
|
|
94
|
+
# reason `_assemble` imports the predicate lazily.
|
|
95
|
+
from snowflake.sandbox._upload_plan import plan_directory
|
|
96
|
+
|
|
97
|
+
# `*.pyc` was a skip the old walk applied by suffix and the engine does not: its
|
|
98
|
+
# defaults prune `__pycache__` as a directory, which covers the common case but not
|
|
99
|
+
# a stray or legacy-layout `.pyc`. Kept as a bundle-scoped exclude rather than added
|
|
100
|
+
# to the engine's defaults, so this consolidation does not quietly change what
|
|
101
|
+
# `snow sandbox run` uploads. It leads, so a caller's own patterns can still
|
|
102
|
+
# re-admit it with `include=["*.pyc"]`.
|
|
103
|
+
plan = plan_directory(
|
|
104
|
+
src,
|
|
105
|
+
dest_root="/",
|
|
106
|
+
exclude=["*.pyc", *(exclude or [])],
|
|
107
|
+
include=include,
|
|
108
|
+
strip_root=True,
|
|
109
|
+
)
|
|
110
|
+
files: list[tuple[str, bytes]] = []
|
|
111
|
+
for item in plan.selected:
|
|
112
|
+
try:
|
|
113
|
+
files.append((item.rel, item.source.read_bytes()))
|
|
114
|
+
except OSError:
|
|
115
|
+
# Readable at stat time, gone or unreadable now. Recorded as a skip so the
|
|
116
|
+
# bundle still reports it rather than silently shrinking.
|
|
117
|
+
plan = _with_extra_skip(plan, item.rel, "unreadable")
|
|
118
|
+
files.sort(key=lambda pair: pair[0])
|
|
119
|
+
if not files:
|
|
120
|
+
# The CLI refuses this as "nothing to upload"; `from_local` used to build a
|
|
121
|
+
# zero-entry zip and hand back a sandbox with no code in it, which only shows
|
|
122
|
+
# up as an import failure inside the container.
|
|
123
|
+
raise SandboxValidationError(
|
|
124
|
+
f"no files to bundle from {src}: all {len(plan.skipped)} path(s) were "
|
|
125
|
+
f"filtered out ({_reason_summary(plan)}). Pass include= to re-admit a "
|
|
126
|
+
f"filtered path, or point from_local at the directory holding your code."
|
|
127
|
+
)
|
|
128
|
+
return files, plan
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _reason_summary(plan: UploadPlan) -> str:
|
|
132
|
+
"""``"91 default, 2 symlink-dir"`` -- the why-line for an empty-bundle error."""
|
|
133
|
+
counts = plan.skipped_by_reason()
|
|
134
|
+
return ", ".join(f"{n} {reason}" for reason, n in sorted(counts.items())) or "none"
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _with_extra_skip(plan: UploadPlan, rel: str, reason: SkipReason) -> UploadPlan:
|
|
138
|
+
"""A copy of *plan* with one more skip, and that file dropped from ``selected``."""
|
|
139
|
+
from snowflake.sandbox._upload_plan import Skipped, UploadPlan
|
|
140
|
+
|
|
141
|
+
return UploadPlan(
|
|
142
|
+
targets=plan.targets,
|
|
143
|
+
selected=tuple(s for s in plan.selected if s.rel != rel),
|
|
144
|
+
skipped=(*plan.skipped, Skipped(rel, reason)),
|
|
145
|
+
total_bytes=sum(s.size for s in plan.selected if s.rel != rel),
|
|
146
|
+
cwd=plan.cwd,
|
|
147
|
+
oversized=plan.oversized,
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _hash_files(files: Sequence[tuple[str, bytes]]) -> str:
|
|
152
|
+
"""Deterministic 16-hex content hash over ``(relpath, content)`` pairs, in the
|
|
153
|
+
order given. Independent of zip build time, so an unchanged tree hashes the
|
|
154
|
+
same and the image layer key is stable."""
|
|
155
|
+
import hashlib
|
|
156
|
+
|
|
157
|
+
hasher = hashlib.sha256()
|
|
158
|
+
for rel, data in files:
|
|
159
|
+
hasher.update(rel.encode())
|
|
160
|
+
hasher.update(data)
|
|
161
|
+
return hasher.hexdigest()[:16]
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _zip_files(files: Sequence[tuple[str, bytes]]) -> bytes:
|
|
165
|
+
"""Reproducible zip (fixed timestamps) of ``(relpath, content)`` pairs."""
|
|
166
|
+
import io
|
|
167
|
+
import zipfile
|
|
168
|
+
|
|
169
|
+
buf = io.BytesIO()
|
|
170
|
+
fixed_date = (2020, 1, 1, 0, 0, 0)
|
|
171
|
+
with zipfile.ZipFile(buf, "w", compression=zipfile.ZIP_DEFLATED) as zf:
|
|
172
|
+
for rel, data in files:
|
|
173
|
+
# compress_type has to be set on the ZipInfo, not just on the ZipFile:
|
|
174
|
+
# a hand-built ZipInfo defaults to ZIP_STORED and that default wins, so
|
|
175
|
+
# the ZipFile's compression= was silently ignored and every bundle went
|
|
176
|
+
# to the stage uncompressed. Only a ZipInfo built by the ZipFile itself
|
|
177
|
+
# (i.e. writestr with a str arcname) inherits it, and that path cannot
|
|
178
|
+
# pin the timestamp this function exists to fix.
|
|
179
|
+
entry = zipfile.ZipInfo(rel, date_time=fixed_date)
|
|
180
|
+
entry.compress_type = zipfile.ZIP_DEFLATED
|
|
181
|
+
zf.writestr(entry, data)
|
|
182
|
+
return buf.getvalue()
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
# Reasons a bundle's caller wants to hear about. Ordinary filtering (`default`, an
|
|
186
|
+
# explicit `rule`) is the feature working; these four mean the bundle may be missing
|
|
187
|
+
# something the caller believes is in it.
|
|
188
|
+
_NOTABLE_SKIPS: frozenset[str] = frozenset({"symlink-dir", "unreadable", "not-regular", "bad-name"})
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
# Default-skipped directory names that are *also* plausible hand-written source
|
|
192
|
+
# directories. Dropping `.git` or `__pycache__` needs no announcement; dropping
|
|
193
|
+
# `build/` or `env/` might have dropped the project. `from_local` inherited these
|
|
194
|
+
# from the shared engine's SKIP_DIRS, where they were only ever vetted against
|
|
195
|
+
# `snow sandbox run`'s TARGETs, so a bundle can now lose a tree the old four-entry
|
|
196
|
+
# skip set kept -- and `default` skips are otherwise silent by design.
|
|
197
|
+
_AMBIGUOUS_SKIP_DIRS: frozenset[str] = frozenset({"env", "venv", "build", "dist", "target"})
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _warn_notable_bundle_skips(src: Path, plan: UploadPlan) -> None:
|
|
201
|
+
"""Warn when a bundle dropped something its author probably expected to ship.
|
|
202
|
+
|
|
203
|
+
`from_local` returns a `Sandbox`, so there is nowhere to hand a plan back to --
|
|
204
|
+
and a bundle missing a symlinked package tree is indistinguishable from a
|
|
205
|
+
complete one until the container fails to import it. A warning is the only
|
|
206
|
+
channel that reaches the caller without changing the return type.
|
|
207
|
+
|
|
208
|
+
Silent about `rule` skips, and about `default` skips other than the ambiguous
|
|
209
|
+
directory names above: filtering out ``node_modules`` is the intended behaviour,
|
|
210
|
+
and warning about it every time would train people to ignore the warning that
|
|
211
|
+
matters.
|
|
212
|
+
"""
|
|
213
|
+
import warnings
|
|
214
|
+
|
|
215
|
+
ambiguous = sorted(
|
|
216
|
+
{k.rel for k in plan.skipped if k.reason == "default" and k.detail in _AMBIGUOUS_SKIP_DIRS}
|
|
217
|
+
)
|
|
218
|
+
if ambiguous:
|
|
219
|
+
warnings.warn(
|
|
220
|
+
f"code bundle from {src} skipped {len(ambiguous)} directory(ies) whose "
|
|
221
|
+
f"names are build output by default: {', '.join(ambiguous)}. If any of "
|
|
222
|
+
f"those hold source, re-admit them with include=.",
|
|
223
|
+
stacklevel=3,
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
notable = [k for k in plan.skipped if k.reason in _NOTABLE_SKIPS]
|
|
227
|
+
if not notable:
|
|
228
|
+
return
|
|
229
|
+
counts: dict[str, int] = {}
|
|
230
|
+
for item in notable:
|
|
231
|
+
counts[item.reason] = counts.get(item.reason, 0) + 1
|
|
232
|
+
summary = ", ".join(f"{n} {reason}" for reason, n in sorted(counts.items()))
|
|
233
|
+
sample = ", ".join(k.rel for k in notable[:3])
|
|
234
|
+
more = f" (and {len(notable) - 3} more)" if len(notable) > 3 else ""
|
|
235
|
+
warnings.warn(
|
|
236
|
+
f"code bundle from {src} left out {summary}: {sample}{more}. Symlinks are "
|
|
237
|
+
f"never followed; pass include= to re-admit a filtered path, or copy the "
|
|
238
|
+
f"target in place of the link.",
|
|
239
|
+
stacklevel=3,
|
|
240
|
+
)
|
|
@@ -0,0 +1,328 @@
|
|
|
1
|
+
"""Connection resolution for ``snowflake.sandbox`` — split out of ``config.py``.
|
|
2
|
+
|
|
3
|
+
We delegate connection handling to snowflake-connector-python end to end. The
|
|
4
|
+
connector owns the file format (``config_manager.CONFIG_MANAGER`` — tomlkit type
|
|
5
|
+
fidelity, file-permission checks, the config.toml/connections.toml merge) AND
|
|
6
|
+
the resolution of a named connection into a live session: ``connect(
|
|
7
|
+
connection_name=...)`` re-reads the connection, applies its own key aliases, and
|
|
8
|
+
authenticates with whatever method the connection declares (password, key-pair,
|
|
9
|
+
oauth token file, external browser). We hand it the connection name + file and
|
|
10
|
+
take the session's REST bearer token — we do not rebuild connect() kwargs or
|
|
11
|
+
re-layer env vars ourselves (that is snow-CLI behaviour the connector does not
|
|
12
|
+
provide; the ``snow sandbox`` plugin gets it for free by using the CLI's own
|
|
13
|
+
connection). The connector import stays lazy: ``import snowflake.sandbox`` never
|
|
14
|
+
loads it — resolution happens on first use.
|
|
15
|
+
|
|
16
|
+
Every name here is re-exported from ``config`` (``from ._connection_resolve
|
|
17
|
+
import ...``) so ``snowflake.sandbox.config.<name>`` keeps resolving. The
|
|
18
|
+
handful of things that stay in ``config`` (``Config``, ``_from_env``,
|
|
19
|
+
``_config_files``, ``_warn_pat_fallback``, the ``_session_registry`` list) are
|
|
20
|
+
imported lazily inside the functions that need them — a module-scope import back
|
|
21
|
+
into ``config`` would close a cycle (``config`` imports this module to re-export
|
|
22
|
+
it).
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import os
|
|
28
|
+
from typing import TYPE_CHECKING, Any
|
|
29
|
+
|
|
30
|
+
if TYPE_CHECKING:
|
|
31
|
+
from snowflake.sandbox.config import Config
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _append_port(host: str, raw_port: Any) -> str:
|
|
35
|
+
"""Append ``:port`` to *host* when *raw_port* is a non-default (non-443) port.
|
|
36
|
+
|
|
37
|
+
A dev/regtest deployment reaches Snowflake on 8082, say, and that port belongs
|
|
38
|
+
in the REST endpoint too (``Config.base_url`` accepts a bare ``hostname[:port]``).
|
|
39
|
+
An absent, non-numeric, or 443 port is left off — 443 is the https default and a
|
|
40
|
+
junk value (a connector that reports a non-int ``port``) must not become a bogus
|
|
41
|
+
``host:port``. One rule, shared by every place that turns connection/connector
|
|
42
|
+
fields into an endpoint.
|
|
43
|
+
"""
|
|
44
|
+
try:
|
|
45
|
+
port_i = int(raw_port) if raw_port is not None else None
|
|
46
|
+
except (TypeError, ValueError):
|
|
47
|
+
port_i = None
|
|
48
|
+
if port_i and port_i != 443:
|
|
49
|
+
return f"{host}:{port_i}"
|
|
50
|
+
return host
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _connector_config_manager() -> Any:
|
|
54
|
+
"""The connector's ``CONFIG_MANAGER``, re-pointed at the current home and
|
|
55
|
+
re-read. Re-pointing mirrors what the connector itself does for a
|
|
56
|
+
``connections_file_path=`` (see ``connection.py``) and picks up a
|
|
57
|
+
``SNOWFLAKE_HOME`` (or a test's tmp home) set after the connector was first
|
|
58
|
+
imported. ``read_config`` also runs the file-permission checks we lacked."""
|
|
59
|
+
from snowflake.connector.config_manager import CONFIG_MANAGER
|
|
60
|
+
|
|
61
|
+
from snowflake.sandbox.config import _config_files
|
|
62
|
+
|
|
63
|
+
config_file, connections_file = _config_files()
|
|
64
|
+
CONFIG_MANAGER.file_path = config_file
|
|
65
|
+
for i, sl in enumerate(CONFIG_MANAGER._slices):
|
|
66
|
+
if sl.section == "connections":
|
|
67
|
+
CONFIG_MANAGER._slices[i] = sl._replace(path=connections_file)
|
|
68
|
+
CONFIG_MANAGER.read_config()
|
|
69
|
+
return CONFIG_MANAGER
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _connection_fields(name: str) -> dict[str, Any]:
|
|
73
|
+
"""The raw fields of connection *name*, straight from the connector's config
|
|
74
|
+
manager with native TOML types. Used only to read the account/host and to
|
|
75
|
+
detect a PAT before deciding how to authenticate — the login itself is handed
|
|
76
|
+
to ``connect(connection_name=...)``, which re-reads and normalises the file."""
|
|
77
|
+
from snowflake.sandbox.config import _config_files
|
|
78
|
+
|
|
79
|
+
cm = _connector_config_manager()
|
|
80
|
+
connections = cm["connections"]
|
|
81
|
+
if name not in connections:
|
|
82
|
+
raise ValueError(
|
|
83
|
+
f"connection {name!r} not found in {_config_files()[1]}; "
|
|
84
|
+
f"known connections: {sorted(connections)}"
|
|
85
|
+
)
|
|
86
|
+
return dict(connections[name])
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _host_from_connection_fields(fields: dict[str, Any]) -> str:
|
|
90
|
+
"""The endpoint for a connection's fields, without hand-rolling one.
|
|
91
|
+
|
|
92
|
+
An explicit ``host`` in the connection wins; otherwise the CONNECTOR's own
|
|
93
|
+
``construct_hostname`` builds it, so ``region =`` is honored and a China region
|
|
94
|
+
gets the ``.cn`` top-level domain. Hand-rolling
|
|
95
|
+
``f"{account}.snowflakecomputing.com"`` instead drops ``region`` (wrong account
|
|
96
|
+
endpoint) and forces ``.com`` (wrong TLD for cn- regions).
|
|
97
|
+
|
|
98
|
+
Shared by ``_config_from_connection`` and ``_dev.snowhouse_oauth_env`` so there
|
|
99
|
+
is exactly one host-derivation rule in the SDK. Note ``construct_hostname``
|
|
100
|
+
does not map ``_``->``-``; ``Config.base_url`` owns that normalisation, so a
|
|
101
|
+
caller that dials this host directly should go through ``Config``.
|
|
102
|
+
"""
|
|
103
|
+
account = fields.get("account")
|
|
104
|
+
explicit = fields.get("host")
|
|
105
|
+
if explicit:
|
|
106
|
+
derived = str(explicit)
|
|
107
|
+
elif account:
|
|
108
|
+
from snowflake.connector.util_text import construct_hostname
|
|
109
|
+
|
|
110
|
+
region = fields.get("region")
|
|
111
|
+
derived = construct_hostname(str(region) if region else None, str(account))
|
|
112
|
+
else:
|
|
113
|
+
raise ValueError("connection has neither host nor account")
|
|
114
|
+
return _append_port(derived, fields.get("port"))
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _default_connection_name() -> str | None:
|
|
118
|
+
"""The connector's default connection name — but only when a matching
|
|
119
|
+
connection actually exists, so an absent implicit ``default`` falls back to
|
|
120
|
+
the environment instead of erroring. An explicit
|
|
121
|
+
``SNOWFLAKE_DEFAULT_CONNECTION_NAME`` is returned even if missing, so the
|
|
122
|
+
downstream resolution failure is loud and actionable."""
|
|
123
|
+
cm = _connector_config_manager()
|
|
124
|
+
name = cm["default_connection_name"] # env > config.toml > "default"
|
|
125
|
+
if os.environ.get("SNOWFLAKE_DEFAULT_CONNECTION_NAME"):
|
|
126
|
+
return str(name)
|
|
127
|
+
return str(name) if name in cm["connections"] else None
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _underlying_connector_connection(obj: Any) -> Any:
|
|
131
|
+
"""Return the ``snowflake.connector`` connection inside *obj*, or *obj* itself.
|
|
132
|
+
|
|
133
|
+
Accepts a Snowpark ``Session`` without importing snowpark: a Session is not a
|
|
134
|
+
connector connection (it has no ``.rest``) but exposes the one it wraps. Users
|
|
135
|
+
inside Streamlit-in-Snowflake and stored procedures hold a Session, not a
|
|
136
|
+
connection, and would otherwise have to reach into private attributes.
|
|
137
|
+
|
|
138
|
+
Duck-typed on purpose. Importing ``snowflake.snowpark`` to isinstance-check would
|
|
139
|
+
make an optional dependency mandatory and cost import time for everyone; the
|
|
140
|
+
attribute we need is the one Snowpark documents.
|
|
141
|
+
"""
|
|
142
|
+
if getattr(obj, "rest", None) is not None:
|
|
143
|
+
return obj # already a connector connection
|
|
144
|
+
inner = getattr(obj, "connection", None)
|
|
145
|
+
if inner is not None and getattr(inner, "rest", None) is not None:
|
|
146
|
+
return inner # a Snowpark Session (or any wrapper exposing .connection)
|
|
147
|
+
return obj
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _config_from_live_connection(conn: Any, role: str | None = None) -> Config:
|
|
151
|
+
"""Build a Config from a connection the CALLER already opened.
|
|
152
|
+
|
|
153
|
+
The ``snow`` CLI resolves and opens the connection with its own machinery (its
|
|
154
|
+
global flags, its env overrides, every auth method) and hands the live object
|
|
155
|
+
here. The SDK then reuses this one connection for everything: the REST bearer is
|
|
156
|
+
``connection.rest.token`` (read live via ``resolve_pat``, so a connector-renewed
|
|
157
|
+
token is picked up), and a stage PUT runs on the same connection instead of
|
|
158
|
+
opening a second one and re-deriving credentials.
|
|
159
|
+
|
|
160
|
+
Unlike ``_config_from_connection`` this does NOT append to ``_session_registry``:
|
|
161
|
+
the caller owns the connection's lifetime, and ``close_connections`` must not close a
|
|
162
|
+
connection it did not open. Holding it on ``Config.connection`` keeps a reference
|
|
163
|
+
without adopting it. ``connector_kwargs`` is left None — there is no name to
|
|
164
|
+
re-open by, and none is needed since the connection is reused directly.
|
|
165
|
+
"""
|
|
166
|
+
import dataclasses
|
|
167
|
+
|
|
168
|
+
from snowflake.sandbox.config import _from_env
|
|
169
|
+
|
|
170
|
+
if role is not None:
|
|
171
|
+
raise ValueError(
|
|
172
|
+
"role= cannot be applied to an already-open connection: the role is "
|
|
173
|
+
"chosen at login and this session already exists. Pass connection=<name> "
|
|
174
|
+
"to mint a session at that role, or open the connection at that role."
|
|
175
|
+
)
|
|
176
|
+
conn = _underlying_connector_connection(conn)
|
|
177
|
+
rest = getattr(conn, "rest", None)
|
|
178
|
+
token = getattr(rest, "token", None) if rest is not None else None
|
|
179
|
+
if not token:
|
|
180
|
+
raise ValueError(
|
|
181
|
+
"connection= did not yield a session token. Pass a connection NAME from "
|
|
182
|
+
"~/.snowflake/connections.toml, a live snowflake.connector connection, a "
|
|
183
|
+
"Snowpark Session, or a Config built from raw credentials. If you passed an "
|
|
184
|
+
"open connection, check it has not been closed."
|
|
185
|
+
)
|
|
186
|
+
resolved = getattr(conn, "host", None)
|
|
187
|
+
if not resolved:
|
|
188
|
+
raise ValueError("the provided connection has no host")
|
|
189
|
+
resolved = _append_port(str(resolved), getattr(conn, "port", None))
|
|
190
|
+
base = _from_env()
|
|
191
|
+
return dataclasses.replace(
|
|
192
|
+
base,
|
|
193
|
+
account=getattr(conn, "account", None) or base.account,
|
|
194
|
+
host=resolved,
|
|
195
|
+
pat=str(token),
|
|
196
|
+
connector_kwargs=None,
|
|
197
|
+
connection=conn,
|
|
198
|
+
)
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _config_from_connection(name: str, role: str | None = None) -> Config:
|
|
202
|
+
"""Resolve a named connection into a Config by delegating to the connector.
|
|
203
|
+
|
|
204
|
+
Every non-PAT connection (password / oauth-token-file / key-pair / external
|
|
205
|
+
browser) is authenticated by ``connect(connection_name=...)``: the connector
|
|
206
|
+
re-reads the connection from the file, applies its own key aliases, and logs
|
|
207
|
+
in with the connection's declared method. We keep the session alive for the
|
|
208
|
+
process and use its REST bearer token. A PAT (``token`` field, or
|
|
209
|
+
``authenticator = PROGRAMMATIC_ACCESS_TOKEN`` with the token in ``password``)
|
|
210
|
+
is exchanged for a session token instead, because a PAT handed through
|
|
211
|
+
verbatim comes back as 390303 "Invalid OAuth access token" from inside a
|
|
212
|
+
sandbox (whose mounted config authenticates ``authenticator=oauth``).
|
|
213
|
+
|
|
214
|
+
``role`` overrides the connection's own ``role`` and is applied at login,
|
|
215
|
+
because a sandbox runs as the primary role of the session that created it —
|
|
216
|
+
see ``resolve_config``.
|
|
217
|
+
"""
|
|
218
|
+
import dataclasses
|
|
219
|
+
|
|
220
|
+
import snowflake.connector # lazy: loaded only when a connection is resolved
|
|
221
|
+
|
|
222
|
+
from snowflake.sandbox.config import _from_env, _session_registry, _warn_pat_fallback
|
|
223
|
+
|
|
224
|
+
fields = _connection_fields(name) # also re-points CONFIG_MANAGER at the home
|
|
225
|
+
account = fields.get("account")
|
|
226
|
+
if not account:
|
|
227
|
+
raise ValueError(f"connection {name!r} has no account")
|
|
228
|
+
auth = str(fields.get("authenticator", "")).lower()
|
|
229
|
+
base = _from_env()
|
|
230
|
+
|
|
231
|
+
def _derived_host() -> str:
|
|
232
|
+
return _host_from_connection_fields(fields)
|
|
233
|
+
|
|
234
|
+
# Carried on the Config so a later stage PUT re-opens the SAME connection (its
|
|
235
|
+
# own credential + role) rather than replaying the minted session token (which
|
|
236
|
+
# fails 250001). We pass the connection NAME only — not secrets, and not a host
|
|
237
|
+
# or connections_file_path: the connector re-reads the file itself (config.toml
|
|
238
|
+
# or connections.toml) and derives/normalises the host. refresh_connection_config
|
|
239
|
+
# re-points CONFIG_MANAGER before that reuse.
|
|
240
|
+
# client_session_keep_alive: let the CONNECTOR keep this session alive
|
|
241
|
+
# server-side (a background heartbeat every master_validity/4 s) instead of a
|
|
242
|
+
# caller-side keepalive thread. Carried in connector_kwargs so a reconnect (and
|
|
243
|
+
# the stage-PUT reuse) inherit it. Not a silver bullet — a heartbeat cannot
|
|
244
|
+
# outlive the master token / OAuth grant, and a sub-heartbeat-interval session
|
|
245
|
+
# can still lapse — which is why the transport also re-mints reactively on a
|
|
246
|
+
# reauth response; but it removes the routine idle-expiry (390112) that made a
|
|
247
|
+
# snapshot go stale mid-workload.
|
|
248
|
+
reconnect: dict[str, Any] = {
|
|
249
|
+
"connection_name": name,
|
|
250
|
+
"login_timeout": 30,
|
|
251
|
+
"client_session_keep_alive": True,
|
|
252
|
+
}
|
|
253
|
+
if role is not None:
|
|
254
|
+
reconnect["role"] = role
|
|
255
|
+
|
|
256
|
+
def _with(pat: str, resolved_host: str, conn: Any | None = None) -> Config:
|
|
257
|
+
return dataclasses.replace(
|
|
258
|
+
base,
|
|
259
|
+
account=account,
|
|
260
|
+
host=resolved_host,
|
|
261
|
+
pat=pat,
|
|
262
|
+
connector_kwargs=dict(reconnect),
|
|
263
|
+
connection=conn,
|
|
264
|
+
)
|
|
265
|
+
|
|
266
|
+
def _session_token_from(conn: Any) -> Config:
|
|
267
|
+
_session_registry.append(conn) # keep alive so the token stays valid
|
|
268
|
+
rest = getattr(conn, "rest", None)
|
|
269
|
+
if rest is None or rest.token is None:
|
|
270
|
+
raise ValueError(f"connection {name!r} did not yield a session token")
|
|
271
|
+
# Adopt the endpoint the connector actually connected to, rather than
|
|
272
|
+
# re-deriving one: it has already applied region, an explicit host, and an
|
|
273
|
+
# account identifier the REST TLS path can use.
|
|
274
|
+
resolved = _append_port(
|
|
275
|
+
str(getattr(conn, "host", None) or _derived_host()), getattr(conn, "port", None)
|
|
276
|
+
)
|
|
277
|
+
# Carry the live connection so ``refresh_config`` can renew the session
|
|
278
|
+
# token off it when this snapshot expires.
|
|
279
|
+
return _with(rest.token, resolved, conn)
|
|
280
|
+
|
|
281
|
+
pat = fields.get("token") or (
|
|
282
|
+
fields.get("password") if auth in ("programmatic_access_token", "pat") else None
|
|
283
|
+
)
|
|
284
|
+
if pat:
|
|
285
|
+
# Exchange the PAT for a session token via the connector's native PAT auth
|
|
286
|
+
# (AuthByPAT), which takes the token from ``token=`` with
|
|
287
|
+
# ``authenticator=PROGRAMMATIC_ACCESS_TOKEN`` — so we normalise the PAT into
|
|
288
|
+
# ``token`` regardless of which field the connection stored it in. Minting a
|
|
289
|
+
# real session (rather than replaying the PAT as the bearer) is what keeps
|
|
290
|
+
# Snowflake access from inside the sandbox from failing 390303, and lets an
|
|
291
|
+
# explicit role be applied.
|
|
292
|
+
# connection_name= so the CONNECTOR reads account/user/host/port/region off
|
|
293
|
+
# the file itself (verified: it honors the file's ``region`` alongside these
|
|
294
|
+
# overrides). Only the authenticator + token are overridden here; nothing
|
|
295
|
+
# about the endpoint is reassembled by us.
|
|
296
|
+
pat_kw: dict[str, Any] = {
|
|
297
|
+
"connection_name": name,
|
|
298
|
+
"login_timeout": 30,
|
|
299
|
+
"token": pat,
|
|
300
|
+
"authenticator": "PROGRAMMATIC_ACCESS_TOKEN",
|
|
301
|
+
"client_session_keep_alive": True,
|
|
302
|
+
}
|
|
303
|
+
if role is not None:
|
|
304
|
+
pat_kw["role"] = role
|
|
305
|
+
try:
|
|
306
|
+
return _session_token_from(snowflake.connector.connect(**pat_kw))
|
|
307
|
+
except Exception as exc: # noqa: BLE001 - fall back to the PAT as bearer
|
|
308
|
+
if role is not None:
|
|
309
|
+
raise ValueError(
|
|
310
|
+
f"connection {name!r}: could not mint a session at role "
|
|
311
|
+
f"{role!r} from its programmatic access token "
|
|
312
|
+
f"({type(exc).__name__}: {exc}). Scope the PAT to the role "
|
|
313
|
+
"instead (ROLE_RESTRICTION), or use a password connection."
|
|
314
|
+
) from exc
|
|
315
|
+
_warn_pat_fallback(name, exc)
|
|
316
|
+
return _with(pat, _derived_host())
|
|
317
|
+
|
|
318
|
+
# No credential pre-check of our own: the connector owns which fields and
|
|
319
|
+
# authenticators constitute a usable credential, and an allowlist here could
|
|
320
|
+
# only ever be a stale subset of that. The previous one rejected configs the
|
|
321
|
+
# connector accepts outright -- an inline ``private_key`` (it checked only
|
|
322
|
+
# ``private_key_file``/``private_key_path`` plus a ``private_key_raw`` that is
|
|
323
|
+
# not a connector field at all), the OAuth flow authenticators
|
|
324
|
+
# (``OAUTH_AUTHORIZATION_CODE`` / ``OAUTH_CLIENT_CREDENTIALS``, whose credential
|
|
325
|
+
# lives in ``oauth_client_id``/``oauth_client_secret``), ``ID_TOKEN``, and
|
|
326
|
+
# ``session_token``/``master_token``. Let ``connect()`` decide and report; its
|
|
327
|
+
# error names the actual missing field.
|
|
328
|
+
return _session_token_from(snowflake.connector.connect(**reconnect))
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""``DeploySpec`` — the internal deploy IR.
|
|
2
|
+
|
|
3
|
+
There is no manifest file: the decorator API (``@app.function``) and the CLI both
|
|
4
|
+
produce a ``DeploySpec``, and ``deploy_spec``/``deploy_async`` both consume one.
|
|
5
|
+
Kept in its own module so ``deploy`` can import it without a cycle.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from collections.abc import Mapping
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
|
|
13
|
+
from snowflake.sandbox._assemble import Bundle
|
|
14
|
+
from snowflake.sandbox.egress import Egress, compile_egress
|
|
15
|
+
from snowflake.sandbox.mount import StageMount
|
|
16
|
+
from snowflake.sandbox.secret import Secret
|
|
17
|
+
|
|
18
|
+
__all__ = ["DeploySpec"]
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(frozen=True)
|
|
22
|
+
class DeploySpec:
|
|
23
|
+
"""Everything a deploy needs, in one object — the internal IR.
|
|
24
|
+
|
|
25
|
+
There is no manifest file: the decorator API (``@app.function``) and the CLI
|
|
26
|
+
both produce a ``DeploySpec``, and ``deploy_spec`` / ``deploy_async`` both
|
|
27
|
+
consume one. Keeping it here (rather than in ``app``) lets ``deploy`` import
|
|
28
|
+
it without a cycle.
|
|
29
|
+
|
|
30
|
+
``entry`` is the command the container runs. ``egress``/``secrets`` are
|
|
31
|
+
combined by `egress_body()` — secrets are authored flat but nest on the wire.
|
|
32
|
+
``stage_mounts`` are FUSE stage/workspace mounts passed through verbatim.
|
|
33
|
+
|
|
34
|
+
``image`` is the catalog base name used for display + prod (SPCS) codegen and
|
|
35
|
+
launched directly (an ``Image`` is a reference to a catalog base).
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
image: str
|
|
39
|
+
entry: tuple[str, ...]
|
|
40
|
+
memory: str = "4g"
|
|
41
|
+
# Explicit CPU in cores, overriding what the memory tier would imply.
|
|
42
|
+
# None inherits the tier default, keeping the request byte-identical to one
|
|
43
|
+
# made before cpu existed.
|
|
44
|
+
cpu: float | None = None
|
|
45
|
+
code_stage: str | None = None
|
|
46
|
+
env: Mapping[str, str] = field(default_factory=dict)
|
|
47
|
+
egress: Egress | None = None
|
|
48
|
+
secrets: tuple[Secret, ...] = field(default_factory=tuple)
|
|
49
|
+
bundle: Bundle | None = None
|
|
50
|
+
stage_mounts: tuple[StageMount, ...] = field(default_factory=tuple)
|
|
51
|
+
project_name: str = "sandbox"
|
|
52
|
+
timeout_s: float = 3600.0
|
|
53
|
+
|
|
54
|
+
def egress_body(self) -> dict[str, object] | None:
|
|
55
|
+
"""The ``egress`` request object, with ``secrets`` folded in."""
|
|
56
|
+
return compile_egress(self.egress, self.secrets)
|