snowflake-sandbox-python 0.2.1a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- snowflake/cli_sandbox/__init__.py +13 -0
- snowflake/cli_sandbox/_adapter.py +170 -0
- snowflake/cli_sandbox/_common.py +77 -0
- snowflake/cli_sandbox/_egress_flags.py +121 -0
- snowflake/cli_sandbox/_get_command.py +109 -0
- snowflake/cli_sandbox/_run_command.py +1091 -0
- snowflake/cli_sandbox/_shell_command.py +666 -0
- snowflake/cli_sandbox/_upload_plan.py +187 -0
- snowflake/cli_sandbox/commands.py +556 -0
- snowflake/cli_sandbox/plugin_spec.py +28 -0
- snowflake/cli_sandbox/py.typed +0 -0
- snowflake/sandbox/__init__.py +317 -0
- snowflake/sandbox/__main__.py +225 -0
- snowflake/sandbox/_ansi.py +206 -0
- snowflake/sandbox/_args.py +208 -0
- snowflake/sandbox/_assemble.py +256 -0
- snowflake/sandbox/_bundle.py +240 -0
- snowflake/sandbox/_connection_resolve.py +328 -0
- snowflake/sandbox/_deploy_spec.py +56 -0
- snowflake/sandbox/_diagnostics.py +501 -0
- snowflake/sandbox/_env.py +143 -0
- snowflake/sandbox/_files_mixin.py +280 -0
- snowflake/sandbox/_fs_ops.py +304 -0
- snowflake/sandbox/_globs.py +176 -0
- snowflake/sandbox/_hosts.py +110 -0
- snowflake/sandbox/_mcp_discovery.py +288 -0
- snowflake/sandbox/_mcp_status.py +183 -0
- snowflake/sandbox/_retry.py +94 -0
- snowflake/sandbox/_runtime/__init__.py +42 -0
- snowflake/sandbox/_runtime/_fs_helper.py +93 -0
- snowflake/sandbox/_runtime/_job_runner.py +111 -0
- snowflake/sandbox/_runtime/_protocol.py +53 -0
- snowflake/sandbox/_runtime/_shims.py +267 -0
- snowflake/sandbox/_sandbox_state.py +303 -0
- snowflake/sandbox/_session_registry.py +222 -0
- snowflake/sandbox/_sse.py +160 -0
- snowflake/sandbox/_stage.py +270 -0
- snowflake/sandbox/_sync_files_mixin.py +272 -0
- snowflake/sandbox/_sync_fs_ops.py +185 -0
- snowflake/sandbox/_sync_transport.py +737 -0
- snowflake/sandbox/_sync_watch.py +99 -0
- snowflake/sandbox/_transport.py +1366 -0
- snowflake/sandbox/_transport_errors.py +270 -0
- snowflake/sandbox/_upload_plan.py +497 -0
- snowflake/sandbox/_version.py +37 -0
- snowflake/sandbox/_watch.py +164 -0
- snowflake/sandbox/_wire.py +348 -0
- snowflake/sandbox/app.py +256 -0
- snowflake/sandbox/client.py +2356 -0
- snowflake/sandbox/config.py +1133 -0
- snowflake/sandbox/connect.py +288 -0
- snowflake/sandbox/deploy.py +499 -0
- snowflake/sandbox/egress.py +388 -0
- snowflake/sandbox/exceptions.py +253 -0
- snowflake/sandbox/exec_stream.py +264 -0
- snowflake/sandbox/files.py +547 -0
- snowflake/sandbox/function.py +567 -0
- snowflake/sandbox/image.py +46 -0
- snowflake/sandbox/jobs.py +649 -0
- snowflake/sandbox/lifecycle.py +67 -0
- snowflake/sandbox/log_stream.py +219 -0
- snowflake/sandbox/mcp.py +480 -0
- snowflake/sandbox/mount.py +161 -0
- snowflake/sandbox/py.typed +0 -0
- snowflake/sandbox/secret.py +244 -0
- snowflake/sandbox/session_app.py +244 -0
- snowflake/sandbox/shell.py +556 -0
- snowflake/sandbox/sync_client.py +2245 -0
- snowflake/sandbox/sync_exec_stream.py +238 -0
- snowflake/sandbox/sync_files.py +377 -0
- snowflake/sandbox/sync_log_stream.py +142 -0
- snowflake/sandbox/sync_shell.py +413 -0
- snowflake/sandbox/types.py +193 -0
- snowflake/sandbox/warm_session.py +700 -0
- snowflake_sandbox_python-0.2.1a1.dist-info/METADATA +339 -0
- snowflake_sandbox_python-0.2.1a1.dist-info/RECORD +80 -0
- snowflake_sandbox_python-0.2.1a1.dist-info/WHEEL +5 -0
- snowflake_sandbox_python-0.2.1a1.dist-info/entry_points.txt +2 -0
- snowflake_sandbox_python-0.2.1a1.dist-info/licenses/LICENSE +202 -0
- snowflake_sandbox_python-0.2.1a1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
"""Connector-session registry + credential renewal — split out of ``config.py``.
|
|
2
|
+
|
|
3
|
+
Lifecycle is delegated to snowflake-connector-python. The token is read LIVE
|
|
4
|
+
from ``connection.rest.token`` (``Config.resolve_pat``), and the connection is
|
|
5
|
+
minted with ``client_session_keep_alive=True`` so the connector heartbeats the
|
|
6
|
+
session alive server-side and renews it in place — the SDK picks that up on the
|
|
7
|
+
next request without any snapshot going stale.
|
|
8
|
+
|
|
9
|
+
When a request still comes back with a reauth signal (the session was truly
|
|
10
|
+
gone — 390111 — or the master token / OAuth grant lapsed faster than the
|
|
11
|
+
heartbeat), the transport calls the reactive path below as a backstop:
|
|
12
|
+
``refresh_config`` renews off the master token first (one
|
|
13
|
+
``/session/token-request``, no re-login) and falls back to a full reconnect via
|
|
14
|
+
the carried ``connector_kwargs``. ``refresh_credential`` is the process-wide
|
|
15
|
+
entry point; it de-duplicates a concurrent burst (via ``_refresh_lock``) so the
|
|
16
|
+
fleet of failing requests triggers exactly ONE re-mint and the rest adopt its
|
|
17
|
+
result. Crucially it does the connector I/O WITHOUT holding ``_lock`` — that
|
|
18
|
+
lock only guards the resolution memo's read/swap — so a refresh (up to a 30 s
|
|
19
|
+
reconnect) never blocks ``current_config()`` on the request hot path.
|
|
20
|
+
|
|
21
|
+
Every name here is re-exported from ``config`` (``from ._session_registry
|
|
22
|
+
import ...``) so ``snowflake.sandbox.config.<name>`` keeps resolving. The shared
|
|
23
|
+
state these touch (the ``_session_registry`` list, ``_lock``, ``_resolution_cache``,
|
|
24
|
+
``_refresh_lock``) and ``Config`` / ``refresh_connection_config`` stay in
|
|
25
|
+
``config`` and are imported lazily inside the functions that need them — a
|
|
26
|
+
module-scope import back into ``config`` would close a cycle (``config`` imports
|
|
27
|
+
this module to re-export it).
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
from __future__ import annotations
|
|
31
|
+
|
|
32
|
+
from typing import TYPE_CHECKING
|
|
33
|
+
|
|
34
|
+
if TYPE_CHECKING:
|
|
35
|
+
from snowflake.sandbox.config import Config
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _close_session_registry() -> None:
|
|
39
|
+
from snowflake.sandbox.config import _session_registry
|
|
40
|
+
|
|
41
|
+
conns = list(_session_registry)
|
|
42
|
+
_session_registry.clear()
|
|
43
|
+
for conn in conns:
|
|
44
|
+
close = getattr(conn, "close", None)
|
|
45
|
+
if close is None:
|
|
46
|
+
continue
|
|
47
|
+
try:
|
|
48
|
+
close()
|
|
49
|
+
except Exception: # noqa: BLE001 - best-effort; a dead session is fine
|
|
50
|
+
pass
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _close_one_session(conn: object) -> bool:
|
|
54
|
+
"""Close *conn* only if the SDK opened it. Returns whether it was closed.
|
|
55
|
+
|
|
56
|
+
Ownership is already recorded by how the Config was built:
|
|
57
|
+
``_config_from_connection`` appends the session it minted to ``_session_registry``,
|
|
58
|
+
while ``_config_from_live_connection`` deliberately does not -- a connection or
|
|
59
|
+
Snowpark ``Session`` the caller opened stays theirs. So membership in the registry IS
|
|
60
|
+
the ownership test, and a caller-supplied connection is never closed here no matter
|
|
61
|
+
what a ``close=True`` asks for.
|
|
62
|
+
"""
|
|
63
|
+
from snowflake.sandbox.config import _session_registry
|
|
64
|
+
|
|
65
|
+
if conn is None:
|
|
66
|
+
return False
|
|
67
|
+
for i, known in enumerate(_session_registry):
|
|
68
|
+
if known is conn:
|
|
69
|
+
del _session_registry[i]
|
|
70
|
+
_close_connection(conn)
|
|
71
|
+
return True
|
|
72
|
+
return False
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _renew_or_reconnect(cfg: Config) -> Config | None:
|
|
76
|
+
"""Re-mint *cfg*'s token. Returns the Config to install, or ``None``.
|
|
77
|
+
|
|
78
|
+
Returns *cfg* UNCHANGED when the token was renewed in place on the existing
|
|
79
|
+
connection (the cheap master-token path — ``resolve_pat`` already reflects the
|
|
80
|
+
new token, so nothing needs replacing). Returns a NEW Config carrying a fresh
|
|
81
|
+
connection when a full reconnect was required (390111 / master dead). Returns
|
|
82
|
+
``None`` when neither was possible, so the caller surfaces the original auth
|
|
83
|
+
error instead of retrying an identical credential.
|
|
84
|
+
"""
|
|
85
|
+
conn = cfg.connection
|
|
86
|
+
if conn is not None:
|
|
87
|
+
rest = getattr(conn, "rest", None)
|
|
88
|
+
renew = getattr(rest, "_renew_session", None)
|
|
89
|
+
if callable(renew):
|
|
90
|
+
before = getattr(rest, "token", None)
|
|
91
|
+
try:
|
|
92
|
+
# Master-token round-trip; mutates rest.token in place. Raises
|
|
93
|
+
# ReauthenticationRequest for 390111/390114/390115 (non-renewable)
|
|
94
|
+
# — caught here so we fall through to a full reconnect.
|
|
95
|
+
renew()
|
|
96
|
+
except Exception: # noqa: BLE001 - non-renewable / connector API drift
|
|
97
|
+
pass
|
|
98
|
+
else:
|
|
99
|
+
after = getattr(rest, "token", None)
|
|
100
|
+
if after and after != before:
|
|
101
|
+
return cfg # renewed in place; resolve_pat() sees it live
|
|
102
|
+
return _reconnect(cfg)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _reconnect(cfg: Config) -> Config | None:
|
|
106
|
+
"""Full re-login via the carried ``connector_kwargs``; ``None`` if unavailable.
|
|
107
|
+
|
|
108
|
+
This is what naming the connection again would do — re-read the TOML and
|
|
109
|
+
authenticate with the connection's declared method — but on demand, keyed to a
|
|
110
|
+
single expired session rather than run on a timer. The old connection is closed
|
|
111
|
+
and dropped from the registry (its keep-alive heartbeat thread and sockets
|
|
112
|
+
would otherwise accumulate one-per-reconnect for the process lifetime); the
|
|
113
|
+
fresh one replaces ``cfg.connection`` so ``resolve_pat`` reads it and the next
|
|
114
|
+
renewal has a live master token again.
|
|
115
|
+
"""
|
|
116
|
+
kwargs = cfg.connector_kwargs
|
|
117
|
+
if not kwargs:
|
|
118
|
+
return None
|
|
119
|
+
import dataclasses
|
|
120
|
+
|
|
121
|
+
import snowflake.connector
|
|
122
|
+
|
|
123
|
+
from snowflake.sandbox.config import _session_registry, refresh_connection_config
|
|
124
|
+
|
|
125
|
+
refresh_connection_config() # re-point CONFIG_MANAGER at the right home first
|
|
126
|
+
conn = snowflake.connector.connect(**kwargs)
|
|
127
|
+
rest = getattr(conn, "rest", None)
|
|
128
|
+
token = getattr(rest, "token", None) if rest is not None else None
|
|
129
|
+
if not token:
|
|
130
|
+
# Do not retain a connection we cannot use; close it so its heartbeat
|
|
131
|
+
# thread does not linger.
|
|
132
|
+
_close_connection(conn)
|
|
133
|
+
return None
|
|
134
|
+
_session_registry.append(conn) # keep alive so its token stays valid
|
|
135
|
+
_retire_connection(cfg.connection) # close+drop the one we are replacing
|
|
136
|
+
return dataclasses.replace(cfg, pat=token, connection=conn)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _close_connection(conn: object) -> None:
|
|
140
|
+
close = getattr(conn, "close", None)
|
|
141
|
+
if close is None:
|
|
142
|
+
return
|
|
143
|
+
try:
|
|
144
|
+
close()
|
|
145
|
+
except Exception: # noqa: BLE001 - best-effort; a dead session is fine
|
|
146
|
+
pass
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _retire_connection(conn: object | None) -> None:
|
|
150
|
+
"""Close *conn* and drop it from the registry (bounds reconnect growth)."""
|
|
151
|
+
from snowflake.sandbox.config import _session_registry
|
|
152
|
+
|
|
153
|
+
if conn is None:
|
|
154
|
+
return
|
|
155
|
+
try:
|
|
156
|
+
_session_registry.remove(conn)
|
|
157
|
+
except ValueError:
|
|
158
|
+
pass
|
|
159
|
+
_close_connection(conn)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def refresh_config(cfg: Config, stale_pat: str) -> Config:
|
|
163
|
+
"""Return *cfg* with its token re-minted after ``stale_pat`` stopped working.
|
|
164
|
+
|
|
165
|
+
Returns *cfg* UNCHANGED when its live token is no longer ``stale_pat`` (another
|
|
166
|
+
path already refreshed it), when it was renewed in place, or when there is
|
|
167
|
+
nothing to refresh (an env/PAT Config with no connection and no reconnect
|
|
168
|
+
kwargs). The caller detects a real reconnect by identity (``result is not cfg``)
|
|
169
|
+
and/or a changed ``resolve_pat()``.
|
|
170
|
+
|
|
171
|
+
Does the connector I/O (a token-request round-trip, or a reconnect) itself, so
|
|
172
|
+
an ``async`` caller must run it off the event loop (``asyncio.to_thread``), and
|
|
173
|
+
a caller sharing it across threads must serialize (see ``refresh_credential`` /
|
|
174
|
+
the transport's per-instance refresh lock).
|
|
175
|
+
"""
|
|
176
|
+
if cfg.resolve_pat() != stale_pat:
|
|
177
|
+
return cfg
|
|
178
|
+
new = _renew_or_reconnect(cfg)
|
|
179
|
+
return new if new is not None else cfg
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def refresh_credential(stale_pat: str) -> str | None:
|
|
183
|
+
"""Re-mint the AMBIENT config's token; the transport's reauth hook.
|
|
184
|
+
|
|
185
|
+
Serves the transport that resolves through `current_config` on every request (the
|
|
186
|
+
one a caller who named no connection gets). A transport bound to an explicit
|
|
187
|
+
``connection=`` carries its own Config and refreshes it through `refresh_config`
|
|
188
|
+
under its own lock instead.
|
|
189
|
+
|
|
190
|
+
De-duplicated against the token the caller actually sent: the first of a
|
|
191
|
+
concurrent burst re-mints (holding ``_refresh_lock``, NOT ``_lock``, so config
|
|
192
|
+
readers are never blocked on the network) and installs the result; every racer
|
|
193
|
+
then sees the live token has changed and adopts it without minting again.
|
|
194
|
+
Returns the fresh token, or ``None`` when nothing could be refreshed (a config with
|
|
195
|
+
no connection/reconnect kwargs) so the transport surfaces the original auth error
|
|
196
|
+
rather than retrying an identical credential.
|
|
197
|
+
"""
|
|
198
|
+
from snowflake.sandbox.config import _refresh_lock, _replace_cached_config, current_config
|
|
199
|
+
|
|
200
|
+
cur = current_config()
|
|
201
|
+
if cur.resolve_pat() != stale_pat:
|
|
202
|
+
return cur.resolve_pat() # already refreshed elsewhere
|
|
203
|
+
with _refresh_lock:
|
|
204
|
+
# Re-read + re-check under the refresh lock: another thread may have
|
|
205
|
+
# re-minted while we waited, in which case adopt its token.
|
|
206
|
+
cur = current_config()
|
|
207
|
+
current = cur.resolve_pat()
|
|
208
|
+
if current != stale_pat:
|
|
209
|
+
return current
|
|
210
|
+
new = _renew_or_reconnect(cur) # connector I/O — no lock held here
|
|
211
|
+
if new is None:
|
|
212
|
+
return None # nothing to refresh — let the auth error surface
|
|
213
|
+
# Install the reconnect where the ambient config is resolved FROM, so the next
|
|
214
|
+
# request serves it instead of re-minting. A miss means the Config we renewed is
|
|
215
|
+
# no longer the one on offer — ``close_connections`` / ``refresh_connection_config``
|
|
216
|
+
# dropped the memo mid-refresh, or the ambient config came from the bare
|
|
217
|
+
# environment and was never memoised — so retire our now-orphan reconnect rather
|
|
218
|
+
# than leak it, and adopt whatever resolves now.
|
|
219
|
+
if new is not cur and not _replace_cached_config(cur, new):
|
|
220
|
+
_retire_connection(new.connection)
|
|
221
|
+
return current_config().resolve_pat()
|
|
222
|
+
return new.resolve_pat()
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
"""Loop-agnostic Server-Sent-Events parsing shared by the transport twins.
|
|
2
|
+
|
|
3
|
+
The parsed-frame type, the single-frame parser, the open-response frame iterator,
|
|
4
|
+
and the initial-open retry policy for a stream that failed on first contact. Used by
|
|
5
|
+
both the async ``_transport`` and the sync ``_sync_transport`` (whose own frame
|
|
6
|
+
iterator is the blocking mirror of ``_iter_sse_frames`` here). A dependency-free
|
|
7
|
+
leaf so both transports import one copy.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
import random
|
|
14
|
+
from collections.abc import AsyncIterator
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
import httpx
|
|
18
|
+
|
|
19
|
+
from snowflake.sandbox._retry import (
|
|
20
|
+
DEFAULT_COLD_START_DELAYS_S,
|
|
21
|
+
DEFAULT_NOT_FOUND_DELAYS_S,
|
|
22
|
+
compute_backoff,
|
|
23
|
+
)
|
|
24
|
+
from snowflake.sandbox._transport_errors import (
|
|
25
|
+
SandboxSSEDecodeError,
|
|
26
|
+
_rate_limit_delay,
|
|
27
|
+
_says_not_ready,
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class SSEEvent:
|
|
32
|
+
"""One parsed Server-Sent-Events frame."""
|
|
33
|
+
|
|
34
|
+
__slots__ = ("data", "event", "id")
|
|
35
|
+
|
|
36
|
+
def __init__(self, event: str, data: str, id: str | None = None) -> None: # noqa: A002
|
|
37
|
+
self.event = event
|
|
38
|
+
self.data = data
|
|
39
|
+
# The frame's `id:` field, when the server sent one. Both resumable
|
|
40
|
+
# streams -- exec (reattached from `last_event_id` after the ~180s
|
|
41
|
+
# ingress GOAWAY; see client._exec_stream_frames) and shell (cut by the
|
|
42
|
+
# gateway at its stream timeout and resumed from the last id seen) --
|
|
43
|
+
# advance a cursor from this id. Optional because non-resumable frames
|
|
44
|
+
# (e.g. the `session`/`exit` bookkeeping frames) may omit it. This parser
|
|
45
|
+
# used to drop the field entirely, which made resumption impossible to
|
|
46
|
+
# implement without the server duplicating the cursor into `data`.
|
|
47
|
+
self.id = id
|
|
48
|
+
|
|
49
|
+
def json(self) -> Any:
|
|
50
|
+
"""Parse ``data`` as JSON.
|
|
51
|
+
|
|
52
|
+
An **empty** ``data`` (a frame that carried no ``data:`` field) returns
|
|
53
|
+
``None`` without attempting a parse -- distinct from the literal string
|
|
54
|
+
``"null"``, which is valid JSON and also decodes to ``None`` but goes
|
|
55
|
+
through the parser. Malformed non-empty ``data`` raises ``SandboxError``
|
|
56
|
+
(a typed member of the hierarchy) rather than the bare
|
|
57
|
+
``json.JSONDecodeError`` a caller doing ``except SandboxError`` would miss.
|
|
58
|
+
"""
|
|
59
|
+
if self.data == "":
|
|
60
|
+
return None
|
|
61
|
+
try:
|
|
62
|
+
return json.loads(self.data)
|
|
63
|
+
except json.JSONDecodeError as exc:
|
|
64
|
+
raise SandboxSSEDecodeError(f"SSE frame data is not valid JSON: {exc}") from exc
|
|
65
|
+
|
|
66
|
+
def __eq__(self, other: object) -> bool:
|
|
67
|
+
if not isinstance(other, SSEEvent):
|
|
68
|
+
return NotImplemented
|
|
69
|
+
return self.event == other.event and self.data == other.data and self.id == other.id
|
|
70
|
+
|
|
71
|
+
def __hash__(self) -> int:
|
|
72
|
+
return hash((self.event, self.data, self.id))
|
|
73
|
+
|
|
74
|
+
def __repr__(self) -> str: # pragma: no cover - debug
|
|
75
|
+
return f"SSEEvent(event={self.event!r}, data={self.data!r}, id={self.id!r})"
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def parse_sse_event(buf: str) -> SSEEvent | None:
|
|
79
|
+
"""Parse a single SSE event (text terminated by a blank line).
|
|
80
|
+
|
|
81
|
+
Returns ``None`` when the block carries no recognized field -- a blank block,
|
|
82
|
+
or one made only of comment/keep-alive lines (``: keep-alive``). A keep-alive
|
|
83
|
+
frame previously produced a phantom ``SSEEvent("message", "")`` that every
|
|
84
|
+
consumer had to filter (or emit a spurious blank line per heartbeat); it is
|
|
85
|
+
now dropped here.
|
|
86
|
+
"""
|
|
87
|
+
if not buf.strip():
|
|
88
|
+
return None
|
|
89
|
+
event = "message"
|
|
90
|
+
data_lines: list[str] = []
|
|
91
|
+
event_id: str | None = None
|
|
92
|
+
saw_field = False
|
|
93
|
+
for line in buf.splitlines():
|
|
94
|
+
if not line or line.startswith(":"):
|
|
95
|
+
continue
|
|
96
|
+
field, _, value = line.partition(":")
|
|
97
|
+
if value.startswith(" "):
|
|
98
|
+
value = value[1:]
|
|
99
|
+
if field == "event":
|
|
100
|
+
event = value
|
|
101
|
+
saw_field = True
|
|
102
|
+
elif field == "data":
|
|
103
|
+
data_lines.append(value)
|
|
104
|
+
saw_field = True
|
|
105
|
+
elif field == "id":
|
|
106
|
+
event_id = value
|
|
107
|
+
saw_field = True
|
|
108
|
+
if not saw_field:
|
|
109
|
+
return None
|
|
110
|
+
return SSEEvent(event=event, data="\n".join(data_lines), id=event_id)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
async def _iter_sse_frames(resp: httpx.Response) -> AsyncIterator[SSEEvent]:
|
|
114
|
+
"""Yield the parsed frames of an open SSE response.
|
|
115
|
+
|
|
116
|
+
Only frames terminated by a blank line are dispatched. A leftover, un-blank-
|
|
117
|
+
terminated buffer at end-of-stream is a **truncated** frame -- the stream was
|
|
118
|
+
cut mid-frame (an h2 GOAWAY at the 180s ingress cut, a 900s shell timeout) --
|
|
119
|
+
and is discarded rather than delivered as if complete. Yielding it corrupted
|
|
120
|
+
the data at exactly the resume seam: the SSE spec says an event is dispatched
|
|
121
|
+
on the blank line, so a partial trailing block is not an event.
|
|
122
|
+
"""
|
|
123
|
+
buf: list[str] = []
|
|
124
|
+
async for line in resp.aiter_lines():
|
|
125
|
+
if line == "":
|
|
126
|
+
if buf:
|
|
127
|
+
evt = parse_sse_event("\n".join(buf))
|
|
128
|
+
buf = []
|
|
129
|
+
if evt is not None:
|
|
130
|
+
yield evt
|
|
131
|
+
else:
|
|
132
|
+
buf.append(line)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _sse_open_retry_delay(
|
|
136
|
+
status: int,
|
|
137
|
+
resp: httpx.Response,
|
|
138
|
+
attempt: int,
|
|
139
|
+
rng: random.Random,
|
|
140
|
+
*,
|
|
141
|
+
retry_not_found: bool = True,
|
|
142
|
+
) -> float | None:
|
|
143
|
+
"""Delay before re-opening a stream that failed on first contact (SSE/download).
|
|
144
|
+
|
|
145
|
+
``None`` means "do not retry" -- the status is terminal, the caller opted out
|
|
146
|
+
of the 404 create-race wait, or the budget is spent. Only statuses that prove
|
|
147
|
+
the server did not start the work are retried, since these routes are POSTs.
|
|
148
|
+
A 429 is capped at the shared rate-limit budget (it used to honor Retry-After
|
|
149
|
+
forever) and, on the last allowed attempt, its ``None`` return sends the caller
|
|
150
|
+
to the ``SandboxRateLimitError`` mapping rather than looping.
|
|
151
|
+
"""
|
|
152
|
+
if status == 404:
|
|
153
|
+
if not retry_not_found:
|
|
154
|
+
return None
|
|
155
|
+
return compute_backoff(attempt, delays=DEFAULT_NOT_FOUND_DELAYS_S, rng=rng)
|
|
156
|
+
if status == 429:
|
|
157
|
+
return _rate_limit_delay(resp, attempt, rng)
|
|
158
|
+
if status in (409, 503) and _says_not_ready(resp):
|
|
159
|
+
return compute_backoff(attempt, delays=DEFAULT_COLD_START_DELAYS_S, rng=rng)
|
|
160
|
+
return None
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
"""Snowflake stage operations for code and image bundles.
|
|
2
|
+
|
|
3
|
+
Resolves credentials, opens a connector connection, and PUTs a bundle. The
|
|
4
|
+
connector PUT is the only upload path that works for every stage type (``@~/``,
|
|
5
|
+
``@%table``, named); the Files REST API is limited to named stages.
|
|
6
|
+
|
|
7
|
+
`_validate_stage_identifier` is the SQL-injection boundary for every path that
|
|
8
|
+
names a stage, so it lives here with the code that interpolates one.
|
|
9
|
+
|
|
10
|
+
Shared by the async and sync clients and by ``image``: the functions need only a
|
|
11
|
+
``.config``, so they take any transport that carries one rather than naming a
|
|
12
|
+
concrete class.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import re
|
|
18
|
+
from typing import TYPE_CHECKING, Any, Protocol
|
|
19
|
+
|
|
20
|
+
from snowflake.sandbox.exceptions import SandboxError
|
|
21
|
+
|
|
22
|
+
if TYPE_CHECKING:
|
|
23
|
+
from snowflake.sandbox.config import Config
|
|
24
|
+
from snowflake.sandbox.mount import StageMount
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class _ConfigCarrier(Protocol):
|
|
28
|
+
"""Anything that can supply a resolved `Config` — either transport does.
|
|
29
|
+
|
|
30
|
+
Stated as a protocol rather than ``Transport | SyncTransport`` because that is
|
|
31
|
+
the whole requirement, and naming both would make this module depend on both
|
|
32
|
+
for nothing.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
@property
|
|
36
|
+
def config(self) -> Config: ...
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
__all__ = [
|
|
40
|
+
"_resolve_stage_auth",
|
|
41
|
+
"_env_has_stage_credential",
|
|
42
|
+
"_connect_for_stage",
|
|
43
|
+
"_validate_stage_identifier",
|
|
44
|
+
"_safe_stage_filename",
|
|
45
|
+
"_resolve_stage_fqn",
|
|
46
|
+
"_put_file_to_stage",
|
|
47
|
+
"_parse_toml_stage_mounts",
|
|
48
|
+
]
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
# A Snowflake identifier part: an unquoted name (letters/digits/_/$, not starting
|
|
52
|
+
# with a digit) or a double-quoted identifier ("" escapes an inner quote). A
|
|
53
|
+
# fully-qualified stage is 1-3 dot-separated parts. Anything else — a space, a
|
|
54
|
+
# quote-and-space (`S LOCATION='s3://attacker/'`), a semicolon, parentheses — is
|
|
55
|
+
# rejected, so a `stage=`/`code_stage=` from an agent or config cannot break out
|
|
56
|
+
# of `CREATE STAGE`/`PUT`/`LIST`/`GET_PRESIGNED_URL` and run attacker SQL as the
|
|
57
|
+
# caller's role.
|
|
58
|
+
_STAGE_ID_PART = re.compile(r'^(?:[A-Za-z_][A-Za-z0-9_$]*|"(?:[^"]|"")*")$')
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
# Chars allowed in the stage-object filename we interpolate into PUT/LIST. The
|
|
62
|
+
# hash16 is the content key; the dir-name prefix is cosmetic, so a name with a
|
|
63
|
+
# quote/slash/space (a local directory basename is attacker-influenceable) is
|
|
64
|
+
# squashed to underscores rather than left to break the `'file://...'` literal.
|
|
65
|
+
_UNSAFE_FILENAME_CHARS = re.compile(r"[^A-Za-z0-9._-]")
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _resolve_stage_auth(transport: _ConfigCarrier | None) -> tuple[str, str, dict[str, Any]]:
|
|
69
|
+
"""``(account, host, connect_kwargs)`` for a connector stage PUT, resolved from
|
|
70
|
+
the environment (what Snowflake injects into a sandbox) first, then the SDK's config.
|
|
71
|
+
Raises `SandboxError` when neither host+token nor host+user/password is set."""
|
|
72
|
+
import os
|
|
73
|
+
|
|
74
|
+
from snowflake.sandbox.config import effective_config
|
|
75
|
+
|
|
76
|
+
_cfg = transport.config if transport else effective_config()
|
|
77
|
+
account_url = os.environ.get("SNOWFLAKE_ACCOUNT_URL", "").rstrip("/")
|
|
78
|
+
if not account_url:
|
|
79
|
+
try:
|
|
80
|
+
account_url = _cfg.base_url.rstrip("/")
|
|
81
|
+
except RuntimeError:
|
|
82
|
+
account_url = ""
|
|
83
|
+
token = os.environ.get("SNOWFLAKE_TOKEN") or os.environ.get("SNOWFLAKE_PAT") or (_cfg.pat or "")
|
|
84
|
+
_user = os.environ.get("SNOWFLAKE_USER", "")
|
|
85
|
+
_password = os.environ.get("SNOWFLAKE_PASSWORD", "")
|
|
86
|
+
if not account_url or not (token or (_user and _password)):
|
|
87
|
+
raise SandboxError(
|
|
88
|
+
"uploading local files to a stage needs a host and a credential: "
|
|
89
|
+
'name a connection (connection="my_connection", a ~/.snowflake default, '
|
|
90
|
+
"or SNOWFLAKE_HOST + SNOWFLAKE_PAT), or set "
|
|
91
|
+
"SNOWFLAKE_ACCOUNT_URL plus SNOWFLAKE_TOKEN (OAuth/Snowflake) or "
|
|
92
|
+
"SNOWFLAKE_USER + SNOWFLAKE_PASSWORD (local dev)"
|
|
93
|
+
)
|
|
94
|
+
_host = account_url.split("//")[-1].split(":")[0].rstrip("/").lower()
|
|
95
|
+
_account = (
|
|
96
|
+
_host[: -len(".snowflakecomputing.com")]
|
|
97
|
+
if _host.endswith(".snowflakecomputing.com")
|
|
98
|
+
else _host
|
|
99
|
+
)
|
|
100
|
+
if token:
|
|
101
|
+
connect_kwargs: dict[str, Any] = {"authenticator": "oauth", "token": token}
|
|
102
|
+
else:
|
|
103
|
+
connect_kwargs = {"user": _user, "password": _password}
|
|
104
|
+
return _account, _host, connect_kwargs
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _env_has_stage_credential() -> bool:
|
|
108
|
+
"""Whether the environment alone carries a usable stage credential.
|
|
109
|
+
|
|
110
|
+
Snowflake injects ``SNOWFLAKE_ACCOUNT_URL`` + ``SNOWFLAKE_TOKEN`` into sandbox
|
|
111
|
+
containers, and that has always taken precedence over the SDK's config; this
|
|
112
|
+
keeps that order intact.
|
|
113
|
+
"""
|
|
114
|
+
import os
|
|
115
|
+
|
|
116
|
+
if not os.environ.get("SNOWFLAKE_ACCOUNT_URL", "").strip():
|
|
117
|
+
return False
|
|
118
|
+
if os.environ.get("SNOWFLAKE_TOKEN") or os.environ.get("SNOWFLAKE_PAT"):
|
|
119
|
+
return True
|
|
120
|
+
return bool(os.environ.get("SNOWFLAKE_USER") and os.environ.get("SNOWFLAKE_PASSWORD"))
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _connect_for_stage(transport: _ConfigCarrier | None) -> tuple[Any, bool]:
|
|
124
|
+
"""Return ``(connection, owns)`` for a stage PUT. The connector PUT is the only
|
|
125
|
+
upload path that works for every stage type (``@~/``, ``@%table``, named); the
|
|
126
|
+
Files REST API is limited to named stages.
|
|
127
|
+
|
|
128
|
+
When the config holds a live connection the connector already opened — a named
|
|
129
|
+
connection the SDK resolved, or one a caller handed to ``connection=<obj>`` (the
|
|
130
|
+
``snow`` CLI does this) — and the environment carries no injected stage
|
|
131
|
+
credential, that connection is REUSED: ``owns`` is False and the caller must NOT
|
|
132
|
+
close it, because closing it would drop the session the REST transport renews its
|
|
133
|
+
bearer token from. This is the single-authority path: the connection that logged
|
|
134
|
+
in does the PUT too, with its own credential and role, so nothing is re-derived.
|
|
135
|
+
|
|
136
|
+
Otherwise a fresh connection is opened and the caller OWNS it (``owns`` True):
|
|
137
|
+
via the named connection's own connect kwargs (``connector_kwargs``), or from the
|
|
138
|
+
environment (what Snowflake injects into a sandbox — ``_resolve_stage_auth``).
|
|
139
|
+
Deriving auth from a bare token as ``authenticator="oauth"`` is correct only for
|
|
140
|
+
such an injected OAuth token, never for a minted *session* token (which fails
|
|
141
|
+
``250001 Invalid OAuth access token``) — which is why a connection-backed config
|
|
142
|
+
is reused directly above and never reaches that derivation.
|
|
143
|
+
"""
|
|
144
|
+
from snowflake.sandbox.config import effective_config
|
|
145
|
+
|
|
146
|
+
# A Transport may carry no config at all, so this cannot assume an object.
|
|
147
|
+
cfg = transport.config if transport is not None else effective_config()
|
|
148
|
+
|
|
149
|
+
# Single-authority reuse: the live connection already authenticated (its role,
|
|
150
|
+
# its credential), and the environment has no injected stage credential that must
|
|
151
|
+
# take precedence. Reuse it directly — the caller must not close it.
|
|
152
|
+
live = cfg.connection if cfg is not None else None
|
|
153
|
+
if live is not None and not _env_has_stage_credential():
|
|
154
|
+
return live, False # owns=False: shared connection, caller must not close it
|
|
155
|
+
|
|
156
|
+
try:
|
|
157
|
+
import snowflake.connector
|
|
158
|
+
except ImportError as exc:
|
|
159
|
+
# snowflake-connector-python is a core dependency, so this should not
|
|
160
|
+
# happen in a normal install; guard defensively and name the real package.
|
|
161
|
+
raise SandboxError(
|
|
162
|
+
"uploading local files requires snowflake-connector-python: "
|
|
163
|
+
"pip install snowflake-connector-python"
|
|
164
|
+
) from exc
|
|
165
|
+
|
|
166
|
+
conn_kwargs = cfg.connector_kwargs if cfg is not None else None
|
|
167
|
+
if conn_kwargs and not _env_has_stage_credential():
|
|
168
|
+
# connector_kwargs re-opens the named connection (connection_name=...), so
|
|
169
|
+
# the connector's CONFIG_MANAGER must point at the same home it was
|
|
170
|
+
# resolved from — it is a process-global that caches its last read.
|
|
171
|
+
from snowflake.sandbox.config import refresh_connection_config
|
|
172
|
+
|
|
173
|
+
refresh_connection_config()
|
|
174
|
+
return snowflake.connector.connect(**conn_kwargs), True
|
|
175
|
+
|
|
176
|
+
account, host, connect_kwargs = _resolve_stage_auth(transport)
|
|
177
|
+
return snowflake.connector.connect(account=account, host=host, **connect_kwargs), True
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _validate_stage_identifier(name: str) -> str:
|
|
181
|
+
"""Return *name* if it is a well-formed (optionally dotted) Snowflake stage
|
|
182
|
+
identifier, else raise `SandboxError`. This is the injection boundary for
|
|
183
|
+
every SQL path that names a stage (see `_STAGE_ID_PART`)."""
|
|
184
|
+
raw = name.strip()
|
|
185
|
+
if not raw:
|
|
186
|
+
raise SandboxError("stage name must not be empty")
|
|
187
|
+
parts = raw.split(".")
|
|
188
|
+
if len(parts) > 3 or not all(_STAGE_ID_PART.match(p) for p in parts):
|
|
189
|
+
raise SandboxError(
|
|
190
|
+
f"stage name {name!r} is not a valid Snowflake identifier: expected "
|
|
191
|
+
"DB.SCHEMA.NAME or a bare NAME (letters, digits, '_' or '$'; a "
|
|
192
|
+
'double-quoted "identifier" is also accepted). Names with spaces, '
|
|
193
|
+
"quotes, or SQL punctuation are rejected to prevent injection."
|
|
194
|
+
)
|
|
195
|
+
return raw
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _safe_stage_filename(name: str) -> str:
|
|
199
|
+
"""Squash *name* to the charset safe to interpolate into a PUT/LIST path."""
|
|
200
|
+
return _UNSAFE_FILENAME_CHARS.sub("_", name)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _resolve_stage_fqn(cur: Any, name: str) -> str:
|
|
204
|
+
"""Resolve *name* to a fully-qualified stage and ensure it exists.
|
|
205
|
+
|
|
206
|
+
A ``DB.SCHEMA.NAME`` is used as-is; a bare name is qualified with the
|
|
207
|
+
connection's current database + schema. ``CREATE STAGE IF NOT EXISTS`` is then
|
|
208
|
+
run (via ``IDENTIFIER(?)`` bind) so the copy staging stage is auto-created.
|
|
209
|
+
*name* is validated as a Snowflake identifier first, so it cannot inject SQL.
|
|
210
|
+
Raises `SandboxError` if a bare name is given without a current db/schema."""
|
|
211
|
+
_validate_stage_identifier(name)
|
|
212
|
+
if name.count(".") >= 2:
|
|
213
|
+
fqn = name
|
|
214
|
+
else:
|
|
215
|
+
cur.execute("SELECT CURRENT_DATABASE(), CURRENT_SCHEMA()")
|
|
216
|
+
row = cur.fetchone() or (None, None)
|
|
217
|
+
db, schema = row[0], row[1]
|
|
218
|
+
if not db or not schema:
|
|
219
|
+
raise SandboxError(
|
|
220
|
+
f"copy_local needs a current database and schema to create the "
|
|
221
|
+
f"staging stage {name!r}. Set them on your connection (in "
|
|
222
|
+
f"connections.toml, or with USE DATABASE/USE SCHEMA), or pass "
|
|
223
|
+
f"stage='DB.SCHEMA.NAME'."
|
|
224
|
+
)
|
|
225
|
+
fqn = f"{db}.{schema}.{name}"
|
|
226
|
+
# IDENTIFIER(%s) binds the (validated) name as data, never as raw SQL text.
|
|
227
|
+
# %s is pyformat, the connector's default paramstyle.
|
|
228
|
+
cur.execute("CREATE STAGE IF NOT EXISTS IDENTIFIER(%s)", (fqn,))
|
|
229
|
+
return fqn
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def _put_file_to_stage(cur: Any, fqn: str, filename: str, zip_bytes: bytes) -> None:
|
|
233
|
+
"""PUT *zip_bytes* as *filename* into stage *fqn* on an already-open cursor,
|
|
234
|
+
LIST-guarded so an already-present content-addressed object is not re-uploaded.
|
|
235
|
+
|
|
236
|
+
*fqn* must already be a validated identifier (see `_resolve_stage_fqn`);
|
|
237
|
+
*filename* is squashed to a safe charset so it cannot break out of the PUT
|
|
238
|
+
``'file://...'`` literal or the LIST path."""
|
|
239
|
+
import os
|
|
240
|
+
import tempfile
|
|
241
|
+
|
|
242
|
+
_validate_stage_identifier(fqn)
|
|
243
|
+
safe_name = _safe_stage_filename(filename)
|
|
244
|
+
cur.execute(f"LIST @{fqn}/{safe_name}")
|
|
245
|
+
if cur.fetchone():
|
|
246
|
+
return
|
|
247
|
+
with tempfile.TemporaryDirectory() as _tmpdir:
|
|
248
|
+
local_zip = os.path.join(_tmpdir, safe_name)
|
|
249
|
+
with open(local_zip, "wb") as _f:
|
|
250
|
+
_f.write(zip_bytes)
|
|
251
|
+
cur.execute(f"PUT 'file://{local_zip}' @{fqn}/ AUTO_COMPRESS=FALSE OVERWRITE=TRUE")
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def _parse_toml_stage_mounts(raw: list[Any]) -> list[StageMount] | None:
|
|
255
|
+
"""Parse ``[[stage_mounts]]`` TOML entries into `StageMount` objects."""
|
|
256
|
+
if not raw:
|
|
257
|
+
return None
|
|
258
|
+
from snowflake.sandbox.mount import StageMount
|
|
259
|
+
|
|
260
|
+
mounts = []
|
|
261
|
+
for entry in raw:
|
|
262
|
+
kind = entry.get("kind", "stage")
|
|
263
|
+
name = entry.get("name") or entry.get("stage_name", "")
|
|
264
|
+
mount_path = entry.get("mount_path", "")
|
|
265
|
+
readonly = bool(entry.get("readonly", False))
|
|
266
|
+
if kind == "workspace":
|
|
267
|
+
mounts.append(StageMount.from_workspace(name, mount_path=mount_path, readonly=readonly))
|
|
268
|
+
else:
|
|
269
|
+
mounts.append(StageMount.from_stage(name, mount_path=mount_path, readonly=readonly))
|
|
270
|
+
return mounts or None
|