snowflake-sandbox-python 0.2.1a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- snowflake/cli_sandbox/__init__.py +13 -0
- snowflake/cli_sandbox/_adapter.py +170 -0
- snowflake/cli_sandbox/_common.py +77 -0
- snowflake/cli_sandbox/_egress_flags.py +121 -0
- snowflake/cli_sandbox/_get_command.py +109 -0
- snowflake/cli_sandbox/_run_command.py +1091 -0
- snowflake/cli_sandbox/_shell_command.py +666 -0
- snowflake/cli_sandbox/_upload_plan.py +187 -0
- snowflake/cli_sandbox/commands.py +556 -0
- snowflake/cli_sandbox/plugin_spec.py +28 -0
- snowflake/cli_sandbox/py.typed +0 -0
- snowflake/sandbox/__init__.py +317 -0
- snowflake/sandbox/__main__.py +225 -0
- snowflake/sandbox/_ansi.py +206 -0
- snowflake/sandbox/_args.py +208 -0
- snowflake/sandbox/_assemble.py +256 -0
- snowflake/sandbox/_bundle.py +240 -0
- snowflake/sandbox/_connection_resolve.py +328 -0
- snowflake/sandbox/_deploy_spec.py +56 -0
- snowflake/sandbox/_diagnostics.py +501 -0
- snowflake/sandbox/_env.py +143 -0
- snowflake/sandbox/_files_mixin.py +280 -0
- snowflake/sandbox/_fs_ops.py +304 -0
- snowflake/sandbox/_globs.py +176 -0
- snowflake/sandbox/_hosts.py +110 -0
- snowflake/sandbox/_mcp_discovery.py +288 -0
- snowflake/sandbox/_mcp_status.py +183 -0
- snowflake/sandbox/_retry.py +94 -0
- snowflake/sandbox/_runtime/__init__.py +42 -0
- snowflake/sandbox/_runtime/_fs_helper.py +93 -0
- snowflake/sandbox/_runtime/_job_runner.py +111 -0
- snowflake/sandbox/_runtime/_protocol.py +53 -0
- snowflake/sandbox/_runtime/_shims.py +267 -0
- snowflake/sandbox/_sandbox_state.py +303 -0
- snowflake/sandbox/_session_registry.py +222 -0
- snowflake/sandbox/_sse.py +160 -0
- snowflake/sandbox/_stage.py +270 -0
- snowflake/sandbox/_sync_files_mixin.py +272 -0
- snowflake/sandbox/_sync_fs_ops.py +185 -0
- snowflake/sandbox/_sync_transport.py +737 -0
- snowflake/sandbox/_sync_watch.py +99 -0
- snowflake/sandbox/_transport.py +1366 -0
- snowflake/sandbox/_transport_errors.py +270 -0
- snowflake/sandbox/_upload_plan.py +497 -0
- snowflake/sandbox/_version.py +37 -0
- snowflake/sandbox/_watch.py +164 -0
- snowflake/sandbox/_wire.py +348 -0
- snowflake/sandbox/app.py +256 -0
- snowflake/sandbox/client.py +2356 -0
- snowflake/sandbox/config.py +1133 -0
- snowflake/sandbox/connect.py +288 -0
- snowflake/sandbox/deploy.py +499 -0
- snowflake/sandbox/egress.py +388 -0
- snowflake/sandbox/exceptions.py +253 -0
- snowflake/sandbox/exec_stream.py +264 -0
- snowflake/sandbox/files.py +547 -0
- snowflake/sandbox/function.py +567 -0
- snowflake/sandbox/image.py +46 -0
- snowflake/sandbox/jobs.py +649 -0
- snowflake/sandbox/lifecycle.py +67 -0
- snowflake/sandbox/log_stream.py +219 -0
- snowflake/sandbox/mcp.py +480 -0
- snowflake/sandbox/mount.py +161 -0
- snowflake/sandbox/py.typed +0 -0
- snowflake/sandbox/secret.py +244 -0
- snowflake/sandbox/session_app.py +244 -0
- snowflake/sandbox/shell.py +556 -0
- snowflake/sandbox/sync_client.py +2245 -0
- snowflake/sandbox/sync_exec_stream.py +238 -0
- snowflake/sandbox/sync_files.py +377 -0
- snowflake/sandbox/sync_log_stream.py +142 -0
- snowflake/sandbox/sync_shell.py +413 -0
- snowflake/sandbox/types.py +193 -0
- snowflake/sandbox/warm_session.py +700 -0
- snowflake_sandbox_python-0.2.1a1.dist-info/METADATA +339 -0
- snowflake_sandbox_python-0.2.1a1.dist-info/RECORD +80 -0
- snowflake_sandbox_python-0.2.1a1.dist-info/WHEEL +5 -0
- snowflake_sandbox_python-0.2.1a1.dist-info/entry_points.txt +2 -0
- snowflake_sandbox_python-0.2.1a1.dist-info/licenses/LICENSE +202 -0
- snowflake_sandbox_python-0.2.1a1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1366 @@
|
|
|
1
|
+
"""HTTP transport for ``snowflake.sandbox``.
|
|
2
|
+
|
|
3
|
+
Wraps a single process-wide ``httpx.AsyncClient`` (HTTP/2, bearer auth,
|
|
4
|
+
JSON content-type), handles the retry policy, maps
|
|
5
|
+
HTTP status codes to the ``SandboxError`` hierarchy, and exposes a
|
|
6
|
+
streaming SSE primitive used by ``exec_stream``.
|
|
7
|
+
|
|
8
|
+
Imports ``httpx`` lazily-at-module-load (i.e. when the module is imported
|
|
9
|
+
from ``__init__``'s ``__getattr__``); ``import snowflake.sandbox`` does
|
|
10
|
+
not pay this cost.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import asyncio
|
|
16
|
+
import contextlib
|
|
17
|
+
import inspect
|
|
18
|
+
import random
|
|
19
|
+
import threading
|
|
20
|
+
import weakref
|
|
21
|
+
from collections.abc import AsyncIterator, Callable, Mapping
|
|
22
|
+
from dataclasses import dataclass
|
|
23
|
+
from typing import Any, BinaryIO, Final
|
|
24
|
+
|
|
25
|
+
import httpx
|
|
26
|
+
|
|
27
|
+
from snowflake.sandbox._retry import (
|
|
28
|
+
DEFAULT_500_DELAYS_S,
|
|
29
|
+
DEFAULT_COLD_START_DELAYS_S,
|
|
30
|
+
DEFAULT_NOT_FOUND_DELAYS_S,
|
|
31
|
+
compute_backoff,
|
|
32
|
+
)
|
|
33
|
+
from snowflake.sandbox._sse import (
|
|
34
|
+
SSEEvent,
|
|
35
|
+
_iter_sse_frames,
|
|
36
|
+
_sse_open_retry_delay,
|
|
37
|
+
parse_sse_event, # noqa: F401 - re-exported so _transport.parse_sse_event resolves
|
|
38
|
+
)
|
|
39
|
+
from snowflake.sandbox._transport_errors import (
|
|
40
|
+
_RATE_LIMIT_MAX_ATTEMPTS, # noqa: F401 - re-exported for callers/tests
|
|
41
|
+
_RETRY_AFTER_MAX_S, # noqa: F401 - re-exported for callers/tests
|
|
42
|
+
SandboxSSEDecodeError, # noqa: F401 - re-exported for callers/tests
|
|
43
|
+
_error_detail,
|
|
44
|
+
_is_clean_goaway,
|
|
45
|
+
_is_cold_start_409,
|
|
46
|
+
_is_reauth_response,
|
|
47
|
+
_rate_limit_delay,
|
|
48
|
+
_retry_after_seconds,
|
|
49
|
+
_says_not_ready,
|
|
50
|
+
_status_to_error,
|
|
51
|
+
)
|
|
52
|
+
from snowflake.sandbox._version import __version__ as _SDK_VERSION
|
|
53
|
+
from snowflake.sandbox.config import (
|
|
54
|
+
Config,
|
|
55
|
+
current_config,
|
|
56
|
+
refresh_config,
|
|
57
|
+
refresh_credential,
|
|
58
|
+
)
|
|
59
|
+
from snowflake.sandbox.exceptions import (
|
|
60
|
+
SandboxAuthError,
|
|
61
|
+
SandboxNotReadyError,
|
|
62
|
+
SandboxRateLimitError,
|
|
63
|
+
SandboxTransportError,
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
# ``shutdown()`` is public API and now has a public home in ``lifecycle``; it is
|
|
67
|
+
# re-exported here so the internal callers (and tests) that reach for
|
|
68
|
+
# ``_transport.shutdown`` keep working. ``lifecycle`` imports this module only inside
|
|
69
|
+
# its function bodies, so this edge does not close a module-scope cycle.
|
|
70
|
+
from snowflake.sandbox.lifecycle import shutdown
|
|
71
|
+
|
|
72
|
+
__all__ = [
|
|
73
|
+
"Transport",
|
|
74
|
+
"get_transport",
|
|
75
|
+
"reset_transport",
|
|
76
|
+
"shutdown",
|
|
77
|
+
"SSEEvent",
|
|
78
|
+
"AsyncStreamAbort",
|
|
79
|
+
"parse_sse_event",
|
|
80
|
+
]
|
|
81
|
+
|
|
82
|
+
_USER_AGENT: Final = f"snowflake-sandbox-python/{_SDK_VERSION}"
|
|
83
|
+
_MCP_PATH: Final = "/api/v2/cortex/v1/mcp"
|
|
84
|
+
# Status codes that indicate the container is still cold-starting.
|
|
85
|
+
_COLD_START_STATUSES: Final = {503}
|
|
86
|
+
# Headers a caller-supplied `extra_headers` (SNOWFLAKE_CWS_EXTRA_HEADERS) may never
|
|
87
|
+
# override -- doing so would swap the bearer token or redirect the request host.
|
|
88
|
+
_RESERVED_HEADER_NAMES: Final = frozenset({"authorization", "host"})
|
|
89
|
+
|
|
90
|
+
# Methods for which re-sending a request that may already have run is harmless.
|
|
91
|
+
# POST is deliberately absent: a re-sent `POST /containers` leaves an orphaned,
|
|
92
|
+
# billed sandbox whose id the caller never learns, and a re-sent
|
|
93
|
+
# `POST /containers/{id}/exec` re-runs the caller's command with its full side
|
|
94
|
+
# effects (`git push`, `dbt run`, a Slack post) while looking identical to a
|
|
95
|
+
# command that never ran.
|
|
96
|
+
_IDEMPOTENT_METHODS: Final = frozenset({"GET", "HEAD", "OPTIONS", "PUT", "DELETE"})
|
|
97
|
+
|
|
98
|
+
# Failures raised before any byte of the request reached the server. Retrying
|
|
99
|
+
# these cannot duplicate a side effect, whatever the method.
|
|
100
|
+
_UNSENT_ERRORS: Final = (
|
|
101
|
+
httpx.ConnectError,
|
|
102
|
+
httpx.ConnectTimeout,
|
|
103
|
+
httpx.PoolTimeout,
|
|
104
|
+
)
|
|
105
|
+
# Failures raised after the request was written. Whether the server ran it is
|
|
106
|
+
# unknowable from the client, so these are retried only for idempotent methods.
|
|
107
|
+
_SENT_ERRORS: Final = (
|
|
108
|
+
httpx.ReadTimeout,
|
|
109
|
+
httpx.ReadError,
|
|
110
|
+
httpx.WriteTimeout,
|
|
111
|
+
httpx.WriteError,
|
|
112
|
+
httpx.RemoteProtocolError,
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
@dataclass
|
|
117
|
+
class _RetryState:
|
|
118
|
+
"""Per-request retry counters, threaded from request()'s send/retry loop into
|
|
119
|
+
the per-status policy in `_status_retry_delay`. One instance per request()."""
|
|
120
|
+
|
|
121
|
+
cold: int = 0
|
|
122
|
+
five_hundred: int = 0
|
|
123
|
+
rate_limit: int = 0
|
|
124
|
+
not_found: int = 0
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
@dataclass
|
|
128
|
+
class _RequestAuth:
|
|
129
|
+
"""The token a request()'s headers carry, tracked so a reauth response can
|
|
130
|
+
re-mint it and retry exactly once. One instance per request(), mutated in
|
|
131
|
+
place by `Transport._reauth` / `SyncTransport._reauth`.
|
|
132
|
+
|
|
133
|
+
``refreshed`` is the one-shot guard: a freshly-minted token that still draws a
|
|
134
|
+
reauth response is a real auth failure, not expiry, so it must not loop."""
|
|
135
|
+
|
|
136
|
+
headers: dict[str, str]
|
|
137
|
+
sent_pat: str | None
|
|
138
|
+
refreshed: bool = False
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _status_retry_delay(
|
|
142
|
+
rng: random.Random,
|
|
143
|
+
resp: httpx.Response,
|
|
144
|
+
*,
|
|
145
|
+
method: str,
|
|
146
|
+
path: str,
|
|
147
|
+
idempotent: bool,
|
|
148
|
+
retry_not_found: bool,
|
|
149
|
+
retry_500_delays: tuple[float, ...],
|
|
150
|
+
state: _RetryState,
|
|
151
|
+
) -> float | None:
|
|
152
|
+
"""The per-status retry POLICY, split out of request()'s send/retry LOOP.
|
|
153
|
+
|
|
154
|
+
For a ``>=400`` response that is NOT a reauth case (the caller handles those),
|
|
155
|
+
returns the seconds to back off before retrying, or ``None`` when the status is
|
|
156
|
+
not one this policy retries (the caller then does reauth-or-terminal-raise).
|
|
157
|
+
Raises the terminal ``SandboxError`` when the method makes a retry unsafe or a
|
|
158
|
+
retry budget is exhausted. Pure and synchronous, so the async and sync
|
|
159
|
+
transports share one implementation. ``state`` carries the attempt counters.
|
|
160
|
+
"""
|
|
161
|
+
sc = resp.status_code
|
|
162
|
+
|
|
163
|
+
# Retryable: 503 or 409 cold-start. For a non-idempotent request the body must
|
|
164
|
+
# say the work was refused, not merely that something went wrong upstream of an
|
|
165
|
+
# unknown outcome.
|
|
166
|
+
if sc in _COLD_START_STATUSES or _is_cold_start_409(resp):
|
|
167
|
+
if not idempotent and not _says_not_ready(resp):
|
|
168
|
+
raise SandboxTransportError(
|
|
169
|
+
f"HTTP {sc} on {method.upper()} {path}: "
|
|
170
|
+
f"{_error_detail(resp.text)} — not retried because the server may "
|
|
171
|
+
"already have executed it"
|
|
172
|
+
)
|
|
173
|
+
state.cold += 1
|
|
174
|
+
delay = compute_backoff(state.cold, delays=DEFAULT_COLD_START_DELAYS_S, rng=rng)
|
|
175
|
+
if delay is None:
|
|
176
|
+
if sc == 409:
|
|
177
|
+
raise SandboxNotReadyError(
|
|
178
|
+
f"sandbox not ready after {len(DEFAULT_COLD_START_DELAYS_S)} retries"
|
|
179
|
+
)
|
|
180
|
+
raise SandboxTransportError(
|
|
181
|
+
f"503 after {len(DEFAULT_COLD_START_DELAYS_S)} retries: {_error_detail(resp.text)}"
|
|
182
|
+
)
|
|
183
|
+
return delay
|
|
184
|
+
|
|
185
|
+
# 429: honor a sanitized Retry-After, retry up to the shared budget. A
|
|
186
|
+
# rate-limited request was rejected without being executed, so this is safe for
|
|
187
|
+
# any method. Jitter is upward only (see _rate_limit_delay) so we never retry
|
|
188
|
+
# before the server said it was safe to.
|
|
189
|
+
if sc == 429:
|
|
190
|
+
state.rate_limit += 1
|
|
191
|
+
delay = _rate_limit_delay(resp, state.rate_limit, rng)
|
|
192
|
+
if delay is None:
|
|
193
|
+
raise SandboxRateLimitError(
|
|
194
|
+
f"rate-limited after {state.rate_limit - 1} retries",
|
|
195
|
+
retry_after=_retry_after_seconds(resp),
|
|
196
|
+
)
|
|
197
|
+
return delay
|
|
198
|
+
|
|
199
|
+
# 500 surfaces as transport-error after one retry — but only for an idempotent
|
|
200
|
+
# method: a 500 means the server accepted the request and failed somewhere
|
|
201
|
+
# inside, which says nothing about whether the side effect landed.
|
|
202
|
+
if sc == 500:
|
|
203
|
+
if not idempotent:
|
|
204
|
+
raise SandboxTransportError(
|
|
205
|
+
f"HTTP 500 on {method.upper()} {path}: {_error_detail(resp.text)} — "
|
|
206
|
+
"not retried because the server may already have executed it"
|
|
207
|
+
)
|
|
208
|
+
state.five_hundred += 1
|
|
209
|
+
delay = compute_backoff(state.five_hundred, delays=retry_500_delays, rng=rng)
|
|
210
|
+
if delay is None:
|
|
211
|
+
raise SandboxTransportError(
|
|
212
|
+
f"500 from server after {len(retry_500_delays)} retries: {_error_detail(resp.text)}"
|
|
213
|
+
)
|
|
214
|
+
return delay
|
|
215
|
+
|
|
216
|
+
# 404 keeps its create-race retry logic; every other terminal status goes
|
|
217
|
+
# through the shared _status_to_error table in the caller.
|
|
218
|
+
if sc == 404:
|
|
219
|
+
# A freshly-created container is briefly unroutable — its id is returned by
|
|
220
|
+
# create before the route is registered, so the first exec/logs can 404.
|
|
221
|
+
# Retry on a tight bounded schedule; a genuinely-missing container still
|
|
222
|
+
# surfaces quickly. Safe for a POST too: a 404 means no handler ran, so
|
|
223
|
+
# there is no side effect a retry could duplicate.
|
|
224
|
+
#
|
|
225
|
+
# retry_not_found=False skips this schedule: a caller probing an optional
|
|
226
|
+
# route (e.g. a cheap /status older servers lack) or looking up a
|
|
227
|
+
# user-supplied id wants the 404 immediately, not after the create-race wait.
|
|
228
|
+
if not retry_not_found:
|
|
229
|
+
raise _status_to_error(404, resp.text)
|
|
230
|
+
state.not_found += 1
|
|
231
|
+
delay = compute_backoff(state.not_found, delays=DEFAULT_NOT_FOUND_DELAYS_S, rng=rng)
|
|
232
|
+
if delay is None:
|
|
233
|
+
raise _status_to_error(404, resp.text)
|
|
234
|
+
return delay
|
|
235
|
+
|
|
236
|
+
return None
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _transport_error_delay(
|
|
240
|
+
exc: Exception,
|
|
241
|
+
*,
|
|
242
|
+
sent: bool,
|
|
243
|
+
idempotent: bool,
|
|
244
|
+
method: str,
|
|
245
|
+
path: str,
|
|
246
|
+
state: _RetryState,
|
|
247
|
+
rng: random.Random,
|
|
248
|
+
) -> float:
|
|
249
|
+
"""Backoff (seconds) for a transport-layer error, or raise if the retry is
|
|
250
|
+
unsafe or the budget is exhausted.
|
|
251
|
+
|
|
252
|
+
A pre-send error is always retryable (nothing reached the server). A post-send
|
|
253
|
+
error (``sent=True``) is retryable only for an idempotent method or a clean
|
|
254
|
+
GOAWAY (error_code=0, no streams processed) — the server told us it closed the
|
|
255
|
+
connection before touching the request. Split from request()'s loop; shared by
|
|
256
|
+
both transports.
|
|
257
|
+
"""
|
|
258
|
+
if sent and not idempotent and not _is_clean_goaway(exc):
|
|
259
|
+
raise SandboxTransportError(
|
|
260
|
+
f"{method.upper()} {path} failed after the request was sent "
|
|
261
|
+
f"({type(exc).__name__}: {exc}); not retried because the "
|
|
262
|
+
"server may already have executed it"
|
|
263
|
+
) from exc
|
|
264
|
+
state.cold += 1
|
|
265
|
+
delay = compute_backoff(state.cold, delays=DEFAULT_COLD_START_DELAYS_S, rng=rng)
|
|
266
|
+
if delay is None:
|
|
267
|
+
raise SandboxTransportError(f"transport error after retries: {exc}") from exc
|
|
268
|
+
return delay
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
class AsyncStreamAbort:
|
|
272
|
+
"""`StreamAbort` for the async client: interrupt a pending SSE read.
|
|
273
|
+
|
|
274
|
+
Different mechanism, same contract. The sync reader blocks in ``recv()`` on
|
|
275
|
+
another thread, so a socket shutdown is what reaches it; here the read is an
|
|
276
|
+
``await`` in the same event loop, and reaching into the transport's socket would
|
|
277
|
+
bypass the loop's bookkeeping. Closing the httpcore stream is the supported
|
|
278
|
+
equivalent -- measured: the pending read resolves immediately.
|
|
279
|
+
|
|
280
|
+
It resolves by RAISING ``httpx.ReadError``, though, where the sync shutdown ends
|
|
281
|
+
at EOF. So an abort arrives at the caller's ``except SandboxTransportError``
|
|
282
|
+
branch rather than its clean-EOF branch, and that branch resumes -- which is why
|
|
283
|
+
`aborted` has to be checked there or terminate() is followed by a reconnect to a
|
|
284
|
+
dead sandbox.
|
|
285
|
+
"""
|
|
286
|
+
|
|
287
|
+
__slots__ = ("_aborted", "_stream")
|
|
288
|
+
|
|
289
|
+
def __init__(self) -> None:
|
|
290
|
+
self._aborted = False
|
|
291
|
+
self._stream: Any = None
|
|
292
|
+
|
|
293
|
+
async def bind(self, resp: httpx.Response) -> None:
|
|
294
|
+
"""Remember the network stream under *resp* for the duration of the read.
|
|
295
|
+
|
|
296
|
+
Best-effort, as in the sync twin: ``network_stream`` is not public httpx API
|
|
297
|
+
and is absent on some transports. Without it the read ends at its timeout, as
|
|
298
|
+
before -- degraded, not broken.
|
|
299
|
+
"""
|
|
300
|
+
stream = None
|
|
301
|
+
try:
|
|
302
|
+
stream = resp.extensions.get("network_stream")
|
|
303
|
+
except Exception: # pragma: no cover - transport-shape dependent
|
|
304
|
+
stream = None
|
|
305
|
+
self._stream = stream
|
|
306
|
+
if self._aborted:
|
|
307
|
+
# abort() ran between the open and this bind, so it saw no stream.
|
|
308
|
+
await self._close()
|
|
309
|
+
|
|
310
|
+
def unbind(self) -> None:
|
|
311
|
+
self._stream = None
|
|
312
|
+
|
|
313
|
+
async def abort(self) -> None:
|
|
314
|
+
self._aborted = True
|
|
315
|
+
await self._close()
|
|
316
|
+
|
|
317
|
+
async def _close(self) -> None:
|
|
318
|
+
stream, self._stream = self._stream, None
|
|
319
|
+
if stream is None:
|
|
320
|
+
return
|
|
321
|
+
try:
|
|
322
|
+
await stream.aclose()
|
|
323
|
+
except Exception: # pragma: no cover - already closed, or never connected
|
|
324
|
+
pass
|
|
325
|
+
|
|
326
|
+
@property
|
|
327
|
+
def aborted(self) -> bool:
|
|
328
|
+
return self._aborted
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
_FILE_CHUNK_BYTES: Final = 64 * 1024
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def _seekable_size(f: BinaryIO) -> int:
|
|
335
|
+
"""Byte length of an open file, leaving it rewound."""
|
|
336
|
+
f.seek(0, 2)
|
|
337
|
+
size = f.tell()
|
|
338
|
+
f.seek(0)
|
|
339
|
+
return size
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
async def _aiter_file(f: BinaryIO) -> AsyncIterator[bytes]:
|
|
343
|
+
"""Yield a file's bytes in chunks for an async request body.
|
|
344
|
+
|
|
345
|
+
An ``AsyncClient`` cannot take a plain file object as ``content`` -- httpx wraps
|
|
346
|
+
it in a *sync* byte stream and refuses to send it -- so the file is adapted to an
|
|
347
|
+
async iterator. Reads go through a thread because a large read on the event loop
|
|
348
|
+
would stall every other request sharing it.
|
|
349
|
+
"""
|
|
350
|
+
while True:
|
|
351
|
+
chunk = await asyncio.to_thread(f.read, _FILE_CHUNK_BYTES)
|
|
352
|
+
if not chunk:
|
|
353
|
+
return
|
|
354
|
+
yield chunk
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def _cws_transform(
|
|
358
|
+
method: str,
|
|
359
|
+
endpoint: str,
|
|
360
|
+
json_body: Any,
|
|
361
|
+
timeout: float | None,
|
|
362
|
+
) -> tuple[str, Any]:
|
|
363
|
+
"""CWS dev mode: redirect MCP exec → REST and convert env dict → sandbox_env."""
|
|
364
|
+
if method.upper() == "POST" and endpoint == _MCP_PATH and isinstance(json_body, dict):
|
|
365
|
+
args = json_body.get("params", {}).get("arguments", {})
|
|
366
|
+
cid = args.get("container_id")
|
|
367
|
+
if cid:
|
|
368
|
+
body: dict[str, Any] = {
|
|
369
|
+
"code": args.get("code", ""),
|
|
370
|
+
"language": "python",
|
|
371
|
+
"timeout_seconds": int(timeout or 60),
|
|
372
|
+
}
|
|
373
|
+
exec_env = args.get("env")
|
|
374
|
+
if isinstance(exec_env, dict) and exec_env:
|
|
375
|
+
body["env"] = exec_env
|
|
376
|
+
return f"containers/{cid}/exec", body
|
|
377
|
+
if (
|
|
378
|
+
method.upper() == "POST"
|
|
379
|
+
and "exec" not in str(endpoint)
|
|
380
|
+
and isinstance(json_body, dict)
|
|
381
|
+
and isinstance(json_body.get("env"), dict)
|
|
382
|
+
):
|
|
383
|
+
body = {k: v for k, v in json_body.items() if k != "env"}
|
|
384
|
+
body["sandbox_env"] = [f"{k}={v}" for k, v in json_body["env"].items()]
|
|
385
|
+
json_body = body
|
|
386
|
+
return endpoint, json_body
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
async def _aclose_quietly(client: httpx.AsyncClient) -> None:
|
|
390
|
+
"""Close a pooled client, swallowing whatever the close itself raises.
|
|
391
|
+
|
|
392
|
+
Best-effort on purpose, and shared by every deferred/refcounted close path: they all
|
|
393
|
+
run during teardown -- an ``__aexit__``, a request's ``finally``, an async generator
|
|
394
|
+
being closed -- where a socket-level failure (or a loop that has already gone away)
|
|
395
|
+
must not become the exception a caller's block raises.
|
|
396
|
+
"""
|
|
397
|
+
try:
|
|
398
|
+
await client.aclose()
|
|
399
|
+
except Exception: # noqa: BLE001 - teardown; see docstring
|
|
400
|
+
pass
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
class Transport:
|
|
404
|
+
"""Owns the process-wide ``httpx.AsyncClient`` and the retry loop.
|
|
405
|
+
|
|
406
|
+
All public methods are async. The client is created lazily on first
|
|
407
|
+
use so importing the module is cheap.
|
|
408
|
+
"""
|
|
409
|
+
|
|
410
|
+
def __init__(self, config: Config | None = None, role_key: str | None = None) -> None:
|
|
411
|
+
self._config_override = config
|
|
412
|
+
# With no explicit config this transport is the AMBIENT one: it re-resolves
|
|
413
|
+
# through current_config() on every request. ``role_key`` is the role to resolve
|
|
414
|
+
# AT, so its requests carry that role's token rather than the default's.
|
|
415
|
+
self._role_key = role_key if config is None else None
|
|
416
|
+
# One (client, semaphore) per running event loop. An ``httpx.AsyncClient``
|
|
417
|
+
# -- and an ``asyncio.Semaphore`` -- binds to the loop it is first used on;
|
|
418
|
+
# a single process-global client shared across loops is broken: the
|
|
419
|
+
# documented ``asyncio.run()`` idiom closes its loop on return, so the next
|
|
420
|
+
# ``asyncio.run()`` (a re-run notebook cell, a second CLI invocation, a
|
|
421
|
+
# pytest case, a per-request handler) inherited a client wired to a closed
|
|
422
|
+
# loop and failed intermittently with ``RuntimeError: Event loop is
|
|
423
|
+
# closed``. Keying on the running loop hands each loop its own client, so
|
|
424
|
+
# sequential ``asyncio.run()`` calls -- and concurrent loops on different
|
|
425
|
+
# threads -- each get a working one. A ``WeakKeyDictionary`` lets a
|
|
426
|
+
# finished loop's entry (and its client) be collected once the loop is gone.
|
|
427
|
+
self._clients: weakref.WeakKeyDictionary[
|
|
428
|
+
asyncio.AbstractEventLoop, tuple[httpx.AsyncClient, asyncio.Semaphore]
|
|
429
|
+
] = weakref.WeakKeyDictionary()
|
|
430
|
+
# A plain threading lock (not an ``asyncio.Lock``, which would itself bind
|
|
431
|
+
# to one loop) guards the map. The client is constructed synchronously
|
|
432
|
+
# inside it -- ``httpx.AsyncClient(...)`` needs no running loop.
|
|
433
|
+
self._clients_lock = threading.Lock()
|
|
434
|
+
# How many `async with AsyncSandbox(...)` blocks on each loop are currently
|
|
435
|
+
# using that loop's client (see _retain_loop_user). Keyed by loop for the same
|
|
436
|
+
# reason ``_clients`` is: the thing being refcounted is per-loop, so a
|
|
437
|
+
# process-global count would have one loop's exit decide another's close.
|
|
438
|
+
# Guarded by ``_clients_lock`` -- the count and the client it protects are
|
|
439
|
+
# read and mutated for the same key, so one lock keeps them consistent.
|
|
440
|
+
self._loop_users: weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, int] = (
|
|
441
|
+
weakref.WeakKeyDictionary()
|
|
442
|
+
)
|
|
443
|
+
# How many requests are currently in flight on each loop's client (see
|
|
444
|
+
# _in_flight_request). Separate from ``_loop_users`` because it counts a
|
|
445
|
+
# different thing: a request on a sandbox driven with explicit
|
|
446
|
+
# ``create()``/``terminate()``, or a module helper like ``list_sandboxes()``,
|
|
447
|
+
# is in flight while retaining nothing. Guarded by ``_clients_lock`` with the
|
|
448
|
+
# other two maps, so the close decision reads a consistent snapshot.
|
|
449
|
+
self._in_flight: weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, int] = (
|
|
450
|
+
weakref.WeakKeyDictionary()
|
|
451
|
+
)
|
|
452
|
+
# Loops whose last ``async with`` exited while a request was still in flight,
|
|
453
|
+
# so the close was deferred to whichever request finishes last (#370).
|
|
454
|
+
self._closes_deferred: weakref.WeakSet[asyncio.AbstractEventLoop] = weakref.WeakSet()
|
|
455
|
+
self._rng = random.Random()
|
|
456
|
+
# Serializes reactive re-mints for a Transport built with an explicit
|
|
457
|
+
# config override (the module path has its own lock in config). Keyed off
|
|
458
|
+
# nothing — the re-check of resolve_pat() inside dedups by token.
|
|
459
|
+
self._refresh_lock = threading.Lock()
|
|
460
|
+
|
|
461
|
+
@property
|
|
462
|
+
def config(self) -> Config:
|
|
463
|
+
return self._config_override or current_config(role=self._role_key)
|
|
464
|
+
|
|
465
|
+
async def _get_client(
|
|
466
|
+
self,
|
|
467
|
+
) -> tuple[httpx.AsyncClient, asyncio.Semaphore]:
|
|
468
|
+
"""The (client, semaphore) pair for the currently-running event loop."""
|
|
469
|
+
loop = asyncio.get_running_loop()
|
|
470
|
+
with self._clients_lock:
|
|
471
|
+
entry = self._clients.get(loop)
|
|
472
|
+
if entry is None:
|
|
473
|
+
cfg = self.config
|
|
474
|
+
limits = httpx.Limits(
|
|
475
|
+
max_connections=cfg.max_connections,
|
|
476
|
+
max_keepalive_connections=cfg.max_connections,
|
|
477
|
+
)
|
|
478
|
+
client = httpx.AsyncClient(
|
|
479
|
+
base_url=cfg.base_url,
|
|
480
|
+
http2=not cfg.cws_dev,
|
|
481
|
+
trust_env=not cfg.cws_dev,
|
|
482
|
+
timeout=cfg.timeout_s,
|
|
483
|
+
verify=cfg.verify,
|
|
484
|
+
limits=limits,
|
|
485
|
+
headers={"User-Agent": _USER_AGENT},
|
|
486
|
+
)
|
|
487
|
+
entry = (client, asyncio.Semaphore(cfg.max_connections))
|
|
488
|
+
self._clients[loop] = entry
|
|
489
|
+
return entry
|
|
490
|
+
|
|
491
|
+
def _retain_loop_user(self) -> None:
|
|
492
|
+
"""Register one user of the running loop's client (an ``async with`` entry).
|
|
493
|
+
|
|
494
|
+
Refcounted rather than closed per sandbox: the pool is process-wide on purpose
|
|
495
|
+
so repeated calls skip the TLS handshake, and nested or concurrent sandboxes on
|
|
496
|
+
one loop must keep it open. Refcounted PER LOOP because the client is per loop
|
|
497
|
+
-- with one process-global count, two ``asyncio.run()`` loops on two threads
|
|
498
|
+
each retained, so the first to exit saw ``count > 0``, closed nothing, and its
|
|
499
|
+
loop closed still holding sockets: ``ResourceWarning: unclosed transport``
|
|
500
|
+
(#202) with the refcount in place.
|
|
501
|
+
"""
|
|
502
|
+
loop = asyncio.get_running_loop()
|
|
503
|
+
with self._clients_lock:
|
|
504
|
+
self._loop_users[loop] = self._loop_users.get(loop, 0) + 1
|
|
505
|
+
# This loop's client has a user again, so a close an earlier exit deferred
|
|
506
|
+
# (#370) is off: the block entering here is entitled to keep it open for as
|
|
507
|
+
# long as it runs, and its own exit will close it if it is the last one.
|
|
508
|
+
self._closes_deferred.discard(loop)
|
|
509
|
+
|
|
510
|
+
async def _release_loop_user(self) -> None:
|
|
511
|
+
"""Drop one user of the running loop's client, closing it when it was the last.
|
|
512
|
+
|
|
513
|
+
Scoped to the running loop: closing every loop's client here (what the
|
|
514
|
+
package-level ``shutdown()`` does) would tear down clients belonging to loops
|
|
515
|
+
still running on other threads. Clients on other loops are left alone, and each
|
|
516
|
+
one closes when its own last block exits.
|
|
517
|
+
|
|
518
|
+
A release with nothing retained on this loop is a no-op -- a sandbox driven with
|
|
519
|
+
explicit ``create()``/``terminate()`` never retained, and closing a client it
|
|
520
|
+
never opened would break whoever did.
|
|
521
|
+
|
|
522
|
+
The close also waits for the loop's client to be IDLE (see
|
|
523
|
+
`_in_flight_request`): with requests still in flight the close is deferred to
|
|
524
|
+
whichever of them finishes last, because closing the client under a sibling task
|
|
525
|
+
aborted its request (#370).
|
|
526
|
+
"""
|
|
527
|
+
loop = asyncio.get_running_loop()
|
|
528
|
+
with self._clients_lock:
|
|
529
|
+
users = self._loop_users.get(loop, 0)
|
|
530
|
+
if users == 0:
|
|
531
|
+
return
|
|
532
|
+
if users > 1:
|
|
533
|
+
self._loop_users[loop] = users - 1
|
|
534
|
+
return
|
|
535
|
+
self._loop_users.pop(loop, None)
|
|
536
|
+
if self._in_flight.get(loop, 0) > 0:
|
|
537
|
+
# Another task on this loop is mid-request. Closing now aborts it: an
|
|
538
|
+
# idempotent GET would retry onto a fresh client, but a POST (``exec``)
|
|
539
|
+
# cannot be re-sent, so it surfaced as ``SandboxTransportError`` for a
|
|
540
|
+
# command the server may well have run. Hand the close to whichever
|
|
541
|
+
# request finishes last -- a deferral, not a leak: `_finish_request`
|
|
542
|
+
# releases the sockets as soon as the client goes idle.
|
|
543
|
+
self._closes_deferred.add(loop)
|
|
544
|
+
return
|
|
545
|
+
self._closes_deferred.discard(loop)
|
|
546
|
+
entry = self._clients.pop(loop, None)
|
|
547
|
+
if entry is None:
|
|
548
|
+
return
|
|
549
|
+
await _aclose_quietly(entry[0])
|
|
550
|
+
|
|
551
|
+
@contextlib.asynccontextmanager
|
|
552
|
+
async def _in_flight_request(self) -> AsyncIterator[None]:
|
|
553
|
+
"""Count one in-flight request against the running loop's client.
|
|
554
|
+
|
|
555
|
+
Every send path holds this for as long as it needs the client, so that
|
|
556
|
+
`_release_loop_user` can tell an idle client from a busy one. The last
|
|
557
|
+
``async with`` block on a loop can exit while a SIBLING task on the same loop is
|
|
558
|
+
mid-request -- a sandbox driven with explicit ``create()``/``terminate()``, or a
|
|
559
|
+
module helper like ``list_sandboxes()``, neither of which retains -- and the
|
|
560
|
+
close that exit performs then landed on a live request (#370).
|
|
561
|
+
|
|
562
|
+
Entered BEFORE ``_get_client()`` on purpose: both take ``_clients_lock``, so
|
|
563
|
+
registering first is what makes the pair atomic against a concurrent release,
|
|
564
|
+
which can otherwise pop the client between the two.
|
|
565
|
+
|
|
566
|
+
Held across retries and their backoff sleeps as well, not just the individual
|
|
567
|
+
``client.request`` await: a client closed mid-backoff fails the next attempt with
|
|
568
|
+
a bare ``RuntimeError`` from httpx, which no branch of the retry loop maps.
|
|
569
|
+
"""
|
|
570
|
+
loop = asyncio.get_running_loop()
|
|
571
|
+
with self._clients_lock:
|
|
572
|
+
self._in_flight[loop] = self._in_flight.get(loop, 0) + 1
|
|
573
|
+
try:
|
|
574
|
+
yield
|
|
575
|
+
finally:
|
|
576
|
+
await self._finish_request(loop)
|
|
577
|
+
|
|
578
|
+
async def _finish_request(self, loop: asyncio.AbstractEventLoop) -> None:
|
|
579
|
+
"""Drop one in-flight request, performing a close `_release_loop_user` deferred.
|
|
580
|
+
|
|
581
|
+
Takes *loop* as an argument instead of reading ``get_running_loop()``: this runs
|
|
582
|
+
from a ``finally``, including the one inside `stream_sse`'s async generator,
|
|
583
|
+
which is not guaranteed to be closed on the loop the stream began on.
|
|
584
|
+
"""
|
|
585
|
+
entry = None
|
|
586
|
+
with self._clients_lock:
|
|
587
|
+
remaining = self._in_flight.get(loop, 0) - 1
|
|
588
|
+
if remaining > 0:
|
|
589
|
+
self._in_flight[loop] = remaining
|
|
590
|
+
return
|
|
591
|
+
self._in_flight.pop(loop, None)
|
|
592
|
+
if loop not in self._closes_deferred:
|
|
593
|
+
return
|
|
594
|
+
# The loop's last block exited while this request was in flight, leaving the
|
|
595
|
+
# close to whoever finished last -- this call. A block that retained the
|
|
596
|
+
# client SINCE clears the flag under this same lock (`_retain_loop_user`), so
|
|
597
|
+
# a flag still set here means nothing has taken the client over and the close
|
|
598
|
+
# is still ours to do.
|
|
599
|
+
self._closes_deferred.discard(loop)
|
|
600
|
+
entry = self._clients.pop(loop, None)
|
|
601
|
+
if entry is not None:
|
|
602
|
+
await _aclose_quietly(entry[0])
|
|
603
|
+
|
|
604
|
+
async def aclose(self) -> None:
|
|
605
|
+
"""Close every cached HTTP client and drop them.
|
|
606
|
+
|
|
607
|
+
Awaiting this from a loop closes that loop's client cleanly; clients bound
|
|
608
|
+
to other (possibly already-closed) loops are closed best-effort. Prefer the
|
|
609
|
+
package-level ``shutdown()`` at program/kernel/handler end.
|
|
610
|
+
|
|
611
|
+
Unlike the refcounted release, this does NOT wait for in-flight requests: it is
|
|
612
|
+
the explicit end-of-program teardown, and a caller who asked for it wants the
|
|
613
|
+
sockets gone. A request still running over one of these clients fails, as it
|
|
614
|
+
always has.
|
|
615
|
+
"""
|
|
616
|
+
with self._clients_lock:
|
|
617
|
+
entries = list(self._clients.values())
|
|
618
|
+
self._clients.clear()
|
|
619
|
+
# A deferred close names the client that was mapped when it was deferred, and
|
|
620
|
+
# every one of those is being closed here. Dropping the flags keeps a stale
|
|
621
|
+
# one from closing a LATER client out from under a request using it.
|
|
622
|
+
self._closes_deferred.clear()
|
|
623
|
+
for client, _sem in entries:
|
|
624
|
+
await _aclose_quietly(client)
|
|
625
|
+
|
|
626
|
+
def _close_sync_best_effort(self) -> None:
|
|
627
|
+
"""Close cached clients from a synchronous context (``reset_transport``).
|
|
628
|
+
|
|
629
|
+
If a loop is running we cannot block on ``aclose`` here, so we drop the
|
|
630
|
+
references (above) and let the clients close on garbage collection. With no
|
|
631
|
+
running loop we spin a throwaway loop to close them, swallowing anything a
|
|
632
|
+
client bound to an already-closed loop raises.
|
|
633
|
+
"""
|
|
634
|
+
with self._clients_lock:
|
|
635
|
+
entries = list(self._clients.values())
|
|
636
|
+
self._clients.clear()
|
|
637
|
+
self._closes_deferred.clear() # see aclose()
|
|
638
|
+
if not entries:
|
|
639
|
+
return
|
|
640
|
+
try:
|
|
641
|
+
asyncio.get_running_loop()
|
|
642
|
+
return # a loop is running; GC will close them
|
|
643
|
+
except RuntimeError:
|
|
644
|
+
pass
|
|
645
|
+
|
|
646
|
+
async def _close_all() -> None:
|
|
647
|
+
for client, _sem in entries:
|
|
648
|
+
try:
|
|
649
|
+
await client.aclose()
|
|
650
|
+
except Exception: # noqa: BLE001 - best-effort
|
|
651
|
+
pass
|
|
652
|
+
|
|
653
|
+
import warnings as _warnings
|
|
654
|
+
|
|
655
|
+
with _warnings.catch_warnings():
|
|
656
|
+
_warnings.simplefilter("ignore")
|
|
657
|
+
try:
|
|
658
|
+
asyncio.run(_close_all())
|
|
659
|
+
except Exception: # noqa: BLE001 - best-effort teardown
|
|
660
|
+
pass
|
|
661
|
+
|
|
662
|
+
def _auth_headers(
|
|
663
|
+
self, extra: Mapping[str, str] | None = None, *, cfg: Config | None = None
|
|
664
|
+
) -> dict[str, str]:
|
|
665
|
+
# Accept the caller's config SNAPSHOT so the token in these headers and the
|
|
666
|
+
# ``sent_pat`` the caller tracks come from the same read — otherwise a
|
|
667
|
+
# concurrent re-mint between the two reads could make them disagree and
|
|
668
|
+
# trigger a redundant refresh.
|
|
669
|
+
cfg = cfg if cfg is not None else self.config
|
|
670
|
+
# Read the token LIVE from the connection when there is one (resolve_pat),
|
|
671
|
+
# so a connector-renewed token is used on the next request with no stale
|
|
672
|
+
# snapshot; falls back to the env/PAT snapshot otherwise.
|
|
673
|
+
token = cfg.resolve_pat()
|
|
674
|
+
if not token:
|
|
675
|
+
raise SandboxAuthError(
|
|
676
|
+
"no Snowflake credential resolved: set SNOWFLAKE_PAT or "
|
|
677
|
+
"SNOWFLAKE_TOKEN, name a connection in ~/.snowflake/connections.toml, "
|
|
678
|
+
"or pass connection= on the call"
|
|
679
|
+
)
|
|
680
|
+
if cfg.cws_dev:
|
|
681
|
+
h: dict[str, str] = {
|
|
682
|
+
"Authorization": f'Snowflake Token="{token}"',
|
|
683
|
+
"Accept": "application/json",
|
|
684
|
+
}
|
|
685
|
+
if cfg.cws_gs_host:
|
|
686
|
+
h["Host"] = cfg.cws_gs_host
|
|
687
|
+
else:
|
|
688
|
+
# Session tokens from snowflake.connector start with "ver:"; they
|
|
689
|
+
# require the Snowflake-proprietary header form. OAuth PATs use Bearer.
|
|
690
|
+
if token.startswith("ver:"):
|
|
691
|
+
auth_val = f'Snowflake Token="{token}"'
|
|
692
|
+
else:
|
|
693
|
+
auth_val = f"Bearer {token}"
|
|
694
|
+
h = {
|
|
695
|
+
"Authorization": auth_val,
|
|
696
|
+
"Accept": "application/json",
|
|
697
|
+
}
|
|
698
|
+
# cfg.extra_headers is untrusted -- it comes from the SNOWFLAKE_CWS_EXTRA_HEADERS
|
|
699
|
+
# env var. A stray/hostile value must never be able to replace the bearer
|
|
700
|
+
# token or redirect the request to another host: SNOWFLAKE_CWS_EXTRA_HEADERS=
|
|
701
|
+
# '{"Authorization":"Bearer ATTACKER"}' previously overrode auth in every mode.
|
|
702
|
+
# `extra` is internal (our own Accept/Content-Type) and stays trusted.
|
|
703
|
+
for k, v in cfg.extra_headers.items():
|
|
704
|
+
if k.lower() in _RESERVED_HEADER_NAMES:
|
|
705
|
+
import warnings
|
|
706
|
+
|
|
707
|
+
warnings.warn(
|
|
708
|
+
f"ignoring header {k!r} from SNOWFLAKE_CWS_EXTRA_HEADERS: it may "
|
|
709
|
+
"not override the request's Authorization or Host.",
|
|
710
|
+
RuntimeWarning,
|
|
711
|
+
stacklevel=2,
|
|
712
|
+
)
|
|
713
|
+
continue
|
|
714
|
+
h[k] = v
|
|
715
|
+
if extra:
|
|
716
|
+
for k, v in extra.items():
|
|
717
|
+
h[k] = v
|
|
718
|
+
return h
|
|
719
|
+
|
|
720
|
+
async def _refresh_auth(self, stale_pat: str | None) -> str | None:
|
|
721
|
+
"""Re-mint the token after a reauth response; the new token, or ``None``.
|
|
722
|
+
|
|
723
|
+
The blocking connector round-trip (a token-request renewal, or a full
|
|
724
|
+
reconnect) runs off the event loop. Returns ``None`` — meaning "surface the
|
|
725
|
+
auth error, do not retry" — when the token is unset, there is nothing to
|
|
726
|
+
refresh, or the mint did not change it.
|
|
727
|
+
|
|
728
|
+
A Transport built with an explicit Config refreshes THAT config (serialized
|
|
729
|
+
+ deduped on its own ``_refresh_lock``); the default transport refreshes the
|
|
730
|
+
process-wide active config, where a concurrent burst collapses to a single
|
|
731
|
+
re-mint in the config layer.
|
|
732
|
+
"""
|
|
733
|
+
if stale_pat is None:
|
|
734
|
+
return None
|
|
735
|
+
if self._config_override is None:
|
|
736
|
+
return await asyncio.to_thread(refresh_credential, stale_pat)
|
|
737
|
+
return await asyncio.to_thread(self._refresh_override, stale_pat)
|
|
738
|
+
|
|
739
|
+
def _refresh_override(self, stale_pat: str) -> str | None:
|
|
740
|
+
"""Deduped re-mint of this transport's explicit config override (worker
|
|
741
|
+
thread). Re-reads under ``_refresh_lock`` so two racers don't double-mint
|
|
742
|
+
or clobber a newer token with an older one."""
|
|
743
|
+
with self._refresh_lock:
|
|
744
|
+
ov = self._config_override
|
|
745
|
+
if ov is None:
|
|
746
|
+
return None
|
|
747
|
+
current = ov.resolve_pat()
|
|
748
|
+
if current != stale_pat:
|
|
749
|
+
return current # already refreshed by a racer
|
|
750
|
+
new = refresh_config(ov, stale_pat)
|
|
751
|
+
new_token = new.resolve_pat()
|
|
752
|
+
if new_token is None or new_token == stale_pat:
|
|
753
|
+
return None
|
|
754
|
+
if new is not ov:
|
|
755
|
+
self._config_override = new
|
|
756
|
+
return new_token
|
|
757
|
+
|
|
758
|
+
async def _ensure_pat(self) -> None:
|
|
759
|
+
"""Lazy-mint a CWS dev session token if no PAT is configured.
|
|
760
|
+
|
|
761
|
+
In CWS dev mode (SNOWFLAKE_CWS_DEV=1), if neither SNOWFLAKE_PAT nor
|
|
762
|
+
SNOWFLAKE_TOKEN is set, mint one via /session/v1/login-request using
|
|
763
|
+
the regtest test account credentials and cache it process-wide. Safe
|
|
764
|
+
to call before every request — the cache fast-paths after the first
|
|
765
|
+
mint. Outside CWS dev mode this is a no-op; _auth_headers raises if
|
|
766
|
+
the PAT is still missing.
|
|
767
|
+
"""
|
|
768
|
+
cfg = self.config
|
|
769
|
+
if cfg.pat or not cfg.cws_dev:
|
|
770
|
+
return
|
|
771
|
+
from snowflake.sandbox._cws_dev import mint_testaccount_token
|
|
772
|
+
|
|
773
|
+
await mint_testaccount_token(cfg)
|
|
774
|
+
|
|
775
|
+
def _full_path(self, endpoint: str) -> str:
|
|
776
|
+
if endpoint.startswith("http://") or endpoint.startswith("https://"):
|
|
777
|
+
return endpoint
|
|
778
|
+
if endpoint.startswith("/"):
|
|
779
|
+
return endpoint
|
|
780
|
+
return f"{self.config.base_path}/{endpoint}"
|
|
781
|
+
|
|
782
|
+
async def request(
|
|
783
|
+
self,
|
|
784
|
+
method: str,
|
|
785
|
+
endpoint: str,
|
|
786
|
+
*,
|
|
787
|
+
json_body: Any = None,
|
|
788
|
+
content: bytes | BinaryIO | None = None,
|
|
789
|
+
params: Mapping[str, str] | None = None,
|
|
790
|
+
timeout: float | None = None,
|
|
791
|
+
extra_headers: Mapping[str, str] | None = None,
|
|
792
|
+
retry_not_found: bool = True,
|
|
793
|
+
retry_500_delays: tuple[float, ...] = DEFAULT_500_DELAYS_S,
|
|
794
|
+
) -> httpx.Response:
|
|
795
|
+
"""Issue a request with retry + error mapping.
|
|
796
|
+
|
|
797
|
+
``retry_500_delays`` overrides the (idempotent-only) 500 retry budget for a
|
|
798
|
+
single call — e.g. the read-only logs route passes a longer schedule to ride
|
|
799
|
+
out a transient upstream 500. Default is the one-shot ``DEFAULT_500_DELAYS_S``.
|
|
800
|
+
|
|
801
|
+
Retries are **idempotency-aware**. For a non-idempotent method (anything
|
|
802
|
+
outside `_IDEMPOTENT_METHODS`, i.e. POST) a retry is issued only when the
|
|
803
|
+
failure proves the server did not run the request: a connect-phase
|
|
804
|
+
transport error, a 429, a 404, or a 409/503 whose body says "not ready".
|
|
805
|
+
A read timeout, a 500, or an opaque 503 after the request was written
|
|
806
|
+
surfaces to the caller instead — re-sending would duplicate the side
|
|
807
|
+
effect and, worse, look exactly like a first attempt.
|
|
808
|
+
|
|
809
|
+
``content`` sends a raw body (used by file upload) instead of JSON. Either
|
|
810
|
+
bytes, or an open binary file: httpx reads a file's size and frames the
|
|
811
|
+
request with ``Content-Length`` rather than chunked, which is required here
|
|
812
|
+
because the sandbox container's server cannot decode a chunked body — while
|
|
813
|
+
still streaming it rather than holding the whole file in memory. A file is
|
|
814
|
+
rewound before every retry, since a partly-consumed stream would otherwise
|
|
815
|
+
re-send a truncated body.
|
|
816
|
+
"""
|
|
817
|
+
if self.config.cws_dev:
|
|
818
|
+
endpoint, json_body = _cws_transform(method, endpoint, json_body, timeout)
|
|
819
|
+
await self._ensure_pat()
|
|
820
|
+
# The whole send/retry cycle is ONE in-flight request against this loop's
|
|
821
|
+
# client, registered before the client is fetched (see _in_flight_request) so a
|
|
822
|
+
# concurrent `async with` exit defers its close instead of aborting this.
|
|
823
|
+
async with self._in_flight_request():
|
|
824
|
+
client, semaphore = await self._get_client()
|
|
825
|
+
path = self._full_path(endpoint)
|
|
826
|
+
# One config snapshot for both the headers and the tracked sent token, so
|
|
827
|
+
# they cannot disagree under a concurrent re-mint.
|
|
828
|
+
cfg = self.config
|
|
829
|
+
headers = self._auth_headers(extra_headers, cfg=cfg)
|
|
830
|
+
if json_body is not None:
|
|
831
|
+
headers.setdefault("Content-Type", "application/json")
|
|
832
|
+
|
|
833
|
+
# A file body is framed with an explicit Content-Length and re-wrapped per
|
|
834
|
+
# attempt. Both are required: httpx would otherwise send an async iterator
|
|
835
|
+
# chunked, which the sandbox container cannot decode, and an iterator is
|
|
836
|
+
# single-use so a retry needs a fresh one over a rewound file.
|
|
837
|
+
file_body: BinaryIO | None = None if isinstance(content, bytes) else content
|
|
838
|
+
if file_body is not None:
|
|
839
|
+
headers["Content-Length"] = str(_seekable_size(file_body))
|
|
840
|
+
|
|
841
|
+
idempotent = method.upper() in _IDEMPOTENT_METHODS
|
|
842
|
+
state = _RetryState()
|
|
843
|
+
# The token these headers carry, tracked so a reauth response can re-mint it
|
|
844
|
+
# and retry exactly once (see `_reauth`).
|
|
845
|
+
auth = _RequestAuth(headers=headers, sent_pat=cfg.resolve_pat())
|
|
846
|
+
|
|
847
|
+
while True:
|
|
848
|
+
async with semaphore:
|
|
849
|
+
# Rewind before every attempt: after a failure the file sits wherever
|
|
850
|
+
# it stopped, and re-sending from there is a silently truncated upload
|
|
851
|
+
# -- the declared length still matches, so nothing downstream notices.
|
|
852
|
+
send_content: bytes | AsyncIterator[bytes] | None = None
|
|
853
|
+
if file_body is not None:
|
|
854
|
+
file_body.seek(0)
|
|
855
|
+
send_content = _aiter_file(file_body)
|
|
856
|
+
elif isinstance(content, bytes):
|
|
857
|
+
send_content = content
|
|
858
|
+
try:
|
|
859
|
+
resp = await client.request(
|
|
860
|
+
method,
|
|
861
|
+
path,
|
|
862
|
+
json=json_body,
|
|
863
|
+
content=send_content,
|
|
864
|
+
params=params,
|
|
865
|
+
headers=auth.headers,
|
|
866
|
+
timeout=timeout if timeout is not None else self.config.timeout_s,
|
|
867
|
+
)
|
|
868
|
+
except _UNSENT_ERRORS as exc:
|
|
869
|
+
delay = _transport_error_delay(
|
|
870
|
+
exc,
|
|
871
|
+
sent=False,
|
|
872
|
+
idempotent=idempotent,
|
|
873
|
+
method=method,
|
|
874
|
+
path=path,
|
|
875
|
+
state=state,
|
|
876
|
+
rng=self._rng,
|
|
877
|
+
)
|
|
878
|
+
await asyncio.sleep(delay)
|
|
879
|
+
continue
|
|
880
|
+
except _SENT_ERRORS as exc:
|
|
881
|
+
delay = _transport_error_delay(
|
|
882
|
+
exc,
|
|
883
|
+
sent=True,
|
|
884
|
+
idempotent=idempotent,
|
|
885
|
+
method=method,
|
|
886
|
+
path=path,
|
|
887
|
+
state=state,
|
|
888
|
+
rng=self._rng,
|
|
889
|
+
)
|
|
890
|
+
await asyncio.sleep(delay)
|
|
891
|
+
continue
|
|
892
|
+
|
|
893
|
+
if resp.status_code < 400:
|
|
894
|
+
# A 2xx can still be a legacy reauth envelope (success:false +
|
|
895
|
+
# 390111) for a session token; re-mint once and retry if so.
|
|
896
|
+
if _is_reauth_response(resp) and await self._reauth(
|
|
897
|
+
auth, extra_headers=extra_headers, json_body=json_body, file_body=file_body
|
|
898
|
+
):
|
|
899
|
+
continue
|
|
900
|
+
return resp
|
|
901
|
+
|
|
902
|
+
# The per-status retry POLICY (cold-start 503/409, 429, 500, 404) lives
|
|
903
|
+
# in _status_retry_delay: it returns the seconds to back off, or None
|
|
904
|
+
# when the status is not one it retries (fall through to reauth /
|
|
905
|
+
# terminal below), or raises the terminal error itself.
|
|
906
|
+
retry_delay = _status_retry_delay(
|
|
907
|
+
self._rng,
|
|
908
|
+
resp,
|
|
909
|
+
method=method,
|
|
910
|
+
path=path,
|
|
911
|
+
idempotent=idempotent,
|
|
912
|
+
retry_not_found=retry_not_found,
|
|
913
|
+
retry_500_delays=retry_500_delays,
|
|
914
|
+
state=state,
|
|
915
|
+
)
|
|
916
|
+
if retry_delay is not None:
|
|
917
|
+
await asyncio.sleep(retry_delay)
|
|
918
|
+
continue
|
|
919
|
+
|
|
920
|
+
# A reauth response (401/403, or a 4xx reauth envelope): the session
|
|
921
|
+
# likely lapsed faster than the connector's keep-alive. Re-mint off the
|
|
922
|
+
# live connection and retry once; a concurrent burst is de-duplicated to
|
|
923
|
+
# a single re-mint in the config layer, so all the sharing requests
|
|
924
|
+
# recover together. If there is nothing to refresh, or the fresh token
|
|
925
|
+
# also fails, fall through to the terminal auth error.
|
|
926
|
+
if _is_reauth_response(resp) and await self._reauth(
|
|
927
|
+
auth, extra_headers=extra_headers, json_body=json_body, file_body=file_body
|
|
928
|
+
):
|
|
929
|
+
continue
|
|
930
|
+
raise _status_to_error(resp.status_code, resp.text)
|
|
931
|
+
|
|
932
|
+
async def _reauth(
|
|
933
|
+
self,
|
|
934
|
+
auth: _RequestAuth,
|
|
935
|
+
*,
|
|
936
|
+
extra_headers: Mapping[str, str] | None,
|
|
937
|
+
json_body: Any,
|
|
938
|
+
file_body: BinaryIO | None,
|
|
939
|
+
) -> bool:
|
|
940
|
+
"""Re-mint the token once on a reauth response and rebuild *auth*'s headers.
|
|
941
|
+
|
|
942
|
+
Mutates ``auth`` in place (fresh token + rebuilt headers) and returns True
|
|
943
|
+
when the caller should retry, False when there is nothing to refresh or the
|
|
944
|
+
fresh token is unchanged. ``auth.refreshed`` bounds this to a single re-mint
|
|
945
|
+
per request, so a fresh token that still fails surfaces as a real auth error.
|
|
946
|
+
"""
|
|
947
|
+
if auth.refreshed:
|
|
948
|
+
return False
|
|
949
|
+
auth.refreshed = True
|
|
950
|
+
new_pat = await self._refresh_auth(auth.sent_pat)
|
|
951
|
+
if new_pat is None or new_pat == auth.sent_pat:
|
|
952
|
+
return False
|
|
953
|
+
auth.sent_pat = new_pat
|
|
954
|
+
auth.headers = self._auth_headers(extra_headers) # fresh read: new token
|
|
955
|
+
if json_body is not None:
|
|
956
|
+
auth.headers.setdefault("Content-Type", "application/json")
|
|
957
|
+
if file_body is not None:
|
|
958
|
+
auth.headers["Content-Length"] = str(_seekable_size(file_body))
|
|
959
|
+
return True
|
|
960
|
+
|
|
961
|
+
async def download(
|
|
962
|
+
self,
|
|
963
|
+
endpoint: str,
|
|
964
|
+
sink: Callable[[bytes], object],
|
|
965
|
+
*,
|
|
966
|
+
params: Mapping[str, str] | None = None,
|
|
967
|
+
timeout: float | None = None,
|
|
968
|
+
retry_not_found: bool = True,
|
|
969
|
+
) -> httpx.Response:
|
|
970
|
+
"""GET ``endpoint`` and hand each chunk to ``sink`` as it arrives.
|
|
971
|
+
|
|
972
|
+
Separate from `request()` because that reads the whole body into memory
|
|
973
|
+
before returning, which for a file means holding it twice -- once in the
|
|
974
|
+
response and once on the way to disk. The returned response has been fully
|
|
975
|
+
consumed; only its status and headers are still meaningful.
|
|
976
|
+
|
|
977
|
+
Error mapping is `request()`'s, applied to the response *before* any chunk
|
|
978
|
+
is handed over, so a 4xx never reaches the sink. Retry is the initial open
|
|
979
|
+
only, exactly as `stream_sse` does it: a create-race 404 or a "not ready"
|
|
980
|
+
409/503 is absorbed on the bounded schedule, and a failure after bytes have
|
|
981
|
+
been delivered surfaces rather than restarting a partial file.
|
|
982
|
+
"""
|
|
983
|
+
await self._ensure_pat()
|
|
984
|
+
# One in-flight request for the whole download, re-opens included, so a
|
|
985
|
+
# concurrent `async with` exit cannot close the client mid-file (see
|
|
986
|
+
# _in_flight_request).
|
|
987
|
+
async with self._in_flight_request():
|
|
988
|
+
client, semaphore = await self._get_client()
|
|
989
|
+
path = self._full_path(endpoint)
|
|
990
|
+
cfg = self.config
|
|
991
|
+
headers = self._auth_headers(cfg=cfg)
|
|
992
|
+
sent_pat = cfg.resolve_pat()
|
|
993
|
+
auth_refreshed = False
|
|
994
|
+
open_attempt = 0
|
|
995
|
+
while True:
|
|
996
|
+
async with semaphore:
|
|
997
|
+
try:
|
|
998
|
+
async with client.stream(
|
|
999
|
+
"GET",
|
|
1000
|
+
path,
|
|
1001
|
+
params=params,
|
|
1002
|
+
headers=headers,
|
|
1003
|
+
timeout=timeout if timeout is not None else self.config.timeout_s,
|
|
1004
|
+
) as resp:
|
|
1005
|
+
if resp.status_code >= 400:
|
|
1006
|
+
body = await resp.aread()
|
|
1007
|
+
# Full body: _status_to_error unwraps the error envelope
|
|
1008
|
+
# and truncates any non-envelope fallback itself.
|
|
1009
|
+
text = body.decode("utf-8", errors="replace")
|
|
1010
|
+
open_attempt += 1
|
|
1011
|
+
delay = _sse_open_retry_delay(
|
|
1012
|
+
resp.status_code,
|
|
1013
|
+
resp,
|
|
1014
|
+
open_attempt,
|
|
1015
|
+
self._rng,
|
|
1016
|
+
retry_not_found=retry_not_found,
|
|
1017
|
+
)
|
|
1018
|
+
if delay is not None:
|
|
1019
|
+
await asyncio.sleep(delay)
|
|
1020
|
+
continue # re-open: nothing was delivered
|
|
1021
|
+
# Reauth response: re-mint once and re-open (nothing was
|
|
1022
|
+
# delivered). See request().
|
|
1023
|
+
if _is_reauth_response(resp) and not auth_refreshed:
|
|
1024
|
+
auth_refreshed = True
|
|
1025
|
+
new_pat = await self._refresh_auth(sent_pat)
|
|
1026
|
+
if new_pat is not None and new_pat != sent_pat:
|
|
1027
|
+
sent_pat = new_pat
|
|
1028
|
+
headers = self._auth_headers()
|
|
1029
|
+
continue
|
|
1030
|
+
raise _status_to_error(resp.status_code, text)
|
|
1031
|
+
async for chunk in resp.aiter_bytes():
|
|
1032
|
+
# The type hint accepts an async sink, so mypy approved
|
|
1033
|
+
# one -- but a bare ``sink(chunk)`` merely built a
|
|
1034
|
+
# coroutine and dropped it, delivering ZERO bytes under
|
|
1035
|
+
# a 200 (an empty file, silently). Await it if awaitable.
|
|
1036
|
+
result = sink(chunk)
|
|
1037
|
+
if inspect.isawaitable(result):
|
|
1038
|
+
await result
|
|
1039
|
+
return resp
|
|
1040
|
+
except _UNSENT_ERRORS as exc:
|
|
1041
|
+
open_attempt += 1
|
|
1042
|
+
delay = compute_backoff(
|
|
1043
|
+
open_attempt, delays=DEFAULT_COLD_START_DELAYS_S, rng=self._rng
|
|
1044
|
+
)
|
|
1045
|
+
if delay is None:
|
|
1046
|
+
raise SandboxTransportError(
|
|
1047
|
+
f"transport error after retries: {exc}"
|
|
1048
|
+
) from exc
|
|
1049
|
+
await asyncio.sleep(delay)
|
|
1050
|
+
except _SENT_ERRORS as exc:
|
|
1051
|
+
# A read/protocol failure mid-stream, after chunks may already
|
|
1052
|
+
# have reached the sink. Converted to the transport taxonomy so a
|
|
1053
|
+
# caller catching SandboxError sees it, but never retried: re-opening
|
|
1054
|
+
# would restart a file the sink has partly written, and the caller
|
|
1055
|
+
# cannot tell the resend from a fresh body. Same rule stream_sse
|
|
1056
|
+
# applies to _SENT_ERRORS.
|
|
1057
|
+
raise SandboxTransportError(f"transport error mid-download: {exc}") from exc
|
|
1058
|
+
|
|
1059
|
+
def stream_sse(
|
|
1060
|
+
self,
|
|
1061
|
+
method: str,
|
|
1062
|
+
endpoint: str,
|
|
1063
|
+
*,
|
|
1064
|
+
json_body: Any = None,
|
|
1065
|
+
timeout: float | None = None,
|
|
1066
|
+
retry_not_found: bool = True,
|
|
1067
|
+
abort: AsyncStreamAbort | None = None,
|
|
1068
|
+
) -> AsyncIterator[SSEEvent]:
|
|
1069
|
+
"""Open an SSE stream and yield parsed events.
|
|
1070
|
+
|
|
1071
|
+
*abort* lets terminate() interrupt the read -- see `AsyncStreamAbort`.
|
|
1072
|
+
Without it a reader stays blocked until the HTTP read timeout, which on the
|
|
1073
|
+
default exec budget is 630s.
|
|
1074
|
+
|
|
1075
|
+
The **initial open** follows the same retry policy as `request()`: a
|
|
1076
|
+
create-race 404, a 429 (now capped at the shared budget, not retried
|
|
1077
|
+
forever), or a 409/503 that says "not ready" is absorbed on the bounded
|
|
1078
|
+
schedule, because none of them ran the request. Once any frame has been
|
|
1079
|
+
delivered the stream is never re-opened -- a mid-stream disconnect surfaces
|
|
1080
|
+
as ``SandboxTransportError`` so the caller can decide, and re-POSTing an
|
|
1081
|
+
exec that is already running is exactly the duplicate this SDK must not
|
|
1082
|
+
create.
|
|
1083
|
+
|
|
1084
|
+
``retry_not_found=False`` returns a 404 immediately instead of paying the
|
|
1085
|
+
~8s create-race backoff -- for a caller resuming against a user-supplied or
|
|
1086
|
+
possibly-vanished session id.
|
|
1087
|
+
|
|
1088
|
+
Returns an async iterator (not a coroutine) so callers can
|
|
1089
|
+
``async for evt in t.stream_sse(...):`` without an extra await.
|
|
1090
|
+
"""
|
|
1091
|
+
path = self._full_path(endpoint)
|
|
1092
|
+
|
|
1093
|
+
return _stream_sse_impl(
|
|
1094
|
+
self,
|
|
1095
|
+
method,
|
|
1096
|
+
path,
|
|
1097
|
+
json_body=json_body,
|
|
1098
|
+
timeout=timeout if timeout is not None else self.config.timeout_s,
|
|
1099
|
+
retry_not_found=retry_not_found,
|
|
1100
|
+
abort=abort,
|
|
1101
|
+
)
|
|
1102
|
+
|
|
1103
|
+
|
|
1104
|
+
async def _stream_sse_impl(
|
|
1105
|
+
transport: Transport,
|
|
1106
|
+
method: str,
|
|
1107
|
+
path: str,
|
|
1108
|
+
*,
|
|
1109
|
+
json_body: Any,
|
|
1110
|
+
timeout: float | None,
|
|
1111
|
+
retry_not_found: bool = True,
|
|
1112
|
+
abort: AsyncStreamAbort | None = None,
|
|
1113
|
+
) -> AsyncIterator[SSEEvent]:
|
|
1114
|
+
await transport._ensure_pat()
|
|
1115
|
+
headers = transport._auth_headers({"Accept": "text/event-stream"})
|
|
1116
|
+
if json_body is not None:
|
|
1117
|
+
headers.setdefault("Content-Type", "application/json")
|
|
1118
|
+
# The token these headers carry, for the reauth self-heal (see request()).
|
|
1119
|
+
# getattr keeps this module function usable by a minimal fake transport that
|
|
1120
|
+
# implements only what it touches; a fake with no config just never refreshes.
|
|
1121
|
+
_cfg = getattr(transport, "config", None)
|
|
1122
|
+
sent_pat = _cfg.resolve_pat() if _cfg is not None else None
|
|
1123
|
+
auth_refreshed = False
|
|
1124
|
+
# A live SSE stream is an in-flight request for its whole life: closing the client
|
|
1125
|
+
# under a reader is exactly the tear-down this guard prevents, so a concurrent
|
|
1126
|
+
# `async with` exit waits for the stream to end (see _in_flight_request).
|
|
1127
|
+
async with transport._in_flight_request():
|
|
1128
|
+
client, _semaphore = await transport._get_client()
|
|
1129
|
+
open_attempt = 0
|
|
1130
|
+
while True:
|
|
1131
|
+
if abort is not None and abort.aborted:
|
|
1132
|
+
return
|
|
1133
|
+
try:
|
|
1134
|
+
async with client.stream(
|
|
1135
|
+
method,
|
|
1136
|
+
path,
|
|
1137
|
+
json=json_body,
|
|
1138
|
+
headers=headers,
|
|
1139
|
+
timeout=timeout,
|
|
1140
|
+
) as resp:
|
|
1141
|
+
if resp.status_code >= 400:
|
|
1142
|
+
body = await resp.aread()
|
|
1143
|
+
# Full body: _status_to_error unwraps the error envelope
|
|
1144
|
+
# and truncates any non-envelope fallback itself.
|
|
1145
|
+
text = body.decode("utf-8", errors="replace")
|
|
1146
|
+
open_attempt += 1
|
|
1147
|
+
delay = _sse_open_retry_delay(
|
|
1148
|
+
resp.status_code,
|
|
1149
|
+
resp,
|
|
1150
|
+
open_attempt,
|
|
1151
|
+
transport._rng,
|
|
1152
|
+
retry_not_found=retry_not_found,
|
|
1153
|
+
)
|
|
1154
|
+
if delay is not None:
|
|
1155
|
+
await asyncio.sleep(delay)
|
|
1156
|
+
continue # re-open the stream
|
|
1157
|
+
# Reauth response: re-mint once and re-open the stream (no frame
|
|
1158
|
+
# delivered yet, so no duplicate exec). See request().
|
|
1159
|
+
if (
|
|
1160
|
+
_is_reauth_response(resp)
|
|
1161
|
+
and not auth_refreshed
|
|
1162
|
+
and sent_pat is not None
|
|
1163
|
+
):
|
|
1164
|
+
auth_refreshed = True
|
|
1165
|
+
new_pat = await transport._refresh_auth(sent_pat)
|
|
1166
|
+
if new_pat is not None and new_pat != sent_pat:
|
|
1167
|
+
sent_pat = new_pat
|
|
1168
|
+
headers = transport._auth_headers({"Accept": "text/event-stream"})
|
|
1169
|
+
if json_body is not None:
|
|
1170
|
+
headers.setdefault("Content-Type", "application/json")
|
|
1171
|
+
continue
|
|
1172
|
+
# One shared terminal table (see _status_to_error): a persistent
|
|
1173
|
+
# 429 now raises SandboxRateLimitError instead of the old
|
|
1174
|
+
# SandboxTransportError, and 409/413/unknown match request() and
|
|
1175
|
+
# download() exactly.
|
|
1176
|
+
raise _status_to_error(resp.status_code, text)
|
|
1177
|
+
if abort is not None:
|
|
1178
|
+
await abort.bind(resp)
|
|
1179
|
+
try:
|
|
1180
|
+
async for evt in _iter_sse_frames(resp):
|
|
1181
|
+
yield evt
|
|
1182
|
+
finally:
|
|
1183
|
+
if abort is not None:
|
|
1184
|
+
abort.unbind()
|
|
1185
|
+
return
|
|
1186
|
+
except _UNSENT_ERRORS as exc:
|
|
1187
|
+
# Nothing was written, so re-opening cannot duplicate the exec.
|
|
1188
|
+
open_attempt += 1
|
|
1189
|
+
delay = compute_backoff(
|
|
1190
|
+
open_attempt, delays=DEFAULT_COLD_START_DELAYS_S, rng=transport._rng
|
|
1191
|
+
)
|
|
1192
|
+
if delay is None:
|
|
1193
|
+
raise SandboxTransportError(f"SSE transport error: {exc}") from exc
|
|
1194
|
+
await asyncio.sleep(delay)
|
|
1195
|
+
except _SENT_ERRORS as exc:
|
|
1196
|
+
# The stream was cut after the request was written -- the ordinary end
|
|
1197
|
+
# of a long-lived SSE, and `RemoteProtocolError` wrapping an h2
|
|
1198
|
+
# `ConnectionTerminated` (a graceful GOAWAY) is the common shape.
|
|
1199
|
+
#
|
|
1200
|
+
# Converted, not retried. Re-opening here would re-run a streaming
|
|
1201
|
+
# `exec` with its full side effects, for the reason `_IDEMPOTENT_METHODS`
|
|
1202
|
+
# gives above; the transport cannot know the caller can resume. Callers
|
|
1203
|
+
# that can, do: `shell.py` re-opens from its last frame id, which the
|
|
1204
|
+
# sandbox answers exactly. Callers that cannot get a typed error instead
|
|
1205
|
+
# of an httpx internal.
|
|
1206
|
+
#
|
|
1207
|
+
# All five members on purpose. This clause used to hand-list ReadError
|
|
1208
|
+
# and ReadTimeout, so WriteError, WriteTimeout and RemoteProtocolError
|
|
1209
|
+
# escaped the generator, sailed past shell.py's
|
|
1210
|
+
# `except SandboxTransportError` reconnect loop, and reached the CLI as
|
|
1211
|
+
# `<ConnectionTerminated error_code:0, last_stream_id:2147483647>` --
|
|
1212
|
+
# killing a live terminal that the resume path was built to save.
|
|
1213
|
+
raise SandboxTransportError(f"SSE transport error: {exc}") from exc
|
|
1214
|
+
|
|
1215
|
+
|
|
1216
|
+
_transport_lock = threading.Lock()
|
|
1217
|
+
# Transport pool, keyed ``(role, scope)``: ``role`` is the uppercased Snowflake role
|
|
1218
|
+
# (``None`` for no role override) and ``scope`` is the identity of an enclosing
|
|
1219
|
+
# ``using()`` connection (``None`` outside one). Created lazily on first
|
|
1220
|
+
# get_transport() call.
|
|
1221
|
+
#
|
|
1222
|
+
# ``scope`` is part of the key because an ``httpx.AsyncClient`` bakes ``base_url``
|
|
1223
|
+
# and ``verify`` in at construction (see ``_get_client``): a transport that merely
|
|
1224
|
+
# re-read the config per request would keep talking to whichever host it was first
|
|
1225
|
+
# used against, and would send a scoped account's token to the previous account's
|
|
1226
|
+
# host.
|
|
1227
|
+
_transport_pool: dict[tuple[str | None, str | None], Transport] = {}
|
|
1228
|
+
|
|
1229
|
+
|
|
1230
|
+
def get_transport_for_config(config: Config, role: str | None = None) -> Transport:
|
|
1231
|
+
"""Return the pooled transport for an explicitly-resolved `Config`.
|
|
1232
|
+
|
|
1233
|
+
The single place an explicit connection becomes a transport: both ``connection=`` on
|
|
1234
|
+
an entry point and a ``using()`` block arrive here, so the two cannot drift apart.
|
|
1235
|
+
Keyed on ``(role, credential identity)`` -- see ``config._scope_key`` for why the
|
|
1236
|
+
credential is part of that, and why a scoped transport carries its Config instead of
|
|
1237
|
+
late-binding.
|
|
1238
|
+
"""
|
|
1239
|
+
from snowflake.sandbox.config import _scope_key
|
|
1240
|
+
|
|
1241
|
+
key: str | None = role.upper() if role else None
|
|
1242
|
+
with _transport_lock:
|
|
1243
|
+
entry = (key, _scope_key(config))
|
|
1244
|
+
if entry not in _transport_pool:
|
|
1245
|
+
_transport_pool[entry] = Transport(config=config, role_key=None)
|
|
1246
|
+
return _transport_pool[entry]
|
|
1247
|
+
|
|
1248
|
+
|
|
1249
|
+
def get_transport(role: str | None = None) -> Transport:
|
|
1250
|
+
"""Return the process-wide async transport for the given role, creating it lazily.
|
|
1251
|
+
|
|
1252
|
+
``role=None`` (the default) returns the no-role transport. Passing a role returns
|
|
1253
|
+
a transport that resolves the ambient connection AT that role, so concurrent agents
|
|
1254
|
+
using different roles never share a transport or HTTP client.
|
|
1255
|
+
|
|
1256
|
+
Inside a ``using()`` block, returns that block's transport instead -- the ambient
|
|
1257
|
+
transport is only for callers who named no connection at all.
|
|
1258
|
+
"""
|
|
1259
|
+
from snowflake.sandbox.config import _scoped_config
|
|
1260
|
+
|
|
1261
|
+
cfg = _scoped_config(role)
|
|
1262
|
+
if cfg is not None:
|
|
1263
|
+
return get_transport_for_config(cfg, role)
|
|
1264
|
+
key: str | None = role.upper() if role else None
|
|
1265
|
+
with _transport_lock:
|
|
1266
|
+
entry = (key, None)
|
|
1267
|
+
if entry not in _transport_pool:
|
|
1268
|
+
# The ambient transport re-resolves through ``current_config(role=...)``.
|
|
1269
|
+
_transport_pool[entry] = Transport(role_key=key)
|
|
1270
|
+
return _transport_pool[entry]
|
|
1271
|
+
|
|
1272
|
+
|
|
1273
|
+
def is_pooled_transport(transport: Transport) -> bool:
|
|
1274
|
+
"""True when ``transport`` is one of this module's process-wide pool slots.
|
|
1275
|
+
|
|
1276
|
+
Identity against the pool, not "was the ``transport=`` argument ``None``". The
|
|
1277
|
+
SDK SELF-injects the shared transport on several paths -- ``_hydrate_sandbox()``
|
|
1278
|
+
(so every handle from ``list_sandboxes`` / ``get_sandbox`` / ``get_sandbox_by_name``)
|
|
1279
|
+
and ``create(role=...)``, which passes ``get_transport(role)`` -- so an
|
|
1280
|
+
argument-based test called those caller-owned and excluded them from the refcount.
|
|
1281
|
+
A transport a caller built themselves (``Transport(config=...)``) is not in the
|
|
1282
|
+
pool, so its lifetime stays entirely theirs; a caller who passes
|
|
1283
|
+
``get_transport()`` explicitly is handing us the shared pool and is treated as such.
|
|
1284
|
+
|
|
1285
|
+
Any role slot counts, not just the default one, so a role-scoped sandbox releases
|
|
1286
|
+
the client it actually used.
|
|
1287
|
+
"""
|
|
1288
|
+
with _transport_lock:
|
|
1289
|
+
return any(t is transport for t in _transport_pool.values())
|
|
1290
|
+
|
|
1291
|
+
|
|
1292
|
+
def reset_scoped_transports() -> None:
|
|
1293
|
+
"""Drop every transport bound to an explicit connection, closing their clients.
|
|
1294
|
+
|
|
1295
|
+
Called by ``close_connections()``. An explicitly-bound transport carries its `Config`
|
|
1296
|
+
(that is what keeps it pointed at the right host), and that Config may reference a
|
|
1297
|
+
live connector session. ``close_connections()`` CLOSES those sessions, which would leave
|
|
1298
|
+
such a transport authenticating from a dead session -- its next request fails to
|
|
1299
|
+
auth, and nothing would ever re-resolve it.
|
|
1300
|
+
|
|
1301
|
+
The ambient transport never had this problem: it re-resolves through
|
|
1302
|
+
``current_config()``, so after a reset it simply resolves a fresh config. Explicit
|
|
1303
|
+
binding is what makes this cleanup necessary, so it is done here rather than left
|
|
1304
|
+
to the caller.
|
|
1305
|
+
"""
|
|
1306
|
+
with _transport_lock:
|
|
1307
|
+
doomed = [k for k in _transport_pool if k[1] is not None]
|
|
1308
|
+
entries = [_transport_pool.pop(k) for k in doomed]
|
|
1309
|
+
for t in entries:
|
|
1310
|
+
t._close_sync_best_effort()
|
|
1311
|
+
|
|
1312
|
+
|
|
1313
|
+
def close_scoped_transports_for(scope_key: str) -> None:
|
|
1314
|
+
"""Close only the scoped async transports built for ONE credential scope.
|
|
1315
|
+
|
|
1316
|
+
The narrow counterpart of `reset_scoped_transports`; see
|
|
1317
|
+
`_sync_transport.close_scoped_sync_transports_for` for why
|
|
1318
|
+
``using(..., close=True)`` needs the narrow form rather than the global one.
|
|
1319
|
+
"""
|
|
1320
|
+
with _transport_lock:
|
|
1321
|
+
doomed = [k for k in _transport_pool if k[1] == scope_key]
|
|
1322
|
+
entries = [_transport_pool.pop(k) for k in doomed]
|
|
1323
|
+
for t in entries:
|
|
1324
|
+
t._close_sync_best_effort()
|
|
1325
|
+
|
|
1326
|
+
|
|
1327
|
+
def _take_all_transports() -> list[Transport]:
|
|
1328
|
+
"""Detach and return every pooled transport (all roles, all ``using()`` scopes).
|
|
1329
|
+
|
|
1330
|
+
``shutdown()`` used to close only the default slot, so a role-scoped or scoped
|
|
1331
|
+
connection's client stayed open until interpreter exit -- the one thing
|
|
1332
|
+
``shutdown()`` exists to prevent. Take-once under the lock, as above: a
|
|
1333
|
+
concurrent second caller gets an empty list rather than double-closing.
|
|
1334
|
+
"""
|
|
1335
|
+
with _transport_lock:
|
|
1336
|
+
entries = list(_transport_pool.values())
|
|
1337
|
+
_transport_pool.clear()
|
|
1338
|
+
return entries
|
|
1339
|
+
|
|
1340
|
+
|
|
1341
|
+
def reset_transport(role: str | None = None) -> None:
|
|
1342
|
+
"""Drop one (or all) async transports from the pool, closing their HTTP clients.
|
|
1343
|
+
|
|
1344
|
+
Called with no arguments (or ``role=None``) to close **all** pool entries.
|
|
1345
|
+
|
|
1346
|
+
Called with a specific role, it closes **every** entry for that role — the ambient
|
|
1347
|
+
one and each connection a ``using()`` block or a ``connection=`` argument bound,
|
|
1348
|
+
since the pool is keyed ``(role, credential)``. Other roles are untouched. The
|
|
1349
|
+
broader-than-it-looks blast radius is deliberate: a caller asking for a role's
|
|
1350
|
+
clients to be released means all of them, not whichever one happened to be ambient.
|
|
1351
|
+
|
|
1352
|
+
Best-effort synchronous close: prefer the async ``shutdown()`` where you can
|
|
1353
|
+
await it -- e.g. at the end of a notebook kernel or a request handler.
|
|
1354
|
+
"""
|
|
1355
|
+
key: str | None = role.upper() if role else None
|
|
1356
|
+
with _transport_lock:
|
|
1357
|
+
if role is None:
|
|
1358
|
+
entries = list(_transport_pool.values())
|
|
1359
|
+
_transport_pool.clear()
|
|
1360
|
+
else:
|
|
1361
|
+
# Every scope's slot for this role, not just the ambient one: the caller
|
|
1362
|
+
# asked for this role's clients to be released.
|
|
1363
|
+
doomed = [k for k in _transport_pool if k[0] == key]
|
|
1364
|
+
entries = [_transport_pool.pop(k) for k in doomed]
|
|
1365
|
+
for t in entries:
|
|
1366
|
+
t._close_sync_best_effort()
|