snowflake-sandbox-python 0.2.1a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. snowflake/cli_sandbox/__init__.py +13 -0
  2. snowflake/cli_sandbox/_adapter.py +170 -0
  3. snowflake/cli_sandbox/_common.py +77 -0
  4. snowflake/cli_sandbox/_egress_flags.py +121 -0
  5. snowflake/cli_sandbox/_get_command.py +109 -0
  6. snowflake/cli_sandbox/_run_command.py +1091 -0
  7. snowflake/cli_sandbox/_shell_command.py +666 -0
  8. snowflake/cli_sandbox/_upload_plan.py +187 -0
  9. snowflake/cli_sandbox/commands.py +556 -0
  10. snowflake/cli_sandbox/plugin_spec.py +28 -0
  11. snowflake/cli_sandbox/py.typed +0 -0
  12. snowflake/sandbox/__init__.py +317 -0
  13. snowflake/sandbox/__main__.py +225 -0
  14. snowflake/sandbox/_ansi.py +206 -0
  15. snowflake/sandbox/_args.py +208 -0
  16. snowflake/sandbox/_assemble.py +256 -0
  17. snowflake/sandbox/_bundle.py +240 -0
  18. snowflake/sandbox/_connection_resolve.py +328 -0
  19. snowflake/sandbox/_deploy_spec.py +56 -0
  20. snowflake/sandbox/_diagnostics.py +501 -0
  21. snowflake/sandbox/_env.py +143 -0
  22. snowflake/sandbox/_files_mixin.py +280 -0
  23. snowflake/sandbox/_fs_ops.py +304 -0
  24. snowflake/sandbox/_globs.py +176 -0
  25. snowflake/sandbox/_hosts.py +110 -0
  26. snowflake/sandbox/_mcp_discovery.py +288 -0
  27. snowflake/sandbox/_mcp_status.py +183 -0
  28. snowflake/sandbox/_retry.py +94 -0
  29. snowflake/sandbox/_runtime/__init__.py +42 -0
  30. snowflake/sandbox/_runtime/_fs_helper.py +93 -0
  31. snowflake/sandbox/_runtime/_job_runner.py +111 -0
  32. snowflake/sandbox/_runtime/_protocol.py +53 -0
  33. snowflake/sandbox/_runtime/_shims.py +267 -0
  34. snowflake/sandbox/_sandbox_state.py +303 -0
  35. snowflake/sandbox/_session_registry.py +222 -0
  36. snowflake/sandbox/_sse.py +160 -0
  37. snowflake/sandbox/_stage.py +270 -0
  38. snowflake/sandbox/_sync_files_mixin.py +272 -0
  39. snowflake/sandbox/_sync_fs_ops.py +185 -0
  40. snowflake/sandbox/_sync_transport.py +737 -0
  41. snowflake/sandbox/_sync_watch.py +99 -0
  42. snowflake/sandbox/_transport.py +1366 -0
  43. snowflake/sandbox/_transport_errors.py +270 -0
  44. snowflake/sandbox/_upload_plan.py +497 -0
  45. snowflake/sandbox/_version.py +37 -0
  46. snowflake/sandbox/_watch.py +164 -0
  47. snowflake/sandbox/_wire.py +348 -0
  48. snowflake/sandbox/app.py +256 -0
  49. snowflake/sandbox/client.py +2356 -0
  50. snowflake/sandbox/config.py +1133 -0
  51. snowflake/sandbox/connect.py +288 -0
  52. snowflake/sandbox/deploy.py +499 -0
  53. snowflake/sandbox/egress.py +388 -0
  54. snowflake/sandbox/exceptions.py +253 -0
  55. snowflake/sandbox/exec_stream.py +264 -0
  56. snowflake/sandbox/files.py +547 -0
  57. snowflake/sandbox/function.py +567 -0
  58. snowflake/sandbox/image.py +46 -0
  59. snowflake/sandbox/jobs.py +649 -0
  60. snowflake/sandbox/lifecycle.py +67 -0
  61. snowflake/sandbox/log_stream.py +219 -0
  62. snowflake/sandbox/mcp.py +480 -0
  63. snowflake/sandbox/mount.py +161 -0
  64. snowflake/sandbox/py.typed +0 -0
  65. snowflake/sandbox/secret.py +244 -0
  66. snowflake/sandbox/session_app.py +244 -0
  67. snowflake/sandbox/shell.py +556 -0
  68. snowflake/sandbox/sync_client.py +2245 -0
  69. snowflake/sandbox/sync_exec_stream.py +238 -0
  70. snowflake/sandbox/sync_files.py +377 -0
  71. snowflake/sandbox/sync_log_stream.py +142 -0
  72. snowflake/sandbox/sync_shell.py +413 -0
  73. snowflake/sandbox/types.py +193 -0
  74. snowflake/sandbox/warm_session.py +700 -0
  75. snowflake_sandbox_python-0.2.1a1.dist-info/METADATA +339 -0
  76. snowflake_sandbox_python-0.2.1a1.dist-info/RECORD +80 -0
  77. snowflake_sandbox_python-0.2.1a1.dist-info/WHEEL +5 -0
  78. snowflake_sandbox_python-0.2.1a1.dist-info/entry_points.txt +2 -0
  79. snowflake_sandbox_python-0.2.1a1.dist-info/licenses/LICENSE +202 -0
  80. snowflake_sandbox_python-0.2.1a1.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1366 @@
1
+ """HTTP transport for ``snowflake.sandbox``.
2
+
3
+ Wraps a single process-wide ``httpx.AsyncClient`` (HTTP/2, bearer auth,
4
+ JSON content-type), handles the retry policy, maps
5
+ HTTP status codes to the ``SandboxError`` hierarchy, and exposes a
6
+ streaming SSE primitive used by ``exec_stream``.
7
+
8
+ Imports ``httpx`` lazily-at-module-load (i.e. when the module is imported
9
+ from ``__init__``'s ``__getattr__``); ``import snowflake.sandbox`` does
10
+ not pay this cost.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import asyncio
16
+ import contextlib
17
+ import inspect
18
+ import random
19
+ import threading
20
+ import weakref
21
+ from collections.abc import AsyncIterator, Callable, Mapping
22
+ from dataclasses import dataclass
23
+ from typing import Any, BinaryIO, Final
24
+
25
+ import httpx
26
+
27
+ from snowflake.sandbox._retry import (
28
+ DEFAULT_500_DELAYS_S,
29
+ DEFAULT_COLD_START_DELAYS_S,
30
+ DEFAULT_NOT_FOUND_DELAYS_S,
31
+ compute_backoff,
32
+ )
33
+ from snowflake.sandbox._sse import (
34
+ SSEEvent,
35
+ _iter_sse_frames,
36
+ _sse_open_retry_delay,
37
+ parse_sse_event, # noqa: F401 - re-exported so _transport.parse_sse_event resolves
38
+ )
39
+ from snowflake.sandbox._transport_errors import (
40
+ _RATE_LIMIT_MAX_ATTEMPTS, # noqa: F401 - re-exported for callers/tests
41
+ _RETRY_AFTER_MAX_S, # noqa: F401 - re-exported for callers/tests
42
+ SandboxSSEDecodeError, # noqa: F401 - re-exported for callers/tests
43
+ _error_detail,
44
+ _is_clean_goaway,
45
+ _is_cold_start_409,
46
+ _is_reauth_response,
47
+ _rate_limit_delay,
48
+ _retry_after_seconds,
49
+ _says_not_ready,
50
+ _status_to_error,
51
+ )
52
+ from snowflake.sandbox._version import __version__ as _SDK_VERSION
53
+ from snowflake.sandbox.config import (
54
+ Config,
55
+ current_config,
56
+ refresh_config,
57
+ refresh_credential,
58
+ )
59
+ from snowflake.sandbox.exceptions import (
60
+ SandboxAuthError,
61
+ SandboxNotReadyError,
62
+ SandboxRateLimitError,
63
+ SandboxTransportError,
64
+ )
65
+
66
+ # ``shutdown()`` is public API and now has a public home in ``lifecycle``; it is
67
+ # re-exported here so the internal callers (and tests) that reach for
68
+ # ``_transport.shutdown`` keep working. ``lifecycle`` imports this module only inside
69
+ # its function bodies, so this edge does not close a module-scope cycle.
70
+ from snowflake.sandbox.lifecycle import shutdown
71
+
72
+ __all__ = [
73
+ "Transport",
74
+ "get_transport",
75
+ "reset_transport",
76
+ "shutdown",
77
+ "SSEEvent",
78
+ "AsyncStreamAbort",
79
+ "parse_sse_event",
80
+ ]
81
+
82
+ _USER_AGENT: Final = f"snowflake-sandbox-python/{_SDK_VERSION}"
83
+ _MCP_PATH: Final = "/api/v2/cortex/v1/mcp"
84
+ # Status codes that indicate the container is still cold-starting.
85
+ _COLD_START_STATUSES: Final = {503}
86
+ # Headers a caller-supplied `extra_headers` (SNOWFLAKE_CWS_EXTRA_HEADERS) may never
87
+ # override -- doing so would swap the bearer token or redirect the request host.
88
+ _RESERVED_HEADER_NAMES: Final = frozenset({"authorization", "host"})
89
+
90
+ # Methods for which re-sending a request that may already have run is harmless.
91
+ # POST is deliberately absent: a re-sent `POST /containers` leaves an orphaned,
92
+ # billed sandbox whose id the caller never learns, and a re-sent
93
+ # `POST /containers/{id}/exec` re-runs the caller's command with its full side
94
+ # effects (`git push`, `dbt run`, a Slack post) while looking identical to a
95
+ # command that never ran.
96
+ _IDEMPOTENT_METHODS: Final = frozenset({"GET", "HEAD", "OPTIONS", "PUT", "DELETE"})
97
+
98
+ # Failures raised before any byte of the request reached the server. Retrying
99
+ # these cannot duplicate a side effect, whatever the method.
100
+ _UNSENT_ERRORS: Final = (
101
+ httpx.ConnectError,
102
+ httpx.ConnectTimeout,
103
+ httpx.PoolTimeout,
104
+ )
105
+ # Failures raised after the request was written. Whether the server ran it is
106
+ # unknowable from the client, so these are retried only for idempotent methods.
107
+ _SENT_ERRORS: Final = (
108
+ httpx.ReadTimeout,
109
+ httpx.ReadError,
110
+ httpx.WriteTimeout,
111
+ httpx.WriteError,
112
+ httpx.RemoteProtocolError,
113
+ )
114
+
115
+
116
+ @dataclass
117
+ class _RetryState:
118
+ """Per-request retry counters, threaded from request()'s send/retry loop into
119
+ the per-status policy in `_status_retry_delay`. One instance per request()."""
120
+
121
+ cold: int = 0
122
+ five_hundred: int = 0
123
+ rate_limit: int = 0
124
+ not_found: int = 0
125
+
126
+
127
+ @dataclass
128
+ class _RequestAuth:
129
+ """The token a request()'s headers carry, tracked so a reauth response can
130
+ re-mint it and retry exactly once. One instance per request(), mutated in
131
+ place by `Transport._reauth` / `SyncTransport._reauth`.
132
+
133
+ ``refreshed`` is the one-shot guard: a freshly-minted token that still draws a
134
+ reauth response is a real auth failure, not expiry, so it must not loop."""
135
+
136
+ headers: dict[str, str]
137
+ sent_pat: str | None
138
+ refreshed: bool = False
139
+
140
+
141
+ def _status_retry_delay(
142
+ rng: random.Random,
143
+ resp: httpx.Response,
144
+ *,
145
+ method: str,
146
+ path: str,
147
+ idempotent: bool,
148
+ retry_not_found: bool,
149
+ retry_500_delays: tuple[float, ...],
150
+ state: _RetryState,
151
+ ) -> float | None:
152
+ """The per-status retry POLICY, split out of request()'s send/retry LOOP.
153
+
154
+ For a ``>=400`` response that is NOT a reauth case (the caller handles those),
155
+ returns the seconds to back off before retrying, or ``None`` when the status is
156
+ not one this policy retries (the caller then does reauth-or-terminal-raise).
157
+ Raises the terminal ``SandboxError`` when the method makes a retry unsafe or a
158
+ retry budget is exhausted. Pure and synchronous, so the async and sync
159
+ transports share one implementation. ``state`` carries the attempt counters.
160
+ """
161
+ sc = resp.status_code
162
+
163
+ # Retryable: 503 or 409 cold-start. For a non-idempotent request the body must
164
+ # say the work was refused, not merely that something went wrong upstream of an
165
+ # unknown outcome.
166
+ if sc in _COLD_START_STATUSES or _is_cold_start_409(resp):
167
+ if not idempotent and not _says_not_ready(resp):
168
+ raise SandboxTransportError(
169
+ f"HTTP {sc} on {method.upper()} {path}: "
170
+ f"{_error_detail(resp.text)} — not retried because the server may "
171
+ "already have executed it"
172
+ )
173
+ state.cold += 1
174
+ delay = compute_backoff(state.cold, delays=DEFAULT_COLD_START_DELAYS_S, rng=rng)
175
+ if delay is None:
176
+ if sc == 409:
177
+ raise SandboxNotReadyError(
178
+ f"sandbox not ready after {len(DEFAULT_COLD_START_DELAYS_S)} retries"
179
+ )
180
+ raise SandboxTransportError(
181
+ f"503 after {len(DEFAULT_COLD_START_DELAYS_S)} retries: {_error_detail(resp.text)}"
182
+ )
183
+ return delay
184
+
185
+ # 429: honor a sanitized Retry-After, retry up to the shared budget. A
186
+ # rate-limited request was rejected without being executed, so this is safe for
187
+ # any method. Jitter is upward only (see _rate_limit_delay) so we never retry
188
+ # before the server said it was safe to.
189
+ if sc == 429:
190
+ state.rate_limit += 1
191
+ delay = _rate_limit_delay(resp, state.rate_limit, rng)
192
+ if delay is None:
193
+ raise SandboxRateLimitError(
194
+ f"rate-limited after {state.rate_limit - 1} retries",
195
+ retry_after=_retry_after_seconds(resp),
196
+ )
197
+ return delay
198
+
199
+ # 500 surfaces as transport-error after one retry — but only for an idempotent
200
+ # method: a 500 means the server accepted the request and failed somewhere
201
+ # inside, which says nothing about whether the side effect landed.
202
+ if sc == 500:
203
+ if not idempotent:
204
+ raise SandboxTransportError(
205
+ f"HTTP 500 on {method.upper()} {path}: {_error_detail(resp.text)} — "
206
+ "not retried because the server may already have executed it"
207
+ )
208
+ state.five_hundred += 1
209
+ delay = compute_backoff(state.five_hundred, delays=retry_500_delays, rng=rng)
210
+ if delay is None:
211
+ raise SandboxTransportError(
212
+ f"500 from server after {len(retry_500_delays)} retries: {_error_detail(resp.text)}"
213
+ )
214
+ return delay
215
+
216
+ # 404 keeps its create-race retry logic; every other terminal status goes
217
+ # through the shared _status_to_error table in the caller.
218
+ if sc == 404:
219
+ # A freshly-created container is briefly unroutable — its id is returned by
220
+ # create before the route is registered, so the first exec/logs can 404.
221
+ # Retry on a tight bounded schedule; a genuinely-missing container still
222
+ # surfaces quickly. Safe for a POST too: a 404 means no handler ran, so
223
+ # there is no side effect a retry could duplicate.
224
+ #
225
+ # retry_not_found=False skips this schedule: a caller probing an optional
226
+ # route (e.g. a cheap /status older servers lack) or looking up a
227
+ # user-supplied id wants the 404 immediately, not after the create-race wait.
228
+ if not retry_not_found:
229
+ raise _status_to_error(404, resp.text)
230
+ state.not_found += 1
231
+ delay = compute_backoff(state.not_found, delays=DEFAULT_NOT_FOUND_DELAYS_S, rng=rng)
232
+ if delay is None:
233
+ raise _status_to_error(404, resp.text)
234
+ return delay
235
+
236
+ return None
237
+
238
+
239
+ def _transport_error_delay(
240
+ exc: Exception,
241
+ *,
242
+ sent: bool,
243
+ idempotent: bool,
244
+ method: str,
245
+ path: str,
246
+ state: _RetryState,
247
+ rng: random.Random,
248
+ ) -> float:
249
+ """Backoff (seconds) for a transport-layer error, or raise if the retry is
250
+ unsafe or the budget is exhausted.
251
+
252
+ A pre-send error is always retryable (nothing reached the server). A post-send
253
+ error (``sent=True``) is retryable only for an idempotent method or a clean
254
+ GOAWAY (error_code=0, no streams processed) — the server told us it closed the
255
+ connection before touching the request. Split from request()'s loop; shared by
256
+ both transports.
257
+ """
258
+ if sent and not idempotent and not _is_clean_goaway(exc):
259
+ raise SandboxTransportError(
260
+ f"{method.upper()} {path} failed after the request was sent "
261
+ f"({type(exc).__name__}: {exc}); not retried because the "
262
+ "server may already have executed it"
263
+ ) from exc
264
+ state.cold += 1
265
+ delay = compute_backoff(state.cold, delays=DEFAULT_COLD_START_DELAYS_S, rng=rng)
266
+ if delay is None:
267
+ raise SandboxTransportError(f"transport error after retries: {exc}") from exc
268
+ return delay
269
+
270
+
271
+ class AsyncStreamAbort:
272
+ """`StreamAbort` for the async client: interrupt a pending SSE read.
273
+
274
+ Different mechanism, same contract. The sync reader blocks in ``recv()`` on
275
+ another thread, so a socket shutdown is what reaches it; here the read is an
276
+ ``await`` in the same event loop, and reaching into the transport's socket would
277
+ bypass the loop's bookkeeping. Closing the httpcore stream is the supported
278
+ equivalent -- measured: the pending read resolves immediately.
279
+
280
+ It resolves by RAISING ``httpx.ReadError``, though, where the sync shutdown ends
281
+ at EOF. So an abort arrives at the caller's ``except SandboxTransportError``
282
+ branch rather than its clean-EOF branch, and that branch resumes -- which is why
283
+ `aborted` has to be checked there or terminate() is followed by a reconnect to a
284
+ dead sandbox.
285
+ """
286
+
287
+ __slots__ = ("_aborted", "_stream")
288
+
289
+ def __init__(self) -> None:
290
+ self._aborted = False
291
+ self._stream: Any = None
292
+
293
+ async def bind(self, resp: httpx.Response) -> None:
294
+ """Remember the network stream under *resp* for the duration of the read.
295
+
296
+ Best-effort, as in the sync twin: ``network_stream`` is not public httpx API
297
+ and is absent on some transports. Without it the read ends at its timeout, as
298
+ before -- degraded, not broken.
299
+ """
300
+ stream = None
301
+ try:
302
+ stream = resp.extensions.get("network_stream")
303
+ except Exception: # pragma: no cover - transport-shape dependent
304
+ stream = None
305
+ self._stream = stream
306
+ if self._aborted:
307
+ # abort() ran between the open and this bind, so it saw no stream.
308
+ await self._close()
309
+
310
+ def unbind(self) -> None:
311
+ self._stream = None
312
+
313
+ async def abort(self) -> None:
314
+ self._aborted = True
315
+ await self._close()
316
+
317
+ async def _close(self) -> None:
318
+ stream, self._stream = self._stream, None
319
+ if stream is None:
320
+ return
321
+ try:
322
+ await stream.aclose()
323
+ except Exception: # pragma: no cover - already closed, or never connected
324
+ pass
325
+
326
+ @property
327
+ def aborted(self) -> bool:
328
+ return self._aborted
329
+
330
+
331
+ _FILE_CHUNK_BYTES: Final = 64 * 1024
332
+
333
+
334
+ def _seekable_size(f: BinaryIO) -> int:
335
+ """Byte length of an open file, leaving it rewound."""
336
+ f.seek(0, 2)
337
+ size = f.tell()
338
+ f.seek(0)
339
+ return size
340
+
341
+
342
+ async def _aiter_file(f: BinaryIO) -> AsyncIterator[bytes]:
343
+ """Yield a file's bytes in chunks for an async request body.
344
+
345
+ An ``AsyncClient`` cannot take a plain file object as ``content`` -- httpx wraps
346
+ it in a *sync* byte stream and refuses to send it -- so the file is adapted to an
347
+ async iterator. Reads go through a thread because a large read on the event loop
348
+ would stall every other request sharing it.
349
+ """
350
+ while True:
351
+ chunk = await asyncio.to_thread(f.read, _FILE_CHUNK_BYTES)
352
+ if not chunk:
353
+ return
354
+ yield chunk
355
+
356
+
357
+ def _cws_transform(
358
+ method: str,
359
+ endpoint: str,
360
+ json_body: Any,
361
+ timeout: float | None,
362
+ ) -> tuple[str, Any]:
363
+ """CWS dev mode: redirect MCP exec → REST and convert env dict → sandbox_env."""
364
+ if method.upper() == "POST" and endpoint == _MCP_PATH and isinstance(json_body, dict):
365
+ args = json_body.get("params", {}).get("arguments", {})
366
+ cid = args.get("container_id")
367
+ if cid:
368
+ body: dict[str, Any] = {
369
+ "code": args.get("code", ""),
370
+ "language": "python",
371
+ "timeout_seconds": int(timeout or 60),
372
+ }
373
+ exec_env = args.get("env")
374
+ if isinstance(exec_env, dict) and exec_env:
375
+ body["env"] = exec_env
376
+ return f"containers/{cid}/exec", body
377
+ if (
378
+ method.upper() == "POST"
379
+ and "exec" not in str(endpoint)
380
+ and isinstance(json_body, dict)
381
+ and isinstance(json_body.get("env"), dict)
382
+ ):
383
+ body = {k: v for k, v in json_body.items() if k != "env"}
384
+ body["sandbox_env"] = [f"{k}={v}" for k, v in json_body["env"].items()]
385
+ json_body = body
386
+ return endpoint, json_body
387
+
388
+
389
+ async def _aclose_quietly(client: httpx.AsyncClient) -> None:
390
+ """Close a pooled client, swallowing whatever the close itself raises.
391
+
392
+ Best-effort on purpose, and shared by every deferred/refcounted close path: they all
393
+ run during teardown -- an ``__aexit__``, a request's ``finally``, an async generator
394
+ being closed -- where a socket-level failure (or a loop that has already gone away)
395
+ must not become the exception a caller's block raises.
396
+ """
397
+ try:
398
+ await client.aclose()
399
+ except Exception: # noqa: BLE001 - teardown; see docstring
400
+ pass
401
+
402
+
403
+ class Transport:
404
+ """Owns the process-wide ``httpx.AsyncClient`` and the retry loop.
405
+
406
+ All public methods are async. The client is created lazily on first
407
+ use so importing the module is cheap.
408
+ """
409
+
410
+ def __init__(self, config: Config | None = None, role_key: str | None = None) -> None:
411
+ self._config_override = config
412
+ # With no explicit config this transport is the AMBIENT one: it re-resolves
413
+ # through current_config() on every request. ``role_key`` is the role to resolve
414
+ # AT, so its requests carry that role's token rather than the default's.
415
+ self._role_key = role_key if config is None else None
416
+ # One (client, semaphore) per running event loop. An ``httpx.AsyncClient``
417
+ # -- and an ``asyncio.Semaphore`` -- binds to the loop it is first used on;
418
+ # a single process-global client shared across loops is broken: the
419
+ # documented ``asyncio.run()`` idiom closes its loop on return, so the next
420
+ # ``asyncio.run()`` (a re-run notebook cell, a second CLI invocation, a
421
+ # pytest case, a per-request handler) inherited a client wired to a closed
422
+ # loop and failed intermittently with ``RuntimeError: Event loop is
423
+ # closed``. Keying on the running loop hands each loop its own client, so
424
+ # sequential ``asyncio.run()`` calls -- and concurrent loops on different
425
+ # threads -- each get a working one. A ``WeakKeyDictionary`` lets a
426
+ # finished loop's entry (and its client) be collected once the loop is gone.
427
+ self._clients: weakref.WeakKeyDictionary[
428
+ asyncio.AbstractEventLoop, tuple[httpx.AsyncClient, asyncio.Semaphore]
429
+ ] = weakref.WeakKeyDictionary()
430
+ # A plain threading lock (not an ``asyncio.Lock``, which would itself bind
431
+ # to one loop) guards the map. The client is constructed synchronously
432
+ # inside it -- ``httpx.AsyncClient(...)`` needs no running loop.
433
+ self._clients_lock = threading.Lock()
434
+ # How many `async with AsyncSandbox(...)` blocks on each loop are currently
435
+ # using that loop's client (see _retain_loop_user). Keyed by loop for the same
436
+ # reason ``_clients`` is: the thing being refcounted is per-loop, so a
437
+ # process-global count would have one loop's exit decide another's close.
438
+ # Guarded by ``_clients_lock`` -- the count and the client it protects are
439
+ # read and mutated for the same key, so one lock keeps them consistent.
440
+ self._loop_users: weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, int] = (
441
+ weakref.WeakKeyDictionary()
442
+ )
443
+ # How many requests are currently in flight on each loop's client (see
444
+ # _in_flight_request). Separate from ``_loop_users`` because it counts a
445
+ # different thing: a request on a sandbox driven with explicit
446
+ # ``create()``/``terminate()``, or a module helper like ``list_sandboxes()``,
447
+ # is in flight while retaining nothing. Guarded by ``_clients_lock`` with the
448
+ # other two maps, so the close decision reads a consistent snapshot.
449
+ self._in_flight: weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, int] = (
450
+ weakref.WeakKeyDictionary()
451
+ )
452
+ # Loops whose last ``async with`` exited while a request was still in flight,
453
+ # so the close was deferred to whichever request finishes last (#370).
454
+ self._closes_deferred: weakref.WeakSet[asyncio.AbstractEventLoop] = weakref.WeakSet()
455
+ self._rng = random.Random()
456
+ # Serializes reactive re-mints for a Transport built with an explicit
457
+ # config override (the module path has its own lock in config). Keyed off
458
+ # nothing — the re-check of resolve_pat() inside dedups by token.
459
+ self._refresh_lock = threading.Lock()
460
+
461
+ @property
462
+ def config(self) -> Config:
463
+ return self._config_override or current_config(role=self._role_key)
464
+
465
+ async def _get_client(
466
+ self,
467
+ ) -> tuple[httpx.AsyncClient, asyncio.Semaphore]:
468
+ """The (client, semaphore) pair for the currently-running event loop."""
469
+ loop = asyncio.get_running_loop()
470
+ with self._clients_lock:
471
+ entry = self._clients.get(loop)
472
+ if entry is None:
473
+ cfg = self.config
474
+ limits = httpx.Limits(
475
+ max_connections=cfg.max_connections,
476
+ max_keepalive_connections=cfg.max_connections,
477
+ )
478
+ client = httpx.AsyncClient(
479
+ base_url=cfg.base_url,
480
+ http2=not cfg.cws_dev,
481
+ trust_env=not cfg.cws_dev,
482
+ timeout=cfg.timeout_s,
483
+ verify=cfg.verify,
484
+ limits=limits,
485
+ headers={"User-Agent": _USER_AGENT},
486
+ )
487
+ entry = (client, asyncio.Semaphore(cfg.max_connections))
488
+ self._clients[loop] = entry
489
+ return entry
490
+
491
+ def _retain_loop_user(self) -> None:
492
+ """Register one user of the running loop's client (an ``async with`` entry).
493
+
494
+ Refcounted rather than closed per sandbox: the pool is process-wide on purpose
495
+ so repeated calls skip the TLS handshake, and nested or concurrent sandboxes on
496
+ one loop must keep it open. Refcounted PER LOOP because the client is per loop
497
+ -- with one process-global count, two ``asyncio.run()`` loops on two threads
498
+ each retained, so the first to exit saw ``count > 0``, closed nothing, and its
499
+ loop closed still holding sockets: ``ResourceWarning: unclosed transport``
500
+ (#202) with the refcount in place.
501
+ """
502
+ loop = asyncio.get_running_loop()
503
+ with self._clients_lock:
504
+ self._loop_users[loop] = self._loop_users.get(loop, 0) + 1
505
+ # This loop's client has a user again, so a close an earlier exit deferred
506
+ # (#370) is off: the block entering here is entitled to keep it open for as
507
+ # long as it runs, and its own exit will close it if it is the last one.
508
+ self._closes_deferred.discard(loop)
509
+
510
+ async def _release_loop_user(self) -> None:
511
+ """Drop one user of the running loop's client, closing it when it was the last.
512
+
513
+ Scoped to the running loop: closing every loop's client here (what the
514
+ package-level ``shutdown()`` does) would tear down clients belonging to loops
515
+ still running on other threads. Clients on other loops are left alone, and each
516
+ one closes when its own last block exits.
517
+
518
+ A release with nothing retained on this loop is a no-op -- a sandbox driven with
519
+ explicit ``create()``/``terminate()`` never retained, and closing a client it
520
+ never opened would break whoever did.
521
+
522
+ The close also waits for the loop's client to be IDLE (see
523
+ `_in_flight_request`): with requests still in flight the close is deferred to
524
+ whichever of them finishes last, because closing the client under a sibling task
525
+ aborted its request (#370).
526
+ """
527
+ loop = asyncio.get_running_loop()
528
+ with self._clients_lock:
529
+ users = self._loop_users.get(loop, 0)
530
+ if users == 0:
531
+ return
532
+ if users > 1:
533
+ self._loop_users[loop] = users - 1
534
+ return
535
+ self._loop_users.pop(loop, None)
536
+ if self._in_flight.get(loop, 0) > 0:
537
+ # Another task on this loop is mid-request. Closing now aborts it: an
538
+ # idempotent GET would retry onto a fresh client, but a POST (``exec``)
539
+ # cannot be re-sent, so it surfaced as ``SandboxTransportError`` for a
540
+ # command the server may well have run. Hand the close to whichever
541
+ # request finishes last -- a deferral, not a leak: `_finish_request`
542
+ # releases the sockets as soon as the client goes idle.
543
+ self._closes_deferred.add(loop)
544
+ return
545
+ self._closes_deferred.discard(loop)
546
+ entry = self._clients.pop(loop, None)
547
+ if entry is None:
548
+ return
549
+ await _aclose_quietly(entry[0])
550
+
551
+ @contextlib.asynccontextmanager
552
+ async def _in_flight_request(self) -> AsyncIterator[None]:
553
+ """Count one in-flight request against the running loop's client.
554
+
555
+ Every send path holds this for as long as it needs the client, so that
556
+ `_release_loop_user` can tell an idle client from a busy one. The last
557
+ ``async with`` block on a loop can exit while a SIBLING task on the same loop is
558
+ mid-request -- a sandbox driven with explicit ``create()``/``terminate()``, or a
559
+ module helper like ``list_sandboxes()``, neither of which retains -- and the
560
+ close that exit performs then landed on a live request (#370).
561
+
562
+ Entered BEFORE ``_get_client()`` on purpose: both take ``_clients_lock``, so
563
+ registering first is what makes the pair atomic against a concurrent release,
564
+ which can otherwise pop the client between the two.
565
+
566
+ Held across retries and their backoff sleeps as well, not just the individual
567
+ ``client.request`` await: a client closed mid-backoff fails the next attempt with
568
+ a bare ``RuntimeError`` from httpx, which no branch of the retry loop maps.
569
+ """
570
+ loop = asyncio.get_running_loop()
571
+ with self._clients_lock:
572
+ self._in_flight[loop] = self._in_flight.get(loop, 0) + 1
573
+ try:
574
+ yield
575
+ finally:
576
+ await self._finish_request(loop)
577
+
578
+ async def _finish_request(self, loop: asyncio.AbstractEventLoop) -> None:
579
+ """Drop one in-flight request, performing a close `_release_loop_user` deferred.
580
+
581
+ Takes *loop* as an argument instead of reading ``get_running_loop()``: this runs
582
+ from a ``finally``, including the one inside `stream_sse`'s async generator,
583
+ which is not guaranteed to be closed on the loop the stream began on.
584
+ """
585
+ entry = None
586
+ with self._clients_lock:
587
+ remaining = self._in_flight.get(loop, 0) - 1
588
+ if remaining > 0:
589
+ self._in_flight[loop] = remaining
590
+ return
591
+ self._in_flight.pop(loop, None)
592
+ if loop not in self._closes_deferred:
593
+ return
594
+ # The loop's last block exited while this request was in flight, leaving the
595
+ # close to whoever finished last -- this call. A block that retained the
596
+ # client SINCE clears the flag under this same lock (`_retain_loop_user`), so
597
+ # a flag still set here means nothing has taken the client over and the close
598
+ # is still ours to do.
599
+ self._closes_deferred.discard(loop)
600
+ entry = self._clients.pop(loop, None)
601
+ if entry is not None:
602
+ await _aclose_quietly(entry[0])
603
+
604
+ async def aclose(self) -> None:
605
+ """Close every cached HTTP client and drop them.
606
+
607
+ Awaiting this from a loop closes that loop's client cleanly; clients bound
608
+ to other (possibly already-closed) loops are closed best-effort. Prefer the
609
+ package-level ``shutdown()`` at program/kernel/handler end.
610
+
611
+ Unlike the refcounted release, this does NOT wait for in-flight requests: it is
612
+ the explicit end-of-program teardown, and a caller who asked for it wants the
613
+ sockets gone. A request still running over one of these clients fails, as it
614
+ always has.
615
+ """
616
+ with self._clients_lock:
617
+ entries = list(self._clients.values())
618
+ self._clients.clear()
619
+ # A deferred close names the client that was mapped when it was deferred, and
620
+ # every one of those is being closed here. Dropping the flags keeps a stale
621
+ # one from closing a LATER client out from under a request using it.
622
+ self._closes_deferred.clear()
623
+ for client, _sem in entries:
624
+ await _aclose_quietly(client)
625
+
626
+ def _close_sync_best_effort(self) -> None:
627
+ """Close cached clients from a synchronous context (``reset_transport``).
628
+
629
+ If a loop is running we cannot block on ``aclose`` here, so we drop the
630
+ references (above) and let the clients close on garbage collection. With no
631
+ running loop we spin a throwaway loop to close them, swallowing anything a
632
+ client bound to an already-closed loop raises.
633
+ """
634
+ with self._clients_lock:
635
+ entries = list(self._clients.values())
636
+ self._clients.clear()
637
+ self._closes_deferred.clear() # see aclose()
638
+ if not entries:
639
+ return
640
+ try:
641
+ asyncio.get_running_loop()
642
+ return # a loop is running; GC will close them
643
+ except RuntimeError:
644
+ pass
645
+
646
+ async def _close_all() -> None:
647
+ for client, _sem in entries:
648
+ try:
649
+ await client.aclose()
650
+ except Exception: # noqa: BLE001 - best-effort
651
+ pass
652
+
653
+ import warnings as _warnings
654
+
655
+ with _warnings.catch_warnings():
656
+ _warnings.simplefilter("ignore")
657
+ try:
658
+ asyncio.run(_close_all())
659
+ except Exception: # noqa: BLE001 - best-effort teardown
660
+ pass
661
+
662
+ def _auth_headers(
663
+ self, extra: Mapping[str, str] | None = None, *, cfg: Config | None = None
664
+ ) -> dict[str, str]:
665
+ # Accept the caller's config SNAPSHOT so the token in these headers and the
666
+ # ``sent_pat`` the caller tracks come from the same read — otherwise a
667
+ # concurrent re-mint between the two reads could make them disagree and
668
+ # trigger a redundant refresh.
669
+ cfg = cfg if cfg is not None else self.config
670
+ # Read the token LIVE from the connection when there is one (resolve_pat),
671
+ # so a connector-renewed token is used on the next request with no stale
672
+ # snapshot; falls back to the env/PAT snapshot otherwise.
673
+ token = cfg.resolve_pat()
674
+ if not token:
675
+ raise SandboxAuthError(
676
+ "no Snowflake credential resolved: set SNOWFLAKE_PAT or "
677
+ "SNOWFLAKE_TOKEN, name a connection in ~/.snowflake/connections.toml, "
678
+ "or pass connection= on the call"
679
+ )
680
+ if cfg.cws_dev:
681
+ h: dict[str, str] = {
682
+ "Authorization": f'Snowflake Token="{token}"',
683
+ "Accept": "application/json",
684
+ }
685
+ if cfg.cws_gs_host:
686
+ h["Host"] = cfg.cws_gs_host
687
+ else:
688
+ # Session tokens from snowflake.connector start with "ver:"; they
689
+ # require the Snowflake-proprietary header form. OAuth PATs use Bearer.
690
+ if token.startswith("ver:"):
691
+ auth_val = f'Snowflake Token="{token}"'
692
+ else:
693
+ auth_val = f"Bearer {token}"
694
+ h = {
695
+ "Authorization": auth_val,
696
+ "Accept": "application/json",
697
+ }
698
+ # cfg.extra_headers is untrusted -- it comes from the SNOWFLAKE_CWS_EXTRA_HEADERS
699
+ # env var. A stray/hostile value must never be able to replace the bearer
700
+ # token or redirect the request to another host: SNOWFLAKE_CWS_EXTRA_HEADERS=
701
+ # '{"Authorization":"Bearer ATTACKER"}' previously overrode auth in every mode.
702
+ # `extra` is internal (our own Accept/Content-Type) and stays trusted.
703
+ for k, v in cfg.extra_headers.items():
704
+ if k.lower() in _RESERVED_HEADER_NAMES:
705
+ import warnings
706
+
707
+ warnings.warn(
708
+ f"ignoring header {k!r} from SNOWFLAKE_CWS_EXTRA_HEADERS: it may "
709
+ "not override the request's Authorization or Host.",
710
+ RuntimeWarning,
711
+ stacklevel=2,
712
+ )
713
+ continue
714
+ h[k] = v
715
+ if extra:
716
+ for k, v in extra.items():
717
+ h[k] = v
718
+ return h
719
+
720
+ async def _refresh_auth(self, stale_pat: str | None) -> str | None:
721
+ """Re-mint the token after a reauth response; the new token, or ``None``.
722
+
723
+ The blocking connector round-trip (a token-request renewal, or a full
724
+ reconnect) runs off the event loop. Returns ``None`` — meaning "surface the
725
+ auth error, do not retry" — when the token is unset, there is nothing to
726
+ refresh, or the mint did not change it.
727
+
728
+ A Transport built with an explicit Config refreshes THAT config (serialized
729
+ + deduped on its own ``_refresh_lock``); the default transport refreshes the
730
+ process-wide active config, where a concurrent burst collapses to a single
731
+ re-mint in the config layer.
732
+ """
733
+ if stale_pat is None:
734
+ return None
735
+ if self._config_override is None:
736
+ return await asyncio.to_thread(refresh_credential, stale_pat)
737
+ return await asyncio.to_thread(self._refresh_override, stale_pat)
738
+
739
+ def _refresh_override(self, stale_pat: str) -> str | None:
740
+ """Deduped re-mint of this transport's explicit config override (worker
741
+ thread). Re-reads under ``_refresh_lock`` so two racers don't double-mint
742
+ or clobber a newer token with an older one."""
743
+ with self._refresh_lock:
744
+ ov = self._config_override
745
+ if ov is None:
746
+ return None
747
+ current = ov.resolve_pat()
748
+ if current != stale_pat:
749
+ return current # already refreshed by a racer
750
+ new = refresh_config(ov, stale_pat)
751
+ new_token = new.resolve_pat()
752
+ if new_token is None or new_token == stale_pat:
753
+ return None
754
+ if new is not ov:
755
+ self._config_override = new
756
+ return new_token
757
+
758
+ async def _ensure_pat(self) -> None:
759
+ """Lazy-mint a CWS dev session token if no PAT is configured.
760
+
761
+ In CWS dev mode (SNOWFLAKE_CWS_DEV=1), if neither SNOWFLAKE_PAT nor
762
+ SNOWFLAKE_TOKEN is set, mint one via /session/v1/login-request using
763
+ the regtest test account credentials and cache it process-wide. Safe
764
+ to call before every request — the cache fast-paths after the first
765
+ mint. Outside CWS dev mode this is a no-op; _auth_headers raises if
766
+ the PAT is still missing.
767
+ """
768
+ cfg = self.config
769
+ if cfg.pat or not cfg.cws_dev:
770
+ return
771
+ from snowflake.sandbox._cws_dev import mint_testaccount_token
772
+
773
+ await mint_testaccount_token(cfg)
774
+
775
+ def _full_path(self, endpoint: str) -> str:
776
+ if endpoint.startswith("http://") or endpoint.startswith("https://"):
777
+ return endpoint
778
+ if endpoint.startswith("/"):
779
+ return endpoint
780
+ return f"{self.config.base_path}/{endpoint}"
781
+
782
+ async def request(
783
+ self,
784
+ method: str,
785
+ endpoint: str,
786
+ *,
787
+ json_body: Any = None,
788
+ content: bytes | BinaryIO | None = None,
789
+ params: Mapping[str, str] | None = None,
790
+ timeout: float | None = None,
791
+ extra_headers: Mapping[str, str] | None = None,
792
+ retry_not_found: bool = True,
793
+ retry_500_delays: tuple[float, ...] = DEFAULT_500_DELAYS_S,
794
+ ) -> httpx.Response:
795
+ """Issue a request with retry + error mapping.
796
+
797
+ ``retry_500_delays`` overrides the (idempotent-only) 500 retry budget for a
798
+ single call — e.g. the read-only logs route passes a longer schedule to ride
799
+ out a transient upstream 500. Default is the one-shot ``DEFAULT_500_DELAYS_S``.
800
+
801
+ Retries are **idempotency-aware**. For a non-idempotent method (anything
802
+ outside `_IDEMPOTENT_METHODS`, i.e. POST) a retry is issued only when the
803
+ failure proves the server did not run the request: a connect-phase
804
+ transport error, a 429, a 404, or a 409/503 whose body says "not ready".
805
+ A read timeout, a 500, or an opaque 503 after the request was written
806
+ surfaces to the caller instead — re-sending would duplicate the side
807
+ effect and, worse, look exactly like a first attempt.
808
+
809
+ ``content`` sends a raw body (used by file upload) instead of JSON. Either
810
+ bytes, or an open binary file: httpx reads a file's size and frames the
811
+ request with ``Content-Length`` rather than chunked, which is required here
812
+ because the sandbox container's server cannot decode a chunked body — while
813
+ still streaming it rather than holding the whole file in memory. A file is
814
+ rewound before every retry, since a partly-consumed stream would otherwise
815
+ re-send a truncated body.
816
+ """
817
+ if self.config.cws_dev:
818
+ endpoint, json_body = _cws_transform(method, endpoint, json_body, timeout)
819
+ await self._ensure_pat()
820
+ # The whole send/retry cycle is ONE in-flight request against this loop's
821
+ # client, registered before the client is fetched (see _in_flight_request) so a
822
+ # concurrent `async with` exit defers its close instead of aborting this.
823
+ async with self._in_flight_request():
824
+ client, semaphore = await self._get_client()
825
+ path = self._full_path(endpoint)
826
+ # One config snapshot for both the headers and the tracked sent token, so
827
+ # they cannot disagree under a concurrent re-mint.
828
+ cfg = self.config
829
+ headers = self._auth_headers(extra_headers, cfg=cfg)
830
+ if json_body is not None:
831
+ headers.setdefault("Content-Type", "application/json")
832
+
833
+ # A file body is framed with an explicit Content-Length and re-wrapped per
834
+ # attempt. Both are required: httpx would otherwise send an async iterator
835
+ # chunked, which the sandbox container cannot decode, and an iterator is
836
+ # single-use so a retry needs a fresh one over a rewound file.
837
+ file_body: BinaryIO | None = None if isinstance(content, bytes) else content
838
+ if file_body is not None:
839
+ headers["Content-Length"] = str(_seekable_size(file_body))
840
+
841
+ idempotent = method.upper() in _IDEMPOTENT_METHODS
842
+ state = _RetryState()
843
+ # The token these headers carry, tracked so a reauth response can re-mint it
844
+ # and retry exactly once (see `_reauth`).
845
+ auth = _RequestAuth(headers=headers, sent_pat=cfg.resolve_pat())
846
+
847
+ while True:
848
+ async with semaphore:
849
+ # Rewind before every attempt: after a failure the file sits wherever
850
+ # it stopped, and re-sending from there is a silently truncated upload
851
+ # -- the declared length still matches, so nothing downstream notices.
852
+ send_content: bytes | AsyncIterator[bytes] | None = None
853
+ if file_body is not None:
854
+ file_body.seek(0)
855
+ send_content = _aiter_file(file_body)
856
+ elif isinstance(content, bytes):
857
+ send_content = content
858
+ try:
859
+ resp = await client.request(
860
+ method,
861
+ path,
862
+ json=json_body,
863
+ content=send_content,
864
+ params=params,
865
+ headers=auth.headers,
866
+ timeout=timeout if timeout is not None else self.config.timeout_s,
867
+ )
868
+ except _UNSENT_ERRORS as exc:
869
+ delay = _transport_error_delay(
870
+ exc,
871
+ sent=False,
872
+ idempotent=idempotent,
873
+ method=method,
874
+ path=path,
875
+ state=state,
876
+ rng=self._rng,
877
+ )
878
+ await asyncio.sleep(delay)
879
+ continue
880
+ except _SENT_ERRORS as exc:
881
+ delay = _transport_error_delay(
882
+ exc,
883
+ sent=True,
884
+ idempotent=idempotent,
885
+ method=method,
886
+ path=path,
887
+ state=state,
888
+ rng=self._rng,
889
+ )
890
+ await asyncio.sleep(delay)
891
+ continue
892
+
893
+ if resp.status_code < 400:
894
+ # A 2xx can still be a legacy reauth envelope (success:false +
895
+ # 390111) for a session token; re-mint once and retry if so.
896
+ if _is_reauth_response(resp) and await self._reauth(
897
+ auth, extra_headers=extra_headers, json_body=json_body, file_body=file_body
898
+ ):
899
+ continue
900
+ return resp
901
+
902
+ # The per-status retry POLICY (cold-start 503/409, 429, 500, 404) lives
903
+ # in _status_retry_delay: it returns the seconds to back off, or None
904
+ # when the status is not one it retries (fall through to reauth /
905
+ # terminal below), or raises the terminal error itself.
906
+ retry_delay = _status_retry_delay(
907
+ self._rng,
908
+ resp,
909
+ method=method,
910
+ path=path,
911
+ idempotent=idempotent,
912
+ retry_not_found=retry_not_found,
913
+ retry_500_delays=retry_500_delays,
914
+ state=state,
915
+ )
916
+ if retry_delay is not None:
917
+ await asyncio.sleep(retry_delay)
918
+ continue
919
+
920
+ # A reauth response (401/403, or a 4xx reauth envelope): the session
921
+ # likely lapsed faster than the connector's keep-alive. Re-mint off the
922
+ # live connection and retry once; a concurrent burst is de-duplicated to
923
+ # a single re-mint in the config layer, so all the sharing requests
924
+ # recover together. If there is nothing to refresh, or the fresh token
925
+ # also fails, fall through to the terminal auth error.
926
+ if _is_reauth_response(resp) and await self._reauth(
927
+ auth, extra_headers=extra_headers, json_body=json_body, file_body=file_body
928
+ ):
929
+ continue
930
+ raise _status_to_error(resp.status_code, resp.text)
931
+
932
+ async def _reauth(
933
+ self,
934
+ auth: _RequestAuth,
935
+ *,
936
+ extra_headers: Mapping[str, str] | None,
937
+ json_body: Any,
938
+ file_body: BinaryIO | None,
939
+ ) -> bool:
940
+ """Re-mint the token once on a reauth response and rebuild *auth*'s headers.
941
+
942
+ Mutates ``auth`` in place (fresh token + rebuilt headers) and returns True
943
+ when the caller should retry, False when there is nothing to refresh or the
944
+ fresh token is unchanged. ``auth.refreshed`` bounds this to a single re-mint
945
+ per request, so a fresh token that still fails surfaces as a real auth error.
946
+ """
947
+ if auth.refreshed:
948
+ return False
949
+ auth.refreshed = True
950
+ new_pat = await self._refresh_auth(auth.sent_pat)
951
+ if new_pat is None or new_pat == auth.sent_pat:
952
+ return False
953
+ auth.sent_pat = new_pat
954
+ auth.headers = self._auth_headers(extra_headers) # fresh read: new token
955
+ if json_body is not None:
956
+ auth.headers.setdefault("Content-Type", "application/json")
957
+ if file_body is not None:
958
+ auth.headers["Content-Length"] = str(_seekable_size(file_body))
959
+ return True
960
+
961
+ async def download(
962
+ self,
963
+ endpoint: str,
964
+ sink: Callable[[bytes], object],
965
+ *,
966
+ params: Mapping[str, str] | None = None,
967
+ timeout: float | None = None,
968
+ retry_not_found: bool = True,
969
+ ) -> httpx.Response:
970
+ """GET ``endpoint`` and hand each chunk to ``sink`` as it arrives.
971
+
972
+ Separate from `request()` because that reads the whole body into memory
973
+ before returning, which for a file means holding it twice -- once in the
974
+ response and once on the way to disk. The returned response has been fully
975
+ consumed; only its status and headers are still meaningful.
976
+
977
+ Error mapping is `request()`'s, applied to the response *before* any chunk
978
+ is handed over, so a 4xx never reaches the sink. Retry is the initial open
979
+ only, exactly as `stream_sse` does it: a create-race 404 or a "not ready"
980
+ 409/503 is absorbed on the bounded schedule, and a failure after bytes have
981
+ been delivered surfaces rather than restarting a partial file.
982
+ """
983
+ await self._ensure_pat()
984
+ # One in-flight request for the whole download, re-opens included, so a
985
+ # concurrent `async with` exit cannot close the client mid-file (see
986
+ # _in_flight_request).
987
+ async with self._in_flight_request():
988
+ client, semaphore = await self._get_client()
989
+ path = self._full_path(endpoint)
990
+ cfg = self.config
991
+ headers = self._auth_headers(cfg=cfg)
992
+ sent_pat = cfg.resolve_pat()
993
+ auth_refreshed = False
994
+ open_attempt = 0
995
+ while True:
996
+ async with semaphore:
997
+ try:
998
+ async with client.stream(
999
+ "GET",
1000
+ path,
1001
+ params=params,
1002
+ headers=headers,
1003
+ timeout=timeout if timeout is not None else self.config.timeout_s,
1004
+ ) as resp:
1005
+ if resp.status_code >= 400:
1006
+ body = await resp.aread()
1007
+ # Full body: _status_to_error unwraps the error envelope
1008
+ # and truncates any non-envelope fallback itself.
1009
+ text = body.decode("utf-8", errors="replace")
1010
+ open_attempt += 1
1011
+ delay = _sse_open_retry_delay(
1012
+ resp.status_code,
1013
+ resp,
1014
+ open_attempt,
1015
+ self._rng,
1016
+ retry_not_found=retry_not_found,
1017
+ )
1018
+ if delay is not None:
1019
+ await asyncio.sleep(delay)
1020
+ continue # re-open: nothing was delivered
1021
+ # Reauth response: re-mint once and re-open (nothing was
1022
+ # delivered). See request().
1023
+ if _is_reauth_response(resp) and not auth_refreshed:
1024
+ auth_refreshed = True
1025
+ new_pat = await self._refresh_auth(sent_pat)
1026
+ if new_pat is not None and new_pat != sent_pat:
1027
+ sent_pat = new_pat
1028
+ headers = self._auth_headers()
1029
+ continue
1030
+ raise _status_to_error(resp.status_code, text)
1031
+ async for chunk in resp.aiter_bytes():
1032
+ # The type hint accepts an async sink, so mypy approved
1033
+ # one -- but a bare ``sink(chunk)`` merely built a
1034
+ # coroutine and dropped it, delivering ZERO bytes under
1035
+ # a 200 (an empty file, silently). Await it if awaitable.
1036
+ result = sink(chunk)
1037
+ if inspect.isawaitable(result):
1038
+ await result
1039
+ return resp
1040
+ except _UNSENT_ERRORS as exc:
1041
+ open_attempt += 1
1042
+ delay = compute_backoff(
1043
+ open_attempt, delays=DEFAULT_COLD_START_DELAYS_S, rng=self._rng
1044
+ )
1045
+ if delay is None:
1046
+ raise SandboxTransportError(
1047
+ f"transport error after retries: {exc}"
1048
+ ) from exc
1049
+ await asyncio.sleep(delay)
1050
+ except _SENT_ERRORS as exc:
1051
+ # A read/protocol failure mid-stream, after chunks may already
1052
+ # have reached the sink. Converted to the transport taxonomy so a
1053
+ # caller catching SandboxError sees it, but never retried: re-opening
1054
+ # would restart a file the sink has partly written, and the caller
1055
+ # cannot tell the resend from a fresh body. Same rule stream_sse
1056
+ # applies to _SENT_ERRORS.
1057
+ raise SandboxTransportError(f"transport error mid-download: {exc}") from exc
1058
+
1059
+ def stream_sse(
1060
+ self,
1061
+ method: str,
1062
+ endpoint: str,
1063
+ *,
1064
+ json_body: Any = None,
1065
+ timeout: float | None = None,
1066
+ retry_not_found: bool = True,
1067
+ abort: AsyncStreamAbort | None = None,
1068
+ ) -> AsyncIterator[SSEEvent]:
1069
+ """Open an SSE stream and yield parsed events.
1070
+
1071
+ *abort* lets terminate() interrupt the read -- see `AsyncStreamAbort`.
1072
+ Without it a reader stays blocked until the HTTP read timeout, which on the
1073
+ default exec budget is 630s.
1074
+
1075
+ The **initial open** follows the same retry policy as `request()`: a
1076
+ create-race 404, a 429 (now capped at the shared budget, not retried
1077
+ forever), or a 409/503 that says "not ready" is absorbed on the bounded
1078
+ schedule, because none of them ran the request. Once any frame has been
1079
+ delivered the stream is never re-opened -- a mid-stream disconnect surfaces
1080
+ as ``SandboxTransportError`` so the caller can decide, and re-POSTing an
1081
+ exec that is already running is exactly the duplicate this SDK must not
1082
+ create.
1083
+
1084
+ ``retry_not_found=False`` returns a 404 immediately instead of paying the
1085
+ ~8s create-race backoff -- for a caller resuming against a user-supplied or
1086
+ possibly-vanished session id.
1087
+
1088
+ Returns an async iterator (not a coroutine) so callers can
1089
+ ``async for evt in t.stream_sse(...):`` without an extra await.
1090
+ """
1091
+ path = self._full_path(endpoint)
1092
+
1093
+ return _stream_sse_impl(
1094
+ self,
1095
+ method,
1096
+ path,
1097
+ json_body=json_body,
1098
+ timeout=timeout if timeout is not None else self.config.timeout_s,
1099
+ retry_not_found=retry_not_found,
1100
+ abort=abort,
1101
+ )
1102
+
1103
+
1104
+ async def _stream_sse_impl(
1105
+ transport: Transport,
1106
+ method: str,
1107
+ path: str,
1108
+ *,
1109
+ json_body: Any,
1110
+ timeout: float | None,
1111
+ retry_not_found: bool = True,
1112
+ abort: AsyncStreamAbort | None = None,
1113
+ ) -> AsyncIterator[SSEEvent]:
1114
+ await transport._ensure_pat()
1115
+ headers = transport._auth_headers({"Accept": "text/event-stream"})
1116
+ if json_body is not None:
1117
+ headers.setdefault("Content-Type", "application/json")
1118
+ # The token these headers carry, for the reauth self-heal (see request()).
1119
+ # getattr keeps this module function usable by a minimal fake transport that
1120
+ # implements only what it touches; a fake with no config just never refreshes.
1121
+ _cfg = getattr(transport, "config", None)
1122
+ sent_pat = _cfg.resolve_pat() if _cfg is not None else None
1123
+ auth_refreshed = False
1124
+ # A live SSE stream is an in-flight request for its whole life: closing the client
1125
+ # under a reader is exactly the tear-down this guard prevents, so a concurrent
1126
+ # `async with` exit waits for the stream to end (see _in_flight_request).
1127
+ async with transport._in_flight_request():
1128
+ client, _semaphore = await transport._get_client()
1129
+ open_attempt = 0
1130
+ while True:
1131
+ if abort is not None and abort.aborted:
1132
+ return
1133
+ try:
1134
+ async with client.stream(
1135
+ method,
1136
+ path,
1137
+ json=json_body,
1138
+ headers=headers,
1139
+ timeout=timeout,
1140
+ ) as resp:
1141
+ if resp.status_code >= 400:
1142
+ body = await resp.aread()
1143
+ # Full body: _status_to_error unwraps the error envelope
1144
+ # and truncates any non-envelope fallback itself.
1145
+ text = body.decode("utf-8", errors="replace")
1146
+ open_attempt += 1
1147
+ delay = _sse_open_retry_delay(
1148
+ resp.status_code,
1149
+ resp,
1150
+ open_attempt,
1151
+ transport._rng,
1152
+ retry_not_found=retry_not_found,
1153
+ )
1154
+ if delay is not None:
1155
+ await asyncio.sleep(delay)
1156
+ continue # re-open the stream
1157
+ # Reauth response: re-mint once and re-open the stream (no frame
1158
+ # delivered yet, so no duplicate exec). See request().
1159
+ if (
1160
+ _is_reauth_response(resp)
1161
+ and not auth_refreshed
1162
+ and sent_pat is not None
1163
+ ):
1164
+ auth_refreshed = True
1165
+ new_pat = await transport._refresh_auth(sent_pat)
1166
+ if new_pat is not None and new_pat != sent_pat:
1167
+ sent_pat = new_pat
1168
+ headers = transport._auth_headers({"Accept": "text/event-stream"})
1169
+ if json_body is not None:
1170
+ headers.setdefault("Content-Type", "application/json")
1171
+ continue
1172
+ # One shared terminal table (see _status_to_error): a persistent
1173
+ # 429 now raises SandboxRateLimitError instead of the old
1174
+ # SandboxTransportError, and 409/413/unknown match request() and
1175
+ # download() exactly.
1176
+ raise _status_to_error(resp.status_code, text)
1177
+ if abort is not None:
1178
+ await abort.bind(resp)
1179
+ try:
1180
+ async for evt in _iter_sse_frames(resp):
1181
+ yield evt
1182
+ finally:
1183
+ if abort is not None:
1184
+ abort.unbind()
1185
+ return
1186
+ except _UNSENT_ERRORS as exc:
1187
+ # Nothing was written, so re-opening cannot duplicate the exec.
1188
+ open_attempt += 1
1189
+ delay = compute_backoff(
1190
+ open_attempt, delays=DEFAULT_COLD_START_DELAYS_S, rng=transport._rng
1191
+ )
1192
+ if delay is None:
1193
+ raise SandboxTransportError(f"SSE transport error: {exc}") from exc
1194
+ await asyncio.sleep(delay)
1195
+ except _SENT_ERRORS as exc:
1196
+ # The stream was cut after the request was written -- the ordinary end
1197
+ # of a long-lived SSE, and `RemoteProtocolError` wrapping an h2
1198
+ # `ConnectionTerminated` (a graceful GOAWAY) is the common shape.
1199
+ #
1200
+ # Converted, not retried. Re-opening here would re-run a streaming
1201
+ # `exec` with its full side effects, for the reason `_IDEMPOTENT_METHODS`
1202
+ # gives above; the transport cannot know the caller can resume. Callers
1203
+ # that can, do: `shell.py` re-opens from its last frame id, which the
1204
+ # sandbox answers exactly. Callers that cannot get a typed error instead
1205
+ # of an httpx internal.
1206
+ #
1207
+ # All five members on purpose. This clause used to hand-list ReadError
1208
+ # and ReadTimeout, so WriteError, WriteTimeout and RemoteProtocolError
1209
+ # escaped the generator, sailed past shell.py's
1210
+ # `except SandboxTransportError` reconnect loop, and reached the CLI as
1211
+ # `<ConnectionTerminated error_code:0, last_stream_id:2147483647>` --
1212
+ # killing a live terminal that the resume path was built to save.
1213
+ raise SandboxTransportError(f"SSE transport error: {exc}") from exc
1214
+
1215
+
1216
+ _transport_lock = threading.Lock()
1217
+ # Transport pool, keyed ``(role, scope)``: ``role`` is the uppercased Snowflake role
1218
+ # (``None`` for no role override) and ``scope`` is the identity of an enclosing
1219
+ # ``using()`` connection (``None`` outside one). Created lazily on first
1220
+ # get_transport() call.
1221
+ #
1222
+ # ``scope`` is part of the key because an ``httpx.AsyncClient`` bakes ``base_url``
1223
+ # and ``verify`` in at construction (see ``_get_client``): a transport that merely
1224
+ # re-read the config per request would keep talking to whichever host it was first
1225
+ # used against, and would send a scoped account's token to the previous account's
1226
+ # host.
1227
+ _transport_pool: dict[tuple[str | None, str | None], Transport] = {}
1228
+
1229
+
1230
+ def get_transport_for_config(config: Config, role: str | None = None) -> Transport:
1231
+ """Return the pooled transport for an explicitly-resolved `Config`.
1232
+
1233
+ The single place an explicit connection becomes a transport: both ``connection=`` on
1234
+ an entry point and a ``using()`` block arrive here, so the two cannot drift apart.
1235
+ Keyed on ``(role, credential identity)`` -- see ``config._scope_key`` for why the
1236
+ credential is part of that, and why a scoped transport carries its Config instead of
1237
+ late-binding.
1238
+ """
1239
+ from snowflake.sandbox.config import _scope_key
1240
+
1241
+ key: str | None = role.upper() if role else None
1242
+ with _transport_lock:
1243
+ entry = (key, _scope_key(config))
1244
+ if entry not in _transport_pool:
1245
+ _transport_pool[entry] = Transport(config=config, role_key=None)
1246
+ return _transport_pool[entry]
1247
+
1248
+
1249
+ def get_transport(role: str | None = None) -> Transport:
1250
+ """Return the process-wide async transport for the given role, creating it lazily.
1251
+
1252
+ ``role=None`` (the default) returns the no-role transport. Passing a role returns
1253
+ a transport that resolves the ambient connection AT that role, so concurrent agents
1254
+ using different roles never share a transport or HTTP client.
1255
+
1256
+ Inside a ``using()`` block, returns that block's transport instead -- the ambient
1257
+ transport is only for callers who named no connection at all.
1258
+ """
1259
+ from snowflake.sandbox.config import _scoped_config
1260
+
1261
+ cfg = _scoped_config(role)
1262
+ if cfg is not None:
1263
+ return get_transport_for_config(cfg, role)
1264
+ key: str | None = role.upper() if role else None
1265
+ with _transport_lock:
1266
+ entry = (key, None)
1267
+ if entry not in _transport_pool:
1268
+ # The ambient transport re-resolves through ``current_config(role=...)``.
1269
+ _transport_pool[entry] = Transport(role_key=key)
1270
+ return _transport_pool[entry]
1271
+
1272
+
1273
+ def is_pooled_transport(transport: Transport) -> bool:
1274
+ """True when ``transport`` is one of this module's process-wide pool slots.
1275
+
1276
+ Identity against the pool, not "was the ``transport=`` argument ``None``". The
1277
+ SDK SELF-injects the shared transport on several paths -- ``_hydrate_sandbox()``
1278
+ (so every handle from ``list_sandboxes`` / ``get_sandbox`` / ``get_sandbox_by_name``)
1279
+ and ``create(role=...)``, which passes ``get_transport(role)`` -- so an
1280
+ argument-based test called those caller-owned and excluded them from the refcount.
1281
+ A transport a caller built themselves (``Transport(config=...)``) is not in the
1282
+ pool, so its lifetime stays entirely theirs; a caller who passes
1283
+ ``get_transport()`` explicitly is handing us the shared pool and is treated as such.
1284
+
1285
+ Any role slot counts, not just the default one, so a role-scoped sandbox releases
1286
+ the client it actually used.
1287
+ """
1288
+ with _transport_lock:
1289
+ return any(t is transport for t in _transport_pool.values())
1290
+
1291
+
1292
+ def reset_scoped_transports() -> None:
1293
+ """Drop every transport bound to an explicit connection, closing their clients.
1294
+
1295
+ Called by ``close_connections()``. An explicitly-bound transport carries its `Config`
1296
+ (that is what keeps it pointed at the right host), and that Config may reference a
1297
+ live connector session. ``close_connections()`` CLOSES those sessions, which would leave
1298
+ such a transport authenticating from a dead session -- its next request fails to
1299
+ auth, and nothing would ever re-resolve it.
1300
+
1301
+ The ambient transport never had this problem: it re-resolves through
1302
+ ``current_config()``, so after a reset it simply resolves a fresh config. Explicit
1303
+ binding is what makes this cleanup necessary, so it is done here rather than left
1304
+ to the caller.
1305
+ """
1306
+ with _transport_lock:
1307
+ doomed = [k for k in _transport_pool if k[1] is not None]
1308
+ entries = [_transport_pool.pop(k) for k in doomed]
1309
+ for t in entries:
1310
+ t._close_sync_best_effort()
1311
+
1312
+
1313
+ def close_scoped_transports_for(scope_key: str) -> None:
1314
+ """Close only the scoped async transports built for ONE credential scope.
1315
+
1316
+ The narrow counterpart of `reset_scoped_transports`; see
1317
+ `_sync_transport.close_scoped_sync_transports_for` for why
1318
+ ``using(..., close=True)`` needs the narrow form rather than the global one.
1319
+ """
1320
+ with _transport_lock:
1321
+ doomed = [k for k in _transport_pool if k[1] == scope_key]
1322
+ entries = [_transport_pool.pop(k) for k in doomed]
1323
+ for t in entries:
1324
+ t._close_sync_best_effort()
1325
+
1326
+
1327
+ def _take_all_transports() -> list[Transport]:
1328
+ """Detach and return every pooled transport (all roles, all ``using()`` scopes).
1329
+
1330
+ ``shutdown()`` used to close only the default slot, so a role-scoped or scoped
1331
+ connection's client stayed open until interpreter exit -- the one thing
1332
+ ``shutdown()`` exists to prevent. Take-once under the lock, as above: a
1333
+ concurrent second caller gets an empty list rather than double-closing.
1334
+ """
1335
+ with _transport_lock:
1336
+ entries = list(_transport_pool.values())
1337
+ _transport_pool.clear()
1338
+ return entries
1339
+
1340
+
1341
+ def reset_transport(role: str | None = None) -> None:
1342
+ """Drop one (or all) async transports from the pool, closing their HTTP clients.
1343
+
1344
+ Called with no arguments (or ``role=None``) to close **all** pool entries.
1345
+
1346
+ Called with a specific role, it closes **every** entry for that role — the ambient
1347
+ one and each connection a ``using()`` block or a ``connection=`` argument bound,
1348
+ since the pool is keyed ``(role, credential)``. Other roles are untouched. The
1349
+ broader-than-it-looks blast radius is deliberate: a caller asking for a role's
1350
+ clients to be released means all of them, not whichever one happened to be ambient.
1351
+
1352
+ Best-effort synchronous close: prefer the async ``shutdown()`` where you can
1353
+ await it -- e.g. at the end of a notebook kernel or a request handler.
1354
+ """
1355
+ key: str | None = role.upper() if role else None
1356
+ with _transport_lock:
1357
+ if role is None:
1358
+ entries = list(_transport_pool.values())
1359
+ _transport_pool.clear()
1360
+ else:
1361
+ # Every scope's slot for this role, not just the ambient one: the caller
1362
+ # asked for this role's clients to be released.
1363
+ doomed = [k for k in _transport_pool if k[0] == key]
1364
+ entries = [_transport_pool.pop(k) for k in doomed]
1365
+ for t in entries:
1366
+ t._close_sync_best_effort()