arrowbricks 5.1.2__tar.gz → 5.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/PKG-INFO +33 -1
  2. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/README.md +32 -0
  3. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/pyproject.toml +1 -1
  4. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/Cargo.lock +1 -1
  5. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/Cargo.toml +1 -1
  6. arrowbricks-5.2.2/rust/arrowbricks_core/src/client/call_context.rs +109 -0
  7. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/client/download.rs +60 -1
  8. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/client/error.rs +30 -1
  9. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/client/sea.rs +29 -23
  10. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/client/thrift_rpc.rs +35 -13
  11. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/client.rs +216 -18
  12. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/heartbeat.rs +3 -2
  13. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/json_convert.rs +161 -1
  14. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/lib.rs +296 -142
  15. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/pipeline/ndjson.rs +127 -32
  16. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/pipeline/reorder.rs +6 -1
  17. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/pipeline/sea.rs +12 -0
  18. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/pipeline/stats.rs +13 -5
  19. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/pipeline/thrift_exec.rs +124 -49
  20. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/thrift.rs +28 -4
  21. arrowbricks-5.2.2/rust/arrowbricks_core/tests/common/mod.rs +115 -0
  22. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/tests/wiremock_pipeline.rs +137 -1
  23. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/tests/wiremock_thrift.rs +344 -0
  24. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/tests_py/test_ipc_stream.py +41 -0
  25. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/tests_py/test_token_provider.py +55 -1
  26. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/src/arrowbricks/_core.pyi +2 -0
  27. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/src/arrowbricks/_streaming.py +31 -8
  28. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/src/arrowbricks/cursor.py +14 -7
  29. arrowbricks-5.1.2/rust/arrowbricks_core/tests/common/mod.rs +0 -29
  30. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/LICENSE +0 -0
  31. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/.gitignore +0 -0
  32. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/README.md +0 -0
  33. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/examples/duckdb_query.py +0 -0
  34. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/examples/fastapi_sse.py +0 -0
  35. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/proptest-regressions/json_convert.txt +0 -0
  36. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/rustfmt.toml +0 -0
  37. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/alloc.rs +0 -0
  38. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/arrow_ffi.rs +0 -0
  39. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/client/model.rs +0 -0
  40. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/client/volume.rs +0 -0
  41. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/pipeline/test_support.rs +0 -0
  42. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/pipeline.rs +0 -0
  43. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/tests/wiremock_volume_files.rs +0 -0
  44. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/tests_py/conftest.py +0 -0
  45. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/tests_py/test_parameters.py +0 -0
  46. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/tests_py/test_stream_ndjson_lines.py +0 -0
  47. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/tests_py/test_streaming.py +0 -0
  48. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/tests_py/test_thrift.py +0 -0
  49. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/tests_py/test_volume_files.py +0 -0
  50. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/tests_py/thrift_mock.py +0 -0
  51. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/src/arrowbricks/__init__.py +0 -0
  52. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/src/arrowbricks/client.py +0 -0
  53. {arrowbricks-5.1.2 → arrowbricks-5.2.2}/src/arrowbricks/py.typed +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: arrowbricks
3
- Version: 5.1.2
3
+ Version: 5.2.2
4
4
  Requires-Dist: arro3-core>=0.8 ; extra == 'arro3'
5
5
  Provides-Extra: arro3
6
6
  License-File: LICENSE
@@ -136,6 +136,8 @@ This example takes `sql` straight from the request for brevity -- arrowbricks do
136
136
 
137
137
  arrowbricks has no opinion on *how* you get a token and no cloud-SDK dependency of its own. If your provider is expensive to call, cache/refresh inside it -- arrowbricks does no caching on your behalf.
138
138
 
139
+ A sync provider is called on one of arrowbricks' own worker threads. An async provider's coroutine runs on the event loop of the code that started the call (the `await cursor.execute(...)`, `async for ... in client.stream_query_json(...)`, and so on), with that code's `contextvars`, even when the token is needed by a background chunk download. One client can therefore be shared across event loops: successive `asyncio.run()` calls, or one loop per thread at the same time.
140
+
139
141
  ```python
140
142
  conn = connect(host=..., warehouse_id=..., token_provider=my_token_provider)
141
143
  ```
@@ -343,6 +345,36 @@ can run concurrently on separate Python threads.
343
345
 
344
346
  `write_ipc_stream` (and everything in this package that serializes Arrow-IPC bytes) always writes **uncompressed** bodies. A compressed body (arro3's own default is `compression="LZ4"`) is transparently decompressed by some Arrow readers (e.g. DuckDB's) but not necessarily by every other Arrow IPC reader -- notably, `duckdb-wasm`'s browser-side decoder silently fails to parse LZ4-compressed bodies. Since arrowbricks' bytes might end up read by anything, plain uncompressed is the safe default.
345
347
 
348
+ IPC output is written incrementally, with each Python `write()` call capped at
349
+ 1 MiB. Short writes are handled; the destination remains open and is not flushed
350
+ by arrowbricks. A write failure can leave partial output, so discard it before retrying.
351
+
352
+ ## Real warehouse verification
353
+
354
+ The opt-in live suite generates synthetic data with read-only SQL. It covers
355
+ Thrift and SEA, compression enabled and disabled, Arrow paging, Python values,
356
+ nested NDJSON, empty schemas, concurrent queries, errors, and timeout recovery.
357
+ Set `DATABRICKS_HOST`, `DATABRICKS_WAREHOUSE_ID`, and `DATABRICKS_TOKEN` in the
358
+ environment or the repository's ignored `.env`, then run:
359
+
360
+ ```bash
361
+ ARROWBRICKS_LIVE=1 uv run pytest tests/live -q
362
+ ```
363
+
364
+ To compare two package directories on the same warehouse, use
365
+ `examples/benchmark_versions.py --baseline /path/to/old --candidate /path/to/new`.
366
+ `--mode` selects `arrow`, `paging`, `rows`, `ndjson`, or `ipc`.
367
+ The benchmark alternates execution order, discards warm-ups, checks row counts,
368
+ and measures each version in its own process. `--verify-ipc` additionally compares
369
+ Arrow content for `arrow`/`ipc` modes; use an ordered query with stable results.
370
+ Samples also include process CPU time and the core's warehouse-wait,
371
+ submit-to-ready, and fetch timings. These help distinguish local processing
372
+ overhead from warehouse and download variability; core fetch time is not pure
373
+ network time and excludes time spent consuming rows in Python.
374
+ Peak RSS includes retained allocator memory and is specific to the host platform.
375
+ `--transform-only` isolates row conversion or IPC serialization after downloading
376
+ real data (`--mode rows` or `--mode ipc`), to distinguish CPU work from network noise.
377
+
346
378
  ## License
347
379
 
348
380
  MIT
@@ -124,6 +124,8 @@ This example takes `sql` straight from the request for brevity -- arrowbricks do
124
124
 
125
125
  arrowbricks has no opinion on *how* you get a token and no cloud-SDK dependency of its own. If your provider is expensive to call, cache/refresh inside it -- arrowbricks does no caching on your behalf.
126
126
 
127
+ A sync provider is called on one of arrowbricks' own worker threads. An async provider's coroutine runs on the event loop of the code that started the call (the `await cursor.execute(...)`, `async for ... in client.stream_query_json(...)`, and so on), with that code's `contextvars`, even when the token is needed by a background chunk download. One client can therefore be shared across event loops: successive `asyncio.run()` calls, or one loop per thread at the same time.
128
+
127
129
  ```python
128
130
  conn = connect(host=..., warehouse_id=..., token_provider=my_token_provider)
129
131
  ```
@@ -331,6 +333,36 @@ can run concurrently on separate Python threads.
331
333
 
332
334
  `write_ipc_stream` (and everything in this package that serializes Arrow-IPC bytes) always writes **uncompressed** bodies. A compressed body (arro3's own default is `compression="LZ4"`) is transparently decompressed by some Arrow readers (e.g. DuckDB's) but not necessarily by every other Arrow IPC reader -- notably, `duckdb-wasm`'s browser-side decoder silently fails to parse LZ4-compressed bodies. Since arrowbricks' bytes might end up read by anything, plain uncompressed is the safe default.
333
335
 
336
+ IPC output is written incrementally, with each Python `write()` call capped at
337
+ 1 MiB. Short writes are handled; the destination remains open and is not flushed
338
+ by arrowbricks. A write failure can leave partial output, so discard it before retrying.
339
+
340
+ ## Real warehouse verification
341
+
342
+ The opt-in live suite generates synthetic data with read-only SQL. It covers
343
+ Thrift and SEA, compression enabled and disabled, Arrow paging, Python values,
344
+ nested NDJSON, empty schemas, concurrent queries, errors, and timeout recovery.
345
+ Set `DATABRICKS_HOST`, `DATABRICKS_WAREHOUSE_ID`, and `DATABRICKS_TOKEN` in the
346
+ environment or the repository's ignored `.env`, then run:
347
+
348
+ ```bash
349
+ ARROWBRICKS_LIVE=1 uv run pytest tests/live -q
350
+ ```
351
+
352
+ To compare two package directories on the same warehouse, use
353
+ `examples/benchmark_versions.py --baseline /path/to/old --candidate /path/to/new`.
354
+ `--mode` selects `arrow`, `paging`, `rows`, `ndjson`, or `ipc`.
355
+ The benchmark alternates execution order, discards warm-ups, checks row counts,
356
+ and measures each version in its own process. `--verify-ipc` additionally compares
357
+ Arrow content for `arrow`/`ipc` modes; use an ordered query with stable results.
358
+ Samples also include process CPU time and the core's warehouse-wait,
359
+ submit-to-ready, and fetch timings. These help distinguish local processing
360
+ overhead from warehouse and download variability; core fetch time is not pure
361
+ network time and excludes time spent consuming rows in Python.
362
+ Peak RSS includes retained allocator memory and is specific to the host platform.
363
+ `--transform-only` isolates row conversion or IPC serialization after downloading
364
+ real data (`--mode rows` or `--mode ipc`), to distinguish CPU work from network noise.
365
+
334
366
  ## License
335
367
 
336
368
  MIT
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "arrowbricks"
3
- version = "5.1.2"
3
+ version = "5.2.2"
4
4
  description = "Runs SQL against a Databricks SQL warehouse via the Statement Execution API and hands you the result as Arrow -- a DB-API-ish Cursor (fetchone/fetchmany/fetchall/fetchall_arrow) or NDJSON streaming. Rust/PyO3 core throughout -- zero required runtime dependencies."
5
5
  readme = "README.md"
6
6
  license = "MIT"
@@ -177,7 +177,7 @@ dependencies = [
177
177
 
178
178
  [[package]]
179
179
  name = "arrowbricks_core"
180
- version = "5.1.2"
180
+ version = "5.2.2"
181
181
  dependencies = [
182
182
  "arrow-array",
183
183
  "arrow-buffer",
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "arrowbricks_core"
3
- version = "5.1.2"
3
+ version = "5.2.2"
4
4
  edition = "2024"
5
5
  readme = "README.md"
6
6
 
@@ -0,0 +1,109 @@
1
+ //! Per-call context that follows a statement into every task it spawns.
2
+ //!
3
+ //! A `TokenProvider` (or `EventSink`) backed by a caller's own runtime may
4
+ //! need to know *which call* it is serving -- `lib.rs`'s `PyTokenProvider`
5
+ //! needs the asyncio event loop of the Python coroutine that started the
6
+ //! statement, so it can run an async `token_provider` on that loop. The
7
+ //! caller sets that context once, at the entry point (`with_call_context`),
8
+ //! where it is known. Every task this crate spawns for the call is wrapped
9
+ //! in `in_call_context`, so the same context is still there when a chunk
10
+ //! worker, the Thrift fetch loop, a heartbeat-wrapped submit or a
11
+ //! best-effort cancel asks for a token. Tokio task-locals are not inherited
12
+ //! by spawned tasks on their own; without the wrap, a token request from
13
+ //! one of those tasks had no way to find the caller's loop (the bug fixed in 5.2.1:
14
+ //! `stream_query_json` on a cold client failed with "no running event
15
+ //! loop").
16
+ //!
17
+ //! The context is opaque here (`Arc<dyn Any>`) so this module stays free of
18
+ //! PyO3, like the rest of `client`; `lib.rs` stores a `TaskLocals` in it and
19
+ //! downcasts it back.
20
+
21
+ use std::any::Any;
22
+ use std::future::Future;
23
+ use std::sync::Arc;
24
+
25
+ /// Opaque per-call state, set by the caller at the entry point. See the
26
+ /// module doc comment.
27
+ pub type CallContext = Arc<dyn Any + Send + Sync>;
28
+
29
+ tokio::task_local! {
30
+ static CALL_CONTEXT: Option<CallContext>;
31
+ }
32
+
33
+ /// The context of the call the current task belongs to, if any.
34
+ pub fn current_call_context() -> Option<CallContext> {
35
+ CALL_CONTEXT.try_with(Clone::clone).ok().flatten()
36
+ }
37
+
38
+ /// Runs `fut` with `ctx` as its call context. For entry points: wrap the
39
+ /// whole future of one public call in this.
40
+ pub fn with_call_context<F: Future>(ctx: Option<CallContext>, fut: F) -> impl Future<Output = F::Output> {
41
+ CALL_CONTEXT.scope(ctx, fut)
42
+ }
43
+
44
+ /// Runs the synchronous `f` with `ctx` as the call context -- for an entry
45
+ /// point that builds (and spawns) its work synchronously, such as
46
+ /// `ResultSet.fetchall_arrow_streamed`, so the tasks it spawns and the
47
+ /// cancel hook it builds pick `ctx` up.
48
+ pub fn enter_call_context<R>(ctx: Option<CallContext>, f: impl FnOnce() -> R) -> R {
49
+ CALL_CONTEXT.sync_scope(ctx, f)
50
+ }
51
+
52
+ /// Wraps a future that is about to be spawned so it keeps the call context
53
+ /// of the code spawning it. Captured now, not when the task first runs.
54
+ /// Every `spawn` in this crate that can reach the token provider (directly
55
+ /// or through a task it spawns in turn) goes through this.
56
+ pub fn in_call_context<F: Future>(fut: F) -> impl Future<Output = F::Output> {
57
+ with_call_context(current_call_context(), fut)
58
+ }
59
+
60
+ #[cfg(test)]
61
+ mod tests {
62
+ use super::*;
63
+
64
+ fn ctx_value(ctx: Option<CallContext>) -> Option<u32> {
65
+ ctx.and_then(|c| c.downcast_ref::<u32>().copied())
66
+ }
67
+
68
+ #[tokio::test]
69
+ async fn spawned_tasks_inherit_the_call_context_only_through_in_call_context() {
70
+ let ctx: CallContext = Arc::new(7u32);
71
+ let (wrapped, bare, nested) = with_call_context(Some(ctx), async {
72
+ let wrapped = tokio::spawn(in_call_context(async { ctx_value(current_call_context()) }));
73
+ let bare = tokio::spawn(async { ctx_value(current_call_context()) });
74
+ // Two levels deep, like the SEA fetch driver and its workers.
75
+ let nested = tokio::spawn(in_call_context(async {
76
+ tokio::spawn(in_call_context(async { ctx_value(current_call_context()) }))
77
+ .await
78
+ .unwrap()
79
+ }));
80
+ (wrapped.await.unwrap(), bare.await.unwrap(), nested.await.unwrap())
81
+ })
82
+ .await;
83
+ assert_eq!(wrapped, Some(7));
84
+ assert_eq!(bare, None, "a plain tokio::spawn does not inherit task-locals");
85
+ assert_eq!(nested, Some(7));
86
+ assert_eq!(ctx_value(current_call_context()), None);
87
+ }
88
+
89
+ #[test]
90
+ fn enter_call_context_is_seen_by_futures_wrapped_inside_it() {
91
+ let ctx: CallContext = Arc::new(3u32);
92
+ let fut = enter_call_context(Some(ctx), || {
93
+ in_call_context(async { ctx_value(current_call_context()) })
94
+ });
95
+ let rt = tokio::runtime::Builder::new_current_thread().build().unwrap();
96
+ assert_eq!(rt.block_on(fut), Some(3));
97
+ }
98
+
99
+ #[tokio::test]
100
+ async fn the_innermost_scope_wins() {
101
+ let outer: CallContext = Arc::new(1u32);
102
+ let inner: CallContext = Arc::new(2u32);
103
+ let seen = with_call_context(Some(outer), async move {
104
+ with_call_context(Some(inner), async { ctx_value(current_call_context()) }).await
105
+ })
106
+ .await;
107
+ assert_eq!(seen, Some(2));
108
+ }
109
+ }
@@ -302,7 +302,12 @@ impl DbClient {
302
302
  parts.push((index, bytes?));
303
303
  }
304
304
  parts.sort_unstable_by_key(|(index, _)| *index);
305
- let mut out = bytes::BytesMut::with_capacity(total.unwrap_or(head.len() as u64) as usize);
305
+ // Size from what was actually received, never from `total`: that
306
+ // comes from the server's Content-Range header, and a bogus one
307
+ // would abort the process in `with_capacity` before the length
308
+ // check below could turn it into a normal error.
309
+ let received = head.len() + parts.iter().map(|(_, part)| part.len()).sum::<usize>();
310
+ let mut out = bytes::BytesMut::with_capacity(received);
306
311
  out.extend_from_slice(&head);
307
312
  for (_, part) in parts {
308
313
  out.extend_from_slice(&part);
@@ -389,6 +394,60 @@ mod tests {
389
394
  }
390
395
  }
391
396
 
397
+ /// A bogus `Content-Range` total must become an ordinary error: it used to
398
+ /// size the output buffer up front, which aborted the whole process.
399
+ #[tokio::test]
400
+ async fn split_download_with_a_bogus_content_range_total_errors_instead_of_aborting() {
401
+ use tokio::io::{AsyncReadExt, AsyncWriteExt};
402
+ use tokio::net::TcpListener;
403
+
404
+ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
405
+ let address = listener.local_addr().unwrap();
406
+ tokio::spawn(async move {
407
+ loop {
408
+ let (mut socket, _) = listener.accept().await.unwrap();
409
+ tokio::spawn(async move {
410
+ let mut request = Vec::new();
411
+ while !request.ends_with(b"\r\n\r\n") {
412
+ if socket.read_u8().await.map(|b| request.push(b)).is_err() {
413
+ return;
414
+ }
415
+ }
416
+ let request = String::from_utf8(request).unwrap().to_ascii_lowercase();
417
+ let range = request.lines().find(|s| s.starts_with("range:")).unwrap();
418
+ let start: u64 = range
419
+ .split_once("bytes=")
420
+ .unwrap()
421
+ .1
422
+ .split_once('-')
423
+ .unwrap()
424
+ .0
425
+ .parse()
426
+ .unwrap();
427
+ let headers = format!(
428
+ "HTTP/1.1 206 Partial Content\r\nContent-Length: 4\r\nContent-Range: bytes {start}-{}/999999999999999\r\nConnection: close\r\n\r\n",
429
+ start + 3,
430
+ );
431
+ let _ = socket.write_all(headers.as_bytes()).await;
432
+ let _ = socket.write_all(b"abcd").await;
433
+ });
434
+ }
435
+ });
436
+ let client = Arc::new(
437
+ DbClient::new(&format!("http://{address}"), "wh", "token")
438
+ .with_retry_attempts(1)
439
+ .with_retry_max_wait_s(0.0),
440
+ );
441
+ let stats = Arc::new(QueryStatsAccumulator::default());
442
+ let result = client
443
+ .fetch_link_bytes_split(&format!("http://{address}/blob"), false, 4, 3, &stats)
444
+ .await;
445
+ assert!(
446
+ result.is_err(),
447
+ "a file shorter than its declared size must be an error"
448
+ );
449
+ }
450
+
392
451
  #[tokio::test]
393
452
  async fn split_download_starts_tail_requests_before_the_probe_body_finishes() {
394
453
  use tokio::io::{AsyncReadExt, AsyncWriteExt};
@@ -121,12 +121,24 @@ impl ApiError {
121
121
  // right.
122
122
  let transient = idempotent && (e.is_decode() || (e.is_request() && !e.is_connect() && !e.is_timeout()));
123
123
  Self {
124
- message: e.to_string(),
124
+ // Strip the URL: reqwest appends it to the message, and for a
125
+ // cloud-fetch download it is a presigned link whose query string
126
+ // (`sig=...`) is a bearer capability for the result file.
127
+ message: e.without_url().to_string(),
125
128
  transient,
126
129
  kind: ApiErrorKind::Other,
127
130
  }
128
131
  }
129
132
 
133
+ /// The server rejected a request because the session it names no longer
134
+ /// exists (closed, or invalidated by a warehouse stop/restart). Observed
135
+ /// against a real warehouse: `HTTP 400` with a Thrift exception body
136
+ /// `INVALID_STATE: Invalid SessionHandle: SessionHandle [..]`. Nothing ran,
137
+ /// so the request is safe to repeat on a fresh session.
138
+ pub(crate) fn is_stale_session(&self) -> bool {
139
+ self.message.contains("Invalid SessionHandle")
140
+ }
141
+
130
142
  /// `idempotent` only gates the 5xx case -- 401/403/408/429 mean the
131
143
  /// request was rejected before any processing started (auth failure,
132
144
  /// rate limit, client-side timeout), safe to retry regardless of method.
@@ -165,3 +177,20 @@ pub(crate) fn join_error(e: tokio::task::JoinError) -> ApiError {
165
177
  kind: ApiErrorKind::Other,
166
178
  }
167
179
  }
180
+
181
+ #[cfg(test)]
182
+ mod tests {
183
+ use super::*;
184
+
185
+ #[tokio::test]
186
+ async fn from_reqwest_message_never_contains_the_request_url() {
187
+ let _ = rustls::crypto::ring::default_provider().install_default();
188
+ let err = reqwest::Client::new()
189
+ .get("http://127.0.0.1:9/blob?sv=1&se=2&sig=SECRET")
190
+ .send()
191
+ .await
192
+ .unwrap_err();
193
+ let msg = ApiError::from_reqwest(err, true).message;
194
+ assert!(!msg.contains("SECRET") && !msg.contains("127.0.0.1"), "{msg}");
195
+ }
196
+ }
@@ -19,6 +19,7 @@ use tokio::sync::mpsc;
19
19
 
20
20
  use super::DbClient;
21
21
  use super::POLL_INTERVAL;
22
+ use super::call_context::in_call_context;
22
23
  use super::error::{ApiError, join_error};
23
24
  use super::model::{
24
25
  CancelHandle, ChunkItem, ChunkMeta, ColumnDescription, InlineOrExternal, QueryStatsAccumulator,
@@ -144,10 +145,10 @@ impl DbClient {
144
145
  let url = format!("{}/api/2.0/sql/sessions", self.host);
145
146
  let mut body = json!({ "warehouse_id": self.warehouse_id });
146
147
  if let Some(c) = catalog {
147
- body["catalog_name"] = json!(c);
148
+ body["catalog"] = json!(c);
148
149
  }
149
150
  if let Some(s) = schema {
150
- body["schema_name"] = json!(s);
151
+ body["schema"] = json!(s);
151
152
  }
152
153
  let data: SessionCreateBody = self.authed_json(reqwest::Method::POST, &url, Some(&body), None).await?;
153
154
  Ok(data.session_id)
@@ -173,16 +174,11 @@ impl DbClient {
173
174
  if let Some(id) = self.session_pool.take(&key) {
174
175
  return Some(id);
175
176
  }
176
- if !self.session_pool.reserve(&key) {
177
- return None;
178
- }
179
- match self.create_session(catalog, schema).await {
180
- Ok(id) => Some(id),
181
- Err(_) => {
182
- self.session_pool.release(&key);
183
- None
184
- }
185
- }
177
+ let reservation = self.session_pool.reserve_guard(&key)?;
178
+ // Dropped (releasing the slot) on error or if this future is cancelled.
179
+ let id = self.create_session(catalog, schema).await.ok()?;
180
+ reservation.defuse();
181
+ Some(id)
186
182
  }
187
183
 
188
184
  /// Returns a session to the pool for reuse (`keep = true`, the statement
@@ -216,7 +212,7 @@ impl DbClient {
216
212
  /// verbatim, same as the Python original does no validation of its own
217
213
  /// shape either.
218
214
  pub async fn execute_arrow_statement(
219
- &self,
215
+ self: &Arc<Self>,
220
216
  statement: &str,
221
217
  catalog: Option<&str>,
222
218
  schema: Option<&str>,
@@ -267,7 +263,7 @@ impl DbClient {
267
263
  /// small result pays for two full statement executions on the (common,
268
264
  /// for them) fallback path instead of one.
269
265
  pub async fn execute_arrow_statement_prefer_inline(
270
- &self,
266
+ self: &Arc<Self>,
271
267
  statement: &str,
272
268
  catalog: Option<&str>,
273
269
  schema: Option<&str>,
@@ -346,7 +342,7 @@ impl DbClient {
346
342
  /// field). The session, if any, is returned to the pool on a clean
347
343
  /// terminal state and discarded on any error.
348
344
  async fn submit_and_poll(
349
- &self,
345
+ self: &Arc<Self>,
350
346
  mut body: Value,
351
347
  catalog: Option<&str>,
352
348
  schema: Option<&str>,
@@ -431,7 +427,7 @@ impl DbClient {
431
427
  }
432
428
 
433
429
  async fn execute_statement(
434
- &self,
430
+ self: &Arc<Self>,
435
431
  statement: &str,
436
432
  format: &str,
437
433
  catalog: Option<&str>,
@@ -544,11 +540,13 @@ impl DbClient {
544
540
  compressed: bool,
545
541
  stats: Arc<QueryStatsAccumulator>,
546
542
  ) -> mpsc::Receiver<Result<ChunkItem, ApiError>> {
547
- let concurrency = self.chunk_fetch_concurrency.max(1);
543
+ // Never more workers than chunks: the rest would start, find the
544
+ // queue empty and exit, for nothing.
545
+ let concurrency = self.chunk_fetch_concurrency.clamp(1, chunk_metas.len().max(1));
548
546
  let (tx, rx) = mpsc::channel::<Result<ChunkItem, ApiError>>(concurrency);
549
547
  let queue = std::sync::Arc::new(Mutex::new(VecDeque::from(chunk_metas)));
550
548
 
551
- tokio::spawn(async move {
549
+ tokio::spawn(in_call_context(async move {
552
550
  let mut handles = Vec::with_capacity(concurrency);
553
551
  for _ in 0..concurrency {
554
552
  let client = self.clone();
@@ -556,7 +554,7 @@ impl DbClient {
556
554
  let worker_tx = tx.clone();
557
555
  let statement_id = statement_id.clone();
558
556
  let stats = stats.clone();
559
- handles.push(tokio::spawn(async move {
557
+ handles.push(tokio::spawn(in_call_context(async move {
560
558
  let work_loop = async {
561
559
  loop {
562
560
  let meta = { queue.lock().unwrap().pop_front() };
@@ -601,7 +599,7 @@ impl DbClient {
601
599
  () = worker_tx.closed() => Ok(()),
602
600
  res = work_loop => res,
603
601
  }
604
- }));
602
+ })));
605
603
  }
606
604
 
607
605
  // `tx` itself (not a clone) stays alive across the join below, so
@@ -610,7 +608,7 @@ impl DbClient {
610
608
  if let Some(e) = join_first_error(handles).await {
611
609
  let _ = tx.send(Err(e)).await;
612
610
  }
613
- });
611
+ }));
614
612
 
615
613
  rx
616
614
  }
@@ -627,7 +625,7 @@ impl DbClient {
627
625
  /// reservation for that key leaks, and after `MAX_SESSIONS_PER_KEY` such
628
626
  /// drops every later query for the key runs session-less.
629
627
  struct SessionCheckin<'a> {
630
- client: &'a DbClient,
628
+ client: &'a Arc<DbClient>,
631
629
  catalog: Option<&'a str>,
632
630
  schema: Option<&'a str>,
633
631
  session_id: Option<String>,
@@ -636,7 +634,15 @@ struct SessionCheckin<'a> {
636
634
  impl SessionCheckin<'_> {
637
635
  fn finish(&mut self, keep: bool) {
638
636
  if let Some(id) = self.session_id.take() {
639
- self.client.checkin_session(self.catalog, self.schema, id, keep);
637
+ self.client.checkin_session(self.catalog, self.schema, id.clone(), keep);
638
+ if !keep {
639
+ // Discarded sessions are closed server-side too, not left to
640
+ // the idle TTL (see `session_pool`'s doc comment).
641
+ let client = self.client.clone();
642
+ pyo3_async_runtimes::tokio::get_runtime().spawn(in_call_context(async move {
643
+ client.delete_session(&id).await;
644
+ }));
645
+ }
640
646
  }
641
647
  }
642
648
  }
@@ -106,25 +106,23 @@ impl DbClient {
106
106
  /// `thrift_session_pool`'s doc comment): `None` means the caller must
107
107
  /// open its own throwaway session for this one call (Thrift has no
108
108
  /// session-less submission mode to fall back to).
109
+ ///
110
+ /// The `bool` is `true` for a session taken from the idle pool (it may have
111
+ /// gone stale since it was last used) and `false` for one opened just now.
109
112
  pub(crate) async fn thrift_checkout_session(
110
113
  &self,
111
114
  catalog: Option<&str>,
112
115
  schema: Option<&str>,
113
- ) -> Option<thrift::SessionHandle> {
116
+ ) -> Option<(thrift::SessionHandle, bool)> {
114
117
  let key = (catalog.map(str::to_string), schema.map(str::to_string));
115
118
  if let Some(handle) = self.thrift_session_pool.take(&key) {
116
- return Some(handle);
117
- }
118
- if !self.thrift_session_pool.reserve(&key) {
119
- return None;
120
- }
121
- match self.thrift_open_session_raw(catalog, schema).await {
122
- Ok(handle) => Some(handle),
123
- Err(_) => {
124
- self.thrift_session_pool.release(&key);
125
- None
126
- }
119
+ return Some((handle, true));
127
120
  }
121
+ let reservation = self.thrift_session_pool.reserve_guard(&key)?;
122
+ // Dropped (releasing the slot) on error or if this future is cancelled.
123
+ let handle = self.thrift_open_session_raw(catalog, schema).await.ok()?;
124
+ reservation.defuse();
125
+ Some((handle, false))
128
126
  }
129
127
 
130
128
  pub(crate) fn thrift_checkin_session(
@@ -138,6 +136,26 @@ impl DbClient {
138
136
  self.thrift_session_pool.checkin(key, session, keep);
139
137
  }
140
138
 
139
+ /// Forgets (and closes, best effort) every idle session for this key --
140
+ /// called when one of them turned out to be stale.
141
+ pub(crate) fn thrift_discard_idle_sessions(
142
+ self: &std::sync::Arc<Self>,
143
+ catalog: Option<&str>,
144
+ schema: Option<&str>,
145
+ ) {
146
+ let key = (catalog.map(str::to_string), schema.map(str::to_string));
147
+ let stale = self.thrift_session_pool.drain_key(&key);
148
+ if stale.is_empty() {
149
+ return;
150
+ }
151
+ let client = self.clone();
152
+ pyo3_async_runtimes::tokio::get_runtime().spawn(super::in_call_context(async move {
153
+ for session in stale {
154
+ client.thrift_close_session_raw(&session).await;
155
+ }
156
+ }));
157
+ }
158
+
141
159
  /// Best-effort close of every currently-idle pooled Thrift session --
142
160
  /// same contract as `close_all_sessions` (SEA).
143
161
  pub async fn close_all_thrift_sessions(&self) {
@@ -168,7 +186,11 @@ impl DbClient {
168
186
  let resp_bytes = self.thrift_call(body, false, Some(stats)).await?;
169
187
  let resp = thrift::parse_execute_statement(&resp_bytes).map_err(Self::thrift_parse_error)?;
170
188
  if let Some(e) = resp.status.error() {
171
- return Err(ApiError::permanent(format!("Thrift ExecuteStatement failed: {e}")));
189
+ // A rejected ExecuteStatement is a statement failure (bad SQL, bad
190
+ // parameter, ...), the same `StatementError` SEA raises for it.
191
+ return Err(ApiError::statement_failed(format!(
192
+ "Thrift ExecuteStatement failed: {e}"
193
+ )));
172
194
  }
173
195
  Ok(resp)
174
196
  }