arrowbricks 5.1.2__tar.gz → 5.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/PKG-INFO +33 -1
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/README.md +32 -0
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/pyproject.toml +1 -1
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/Cargo.lock +1 -1
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/Cargo.toml +1 -1
- arrowbricks-5.2.2/rust/arrowbricks_core/src/client/call_context.rs +109 -0
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/client/download.rs +60 -1
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/client/error.rs +30 -1
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/client/sea.rs +29 -23
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/client/thrift_rpc.rs +35 -13
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/client.rs +216 -18
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/heartbeat.rs +3 -2
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/json_convert.rs +161 -1
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/lib.rs +296 -142
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/pipeline/ndjson.rs +127 -32
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/pipeline/reorder.rs +6 -1
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/pipeline/sea.rs +12 -0
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/pipeline/stats.rs +13 -5
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/pipeline/thrift_exec.rs +124 -49
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/thrift.rs +28 -4
- arrowbricks-5.2.2/rust/arrowbricks_core/tests/common/mod.rs +115 -0
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/tests/wiremock_pipeline.rs +137 -1
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/tests/wiremock_thrift.rs +344 -0
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/tests_py/test_ipc_stream.py +41 -0
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/tests_py/test_token_provider.py +55 -1
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/src/arrowbricks/_core.pyi +2 -0
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/src/arrowbricks/_streaming.py +31 -8
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/src/arrowbricks/cursor.py +14 -7
- arrowbricks-5.1.2/rust/arrowbricks_core/tests/common/mod.rs +0 -29
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/LICENSE +0 -0
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/.gitignore +0 -0
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/README.md +0 -0
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/examples/duckdb_query.py +0 -0
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/examples/fastapi_sse.py +0 -0
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/proptest-regressions/json_convert.txt +0 -0
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/rustfmt.toml +0 -0
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/alloc.rs +0 -0
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/arrow_ffi.rs +0 -0
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/client/model.rs +0 -0
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/client/volume.rs +0 -0
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/pipeline/test_support.rs +0 -0
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/pipeline.rs +0 -0
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/tests/wiremock_volume_files.rs +0 -0
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/tests_py/conftest.py +0 -0
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/tests_py/test_parameters.py +0 -0
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/tests_py/test_stream_ndjson_lines.py +0 -0
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/tests_py/test_streaming.py +0 -0
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/tests_py/test_thrift.py +0 -0
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/tests_py/test_volume_files.py +0 -0
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/rust/arrowbricks_core/tests_py/thrift_mock.py +0 -0
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/src/arrowbricks/__init__.py +0 -0
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/src/arrowbricks/client.py +0 -0
- {arrowbricks-5.1.2 → arrowbricks-5.2.2}/src/arrowbricks/py.typed +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: arrowbricks
|
|
3
|
-
Version: 5.
|
|
3
|
+
Version: 5.2.2
|
|
4
4
|
Requires-Dist: arro3-core>=0.8 ; extra == 'arro3'
|
|
5
5
|
Provides-Extra: arro3
|
|
6
6
|
License-File: LICENSE
|
|
@@ -136,6 +136,8 @@ This example takes `sql` straight from the request for brevity -- arrowbricks do
|
|
|
136
136
|
|
|
137
137
|
arrowbricks has no opinion on *how* you get a token and no cloud-SDK dependency of its own. If your provider is expensive to call, cache/refresh inside it -- arrowbricks does no caching on your behalf.
|
|
138
138
|
|
|
139
|
+
A sync provider is called on one of arrowbricks' own worker threads. An async provider's coroutine runs on the event loop of the code that started the call (the `await cursor.execute(...)`, `async for ... in client.stream_query_json(...)`, and so on), with that code's `contextvars`, even when the token is needed by a background chunk download. One client can therefore be shared across event loops: successive `asyncio.run()` calls, or one loop per thread at the same time.
|
|
140
|
+
|
|
139
141
|
```python
|
|
140
142
|
conn = connect(host=..., warehouse_id=..., token_provider=my_token_provider)
|
|
141
143
|
```
|
|
@@ -343,6 +345,36 @@ can run concurrently on separate Python threads.
|
|
|
343
345
|
|
|
344
346
|
`write_ipc_stream` (and everything in this package that serializes Arrow-IPC bytes) always writes **uncompressed** bodies. A compressed body (arro3's own default is `compression="LZ4"`) is transparently decompressed by some Arrow readers (e.g. DuckDB's) but not necessarily by every other Arrow IPC reader -- notably, `duckdb-wasm`'s browser-side decoder silently fails to parse LZ4-compressed bodies. Since arrowbricks' bytes might end up read by anything, plain uncompressed is the safe default.
|
|
345
347
|
|
|
348
|
+
IPC output is written incrementally, with each Python `write()` call capped at
|
|
349
|
+
1 MiB. Short writes are handled; the destination remains open and is not flushed
|
|
350
|
+
by arrowbricks. A write failure can leave partial output, so discard it before retrying.
|
|
351
|
+
|
|
352
|
+
## Real warehouse verification
|
|
353
|
+
|
|
354
|
+
The opt-in live suite generates synthetic data with read-only SQL. It covers
|
|
355
|
+
Thrift and SEA, compression enabled and disabled, Arrow paging, Python values,
|
|
356
|
+
nested NDJSON, empty schemas, concurrent queries, errors, and timeout recovery.
|
|
357
|
+
Set `DATABRICKS_HOST`, `DATABRICKS_WAREHOUSE_ID`, and `DATABRICKS_TOKEN` in the
|
|
358
|
+
environment or the repository's ignored `.env`, then run:
|
|
359
|
+
|
|
360
|
+
```bash
|
|
361
|
+
ARROWBRICKS_LIVE=1 uv run pytest tests/live -q
|
|
362
|
+
```
|
|
363
|
+
|
|
364
|
+
To compare two package directories on the same warehouse, use
|
|
365
|
+
`examples/benchmark_versions.py --baseline /path/to/old --candidate /path/to/new`.
|
|
366
|
+
`--mode` selects `arrow`, `paging`, `rows`, `ndjson`, or `ipc`.
|
|
367
|
+
The benchmark alternates execution order, discards warm-ups, checks row counts,
|
|
368
|
+
and measures each version in its own process. `--verify-ipc` additionally compares
|
|
369
|
+
Arrow content for `arrow`/`ipc` modes; use an ordered query with stable results.
|
|
370
|
+
Samples also include process CPU time and the core's warehouse-wait,
|
|
371
|
+
submit-to-ready, and fetch timings. These help distinguish local processing
|
|
372
|
+
overhead from warehouse and download variability; core fetch time is not pure
|
|
373
|
+
network time and excludes time spent consuming rows in Python.
|
|
374
|
+
Peak RSS includes retained allocator memory and is specific to the host platform.
|
|
375
|
+
`--transform-only` isolates row conversion or IPC serialization after downloading
|
|
376
|
+
real data (`--mode rows` or `--mode ipc`), to distinguish CPU work from network noise.
|
|
377
|
+
|
|
346
378
|
## License
|
|
347
379
|
|
|
348
380
|
MIT
|
|
@@ -124,6 +124,8 @@ This example takes `sql` straight from the request for brevity -- arrowbricks do
|
|
|
124
124
|
|
|
125
125
|
arrowbricks has no opinion on *how* you get a token and no cloud-SDK dependency of its own. If your provider is expensive to call, cache/refresh inside it -- arrowbricks does no caching on your behalf.
|
|
126
126
|
|
|
127
|
+
A sync provider is called on one of arrowbricks' own worker threads. An async provider's coroutine runs on the event loop of the code that started the call (the `await cursor.execute(...)`, `async for ... in client.stream_query_json(...)`, and so on), with that code's `contextvars`, even when the token is needed by a background chunk download. One client can therefore be shared across event loops: successive `asyncio.run()` calls, or one loop per thread at the same time.
|
|
128
|
+
|
|
127
129
|
```python
|
|
128
130
|
conn = connect(host=..., warehouse_id=..., token_provider=my_token_provider)
|
|
129
131
|
```
|
|
@@ -331,6 +333,36 @@ can run concurrently on separate Python threads.
|
|
|
331
333
|
|
|
332
334
|
`write_ipc_stream` (and everything in this package that serializes Arrow-IPC bytes) always writes **uncompressed** bodies. A compressed body (arro3's own default is `compression="LZ4"`) is transparently decompressed by some Arrow readers (e.g. DuckDB's) but not necessarily by every other Arrow IPC reader -- notably, `duckdb-wasm`'s browser-side decoder silently fails to parse LZ4-compressed bodies. Since arrowbricks' bytes might end up read by anything, plain uncompressed is the safe default.
|
|
333
335
|
|
|
336
|
+
IPC output is written incrementally, with each Python `write()` call capped at
|
|
337
|
+
1 MiB. Short writes are handled; the destination remains open and is not flushed
|
|
338
|
+
by arrowbricks. A write failure can leave partial output, so discard it before retrying.
|
|
339
|
+
|
|
340
|
+
## Real warehouse verification
|
|
341
|
+
|
|
342
|
+
The opt-in live suite generates synthetic data with read-only SQL. It covers
|
|
343
|
+
Thrift and SEA, compression enabled and disabled, Arrow paging, Python values,
|
|
344
|
+
nested NDJSON, empty schemas, concurrent queries, errors, and timeout recovery.
|
|
345
|
+
Set `DATABRICKS_HOST`, `DATABRICKS_WAREHOUSE_ID`, and `DATABRICKS_TOKEN` in the
|
|
346
|
+
environment or the repository's ignored `.env`, then run:
|
|
347
|
+
|
|
348
|
+
```bash
|
|
349
|
+
ARROWBRICKS_LIVE=1 uv run pytest tests/live -q
|
|
350
|
+
```
|
|
351
|
+
|
|
352
|
+
To compare two package directories on the same warehouse, use
|
|
353
|
+
`examples/benchmark_versions.py --baseline /path/to/old --candidate /path/to/new`.
|
|
354
|
+
`--mode` selects `arrow`, `paging`, `rows`, `ndjson`, or `ipc`.
|
|
355
|
+
The benchmark alternates execution order, discards warm-ups, checks row counts,
|
|
356
|
+
and measures each version in its own process. `--verify-ipc` additionally compares
|
|
357
|
+
Arrow content for `arrow`/`ipc` modes; use an ordered query with stable results.
|
|
358
|
+
Samples also include process CPU time and the core's warehouse-wait,
|
|
359
|
+
submit-to-ready, and fetch timings. These help distinguish local processing
|
|
360
|
+
overhead from warehouse and download variability; core fetch time is not pure
|
|
361
|
+
network time and excludes time spent consuming rows in Python.
|
|
362
|
+
Peak RSS includes retained allocator memory and is specific to the host platform.
|
|
363
|
+
`--transform-only` isolates row conversion or IPC serialization after downloading
|
|
364
|
+
real data (`--mode rows` or `--mode ipc`), to distinguish CPU work from network noise.
|
|
365
|
+
|
|
334
366
|
## License
|
|
335
367
|
|
|
336
368
|
MIT
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "arrowbricks"
|
|
3
|
-
version = "5.
|
|
3
|
+
version = "5.2.2"
|
|
4
4
|
description = "Runs SQL against a Databricks SQL warehouse via the Statement Execution API and hands you the result as Arrow -- a DB-API-ish Cursor (fetchone/fetchmany/fetchall/fetchall_arrow) or NDJSON streaming. Rust/PyO3 core throughout -- zero required runtime dependencies."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "MIT"
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
//! Per-call context that follows a statement into every task it spawns.
|
|
2
|
+
//!
|
|
3
|
+
//! A `TokenProvider` (or `EventSink`) backed by a caller's own runtime may
|
|
4
|
+
//! need to know *which call* it is serving -- `lib.rs`'s `PyTokenProvider`
|
|
5
|
+
//! needs the asyncio event loop of the Python coroutine that started the
|
|
6
|
+
//! statement, so it can run an async `token_provider` on that loop. The
|
|
7
|
+
//! caller sets that context once, at the entry point (`with_call_context`),
|
|
8
|
+
//! where it is known. Every task this crate spawns for the call is wrapped
|
|
9
|
+
//! in `in_call_context`, so the same context is still there when a chunk
|
|
10
|
+
//! worker, the Thrift fetch loop, a heartbeat-wrapped submit or a
|
|
11
|
+
//! best-effort cancel asks for a token. Tokio task-locals are not inherited
|
|
12
|
+
//! by spawned tasks on their own; without the wrap, a token request from
|
|
13
|
+
//! one of those tasks had no way to find the caller's loop (the bug fixed in 5.2.1:
|
|
14
|
+
//! `stream_query_json` on a cold client failed with "no running event
|
|
15
|
+
//! loop").
|
|
16
|
+
//!
|
|
17
|
+
//! The context is opaque here (`Arc<dyn Any>`) so this module stays free of
|
|
18
|
+
//! PyO3, like the rest of `client`; `lib.rs` stores a `TaskLocals` in it and
|
|
19
|
+
//! downcasts it back.
|
|
20
|
+
|
|
21
|
+
use std::any::Any;
|
|
22
|
+
use std::future::Future;
|
|
23
|
+
use std::sync::Arc;
|
|
24
|
+
|
|
25
|
+
/// Opaque per-call state, set by the caller at the entry point. See the
|
|
26
|
+
/// module doc comment.
|
|
27
|
+
pub type CallContext = Arc<dyn Any + Send + Sync>;
|
|
28
|
+
|
|
29
|
+
tokio::task_local! {
|
|
30
|
+
static CALL_CONTEXT: Option<CallContext>;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/// The context of the call the current task belongs to, if any.
|
|
34
|
+
pub fn current_call_context() -> Option<CallContext> {
|
|
35
|
+
CALL_CONTEXT.try_with(Clone::clone).ok().flatten()
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/// Runs `fut` with `ctx` as its call context. For entry points: wrap the
|
|
39
|
+
/// whole future of one public call in this.
|
|
40
|
+
pub fn with_call_context<F: Future>(ctx: Option<CallContext>, fut: F) -> impl Future<Output = F::Output> {
|
|
41
|
+
CALL_CONTEXT.scope(ctx, fut)
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/// Runs the synchronous `f` with `ctx` as the call context -- for an entry
|
|
45
|
+
/// point that builds (and spawns) its work synchronously, such as
|
|
46
|
+
/// `ResultSet.fetchall_arrow_streamed`, so the tasks it spawns and the
|
|
47
|
+
/// cancel hook it builds pick `ctx` up.
|
|
48
|
+
pub fn enter_call_context<R>(ctx: Option<CallContext>, f: impl FnOnce() -> R) -> R {
|
|
49
|
+
CALL_CONTEXT.sync_scope(ctx, f)
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/// Wraps a future that is about to be spawned so it keeps the call context
|
|
53
|
+
/// of the code spawning it. Captured now, not when the task first runs.
|
|
54
|
+
/// Every `spawn` in this crate that can reach the token provider (directly
|
|
55
|
+
/// or through a task it spawns in turn) goes through this.
|
|
56
|
+
pub fn in_call_context<F: Future>(fut: F) -> impl Future<Output = F::Output> {
|
|
57
|
+
with_call_context(current_call_context(), fut)
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
#[cfg(test)]
|
|
61
|
+
mod tests {
|
|
62
|
+
use super::*;
|
|
63
|
+
|
|
64
|
+
fn ctx_value(ctx: Option<CallContext>) -> Option<u32> {
|
|
65
|
+
ctx.and_then(|c| c.downcast_ref::<u32>().copied())
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
#[tokio::test]
|
|
69
|
+
async fn spawned_tasks_inherit_the_call_context_only_through_in_call_context() {
|
|
70
|
+
let ctx: CallContext = Arc::new(7u32);
|
|
71
|
+
let (wrapped, bare, nested) = with_call_context(Some(ctx), async {
|
|
72
|
+
let wrapped = tokio::spawn(in_call_context(async { ctx_value(current_call_context()) }));
|
|
73
|
+
let bare = tokio::spawn(async { ctx_value(current_call_context()) });
|
|
74
|
+
// Two levels deep, like the SEA fetch driver and its workers.
|
|
75
|
+
let nested = tokio::spawn(in_call_context(async {
|
|
76
|
+
tokio::spawn(in_call_context(async { ctx_value(current_call_context()) }))
|
|
77
|
+
.await
|
|
78
|
+
.unwrap()
|
|
79
|
+
}));
|
|
80
|
+
(wrapped.await.unwrap(), bare.await.unwrap(), nested.await.unwrap())
|
|
81
|
+
})
|
|
82
|
+
.await;
|
|
83
|
+
assert_eq!(wrapped, Some(7));
|
|
84
|
+
assert_eq!(bare, None, "a plain tokio::spawn does not inherit task-locals");
|
|
85
|
+
assert_eq!(nested, Some(7));
|
|
86
|
+
assert_eq!(ctx_value(current_call_context()), None);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
#[test]
|
|
90
|
+
fn enter_call_context_is_seen_by_futures_wrapped_inside_it() {
|
|
91
|
+
let ctx: CallContext = Arc::new(3u32);
|
|
92
|
+
let fut = enter_call_context(Some(ctx), || {
|
|
93
|
+
in_call_context(async { ctx_value(current_call_context()) })
|
|
94
|
+
});
|
|
95
|
+
let rt = tokio::runtime::Builder::new_current_thread().build().unwrap();
|
|
96
|
+
assert_eq!(rt.block_on(fut), Some(3));
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
#[tokio::test]
|
|
100
|
+
async fn the_innermost_scope_wins() {
|
|
101
|
+
let outer: CallContext = Arc::new(1u32);
|
|
102
|
+
let inner: CallContext = Arc::new(2u32);
|
|
103
|
+
let seen = with_call_context(Some(outer), async move {
|
|
104
|
+
with_call_context(Some(inner), async { ctx_value(current_call_context()) }).await
|
|
105
|
+
})
|
|
106
|
+
.await;
|
|
107
|
+
assert_eq!(seen, Some(2));
|
|
108
|
+
}
|
|
109
|
+
}
|
|
@@ -302,7 +302,12 @@ impl DbClient {
|
|
|
302
302
|
parts.push((index, bytes?));
|
|
303
303
|
}
|
|
304
304
|
parts.sort_unstable_by_key(|(index, _)| *index);
|
|
305
|
-
|
|
305
|
+
// Size from what was actually received, never from `total`: that
|
|
306
|
+
// comes from the server's Content-Range header, and a bogus one
|
|
307
|
+
// would abort the process in `with_capacity` before the length
|
|
308
|
+
// check below could turn it into a normal error.
|
|
309
|
+
let received = head.len() + parts.iter().map(|(_, part)| part.len()).sum::<usize>();
|
|
310
|
+
let mut out = bytes::BytesMut::with_capacity(received);
|
|
306
311
|
out.extend_from_slice(&head);
|
|
307
312
|
for (_, part) in parts {
|
|
308
313
|
out.extend_from_slice(&part);
|
|
@@ -389,6 +394,60 @@ mod tests {
|
|
|
389
394
|
}
|
|
390
395
|
}
|
|
391
396
|
|
|
397
|
+
/// A bogus `Content-Range` total must become an ordinary error: it used to
|
|
398
|
+
/// size the output buffer up front, which aborted the whole process.
|
|
399
|
+
#[tokio::test]
|
|
400
|
+
async fn split_download_with_a_bogus_content_range_total_errors_instead_of_aborting() {
|
|
401
|
+
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
|
402
|
+
use tokio::net::TcpListener;
|
|
403
|
+
|
|
404
|
+
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
405
|
+
let address = listener.local_addr().unwrap();
|
|
406
|
+
tokio::spawn(async move {
|
|
407
|
+
loop {
|
|
408
|
+
let (mut socket, _) = listener.accept().await.unwrap();
|
|
409
|
+
tokio::spawn(async move {
|
|
410
|
+
let mut request = Vec::new();
|
|
411
|
+
while !request.ends_with(b"\r\n\r\n") {
|
|
412
|
+
if socket.read_u8().await.map(|b| request.push(b)).is_err() {
|
|
413
|
+
return;
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
let request = String::from_utf8(request).unwrap().to_ascii_lowercase();
|
|
417
|
+
let range = request.lines().find(|s| s.starts_with("range:")).unwrap();
|
|
418
|
+
let start: u64 = range
|
|
419
|
+
.split_once("bytes=")
|
|
420
|
+
.unwrap()
|
|
421
|
+
.1
|
|
422
|
+
.split_once('-')
|
|
423
|
+
.unwrap()
|
|
424
|
+
.0
|
|
425
|
+
.parse()
|
|
426
|
+
.unwrap();
|
|
427
|
+
let headers = format!(
|
|
428
|
+
"HTTP/1.1 206 Partial Content\r\nContent-Length: 4\r\nContent-Range: bytes {start}-{}/999999999999999\r\nConnection: close\r\n\r\n",
|
|
429
|
+
start + 3,
|
|
430
|
+
);
|
|
431
|
+
let _ = socket.write_all(headers.as_bytes()).await;
|
|
432
|
+
let _ = socket.write_all(b"abcd").await;
|
|
433
|
+
});
|
|
434
|
+
}
|
|
435
|
+
});
|
|
436
|
+
let client = Arc::new(
|
|
437
|
+
DbClient::new(&format!("http://{address}"), "wh", "token")
|
|
438
|
+
.with_retry_attempts(1)
|
|
439
|
+
.with_retry_max_wait_s(0.0),
|
|
440
|
+
);
|
|
441
|
+
let stats = Arc::new(QueryStatsAccumulator::default());
|
|
442
|
+
let result = client
|
|
443
|
+
.fetch_link_bytes_split(&format!("http://{address}/blob"), false, 4, 3, &stats)
|
|
444
|
+
.await;
|
|
445
|
+
assert!(
|
|
446
|
+
result.is_err(),
|
|
447
|
+
"a file shorter than its declared size must be an error"
|
|
448
|
+
);
|
|
449
|
+
}
|
|
450
|
+
|
|
392
451
|
#[tokio::test]
|
|
393
452
|
async fn split_download_starts_tail_requests_before_the_probe_body_finishes() {
|
|
394
453
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
|
@@ -121,12 +121,24 @@ impl ApiError {
|
|
|
121
121
|
// right.
|
|
122
122
|
let transient = idempotent && (e.is_decode() || (e.is_request() && !e.is_connect() && !e.is_timeout()));
|
|
123
123
|
Self {
|
|
124
|
-
|
|
124
|
+
// Strip the URL: reqwest appends it to the message, and for a
|
|
125
|
+
// cloud-fetch download it is a presigned link whose query string
|
|
126
|
+
// (`sig=...`) is a bearer capability for the result file.
|
|
127
|
+
message: e.without_url().to_string(),
|
|
125
128
|
transient,
|
|
126
129
|
kind: ApiErrorKind::Other,
|
|
127
130
|
}
|
|
128
131
|
}
|
|
129
132
|
|
|
133
|
+
/// The server rejected a request because the session it names no longer
|
|
134
|
+
/// exists (closed, or invalidated by a warehouse stop/restart). Observed
|
|
135
|
+
/// against a real warehouse: `HTTP 400` with a Thrift exception body
|
|
136
|
+
/// `INVALID_STATE: Invalid SessionHandle: SessionHandle [..]`. Nothing ran,
|
|
137
|
+
/// so the request is safe to repeat on a fresh session.
|
|
138
|
+
pub(crate) fn is_stale_session(&self) -> bool {
|
|
139
|
+
self.message.contains("Invalid SessionHandle")
|
|
140
|
+
}
|
|
141
|
+
|
|
130
142
|
/// `idempotent` only gates the 5xx case -- 401/403/408/429 mean the
|
|
131
143
|
/// request was rejected before any processing started (auth failure,
|
|
132
144
|
/// rate limit, client-side timeout), safe to retry regardless of method.
|
|
@@ -165,3 +177,20 @@ pub(crate) fn join_error(e: tokio::task::JoinError) -> ApiError {
|
|
|
165
177
|
kind: ApiErrorKind::Other,
|
|
166
178
|
}
|
|
167
179
|
}
|
|
180
|
+
|
|
181
|
+
#[cfg(test)]
|
|
182
|
+
mod tests {
|
|
183
|
+
use super::*;
|
|
184
|
+
|
|
185
|
+
#[tokio::test]
|
|
186
|
+
async fn from_reqwest_message_never_contains_the_request_url() {
|
|
187
|
+
let _ = rustls::crypto::ring::default_provider().install_default();
|
|
188
|
+
let err = reqwest::Client::new()
|
|
189
|
+
.get("http://127.0.0.1:9/blob?sv=1&se=2&sig=SECRET")
|
|
190
|
+
.send()
|
|
191
|
+
.await
|
|
192
|
+
.unwrap_err();
|
|
193
|
+
let msg = ApiError::from_reqwest(err, true).message;
|
|
194
|
+
assert!(!msg.contains("SECRET") && !msg.contains("127.0.0.1"), "{msg}");
|
|
195
|
+
}
|
|
196
|
+
}
|
|
@@ -19,6 +19,7 @@ use tokio::sync::mpsc;
|
|
|
19
19
|
|
|
20
20
|
use super::DbClient;
|
|
21
21
|
use super::POLL_INTERVAL;
|
|
22
|
+
use super::call_context::in_call_context;
|
|
22
23
|
use super::error::{ApiError, join_error};
|
|
23
24
|
use super::model::{
|
|
24
25
|
CancelHandle, ChunkItem, ChunkMeta, ColumnDescription, InlineOrExternal, QueryStatsAccumulator,
|
|
@@ -144,10 +145,10 @@ impl DbClient {
|
|
|
144
145
|
let url = format!("{}/api/2.0/sql/sessions", self.host);
|
|
145
146
|
let mut body = json!({ "warehouse_id": self.warehouse_id });
|
|
146
147
|
if let Some(c) = catalog {
|
|
147
|
-
body["
|
|
148
|
+
body["catalog"] = json!(c);
|
|
148
149
|
}
|
|
149
150
|
if let Some(s) = schema {
|
|
150
|
-
body["
|
|
151
|
+
body["schema"] = json!(s);
|
|
151
152
|
}
|
|
152
153
|
let data: SessionCreateBody = self.authed_json(reqwest::Method::POST, &url, Some(&body), None).await?;
|
|
153
154
|
Ok(data.session_id)
|
|
@@ -173,16 +174,11 @@ impl DbClient {
|
|
|
173
174
|
if let Some(id) = self.session_pool.take(&key) {
|
|
174
175
|
return Some(id);
|
|
175
176
|
}
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
Err(_) => {
|
|
182
|
-
self.session_pool.release(&key);
|
|
183
|
-
None
|
|
184
|
-
}
|
|
185
|
-
}
|
|
177
|
+
let reservation = self.session_pool.reserve_guard(&key)?;
|
|
178
|
+
// Dropped (releasing the slot) on error or if this future is cancelled.
|
|
179
|
+
let id = self.create_session(catalog, schema).await.ok()?;
|
|
180
|
+
reservation.defuse();
|
|
181
|
+
Some(id)
|
|
186
182
|
}
|
|
187
183
|
|
|
188
184
|
/// Returns a session to the pool for reuse (`keep = true`, the statement
|
|
@@ -216,7 +212,7 @@ impl DbClient {
|
|
|
216
212
|
/// verbatim, same as the Python original does no validation of its own
|
|
217
213
|
/// shape either.
|
|
218
214
|
pub async fn execute_arrow_statement(
|
|
219
|
-
&
|
|
215
|
+
self: &Arc<Self>,
|
|
220
216
|
statement: &str,
|
|
221
217
|
catalog: Option<&str>,
|
|
222
218
|
schema: Option<&str>,
|
|
@@ -267,7 +263,7 @@ impl DbClient {
|
|
|
267
263
|
/// small result pays for two full statement executions on the (common,
|
|
268
264
|
/// for them) fallback path instead of one.
|
|
269
265
|
pub async fn execute_arrow_statement_prefer_inline(
|
|
270
|
-
&
|
|
266
|
+
self: &Arc<Self>,
|
|
271
267
|
statement: &str,
|
|
272
268
|
catalog: Option<&str>,
|
|
273
269
|
schema: Option<&str>,
|
|
@@ -346,7 +342,7 @@ impl DbClient {
|
|
|
346
342
|
/// field). The session, if any, is returned to the pool on a clean
|
|
347
343
|
/// terminal state and discarded on any error.
|
|
348
344
|
async fn submit_and_poll(
|
|
349
|
-
&
|
|
345
|
+
self: &Arc<Self>,
|
|
350
346
|
mut body: Value,
|
|
351
347
|
catalog: Option<&str>,
|
|
352
348
|
schema: Option<&str>,
|
|
@@ -431,7 +427,7 @@ impl DbClient {
|
|
|
431
427
|
}
|
|
432
428
|
|
|
433
429
|
async fn execute_statement(
|
|
434
|
-
&
|
|
430
|
+
self: &Arc<Self>,
|
|
435
431
|
statement: &str,
|
|
436
432
|
format: &str,
|
|
437
433
|
catalog: Option<&str>,
|
|
@@ -544,11 +540,13 @@ impl DbClient {
|
|
|
544
540
|
compressed: bool,
|
|
545
541
|
stats: Arc<QueryStatsAccumulator>,
|
|
546
542
|
) -> mpsc::Receiver<Result<ChunkItem, ApiError>> {
|
|
547
|
-
|
|
543
|
+
// Never more workers than chunks: the rest would start, find the
|
|
544
|
+
// queue empty and exit, for nothing.
|
|
545
|
+
let concurrency = self.chunk_fetch_concurrency.clamp(1, chunk_metas.len().max(1));
|
|
548
546
|
let (tx, rx) = mpsc::channel::<Result<ChunkItem, ApiError>>(concurrency);
|
|
549
547
|
let queue = std::sync::Arc::new(Mutex::new(VecDeque::from(chunk_metas)));
|
|
550
548
|
|
|
551
|
-
tokio::spawn(async move {
|
|
549
|
+
tokio::spawn(in_call_context(async move {
|
|
552
550
|
let mut handles = Vec::with_capacity(concurrency);
|
|
553
551
|
for _ in 0..concurrency {
|
|
554
552
|
let client = self.clone();
|
|
@@ -556,7 +554,7 @@ impl DbClient {
|
|
|
556
554
|
let worker_tx = tx.clone();
|
|
557
555
|
let statement_id = statement_id.clone();
|
|
558
556
|
let stats = stats.clone();
|
|
559
|
-
handles.push(tokio::spawn(async move {
|
|
557
|
+
handles.push(tokio::spawn(in_call_context(async move {
|
|
560
558
|
let work_loop = async {
|
|
561
559
|
loop {
|
|
562
560
|
let meta = { queue.lock().unwrap().pop_front() };
|
|
@@ -601,7 +599,7 @@ impl DbClient {
|
|
|
601
599
|
() = worker_tx.closed() => Ok(()),
|
|
602
600
|
res = work_loop => res,
|
|
603
601
|
}
|
|
604
|
-
}));
|
|
602
|
+
})));
|
|
605
603
|
}
|
|
606
604
|
|
|
607
605
|
// `tx` itself (not a clone) stays alive across the join below, so
|
|
@@ -610,7 +608,7 @@ impl DbClient {
|
|
|
610
608
|
if let Some(e) = join_first_error(handles).await {
|
|
611
609
|
let _ = tx.send(Err(e)).await;
|
|
612
610
|
}
|
|
613
|
-
});
|
|
611
|
+
}));
|
|
614
612
|
|
|
615
613
|
rx
|
|
616
614
|
}
|
|
@@ -627,7 +625,7 @@ impl DbClient {
|
|
|
627
625
|
/// reservation for that key leaks, and after `MAX_SESSIONS_PER_KEY` such
|
|
628
626
|
/// drops every later query for the key runs session-less.
|
|
629
627
|
struct SessionCheckin<'a> {
|
|
630
|
-
client: &'a DbClient
|
|
628
|
+
client: &'a Arc<DbClient>,
|
|
631
629
|
catalog: Option<&'a str>,
|
|
632
630
|
schema: Option<&'a str>,
|
|
633
631
|
session_id: Option<String>,
|
|
@@ -636,7 +634,15 @@ struct SessionCheckin<'a> {
|
|
|
636
634
|
impl SessionCheckin<'_> {
|
|
637
635
|
fn finish(&mut self, keep: bool) {
|
|
638
636
|
if let Some(id) = self.session_id.take() {
|
|
639
|
-
self.client.checkin_session(self.catalog, self.schema, id, keep);
|
|
637
|
+
self.client.checkin_session(self.catalog, self.schema, id.clone(), keep);
|
|
638
|
+
if !keep {
|
|
639
|
+
// Discarded sessions are closed server-side too, not left to
|
|
640
|
+
// the idle TTL (see `session_pool`'s doc comment).
|
|
641
|
+
let client = self.client.clone();
|
|
642
|
+
pyo3_async_runtimes::tokio::get_runtime().spawn(in_call_context(async move {
|
|
643
|
+
client.delete_session(&id).await;
|
|
644
|
+
}));
|
|
645
|
+
}
|
|
640
646
|
}
|
|
641
647
|
}
|
|
642
648
|
}
|
|
@@ -106,25 +106,23 @@ impl DbClient {
|
|
|
106
106
|
/// `thrift_session_pool`'s doc comment): `None` means the caller must
|
|
107
107
|
/// open its own throwaway session for this one call (Thrift has no
|
|
108
108
|
/// session-less submission mode to fall back to).
|
|
109
|
+
///
|
|
110
|
+
/// The `bool` is `true` for a session taken from the idle pool (it may have
|
|
111
|
+
/// gone stale since it was last used) and `false` for one opened just now.
|
|
109
112
|
pub(crate) async fn thrift_checkout_session(
|
|
110
113
|
&self,
|
|
111
114
|
catalog: Option<&str>,
|
|
112
115
|
schema: Option<&str>,
|
|
113
|
-
) -> Option<thrift::SessionHandle> {
|
|
116
|
+
) -> Option<(thrift::SessionHandle, bool)> {
|
|
114
117
|
let key = (catalog.map(str::to_string), schema.map(str::to_string));
|
|
115
118
|
if let Some(handle) = self.thrift_session_pool.take(&key) {
|
|
116
|
-
return Some(handle);
|
|
117
|
-
}
|
|
118
|
-
if !self.thrift_session_pool.reserve(&key) {
|
|
119
|
-
return None;
|
|
120
|
-
}
|
|
121
|
-
match self.thrift_open_session_raw(catalog, schema).await {
|
|
122
|
-
Ok(handle) => Some(handle),
|
|
123
|
-
Err(_) => {
|
|
124
|
-
self.thrift_session_pool.release(&key);
|
|
125
|
-
None
|
|
126
|
-
}
|
|
119
|
+
return Some((handle, true));
|
|
127
120
|
}
|
|
121
|
+
let reservation = self.thrift_session_pool.reserve_guard(&key)?;
|
|
122
|
+
// Dropped (releasing the slot) on error or if this future is cancelled.
|
|
123
|
+
let handle = self.thrift_open_session_raw(catalog, schema).await.ok()?;
|
|
124
|
+
reservation.defuse();
|
|
125
|
+
Some((handle, false))
|
|
128
126
|
}
|
|
129
127
|
|
|
130
128
|
pub(crate) fn thrift_checkin_session(
|
|
@@ -138,6 +136,26 @@ impl DbClient {
|
|
|
138
136
|
self.thrift_session_pool.checkin(key, session, keep);
|
|
139
137
|
}
|
|
140
138
|
|
|
139
|
+
/// Forgets (and closes, best effort) every idle session for this key --
|
|
140
|
+
/// called when one of them turned out to be stale.
|
|
141
|
+
pub(crate) fn thrift_discard_idle_sessions(
|
|
142
|
+
self: &std::sync::Arc<Self>,
|
|
143
|
+
catalog: Option<&str>,
|
|
144
|
+
schema: Option<&str>,
|
|
145
|
+
) {
|
|
146
|
+
let key = (catalog.map(str::to_string), schema.map(str::to_string));
|
|
147
|
+
let stale = self.thrift_session_pool.drain_key(&key);
|
|
148
|
+
if stale.is_empty() {
|
|
149
|
+
return;
|
|
150
|
+
}
|
|
151
|
+
let client = self.clone();
|
|
152
|
+
pyo3_async_runtimes::tokio::get_runtime().spawn(super::in_call_context(async move {
|
|
153
|
+
for session in stale {
|
|
154
|
+
client.thrift_close_session_raw(&session).await;
|
|
155
|
+
}
|
|
156
|
+
}));
|
|
157
|
+
}
|
|
158
|
+
|
|
141
159
|
/// Best-effort close of every currently-idle pooled Thrift session --
|
|
142
160
|
/// same contract as `close_all_sessions` (SEA).
|
|
143
161
|
pub async fn close_all_thrift_sessions(&self) {
|
|
@@ -168,7 +186,11 @@ impl DbClient {
|
|
|
168
186
|
let resp_bytes = self.thrift_call(body, false, Some(stats)).await?;
|
|
169
187
|
let resp = thrift::parse_execute_statement(&resp_bytes).map_err(Self::thrift_parse_error)?;
|
|
170
188
|
if let Some(e) = resp.status.error() {
|
|
171
|
-
|
|
189
|
+
// A rejected ExecuteStatement is a statement failure (bad SQL, bad
|
|
190
|
+
// parameter, ...), the same `StatementError` SEA raises for it.
|
|
191
|
+
return Err(ApiError::statement_failed(format!(
|
|
192
|
+
"Thrift ExecuteStatement failed: {e}"
|
|
193
|
+
)));
|
|
172
194
|
}
|
|
173
195
|
Ok(resp)
|
|
174
196
|
}
|