arrowbricks 5.1.1__tar.gz → 5.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/PKG-INFO +35 -1
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/README.md +34 -0
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/pyproject.toml +1 -1
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/Cargo.lock +1 -1
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/Cargo.toml +1 -1
- arrowbricks-5.2.1/rust/arrowbricks_core/src/client/call_context.rs +109 -0
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/src/client/download.rs +60 -1
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/src/client/error.rs +21 -1
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/src/client/sea.rs +68 -51
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/src/client/thrift_rpc.rs +10 -11
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/src/client.rs +205 -18
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/src/heartbeat.rs +3 -2
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/src/json_convert.rs +161 -1
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/src/lib.rs +296 -142
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/src/pipeline/ndjson.rs +127 -32
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/src/pipeline/reorder.rs +6 -1
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/src/pipeline/sea.rs +12 -0
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/src/pipeline/stats.rs +13 -5
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/src/pipeline/thrift_exec.rs +132 -32
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/src/thrift.rs +28 -4
- arrowbricks-5.2.1/rust/arrowbricks_core/tests/common/mod.rs +115 -0
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/tests/wiremock_pipeline.rs +250 -1
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/tests/wiremock_thrift.rs +358 -0
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/tests_py/test_ipc_stream.py +41 -0
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/tests_py/test_token_provider.py +55 -1
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/src/arrowbricks/_core.pyi +2 -0
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/src/arrowbricks/_streaming.py +31 -8
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/src/arrowbricks/cursor.py +14 -7
- arrowbricks-5.1.1/rust/arrowbricks_core/tests/common/mod.rs +0 -29
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/LICENSE +0 -0
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/.gitignore +0 -0
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/README.md +0 -0
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/examples/duckdb_query.py +0 -0
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/examples/fastapi_sse.py +0 -0
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/proptest-regressions/json_convert.txt +0 -0
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/rustfmt.toml +0 -0
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/src/alloc.rs +0 -0
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/src/arrow_ffi.rs +0 -0
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/src/client/model.rs +0 -0
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/src/client/volume.rs +0 -0
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/src/pipeline/test_support.rs +0 -0
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/src/pipeline.rs +0 -0
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/tests/wiremock_volume_files.rs +0 -0
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/tests_py/conftest.py +0 -0
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/tests_py/test_parameters.py +0 -0
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/tests_py/test_stream_ndjson_lines.py +0 -0
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/tests_py/test_streaming.py +0 -0
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/tests_py/test_thrift.py +0 -0
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/tests_py/test_volume_files.py +0 -0
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/rust/arrowbricks_core/tests_py/thrift_mock.py +0 -0
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/src/arrowbricks/__init__.py +0 -0
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/src/arrowbricks/client.py +0 -0
- {arrowbricks-5.1.1 → arrowbricks-5.2.1}/src/arrowbricks/py.typed +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: arrowbricks
|
|
3
|
-
Version: 5.
|
|
3
|
+
Version: 5.2.1
|
|
4
4
|
Requires-Dist: arro3-core>=0.8 ; extra == 'arro3'
|
|
5
5
|
Provides-Extra: arro3
|
|
6
6
|
License-File: LICENSE
|
|
@@ -136,6 +136,8 @@ This example takes `sql` straight from the request for brevity -- arrowbricks do
|
|
|
136
136
|
|
|
137
137
|
arrowbricks has no opinion on *how* you get a token and no cloud-SDK dependency of its own. If your provider is expensive to call, cache/refresh inside it -- arrowbricks does no caching on your behalf.
|
|
138
138
|
|
|
139
|
+
A sync provider is called on one of arrowbricks' own worker threads. An async provider's coroutine runs on the event loop of the code that started the call (the `await cursor.execute(...)`, `async for ... in client.stream_query_json(...)`, and so on), with that code's `contextvars`, even when the token is needed by a background chunk download. One client can therefore be shared across event loops: successive `asyncio.run()` calls, or one loop per thread at the same time.
|
|
140
|
+
|
|
139
141
|
```python
|
|
140
142
|
conn = connect(host=..., warehouse_id=..., token_provider=my_token_provider)
|
|
141
143
|
```
|
|
@@ -164,6 +166,8 @@ When `total_timeout_s` elapses or your code cancels the surrounding coroutine (`
|
|
|
164
166
|
|
|
165
167
|
This covers both phases of a query: the submit/poll wait while the statement is still running (`Cursor.execute()`/`execute_streamed()`, and the start of `client.stream_query_json(...)`), and the chunk download afterwards (`Cursor.fetchall_streamed()`/`fetchall_arrow_streamed()`, the rest of `stream_query_json`). `stream_query_json`'s `total_timeout_s` is one budget across both phases. On `protocol="sea"` there is one short blind spot: the submit request itself can wait server-side for up to `wait_timeout` (default 30s) before Databricks returns a statement id, and a query abandoned inside that window can't be cancelled.
|
|
166
168
|
|
|
169
|
+
Abandoning a result also stops the client-side work right away, for every fetch method (plain `fetchall_arrow()`/`fetchmany()` too): cancelling the task, closing a `stream_query_json` iterator early (what a web framework does when the HTTP client disconnects mid-response), or dropping a `Cursor` before its result is drained aborts the chunk downloads in flight, starts no new ones, and on Thrift closes the operation (`CloseOperation`). Nothing to opt into: cancel the task (or let your framework close the generator) and arrowbricks cleans up. Cancelling a FastAPI handler on client disconnect therefore stops the warehouse query (if it is still running) and the downloads (if it is not).
|
|
170
|
+
|
|
167
171
|
## Errors
|
|
168
172
|
|
|
169
173
|
Every exception arrowbricks raises itself is an `ArrowbricksError` -- a `RuntimeError` subclass, so an `except RuntimeError` written before this hierarchy existed keeps catching everything unchanged. Three subclasses split by what actually changes what your code should do next:
|
|
@@ -341,6 +345,36 @@ can run concurrently on separate Python threads.
|
|
|
341
345
|
|
|
342
346
|
`write_ipc_stream` (and everything in this package that serializes Arrow-IPC bytes) always writes **uncompressed** bodies. A compressed body (arro3's own default is `compression="LZ4"`) is transparently decompressed by some Arrow readers (e.g. DuckDB's) but not necessarily by every other Arrow IPC reader -- notably, `duckdb-wasm`'s browser-side decoder silently fails to parse LZ4-compressed bodies. Since arrowbricks' bytes might end up read by anything, plain uncompressed is the safe default.
|
|
343
347
|
|
|
348
|
+
IPC output is written incrementally, with each Python `write()` call capped at
|
|
349
|
+
1 MiB. Short writes are handled; the destination remains open and is not flushed
|
|
350
|
+
by arrowbricks. A write failure can leave partial output, so discard it before retrying.
|
|
351
|
+
|
|
352
|
+
## Real warehouse verification
|
|
353
|
+
|
|
354
|
+
The opt-in live suite generates synthetic data with read-only SQL. It covers
|
|
355
|
+
Thrift and SEA, compression enabled and disabled, Arrow paging, Python values,
|
|
356
|
+
nested NDJSON, empty schemas, concurrent queries, errors, and timeout recovery.
|
|
357
|
+
Set `DATABRICKS_HOST`, `DATABRICKS_WAREHOUSE_ID`, and `DATABRICKS_TOKEN` in the
|
|
358
|
+
environment or the repository's ignored `.env`, then run:
|
|
359
|
+
|
|
360
|
+
```bash
|
|
361
|
+
ARROWBRICKS_LIVE=1 uv run pytest tests/live -q
|
|
362
|
+
```
|
|
363
|
+
|
|
364
|
+
To compare two package directories on the same warehouse, use
|
|
365
|
+
`examples/benchmark_versions.py --baseline /path/to/old --candidate /path/to/new`.
|
|
366
|
+
`--mode` selects `arrow`, `paging`, `rows`, `ndjson`, or `ipc`.
|
|
367
|
+
The benchmark alternates execution order, discards warm-ups, checks row counts,
|
|
368
|
+
and measures each version in its own process. `--verify-ipc` additionally compares
|
|
369
|
+
Arrow content for `arrow`/`ipc` modes; use an ordered query with stable results.
|
|
370
|
+
Samples also include process CPU time and the core's warehouse-wait,
|
|
371
|
+
submit-to-ready, and fetch timings. These help distinguish local processing
|
|
372
|
+
overhead from warehouse and download variability; core fetch time is not pure
|
|
373
|
+
network time and excludes time spent consuming rows in Python.
|
|
374
|
+
Peak RSS includes retained allocator memory and is specific to the host platform.
|
|
375
|
+
`--transform-only` isolates row conversion or IPC serialization after downloading
|
|
376
|
+
real data (`--mode rows` or `--mode ipc`), to distinguish CPU work from network noise.
|
|
377
|
+
|
|
344
378
|
## License
|
|
345
379
|
|
|
346
380
|
MIT
|
|
@@ -124,6 +124,8 @@ This example takes `sql` straight from the request for brevity -- arrowbricks do
|
|
|
124
124
|
|
|
125
125
|
arrowbricks has no opinion on *how* you get a token and no cloud-SDK dependency of its own. If your provider is expensive to call, cache/refresh inside it -- arrowbricks does no caching on your behalf.
|
|
126
126
|
|
|
127
|
+
A sync provider is called on one of arrowbricks' own worker threads. An async provider's coroutine runs on the event loop of the code that started the call (the `await cursor.execute(...)`, `async for ... in client.stream_query_json(...)`, and so on), with that code's `contextvars`, even when the token is needed by a background chunk download. One client can therefore be shared across event loops: successive `asyncio.run()` calls, or one loop per thread at the same time.
|
|
128
|
+
|
|
127
129
|
```python
|
|
128
130
|
conn = connect(host=..., warehouse_id=..., token_provider=my_token_provider)
|
|
129
131
|
```
|
|
@@ -152,6 +154,8 @@ When `total_timeout_s` elapses or your code cancels the surrounding coroutine (`
|
|
|
152
154
|
|
|
153
155
|
This covers both phases of a query: the submit/poll wait while the statement is still running (`Cursor.execute()`/`execute_streamed()`, and the start of `client.stream_query_json(...)`), and the chunk download afterwards (`Cursor.fetchall_streamed()`/`fetchall_arrow_streamed()`, the rest of `stream_query_json`). `stream_query_json`'s `total_timeout_s` is one budget across both phases. On `protocol="sea"` there is one short blind spot: the submit request itself can wait server-side for up to `wait_timeout` (default 30s) before Databricks returns a statement id, and a query abandoned inside that window can't be cancelled.
|
|
154
156
|
|
|
157
|
+
Abandoning a result also stops the client-side work right away, for every fetch method (plain `fetchall_arrow()`/`fetchmany()` too): cancelling the task, closing a `stream_query_json` iterator early (what a web framework does when the HTTP client disconnects mid-response), or dropping a `Cursor` before its result is drained aborts the chunk downloads in flight, starts no new ones, and on Thrift closes the operation (`CloseOperation`). Nothing to opt into: cancel the task (or let your framework close the generator) and arrowbricks cleans up. Cancelling a FastAPI handler on client disconnect therefore stops the warehouse query (if it is still running) and the downloads (if it is not).
|
|
158
|
+
|
|
155
159
|
## Errors
|
|
156
160
|
|
|
157
161
|
Every exception arrowbricks raises itself is an `ArrowbricksError` -- a `RuntimeError` subclass, so an `except RuntimeError` written before this hierarchy existed keeps catching everything unchanged. Three subclasses split by what actually changes what your code should do next:
|
|
@@ -329,6 +333,36 @@ can run concurrently on separate Python threads.
|
|
|
329
333
|
|
|
330
334
|
`write_ipc_stream` (and everything in this package that serializes Arrow-IPC bytes) always writes **uncompressed** bodies. A compressed body (arro3's own default is `compression="LZ4"`) is transparently decompressed by some Arrow readers (e.g. DuckDB's) but not necessarily by every other Arrow IPC reader -- notably, `duckdb-wasm`'s browser-side decoder silently fails to parse LZ4-compressed bodies. Since arrowbricks' bytes might end up read by anything, plain uncompressed is the safe default.
|
|
331
335
|
|
|
336
|
+
IPC output is written incrementally, with each Python `write()` call capped at
|
|
337
|
+
1 MiB. Short writes are handled; the destination remains open and is not flushed
|
|
338
|
+
by arrowbricks. A write failure can leave partial output, so discard it before retrying.
|
|
339
|
+
|
|
340
|
+
## Real warehouse verification
|
|
341
|
+
|
|
342
|
+
The opt-in live suite generates synthetic data with read-only SQL. It covers
|
|
343
|
+
Thrift and SEA, compression enabled and disabled, Arrow paging, Python values,
|
|
344
|
+
nested NDJSON, empty schemas, concurrent queries, errors, and timeout recovery.
|
|
345
|
+
Set `DATABRICKS_HOST`, `DATABRICKS_WAREHOUSE_ID`, and `DATABRICKS_TOKEN` in the
|
|
346
|
+
environment or the repository's ignored `.env`, then run:
|
|
347
|
+
|
|
348
|
+
```bash
|
|
349
|
+
ARROWBRICKS_LIVE=1 uv run pytest tests/live -q
|
|
350
|
+
```
|
|
351
|
+
|
|
352
|
+
To compare two package directories on the same warehouse, use
|
|
353
|
+
`examples/benchmark_versions.py --baseline /path/to/old --candidate /path/to/new`.
|
|
354
|
+
`--mode` selects `arrow`, `paging`, `rows`, `ndjson`, or `ipc`.
|
|
355
|
+
The benchmark alternates execution order, discards warm-ups, checks row counts,
|
|
356
|
+
and measures each version in its own process. `--verify-ipc` additionally compares
|
|
357
|
+
Arrow content for `arrow`/`ipc` modes; use an ordered query with stable results.
|
|
358
|
+
Samples also include process CPU time and the core's warehouse-wait,
|
|
359
|
+
submit-to-ready, and fetch timings. These help distinguish local processing
|
|
360
|
+
overhead from warehouse and download variability; core fetch time is not pure
|
|
361
|
+
network time and excludes time spent consuming rows in Python.
|
|
362
|
+
Peak RSS includes retained allocator memory and is specific to the host platform.
|
|
363
|
+
`--transform-only` isolates row conversion or IPC serialization after downloading
|
|
364
|
+
real data (`--mode rows` or `--mode ipc`), to distinguish CPU work from network noise.
|
|
365
|
+
|
|
332
366
|
## License
|
|
333
367
|
|
|
334
368
|
MIT
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "arrowbricks"
|
|
3
|
-
version = "5.
|
|
3
|
+
version = "5.2.1"
|
|
4
4
|
description = "Runs SQL against a Databricks SQL warehouse via the Statement Execution API and hands you the result as Arrow -- a DB-API-ish Cursor (fetchone/fetchmany/fetchall/fetchall_arrow) or NDJSON streaming. Rust/PyO3 core throughout -- zero required runtime dependencies."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "MIT"
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
//! Per-call context that follows a statement into every task it spawns.
|
|
2
|
+
//!
|
|
3
|
+
//! A `TokenProvider` (or `EventSink`) backed by a caller's own runtime may
|
|
4
|
+
//! need to know *which call* it is serving -- `lib.rs`'s `PyTokenProvider`
|
|
5
|
+
//! needs the asyncio event loop of the Python coroutine that started the
|
|
6
|
+
//! statement, so it can run an async `token_provider` on that loop. The
|
|
7
|
+
//! caller sets that context once, at the entry point (`with_call_context`),
|
|
8
|
+
//! where it is known. Every task this crate spawns for the call is wrapped
|
|
9
|
+
//! in `in_call_context`, so the same context is still there when a chunk
|
|
10
|
+
//! worker, the Thrift fetch loop, a heartbeat-wrapped submit or a
|
|
11
|
+
//! best-effort cancel asks for a token. Tokio task-locals are not inherited
|
|
12
|
+
//! by spawned tasks on their own; without the wrap, a token request from
|
|
13
|
+
//! one of those tasks had no way to find the caller's loop (the bug fixed in 5.2.1:
|
|
14
|
+
//! `stream_query_json` on a cold client failed with "no running event
|
|
15
|
+
//! loop").
|
|
16
|
+
//!
|
|
17
|
+
//! The context is opaque here (`Arc<dyn Any>`) so this module stays free of
|
|
18
|
+
//! PyO3, like the rest of `client`; `lib.rs` stores a `TaskLocals` in it and
|
|
19
|
+
//! downcasts it back.
|
|
20
|
+
|
|
21
|
+
use std::any::Any;
|
|
22
|
+
use std::future::Future;
|
|
23
|
+
use std::sync::Arc;
|
|
24
|
+
|
|
25
|
+
/// Opaque per-call state, set by the caller at the entry point. See the
|
|
26
|
+
/// module doc comment.
|
|
27
|
+
pub type CallContext = Arc<dyn Any + Send + Sync>;
|
|
28
|
+
|
|
29
|
+
tokio::task_local! {
|
|
30
|
+
static CALL_CONTEXT: Option<CallContext>;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/// The context of the call the current task belongs to, if any.
|
|
34
|
+
pub fn current_call_context() -> Option<CallContext> {
|
|
35
|
+
CALL_CONTEXT.try_with(Clone::clone).ok().flatten()
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/// Runs `fut` with `ctx` as its call context. For entry points: wrap the
|
|
39
|
+
/// whole future of one public call in this.
|
|
40
|
+
pub fn with_call_context<F: Future>(ctx: Option<CallContext>, fut: F) -> impl Future<Output = F::Output> {
|
|
41
|
+
CALL_CONTEXT.scope(ctx, fut)
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/// Runs the synchronous `f` with `ctx` as the call context -- for an entry
|
|
45
|
+
/// point that builds (and spawns) its work synchronously, such as
|
|
46
|
+
/// `ResultSet.fetchall_arrow_streamed`, so the tasks it spawns and the
|
|
47
|
+
/// cancel hook it builds pick `ctx` up.
|
|
48
|
+
pub fn enter_call_context<R>(ctx: Option<CallContext>, f: impl FnOnce() -> R) -> R {
|
|
49
|
+
CALL_CONTEXT.sync_scope(ctx, f)
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/// Wraps a future that is about to be spawned so it keeps the call context
|
|
53
|
+
/// of the code spawning it. Captured now, not when the task first runs.
|
|
54
|
+
/// Every `spawn` in this crate that can reach the token provider (directly
|
|
55
|
+
/// or through a task it spawns in turn) goes through this.
|
|
56
|
+
pub fn in_call_context<F: Future>(fut: F) -> impl Future<Output = F::Output> {
|
|
57
|
+
with_call_context(current_call_context(), fut)
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
#[cfg(test)]
|
|
61
|
+
mod tests {
|
|
62
|
+
use super::*;
|
|
63
|
+
|
|
64
|
+
fn ctx_value(ctx: Option<CallContext>) -> Option<u32> {
|
|
65
|
+
ctx.and_then(|c| c.downcast_ref::<u32>().copied())
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
#[tokio::test]
|
|
69
|
+
async fn spawned_tasks_inherit_the_call_context_only_through_in_call_context() {
|
|
70
|
+
let ctx: CallContext = Arc::new(7u32);
|
|
71
|
+
let (wrapped, bare, nested) = with_call_context(Some(ctx), async {
|
|
72
|
+
let wrapped = tokio::spawn(in_call_context(async { ctx_value(current_call_context()) }));
|
|
73
|
+
let bare = tokio::spawn(async { ctx_value(current_call_context()) });
|
|
74
|
+
// Two levels deep, like the SEA fetch driver and its workers.
|
|
75
|
+
let nested = tokio::spawn(in_call_context(async {
|
|
76
|
+
tokio::spawn(in_call_context(async { ctx_value(current_call_context()) }))
|
|
77
|
+
.await
|
|
78
|
+
.unwrap()
|
|
79
|
+
}));
|
|
80
|
+
(wrapped.await.unwrap(), bare.await.unwrap(), nested.await.unwrap())
|
|
81
|
+
})
|
|
82
|
+
.await;
|
|
83
|
+
assert_eq!(wrapped, Some(7));
|
|
84
|
+
assert_eq!(bare, None, "a plain tokio::spawn does not inherit task-locals");
|
|
85
|
+
assert_eq!(nested, Some(7));
|
|
86
|
+
assert_eq!(ctx_value(current_call_context()), None);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
#[test]
|
|
90
|
+
fn enter_call_context_is_seen_by_futures_wrapped_inside_it() {
|
|
91
|
+
let ctx: CallContext = Arc::new(3u32);
|
|
92
|
+
let fut = enter_call_context(Some(ctx), || {
|
|
93
|
+
in_call_context(async { ctx_value(current_call_context()) })
|
|
94
|
+
});
|
|
95
|
+
let rt = tokio::runtime::Builder::new_current_thread().build().unwrap();
|
|
96
|
+
assert_eq!(rt.block_on(fut), Some(3));
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
#[tokio::test]
|
|
100
|
+
async fn the_innermost_scope_wins() {
|
|
101
|
+
let outer: CallContext = Arc::new(1u32);
|
|
102
|
+
let inner: CallContext = Arc::new(2u32);
|
|
103
|
+
let seen = with_call_context(Some(outer), async move {
|
|
104
|
+
with_call_context(Some(inner), async { ctx_value(current_call_context()) }).await
|
|
105
|
+
})
|
|
106
|
+
.await;
|
|
107
|
+
assert_eq!(seen, Some(2));
|
|
108
|
+
}
|
|
109
|
+
}
|
|
@@ -302,7 +302,12 @@ impl DbClient {
|
|
|
302
302
|
parts.push((index, bytes?));
|
|
303
303
|
}
|
|
304
304
|
parts.sort_unstable_by_key(|(index, _)| *index);
|
|
305
|
-
|
|
305
|
+
// Size from what was actually received, never from `total`: that
|
|
306
|
+
// comes from the server's Content-Range header, and a bogus one
|
|
307
|
+
// would abort the process in `with_capacity` before the length
|
|
308
|
+
// check below could turn it into a normal error.
|
|
309
|
+
let received = head.len() + parts.iter().map(|(_, part)| part.len()).sum::<usize>();
|
|
310
|
+
let mut out = bytes::BytesMut::with_capacity(received);
|
|
306
311
|
out.extend_from_slice(&head);
|
|
307
312
|
for (_, part) in parts {
|
|
308
313
|
out.extend_from_slice(&part);
|
|
@@ -389,6 +394,60 @@ mod tests {
|
|
|
389
394
|
}
|
|
390
395
|
}
|
|
391
396
|
|
|
397
|
+
/// A bogus `Content-Range` total must become an ordinary error: it used to
|
|
398
|
+
/// size the output buffer up front, which aborted the whole process.
|
|
399
|
+
#[tokio::test]
|
|
400
|
+
async fn split_download_with_a_bogus_content_range_total_errors_instead_of_aborting() {
|
|
401
|
+
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
|
402
|
+
use tokio::net::TcpListener;
|
|
403
|
+
|
|
404
|
+
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
405
|
+
let address = listener.local_addr().unwrap();
|
|
406
|
+
tokio::spawn(async move {
|
|
407
|
+
loop {
|
|
408
|
+
let (mut socket, _) = listener.accept().await.unwrap();
|
|
409
|
+
tokio::spawn(async move {
|
|
410
|
+
let mut request = Vec::new();
|
|
411
|
+
while !request.ends_with(b"\r\n\r\n") {
|
|
412
|
+
if socket.read_u8().await.map(|b| request.push(b)).is_err() {
|
|
413
|
+
return;
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
let request = String::from_utf8(request).unwrap().to_ascii_lowercase();
|
|
417
|
+
let range = request.lines().find(|s| s.starts_with("range:")).unwrap();
|
|
418
|
+
let start: u64 = range
|
|
419
|
+
.split_once("bytes=")
|
|
420
|
+
.unwrap()
|
|
421
|
+
.1
|
|
422
|
+
.split_once('-')
|
|
423
|
+
.unwrap()
|
|
424
|
+
.0
|
|
425
|
+
.parse()
|
|
426
|
+
.unwrap();
|
|
427
|
+
let headers = format!(
|
|
428
|
+
"HTTP/1.1 206 Partial Content\r\nContent-Length: 4\r\nContent-Range: bytes {start}-{}/999999999999999\r\nConnection: close\r\n\r\n",
|
|
429
|
+
start + 3,
|
|
430
|
+
);
|
|
431
|
+
let _ = socket.write_all(headers.as_bytes()).await;
|
|
432
|
+
let _ = socket.write_all(b"abcd").await;
|
|
433
|
+
});
|
|
434
|
+
}
|
|
435
|
+
});
|
|
436
|
+
let client = Arc::new(
|
|
437
|
+
DbClient::new(&format!("http://{address}"), "wh", "token")
|
|
438
|
+
.with_retry_attempts(1)
|
|
439
|
+
.with_retry_max_wait_s(0.0),
|
|
440
|
+
);
|
|
441
|
+
let stats = Arc::new(QueryStatsAccumulator::default());
|
|
442
|
+
let result = client
|
|
443
|
+
.fetch_link_bytes_split(&format!("http://{address}/blob"), false, 4, 3, &stats)
|
|
444
|
+
.await;
|
|
445
|
+
assert!(
|
|
446
|
+
result.is_err(),
|
|
447
|
+
"a file shorter than its declared size must be an error"
|
|
448
|
+
);
|
|
449
|
+
}
|
|
450
|
+
|
|
392
451
|
#[tokio::test]
|
|
393
452
|
async fn split_download_starts_tail_requests_before_the_probe_body_finishes() {
|
|
394
453
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
|
@@ -121,7 +121,10 @@ impl ApiError {
|
|
|
121
121
|
// right.
|
|
122
122
|
let transient = idempotent && (e.is_decode() || (e.is_request() && !e.is_connect() && !e.is_timeout()));
|
|
123
123
|
Self {
|
|
124
|
-
|
|
124
|
+
// Strip the URL: reqwest appends it to the message, and for a
|
|
125
|
+
// cloud-fetch download it is a presigned link whose query string
|
|
126
|
+
// (`sig=...`) is a bearer capability for the result file.
|
|
127
|
+
message: e.without_url().to_string(),
|
|
125
128
|
transient,
|
|
126
129
|
kind: ApiErrorKind::Other,
|
|
127
130
|
}
|
|
@@ -165,3 +168,20 @@ pub(crate) fn join_error(e: tokio::task::JoinError) -> ApiError {
|
|
|
165
168
|
kind: ApiErrorKind::Other,
|
|
166
169
|
}
|
|
167
170
|
}
|
|
171
|
+
|
|
172
|
+
#[cfg(test)]
|
|
173
|
+
mod tests {
|
|
174
|
+
use super::*;
|
|
175
|
+
|
|
176
|
+
#[tokio::test]
|
|
177
|
+
async fn from_reqwest_message_never_contains_the_request_url() {
|
|
178
|
+
let _ = rustls::crypto::ring::default_provider().install_default();
|
|
179
|
+
let err = reqwest::Client::new()
|
|
180
|
+
.get("http://127.0.0.1:9/blob?sv=1&se=2&sig=SECRET")
|
|
181
|
+
.send()
|
|
182
|
+
.await
|
|
183
|
+
.unwrap_err();
|
|
184
|
+
let msg = ApiError::from_reqwest(err, true).message;
|
|
185
|
+
assert!(!msg.contains("SECRET") && !msg.contains("127.0.0.1"), "{msg}");
|
|
186
|
+
}
|
|
187
|
+
}
|
|
@@ -19,6 +19,7 @@ use tokio::sync::mpsc;
|
|
|
19
19
|
|
|
20
20
|
use super::DbClient;
|
|
21
21
|
use super::POLL_INTERVAL;
|
|
22
|
+
use super::call_context::in_call_context;
|
|
22
23
|
use super::error::{ApiError, join_error};
|
|
23
24
|
use super::model::{
|
|
24
25
|
CancelHandle, ChunkItem, ChunkMeta, ColumnDescription, InlineOrExternal, QueryStatsAccumulator,
|
|
@@ -144,10 +145,10 @@ impl DbClient {
|
|
|
144
145
|
let url = format!("{}/api/2.0/sql/sessions", self.host);
|
|
145
146
|
let mut body = json!({ "warehouse_id": self.warehouse_id });
|
|
146
147
|
if let Some(c) = catalog {
|
|
147
|
-
body["
|
|
148
|
+
body["catalog"] = json!(c);
|
|
148
149
|
}
|
|
149
150
|
if let Some(s) = schema {
|
|
150
|
-
body["
|
|
151
|
+
body["schema"] = json!(s);
|
|
151
152
|
}
|
|
152
153
|
let data: SessionCreateBody = self.authed_json(reqwest::Method::POST, &url, Some(&body), None).await?;
|
|
153
154
|
Ok(data.session_id)
|
|
@@ -173,16 +174,11 @@ impl DbClient {
|
|
|
173
174
|
if let Some(id) = self.session_pool.take(&key) {
|
|
174
175
|
return Some(id);
|
|
175
176
|
}
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
Err(_) => {
|
|
182
|
-
self.session_pool.release(&key);
|
|
183
|
-
None
|
|
184
|
-
}
|
|
185
|
-
}
|
|
177
|
+
let reservation = self.session_pool.reserve_guard(&key)?;
|
|
178
|
+
// Dropped (releasing the slot) on error or if this future is cancelled.
|
|
179
|
+
let id = self.create_session(catalog, schema).await.ok()?;
|
|
180
|
+
reservation.defuse();
|
|
181
|
+
Some(id)
|
|
186
182
|
}
|
|
187
183
|
|
|
188
184
|
/// Returns a session to the pool for reuse (`keep = true`, the statement
|
|
@@ -216,7 +212,7 @@ impl DbClient {
|
|
|
216
212
|
/// verbatim, same as the Python original does no validation of its own
|
|
217
213
|
/// shape either.
|
|
218
214
|
pub async fn execute_arrow_statement(
|
|
219
|
-
&
|
|
215
|
+
self: &Arc<Self>,
|
|
220
216
|
statement: &str,
|
|
221
217
|
catalog: Option<&str>,
|
|
222
218
|
schema: Option<&str>,
|
|
@@ -267,7 +263,7 @@ impl DbClient {
|
|
|
267
263
|
/// small result pays for two full statement executions on the (common,
|
|
268
264
|
/// for them) fallback path instead of one.
|
|
269
265
|
pub async fn execute_arrow_statement_prefer_inline(
|
|
270
|
-
&
|
|
266
|
+
self: &Arc<Self>,
|
|
271
267
|
statement: &str,
|
|
272
268
|
catalog: Option<&str>,
|
|
273
269
|
schema: Option<&str>,
|
|
@@ -346,7 +342,7 @@ impl DbClient {
|
|
|
346
342
|
/// field). The session, if any, is returned to the pool on a clean
|
|
347
343
|
/// terminal state and discarded on any error.
|
|
348
344
|
async fn submit_and_poll(
|
|
349
|
-
&
|
|
345
|
+
self: &Arc<Self>,
|
|
350
346
|
mut body: Value,
|
|
351
347
|
catalog: Option<&str>,
|
|
352
348
|
schema: Option<&str>,
|
|
@@ -431,7 +427,7 @@ impl DbClient {
|
|
|
431
427
|
}
|
|
432
428
|
|
|
433
429
|
async fn execute_statement(
|
|
434
|
-
&
|
|
430
|
+
self: &Arc<Self>,
|
|
435
431
|
statement: &str,
|
|
436
432
|
format: &str,
|
|
437
433
|
catalog: Option<&str>,
|
|
@@ -544,11 +540,13 @@ impl DbClient {
|
|
|
544
540
|
compressed: bool,
|
|
545
541
|
stats: Arc<QueryStatsAccumulator>,
|
|
546
542
|
) -> mpsc::Receiver<Result<ChunkItem, ApiError>> {
|
|
547
|
-
|
|
543
|
+
// Never more workers than chunks: the rest would start, find the
|
|
544
|
+
// queue empty and exit, for nothing.
|
|
545
|
+
let concurrency = self.chunk_fetch_concurrency.clamp(1, chunk_metas.len().max(1));
|
|
548
546
|
let (tx, rx) = mpsc::channel::<Result<ChunkItem, ApiError>>(concurrency);
|
|
549
547
|
let queue = std::sync::Arc::new(Mutex::new(VecDeque::from(chunk_metas)));
|
|
550
548
|
|
|
551
|
-
tokio::spawn(async move {
|
|
549
|
+
tokio::spawn(in_call_context(async move {
|
|
552
550
|
let mut handles = Vec::with_capacity(concurrency);
|
|
553
551
|
for _ in 0..concurrency {
|
|
554
552
|
let client = self.clone();
|
|
@@ -556,41 +554,52 @@ impl DbClient {
|
|
|
556
554
|
let worker_tx = tx.clone();
|
|
557
555
|
let statement_id = statement_id.clone();
|
|
558
556
|
let stats = stats.clone();
|
|
559
|
-
handles.push(tokio::spawn(async move {
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
557
|
+
handles.push(tokio::spawn(in_call_context(async move {
|
|
558
|
+
let work_loop = async {
|
|
559
|
+
loop {
|
|
560
|
+
let meta = { queue.lock().unwrap().pop_front() };
|
|
561
|
+
let Some(meta) = meta else { return Ok(()) };
|
|
562
|
+
let fetched = if meta.pre_resolved_links.is_empty() {
|
|
563
|
+
client
|
|
564
|
+
.fetch_chunk_index(&statement_id, meta.chunk_index, compressed, &stats)
|
|
565
|
+
.await
|
|
566
|
+
} else {
|
|
567
|
+
client
|
|
568
|
+
.fetch_pre_resolved_links(&meta.pre_resolved_links, compressed, &stats)
|
|
569
|
+
.await
|
|
570
|
+
};
|
|
571
|
+
match fetched {
|
|
572
|
+
Ok(blobs) => {
|
|
573
|
+
// One blob ⇒ `meta.row_count` (the whole chunk_index's
|
|
574
|
+
// declared count) and "this blob's count" are the same
|
|
575
|
+
// number -- see `ChunkItem::truncate_to`'s own doc comment.
|
|
576
|
+
let truncate_to = if blobs.len() == 1 { meta.row_count } else { None };
|
|
577
|
+
for blob in blobs {
|
|
578
|
+
let item = ChunkItem {
|
|
579
|
+
blob,
|
|
580
|
+
row_count: meta.row_count,
|
|
581
|
+
chunk_index: meta.chunk_index,
|
|
582
|
+
truncate_to,
|
|
583
|
+
};
|
|
584
|
+
if worker_tx.send(Ok(item)).await.is_err() {
|
|
585
|
+
return Ok(());
|
|
586
|
+
}
|
|
587
587
|
}
|
|
588
588
|
}
|
|
589
|
+
Err(e) => return Err(e),
|
|
589
590
|
}
|
|
590
|
-
Err(e) => return Err(e),
|
|
591
591
|
}
|
|
592
|
+
};
|
|
593
|
+
// The consumer dropping its receiver (a cancelled,
|
|
594
|
+
// timed-out or abandoned fetch) stops this worker at
|
|
595
|
+
// once, aborting its in-flight download instead of
|
|
596
|
+
// finishing -- and then starting -- chunks nobody reads.
|
|
597
|
+
tokio::select! {
|
|
598
|
+
biased;
|
|
599
|
+
() = worker_tx.closed() => Ok(()),
|
|
600
|
+
res = work_loop => res,
|
|
592
601
|
}
|
|
593
|
-
}));
|
|
602
|
+
})));
|
|
594
603
|
}
|
|
595
604
|
|
|
596
605
|
// `tx` itself (not a clone) stays alive across the join below, so
|
|
@@ -599,7 +608,7 @@ impl DbClient {
|
|
|
599
608
|
if let Some(e) = join_first_error(handles).await {
|
|
600
609
|
let _ = tx.send(Err(e)).await;
|
|
601
610
|
}
|
|
602
|
-
});
|
|
611
|
+
}));
|
|
603
612
|
|
|
604
613
|
rx
|
|
605
614
|
}
|
|
@@ -616,7 +625,7 @@ impl DbClient {
|
|
|
616
625
|
/// reservation for that key leaks, and after `MAX_SESSIONS_PER_KEY` such
|
|
617
626
|
/// drops every later query for the key runs session-less.
|
|
618
627
|
struct SessionCheckin<'a> {
|
|
619
|
-
client: &'a DbClient
|
|
628
|
+
client: &'a Arc<DbClient>,
|
|
620
629
|
catalog: Option<&'a str>,
|
|
621
630
|
schema: Option<&'a str>,
|
|
622
631
|
session_id: Option<String>,
|
|
@@ -625,7 +634,15 @@ struct SessionCheckin<'a> {
|
|
|
625
634
|
impl SessionCheckin<'_> {
|
|
626
635
|
fn finish(&mut self, keep: bool) {
|
|
627
636
|
if let Some(id) = self.session_id.take() {
|
|
628
|
-
self.client.checkin_session(self.catalog, self.schema, id, keep);
|
|
637
|
+
self.client.checkin_session(self.catalog, self.schema, id.clone(), keep);
|
|
638
|
+
if !keep {
|
|
639
|
+
// Discarded sessions are closed server-side too, not left to
|
|
640
|
+
// the idle TTL (see `session_pool`'s doc comment).
|
|
641
|
+
let client = self.client.clone();
|
|
642
|
+
pyo3_async_runtimes::tokio::get_runtime().spawn(in_call_context(async move {
|
|
643
|
+
client.delete_session(&id).await;
|
|
644
|
+
}));
|
|
645
|
+
}
|
|
629
646
|
}
|
|
630
647
|
}
|
|
631
648
|
}
|
|
@@ -115,16 +115,11 @@ impl DbClient {
|
|
|
115
115
|
if let Some(handle) = self.thrift_session_pool.take(&key) {
|
|
116
116
|
return Some(handle);
|
|
117
117
|
}
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
Err(_) => {
|
|
124
|
-
self.thrift_session_pool.release(&key);
|
|
125
|
-
None
|
|
126
|
-
}
|
|
127
|
-
}
|
|
118
|
+
let reservation = self.thrift_session_pool.reserve_guard(&key)?;
|
|
119
|
+
// Dropped (releasing the slot) on error or if this future is cancelled.
|
|
120
|
+
let handle = self.thrift_open_session_raw(catalog, schema).await.ok()?;
|
|
121
|
+
reservation.defuse();
|
|
122
|
+
Some(handle)
|
|
128
123
|
}
|
|
129
124
|
|
|
130
125
|
pub(crate) fn thrift_checkin_session(
|
|
@@ -168,7 +163,11 @@ impl DbClient {
|
|
|
168
163
|
let resp_bytes = self.thrift_call(body, false, Some(stats)).await?;
|
|
169
164
|
let resp = thrift::parse_execute_statement(&resp_bytes).map_err(Self::thrift_parse_error)?;
|
|
170
165
|
if let Some(e) = resp.status.error() {
|
|
171
|
-
|
|
166
|
+
// A rejected ExecuteStatement is a statement failure (bad SQL, bad
|
|
167
|
+
// parameter, ...), the same `StatementError` SEA raises for it.
|
|
168
|
+
return Err(ApiError::statement_failed(format!(
|
|
169
|
+
"Thrift ExecuteStatement failed: {e}"
|
|
170
|
+
)));
|
|
172
171
|
}
|
|
173
172
|
Ok(resp)
|
|
174
173
|
}
|