arrowbricks 5.1.0__tar.gz → 5.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- arrowbricks-5.1.0/README.md → arrowbricks-5.1.2/PKG-INFO +15 -0
- arrowbricks-5.1.0/PKG-INFO → arrowbricks-5.1.2/README.md +2 -13
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/pyproject.toml +1 -1
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/Cargo.lock +1 -1
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/Cargo.toml +6 -2
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/src/alloc.rs +54 -2
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/src/client/download.rs +82 -4
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/src/client/sea.rs +39 -28
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/src/lib.rs +5 -5
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/src/pipeline/ndjson.rs +8 -12
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/src/pipeline/thrift_exec.rs +93 -20
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/tests/wiremock_pipeline.rs +113 -0
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/tests/wiremock_thrift.rs +114 -0
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/src/arrowbricks/_streaming.py +1 -4
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/LICENSE +0 -0
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/.gitignore +0 -0
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/README.md +0 -0
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/examples/duckdb_query.py +0 -0
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/examples/fastapi_sse.py +0 -0
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/proptest-regressions/json_convert.txt +0 -0
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/rustfmt.toml +0 -0
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/src/arrow_ffi.rs +0 -0
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/src/client/error.rs +0 -0
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/src/client/model.rs +0 -0
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/src/client/thrift_rpc.rs +0 -0
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/src/client/volume.rs +0 -0
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/src/client.rs +0 -0
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/src/heartbeat.rs +0 -0
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/src/json_convert.rs +0 -0
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/src/pipeline/reorder.rs +0 -0
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/src/pipeline/sea.rs +0 -0
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/src/pipeline/stats.rs +0 -0
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/src/pipeline/test_support.rs +0 -0
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/src/pipeline.rs +0 -0
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/src/thrift.rs +0 -0
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/tests/common/mod.rs +0 -0
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/tests/wiremock_volume_files.rs +0 -0
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/tests_py/conftest.py +0 -0
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/tests_py/test_ipc_stream.py +0 -0
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/tests_py/test_parameters.py +0 -0
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/tests_py/test_stream_ndjson_lines.py +0 -0
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/tests_py/test_streaming.py +0 -0
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/tests_py/test_thrift.py +0 -0
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/tests_py/test_token_provider.py +0 -0
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/tests_py/test_volume_files.py +0 -0
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/tests_py/thrift_mock.py +0 -0
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/src/arrowbricks/__init__.py +0 -0
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/src/arrowbricks/_core.pyi +0 -0
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/src/arrowbricks/client.py +0 -0
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/src/arrowbricks/cursor.py +0 -0
- {arrowbricks-5.1.0 → arrowbricks-5.1.2}/src/arrowbricks/py.typed +0 -0
|
@@ -1,3 +1,15 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: arrowbricks
|
|
3
|
+
Version: 5.1.2
|
|
4
|
+
Requires-Dist: arro3-core>=0.8 ; extra == 'arro3'
|
|
5
|
+
Provides-Extra: arro3
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
Summary: Runs SQL against a Databricks SQL warehouse via the Statement Execution API and hands you the result as Arrow -- a DB-API-ish Cursor (fetchone/fetchmany/fetchall/fetchall_arrow) or NDJSON streaming. Rust/PyO3 core throughout -- zero required runtime dependencies.
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
Requires-Python: >=3.11
|
|
10
|
+
Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
|
|
11
|
+
Project-URL: Repository, https://github.com/bmsuisse/arrowbricks
|
|
12
|
+
|
|
1
13
|
<p align="center">
|
|
2
14
|
<img src="assets/logo.svg" width="72" height="72" alt="arrowbricks logo">
|
|
3
15
|
</p>
|
|
@@ -152,6 +164,8 @@ When `total_timeout_s` elapses or your code cancels the surrounding coroutine (`
|
|
|
152
164
|
|
|
153
165
|
This covers both phases of a query: the submit/poll wait while the statement is still running (`Cursor.execute()`/`execute_streamed()`, and the start of `client.stream_query_json(...)`), and the chunk download afterwards (`Cursor.fetchall_streamed()`/`fetchall_arrow_streamed()`, the rest of `stream_query_json`). `stream_query_json`'s `total_timeout_s` is one budget across both phases. On `protocol="sea"` there is one short blind spot: the submit request itself can wait server-side for up to `wait_timeout` (default 30s) before Databricks returns a statement id, and a query abandoned inside that window can't be cancelled.
|
|
154
166
|
|
|
167
|
+
Abandoning a result also stops the client-side work right away, for every fetch method (plain `fetchall_arrow()`/`fetchmany()` too): cancelling the task, closing a `stream_query_json` iterator early (what a web framework does when the HTTP client disconnects mid-response), or dropping a `Cursor` before its result is drained aborts the chunk downloads in flight, starts no new ones, and on Thrift closes the operation (`CloseOperation`). Nothing to opt into: cancel the task (or let your framework close the generator) and arrowbricks cleans up. Cancelling a FastAPI handler on client disconnect therefore stops the warehouse query (if it is still running) and the downloads (if it is not).
|
|
168
|
+
|
|
155
169
|
## Errors
|
|
156
170
|
|
|
157
171
|
Every exception arrowbricks raises itself is an `ArrowbricksError` -- a `RuntimeError` subclass, so an `except RuntimeError` written before this hierarchy existed keeps catching everything unchanged. Three subclasses split by what actually changes what your code should do next:
|
|
@@ -332,3 +346,4 @@ can run concurrently on separate Python threads.
|
|
|
332
346
|
## License
|
|
333
347
|
|
|
334
348
|
MIT
|
|
349
|
+
|
|
@@ -1,15 +1,3 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: arrowbricks
|
|
3
|
-
Version: 5.1.0
|
|
4
|
-
Requires-Dist: arro3-core>=0.8 ; extra == 'arro3'
|
|
5
|
-
Provides-Extra: arro3
|
|
6
|
-
License-File: LICENSE
|
|
7
|
-
Summary: Runs SQL against a Databricks SQL warehouse via the Statement Execution API and hands you the result as Arrow -- a DB-API-ish Cursor (fetchone/fetchmany/fetchall/fetchall_arrow) or NDJSON streaming. Rust/PyO3 core throughout -- zero required runtime dependencies.
|
|
8
|
-
License-Expression: MIT
|
|
9
|
-
Requires-Python: >=3.11
|
|
10
|
-
Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
|
|
11
|
-
Project-URL: Repository, https://github.com/bmsuisse/arrowbricks
|
|
12
|
-
|
|
13
1
|
<p align="center">
|
|
14
2
|
<img src="assets/logo.svg" width="72" height="72" alt="arrowbricks logo">
|
|
15
3
|
</p>
|
|
@@ -164,6 +152,8 @@ When `total_timeout_s` elapses or your code cancels the surrounding coroutine (`
|
|
|
164
152
|
|
|
165
153
|
This covers both phases of a query: the submit/poll wait while the statement is still running (`Cursor.execute()`/`execute_streamed()`, and the start of `client.stream_query_json(...)`), and the chunk download afterwards (`Cursor.fetchall_streamed()`/`fetchall_arrow_streamed()`, the rest of `stream_query_json`). `stream_query_json`'s `total_timeout_s` is one budget across both phases. On `protocol="sea"` there is one short blind spot: the submit request itself can wait server-side for up to `wait_timeout` (default 30s) before Databricks returns a statement id, and a query abandoned inside that window can't be cancelled.
|
|
166
154
|
|
|
155
|
+
Abandoning a result also stops the client-side work right away, for every fetch method (plain `fetchall_arrow()`/`fetchmany()` too): cancelling the task, closing a `stream_query_json` iterator early (what a web framework does when the HTTP client disconnects mid-response), or dropping a `Cursor` before its result is drained aborts the chunk downloads in flight, starts no new ones, and on Thrift closes the operation (`CloseOperation`). Nothing to opt into: cancel the task (or let your framework close the generator) and arrowbricks cleans up. Cancelling a FastAPI handler on client disconnect therefore stops the warehouse query (if it is still running) and the downloads (if it is not).
|
|
156
|
+
|
|
167
157
|
## Errors
|
|
168
158
|
|
|
169
159
|
Every exception arrowbricks raises itself is an `ArrowbricksError` -- a `RuntimeError` subclass, so an `except RuntimeError` written before this hierarchy existed keeps catching everything unchanged. Three subclasses split by what actually changes what your code should do next:
|
|
@@ -344,4 +334,3 @@ can run concurrently on separate Python threads.
|
|
|
344
334
|
## License
|
|
345
335
|
|
|
346
336
|
MIT
|
|
347
|
-
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "arrowbricks"
|
|
3
|
-
version = "5.1.
|
|
3
|
+
version = "5.1.2"
|
|
4
4
|
description = "Runs SQL against a Databricks SQL warehouse via the Statement Execution API and hands you the result as Arrow -- a DB-API-ish Cursor (fetchone/fetchmany/fetchall/fetchall_arrow) or NDJSON streaming. Rust/PyO3 core throughout -- zero required runtime dependencies."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "MIT"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[package]
|
|
2
2
|
name = "arrowbricks_core"
|
|
3
|
-
version = "5.1.
|
|
3
|
+
version = "5.1.2"
|
|
4
4
|
edition = "2024"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
|
|
@@ -74,13 +74,17 @@ tokio = { version = "1.53.1", features = ["rt-multi-thread", "macros"] }
|
|
|
74
74
|
libc = "0.2"
|
|
75
75
|
|
|
76
76
|
[features]
|
|
77
|
-
default = ["extension-module"]
|
|
77
|
+
default = ["extension-module", "large-block-alloc"]
|
|
78
78
|
# `extension-module` skips linking against libpython -- correct for the
|
|
79
79
|
# cdylib maturin actually ships (Python provides those symbols at import
|
|
80
80
|
# time), but it breaks a plain `cargo test`/`cargo bench` binary, which does
|
|
81
81
|
# need to link libpython itself. Tests/benches build with
|
|
82
82
|
# `--no-default-features` to get a normal link.
|
|
83
83
|
extension-module = ["pyo3/extension-module"]
|
|
84
|
+
# Linux/glibc only: returns buffers of 1 MiB or more to the OS when freed (src/alloc.rs).
|
|
85
|
+
# A Rust crate depending on this one with its own global allocator opts out with
|
|
86
|
+
# `default-features = false`.
|
|
87
|
+
large-block-alloc = []
|
|
84
88
|
|
|
85
89
|
[dev-dependencies]
|
|
86
90
|
tokio = { version = "1.53.1", features = ["rt-multi-thread", "macros", "test-util", "net", "io-util"] }
|
|
@@ -6,7 +6,14 @@
|
|
|
6
6
|
//! idle RSS from 200 MB to 1.4 GB. Blocks of `LARGE` bytes or more bypass
|
|
7
7
|
//! malloc here (`mmap`/`munmap`, `mremap` to grow without copying); smaller
|
|
8
8
|
//! ones use the system allocator. Only Rust allocations in this extension are
|
|
9
|
-
//! affected, not the host process's malloc. Linux only
|
|
9
|
+
//! affected, not the host process's malloc. Linux with glibc only (the one
|
|
10
|
+
//! configuration this was measured on); see `lib.rs` for the gate.
|
|
11
|
+
//!
|
|
12
|
+
//! Failure mode: `mmap`/`mremap` return null when the address-space limit is
|
|
13
|
+
//! hit (`RLIMIT_AS`, strict overcommit, `vm.max_map_count`), which Rust turns
|
|
14
|
+
//! into an abort like any other failed allocation.
|
|
15
|
+
|
|
16
|
+
#![warn(clippy::undocumented_unsafe_blocks)]
|
|
10
17
|
|
|
11
18
|
use std::alloc::{GlobalAlloc, Layout, System};
|
|
12
19
|
use std::ptr;
|
|
@@ -16,19 +23,32 @@ const PAGE: usize = 4096;
|
|
|
16
23
|
|
|
17
24
|
pub struct LargeBlockAlloc;
|
|
18
25
|
|
|
26
|
+
/// Must stay a pure function of the `Layout`: `alloc`, `realloc` and `dealloc`
|
|
27
|
+
/// each call it on the same block and must get the same answer, otherwise a
|
|
28
|
+
/// block is freed by the wrong allocator.
|
|
19
29
|
fn is_large(layout: Layout) -> bool {
|
|
20
30
|
layout.size() >= LARGE && layout.align() <= PAGE
|
|
21
31
|
}
|
|
22
32
|
|
|
33
|
+
/// The length the kernel maps for `size` bytes. Cannot overflow: a `Layout`
|
|
34
|
+
/// size is at most `isize::MAX`.
|
|
23
35
|
fn mapped_len(size: usize) -> usize {
|
|
24
36
|
size.next_multiple_of(PAGE)
|
|
25
37
|
}
|
|
26
38
|
|
|
39
|
+
// SAFETY: every method routes a block to `mmap` or to `System` based only on
|
|
40
|
+
// `is_large(layout)`, and the caller must pass the block's original layout to
|
|
41
|
+
// `dealloc`/`realloc` (the `GlobalAlloc` contract), so a block is always
|
|
42
|
+
// released by the allocator that produced it. Mappings are page-aligned, which
|
|
43
|
+
// satisfies every `align <= PAGE`; larger alignments go to `System`.
|
|
27
44
|
unsafe impl GlobalAlloc for LargeBlockAlloc {
|
|
28
45
|
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
|
|
29
46
|
if !is_large(layout) {
|
|
47
|
+
// SAFETY: `layout` is forwarded unchanged; the caller upholds `alloc`'s contract.
|
|
30
48
|
return unsafe { System.alloc(layout) };
|
|
31
49
|
}
|
|
50
|
+
// SAFETY: an anonymous private mapping with no address hint; the length is
|
|
51
|
+
// non-zero because `layout.size() >= LARGE`.
|
|
32
52
|
let p = unsafe {
|
|
33
53
|
libc::mmap(
|
|
34
54
|
ptr::null_mut(),
|
|
@@ -49,29 +69,40 @@ unsafe impl GlobalAlloc for LargeBlockAlloc {
|
|
|
49
69
|
// Fresh anonymous mappings are already zeroed.
|
|
50
70
|
unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
|
|
51
71
|
if is_large(layout) {
|
|
72
|
+
// SAFETY: same contract as `alloc`, which returns zeroed pages for large layouts.
|
|
52
73
|
unsafe { self.alloc(layout) }
|
|
53
74
|
} else {
|
|
75
|
+
// SAFETY: `layout` is forwarded unchanged to the system allocator.
|
|
54
76
|
unsafe { System.alloc_zeroed(layout) }
|
|
55
77
|
}
|
|
56
78
|
}
|
|
57
79
|
|
|
58
80
|
unsafe fn dealloc(&self, p: *mut u8, layout: Layout) {
|
|
59
81
|
if is_large(layout) {
|
|
60
|
-
|
|
82
|
+
// SAFETY: `p` and `layout` came from this allocator, so a large layout means `p`
|
|
83
|
+
// is the start of a mapping of exactly `mapped_len(layout.size())` bytes.
|
|
84
|
+
let rc = unsafe { libc::munmap(p.cast(), mapped_len(layout.size())) };
|
|
85
|
+
debug_assert_eq!(rc, 0, "munmap failed: the layout does not match the block");
|
|
61
86
|
} else {
|
|
87
|
+
// SAFETY: a small layout means `p` came from `System` with this same layout.
|
|
62
88
|
unsafe { System.dealloc(p, layout) };
|
|
63
89
|
}
|
|
64
90
|
}
|
|
65
91
|
|
|
66
92
|
unsafe fn realloc(&self, p: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
|
|
93
|
+
// SAFETY: the caller guarantees `new_size`, rounded up to `layout.align()`, does not
|
|
94
|
+
// overflow `isize::MAX`, and `layout.align()` is already a valid alignment.
|
|
67
95
|
let new_layout = unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) };
|
|
68
96
|
match (is_large(layout), is_large(new_layout)) {
|
|
97
|
+
// SAFETY: both layouts are small, so `p` is a `System` block of `layout`.
|
|
69
98
|
(false, false) => unsafe { System.realloc(p, layout, new_size) },
|
|
70
99
|
(true, true) => {
|
|
71
100
|
let (old_len, new_len) = (mapped_len(layout.size()), mapped_len(new_size));
|
|
72
101
|
if old_len == new_len {
|
|
73
102
|
return p;
|
|
74
103
|
}
|
|
104
|
+
// SAFETY: `p` is the start of a mapping of `old_len` bytes; on failure
|
|
105
|
+
// `mremap` leaves that mapping intact and the null result tells the caller.
|
|
75
106
|
let q = unsafe { libc::mremap(p.cast(), old_len, new_len, libc::MREMAP_MAYMOVE) };
|
|
76
107
|
if q == libc::MAP_FAILED {
|
|
77
108
|
ptr::null_mut()
|
|
@@ -80,8 +111,12 @@ unsafe impl GlobalAlloc for LargeBlockAlloc {
|
|
|
80
111
|
}
|
|
81
112
|
}
|
|
82
113
|
_ => {
|
|
114
|
+
// Crossing the threshold: new block from the matching allocator, copy, free old.
|
|
115
|
+
// SAFETY: `new_layout` is valid (see above) and non-zero sized.
|
|
83
116
|
let q = unsafe { self.alloc(new_layout) };
|
|
84
117
|
if !q.is_null() {
|
|
118
|
+
// SAFETY: `p` is valid for `layout.size()` bytes and `q` for `new_size`, so
|
|
119
|
+
// both are valid for the smaller of the two; they are distinct blocks.
|
|
85
120
|
unsafe {
|
|
86
121
|
ptr::copy_nonoverlapping(p, q, layout.size().min(new_size));
|
|
87
122
|
self.dealloc(p, layout);
|
|
@@ -94,6 +129,8 @@ unsafe impl GlobalAlloc for LargeBlockAlloc {
|
|
|
94
129
|
}
|
|
95
130
|
|
|
96
131
|
#[cfg(test)]
|
|
132
|
+
// The tests drive the allocator's raw interface directly, so each block would repeat the same note.
|
|
133
|
+
#[allow(clippy::undocumented_unsafe_blocks)]
|
|
97
134
|
mod tests {
|
|
98
135
|
use super::*;
|
|
99
136
|
|
|
@@ -109,6 +146,21 @@ mod tests {
|
|
|
109
146
|
}
|
|
110
147
|
}
|
|
111
148
|
|
|
149
|
+
/// Fails if the `#[global_allocator]` static in `lib.rs` is removed or mis-gated: only
|
|
150
|
+
/// this allocator hands out page-aligned multi-MiB blocks (glibc's mmap path returns
|
|
151
|
+
/// a 16-byte header offset).
|
|
152
|
+
#[test]
|
|
153
|
+
fn global_allocator_serves_large_vecs_from_page_aligned_mappings() {
|
|
154
|
+
let blocks: Vec<Vec<u8>> = (0..8).map(|_| Vec::with_capacity(2 * LARGE)).collect();
|
|
155
|
+
for b in &blocks {
|
|
156
|
+
assert_eq!(
|
|
157
|
+
b.as_ptr() as usize % PAGE,
|
|
158
|
+
0,
|
|
159
|
+
"LargeBlockAlloc is not the global allocator"
|
|
160
|
+
);
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
|
|
112
164
|
#[test]
|
|
113
165
|
fn realloc_keeps_contents_across_the_large_threshold() {
|
|
114
166
|
let a = LargeBlockAlloc;
|
|
@@ -74,15 +74,25 @@ pub(crate) fn lz4_frame_decode_into(src: &[u8], out: &mut Vec<u8>) -> Result<(),
|
|
|
74
74
|
Ok(())
|
|
75
75
|
}
|
|
76
76
|
|
|
77
|
+
/// Most a download reserves up front from `Content-Length`; a longer body
|
|
78
|
+
/// simply grows the buffer.
|
|
79
|
+
const MAX_PREALLOC: usize = 256 << 20;
|
|
80
|
+
|
|
77
81
|
/// Reads the body into one buffer sized from `Content-Length`. Frames are
|
|
78
82
|
/// freed as they are consumed, so a chunk peaks at about its own size instead
|
|
79
83
|
/// of the ~2x `Response::bytes()` reaches while joining collected frames.
|
|
80
84
|
async fn read_body(mut resp: reqwest::Response) -> Result<Bytes, reqwest::Error> {
|
|
81
|
-
//
|
|
82
|
-
|
|
85
|
+
// The header is untrusted: reserve at most `MAX_PREALLOC`, and if that
|
|
86
|
+
// reservation fails (address-space limits) or there is no length, fall
|
|
87
|
+
// back to `bytes()` instead of aborting the process on allocation failure.
|
|
88
|
+
let mut buf = Vec::new();
|
|
89
|
+
let reserved = resp
|
|
90
|
+
.content_length()
|
|
91
|
+
.map(|len| usize::try_from(len).unwrap_or(usize::MAX).min(MAX_PREALLOC))
|
|
92
|
+
.is_some_and(|len| buf.try_reserve_exact(len).is_ok());
|
|
93
|
+
if !reserved {
|
|
83
94
|
return resp.bytes().await;
|
|
84
|
-
}
|
|
85
|
-
let mut buf = Vec::with_capacity(len);
|
|
95
|
+
}
|
|
86
96
|
while let Some(frame) = resp.chunk().await? {
|
|
87
97
|
buf.extend_from_slice(&frame);
|
|
88
98
|
}
|
|
@@ -622,3 +632,71 @@ mod proptests {
|
|
|
622
632
|
}
|
|
623
633
|
}
|
|
624
634
|
}
|
|
635
|
+
|
|
636
|
+
#[cfg(test)]
|
|
637
|
+
mod read_body_tests {
|
|
638
|
+
use std::io::{Read, Write};
|
|
639
|
+
use std::net::TcpListener;
|
|
640
|
+
|
|
641
|
+
use super::*;
|
|
642
|
+
|
|
643
|
+
/// The production HTTP client (`DbClient::new` installs the rustls crypto provider).
|
|
644
|
+
async fn get(url: String) -> reqwest::Response {
|
|
645
|
+
DbClient::new("http://unused.invalid", "wh", "token")
|
|
646
|
+
.http
|
|
647
|
+
.get(url)
|
|
648
|
+
.send()
|
|
649
|
+
.await
|
|
650
|
+
.unwrap()
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
/// Serves `response` verbatim to the first connection, then closes it.
|
|
654
|
+
fn serve_once(response: &'static [u8]) -> String {
|
|
655
|
+
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
|
656
|
+
let url = format!("http://{}/", listener.local_addr().unwrap());
|
|
657
|
+
std::thread::spawn(move || {
|
|
658
|
+
let (mut stream, _) = listener.accept().unwrap();
|
|
659
|
+
let mut request = [0u8; 1024];
|
|
660
|
+
let _ = stream.read(&mut request);
|
|
661
|
+
stream.write_all(response).unwrap();
|
|
662
|
+
});
|
|
663
|
+
url
|
|
664
|
+
}
|
|
665
|
+
|
|
666
|
+
#[tokio::test]
|
|
667
|
+
async fn read_body_reads_a_content_length_body() {
|
|
668
|
+
let url = serve_once(b"HTTP/1.1 200 OK\r\nContent-Length: 11\r\n\r\nhello world");
|
|
669
|
+
let resp = get(url).await;
|
|
670
|
+
assert_eq!(read_body(resp).await.unwrap(), Bytes::from_static(b"hello world"));
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
#[tokio::test]
|
|
674
|
+
async fn read_body_falls_back_when_there_is_no_content_length() {
|
|
675
|
+
let url =
|
|
676
|
+
serve_once(b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nhello\r\n6\r\n world\r\n0\r\n\r\n");
|
|
677
|
+
let resp = get(url).await;
|
|
678
|
+
assert_eq!(read_body(resp).await.unwrap(), Bytes::from_static(b"hello world"));
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
#[tokio::test]
|
|
682
|
+
async fn read_body_errors_on_a_body_shorter_than_content_length() {
|
|
683
|
+
let url = serve_once(b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\nabc");
|
|
684
|
+
let resp = get(url).await;
|
|
685
|
+
assert!(read_body(resp).await.is_err());
|
|
686
|
+
}
|
|
687
|
+
|
|
688
|
+
#[test]
|
|
689
|
+
fn lz4_frame_decode_into_appends_every_frame_including_an_empty_one() {
|
|
690
|
+
fn frame(data: &[u8]) -> Vec<u8> {
|
|
691
|
+
let mut encoder = lz4_flex::frame::FrameEncoder::new(Vec::new());
|
|
692
|
+
encoder.write_all(data).unwrap();
|
|
693
|
+
encoder.finish().unwrap()
|
|
694
|
+
}
|
|
695
|
+
let mut src = frame(b"abc");
|
|
696
|
+
src.extend(frame(b""));
|
|
697
|
+
src.extend(frame(b"def"));
|
|
698
|
+
let mut out = b"prefix-".to_vec();
|
|
699
|
+
lz4_frame_decode_into(&src, &mut out).unwrap();
|
|
700
|
+
assert_eq!(out, b"prefix-abcdef");
|
|
701
|
+
}
|
|
702
|
+
}
|
|
@@ -557,38 +557,49 @@ impl DbClient {
|
|
|
557
557
|
let statement_id = statement_id.clone();
|
|
558
558
|
let stats = stats.clone();
|
|
559
559
|
handles.push(tokio::spawn(async move {
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
560
|
+
let work_loop = async {
|
|
561
|
+
loop {
|
|
562
|
+
let meta = { queue.lock().unwrap().pop_front() };
|
|
563
|
+
let Some(meta) = meta else { return Ok(()) };
|
|
564
|
+
let fetched = if meta.pre_resolved_links.is_empty() {
|
|
565
|
+
client
|
|
566
|
+
.fetch_chunk_index(&statement_id, meta.chunk_index, compressed, &stats)
|
|
567
|
+
.await
|
|
568
|
+
} else {
|
|
569
|
+
client
|
|
570
|
+
.fetch_pre_resolved_links(&meta.pre_resolved_links, compressed, &stats)
|
|
571
|
+
.await
|
|
572
|
+
};
|
|
573
|
+
match fetched {
|
|
574
|
+
Ok(blobs) => {
|
|
575
|
+
// One blob ⇒ `meta.row_count` (the whole chunk_index's
|
|
576
|
+
// declared count) and "this blob's count" are the same
|
|
577
|
+
// number -- see `ChunkItem::truncate_to`'s own doc comment.
|
|
578
|
+
let truncate_to = if blobs.len() == 1 { meta.row_count } else { None };
|
|
579
|
+
for blob in blobs {
|
|
580
|
+
let item = ChunkItem {
|
|
581
|
+
blob,
|
|
582
|
+
row_count: meta.row_count,
|
|
583
|
+
chunk_index: meta.chunk_index,
|
|
584
|
+
truncate_to,
|
|
585
|
+
};
|
|
586
|
+
if worker_tx.send(Ok(item)).await.is_err() {
|
|
587
|
+
return Ok(());
|
|
588
|
+
}
|
|
587
589
|
}
|
|
588
590
|
}
|
|
591
|
+
Err(e) => return Err(e),
|
|
589
592
|
}
|
|
590
|
-
Err(e) => return Err(e),
|
|
591
593
|
}
|
|
594
|
+
};
|
|
595
|
+
// The consumer dropping its receiver (a cancelled,
|
|
596
|
+
// timed-out or abandoned fetch) stops this worker at
|
|
597
|
+
// once, aborting its in-flight download instead of
|
|
598
|
+
// finishing -- and then starting -- chunks nobody reads.
|
|
599
|
+
tokio::select! {
|
|
600
|
+
biased;
|
|
601
|
+
() = worker_tx.closed() => Ok(()),
|
|
602
|
+
res = work_loop => res,
|
|
592
603
|
}
|
|
593
604
|
}));
|
|
594
605
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
#[cfg(target_os = "linux")]
|
|
1
|
+
#[cfg(all(target_os = "linux", target_env = "gnu", feature = "large-block-alloc"))]
|
|
2
2
|
mod alloc;
|
|
3
3
|
mod arrow_ffi;
|
|
4
4
|
pub mod client;
|
|
@@ -9,10 +9,6 @@ pub mod thrift;
|
|
|
9
9
|
|
|
10
10
|
use std::sync::{Arc, Mutex};
|
|
11
11
|
|
|
12
|
-
#[cfg(target_os = "linux")]
|
|
13
|
-
#[global_allocator]
|
|
14
|
-
static GLOBAL: alloc::LargeBlockAlloc = alloc::LargeBlockAlloc;
|
|
15
|
-
|
|
16
12
|
use arrow_array::RecordBatch;
|
|
17
13
|
use arrow_schema::{Schema, SchemaRef};
|
|
18
14
|
use pyo3::exceptions::{PyRuntimeError, PyStopAsyncIteration, PyValueError};
|
|
@@ -30,6 +26,10 @@ use client::{
|
|
|
30
26
|
use heartbeat::{HeartbeatStream, HeartbeatWait, Tick};
|
|
31
27
|
use pipeline::{NdjsonStream, ResultStream};
|
|
32
28
|
|
|
29
|
+
#[cfg(all(target_os = "linux", target_env = "gnu", feature = "large-block-alloc"))]
|
|
30
|
+
#[global_allocator]
|
|
31
|
+
static GLOBAL: alloc::LargeBlockAlloc = alloc::LargeBlockAlloc;
|
|
32
|
+
|
|
33
33
|
/// Writes any object implementing `__arrow_c_stream__` (a `Table`/
|
|
34
34
|
/// `RecordBatchReader` from this crate, arro3, pyarrow, or anything else
|
|
35
35
|
/// Arrow-C-Data-Interface-compatible) as Arrow-IPC stream bytes to `buf` (a
|
|
@@ -54,6 +54,10 @@ impl Write for LineCollector {
|
|
|
54
54
|
}
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
+
fn encode_err(e: impl std::fmt::Display) -> ApiError {
|
|
58
|
+
ApiError::permanent(format!("NDJSON encode error: {e}"))
|
|
59
|
+
}
|
|
60
|
+
|
|
57
61
|
/// Converts one chunk's decoded batches into NDJSON lines, one per row, in
|
|
58
62
|
/// arro3-`write_ndjson(explicit_nulls=True)`-compatible format: null-valued
|
|
59
63
|
/// keys stay present as JSON `null` rather than being omitted, and a
|
|
@@ -83,12 +87,8 @@ fn encode_ndjson_lines(batches: &[RecordBatch], non_finite_as_string: bool) -> R
|
|
|
83
87
|
let builder = arrow_json::WriterBuilder::new().with_explicit_nulls(true);
|
|
84
88
|
let mut writer = builder.build::<_, arrow_json::writer::LineDelimited>(&mut output);
|
|
85
89
|
let refs: Vec<&RecordBatch> = batches.iter().collect();
|
|
86
|
-
writer
|
|
87
|
-
|
|
88
|
-
.map_err(|e| ApiError::permanent(format!("NDJSON encode error: {e}")))?;
|
|
89
|
-
writer
|
|
90
|
-
.finish()
|
|
91
|
-
.map_err(|e| ApiError::permanent(format!("NDJSON encode error: {e}")))?;
|
|
90
|
+
writer.write_batches(&refs).map_err(encode_err)?;
|
|
91
|
+
writer.finish().map_err(encode_err)?;
|
|
92
92
|
}
|
|
93
93
|
if !output.pending.is_empty() {
|
|
94
94
|
return Err(ApiError::permanent("NDJSON encode produced an unterminated row"));
|
|
@@ -403,12 +403,8 @@ mod tests {
|
|
|
403
403
|
let builder = arrow_json::WriterBuilder::new().with_explicit_nulls(true);
|
|
404
404
|
let mut writer = builder.build::<_, arrow_json::writer::LineDelimited>(&mut buf);
|
|
405
405
|
let refs: Vec<&RecordBatch> = batches.iter().collect();
|
|
406
|
-
writer
|
|
407
|
-
|
|
408
|
-
.map_err(|e| ApiError::permanent(format!("NDJSON encode error: {e}")))?;
|
|
409
|
-
writer
|
|
410
|
-
.finish()
|
|
411
|
-
.map_err(|e| ApiError::permanent(format!("NDJSON encode error: {e}")))?;
|
|
406
|
+
writer.write_batches(&refs).map_err(encode_err)?;
|
|
407
|
+
writer.finish().map_err(encode_err)?;
|
|
412
408
|
}
|
|
413
409
|
let mut lines: Vec<String> = String::from_utf8(buf)
|
|
414
410
|
.map_err(|e| ApiError::permanent(format!("NDJSON encode produced invalid UTF-8: {e}")))?
|
|
@@ -626,17 +626,35 @@ async fn run_thrift_fetch_loop(
|
|
|
626
626
|
let compressed_flag = compressed_flag.clone();
|
|
627
627
|
let worker_stats = stats.clone();
|
|
628
628
|
worker_handles.push(tokio::spawn(async move {
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
629
|
+
let work_loop = async {
|
|
630
|
+
loop {
|
|
631
|
+
let work = { link_rx.lock().await.recv().await };
|
|
632
|
+
let Some(work) = work else { return };
|
|
633
|
+
let compressed = compressed_flag.load(std::sync::atomic::Ordering::Relaxed);
|
|
634
|
+
let result = fetch_thrift_link(&client, work, compressed, &worker_stats).await;
|
|
635
|
+
if out_tx.send(result).await.is_err() {
|
|
636
|
+
return;
|
|
637
|
+
}
|
|
636
638
|
}
|
|
639
|
+
};
|
|
640
|
+
// The consumer (`ResultStream`/`NdjsonStream`) dropping its
|
|
641
|
+
// receiver -- a cancelled or abandoned fetch -- stops this worker
|
|
642
|
+
// at once, aborting its in-flight download, instead of finishing
|
|
643
|
+
// a chunk nobody will read. See `queue_link`.
|
|
644
|
+
tokio::select! {
|
|
645
|
+
biased;
|
|
646
|
+
() = out_tx.closed() => {}
|
|
647
|
+
() = work_loop => {}
|
|
637
648
|
}
|
|
638
649
|
}));
|
|
639
650
|
}
|
|
651
|
+
// Only the workers hold the receiving end from here on: if every worker
|
|
652
|
+
// has exited, `link_tx.send` fails instead of waiting forever for room in
|
|
653
|
+
// a queue nobody drains -- which used to hang this task (and skip the
|
|
654
|
+
// `CloseOperation`/`CloseSession` cleanup in `drive_thrift_fetch_loop`)
|
|
655
|
+
// whenever a consumer abandoned a result with more than about
|
|
656
|
+
// `2 * concurrency` links left to queue.
|
|
657
|
+
drop(link_rx);
|
|
640
658
|
|
|
641
659
|
let mut chunk_index: i64 = 0;
|
|
642
660
|
let mut pending = initial_rowset;
|
|
@@ -658,11 +676,20 @@ async fn run_thrift_fetch_loop(
|
|
|
658
676
|
// Direct results may already have confirmed it during submission. In
|
|
659
677
|
// that case downloads can overlap the very first FetchResults RPC too.
|
|
660
678
|
let mut pending_until_confirmed: Vec<ThriftLinkWork> = Vec::new();
|
|
661
|
-
loop {
|
|
679
|
+
'discover: loop {
|
|
662
680
|
let (row_set, has_more) = if let Some(v) = pending.take() {
|
|
663
681
|
v
|
|
664
682
|
} else {
|
|
665
|
-
|
|
683
|
+
// Stop discovering as soon as the consumer is gone (see
|
|
684
|
+
// `queue_link`) -- no further `FetchResults` round trips for a
|
|
685
|
+
// result nobody reads; `drive_thrift_fetch_loop` then closes the
|
|
686
|
+
// operation.
|
|
687
|
+
let fetched = tokio::select! {
|
|
688
|
+
biased;
|
|
689
|
+
() = tx.closed() => break 'discover,
|
|
690
|
+
fetched = client.thrift_fetch_results_raw(operation, stats) => fetched,
|
|
691
|
+
};
|
|
692
|
+
match fetched {
|
|
666
693
|
Ok(fr) => {
|
|
667
694
|
if let Some(e) = fr.status.error() {
|
|
668
695
|
let _ = tx
|
|
@@ -752,14 +779,11 @@ async fn run_thrift_fetch_loop(
|
|
|
752
779
|
// every worker is currently busy, so this await is exactly
|
|
753
780
|
// the same "peak buffered stays at ~concurrency" trade-off
|
|
754
781
|
// `fetch_chunks_with_backpressure`'s own doc comment
|
|
755
|
-
// describes.
|
|
756
|
-
// worker
|
|
757
|
-
//
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
// regardless.
|
|
761
|
-
if link_tx.send(work).await.is_err() {
|
|
762
|
-
break;
|
|
782
|
+
// describes. `false` means the consumer is gone or every
|
|
783
|
+
// worker already exited (e.g. all panicked) -- stop
|
|
784
|
+
// discovering either way, see `queue_link`.
|
|
785
|
+
if !queue_link(&link_tx, work, tx).await {
|
|
786
|
+
break 'discover;
|
|
763
787
|
}
|
|
764
788
|
} else {
|
|
765
789
|
// See this function's own comment above `metadata_confirmed`.
|
|
@@ -769,8 +793,8 @@ async fn run_thrift_fetch_loop(
|
|
|
769
793
|
|
|
770
794
|
if metadata_confirmed {
|
|
771
795
|
for work in pending_until_confirmed.drain(..) {
|
|
772
|
-
if link_tx
|
|
773
|
-
break;
|
|
796
|
+
if !queue_link(&link_tx, work, tx).await {
|
|
797
|
+
break 'discover;
|
|
774
798
|
}
|
|
775
799
|
}
|
|
776
800
|
}
|
|
@@ -783,7 +807,7 @@ async fn run_thrift_fetch_loop(
|
|
|
783
807
|
// Final flush for the never-confirmed case -- see `metadata_confirmed`'s
|
|
784
808
|
// own comment above.
|
|
785
809
|
for work in pending_until_confirmed.drain(..) {
|
|
786
|
-
if link_tx
|
|
810
|
+
if !queue_link(&link_tx, work, tx).await {
|
|
787
811
|
break;
|
|
788
812
|
}
|
|
789
813
|
}
|
|
@@ -794,6 +818,28 @@ async fn run_thrift_fetch_loop(
|
|
|
794
818
|
}
|
|
795
819
|
}
|
|
796
820
|
|
|
821
|
+
/// Hands one discovered link to the download workers, unless the consumer
|
|
822
|
+
/// has dropped its end of `out` (a cancelled, timed-out or abandoned
|
|
823
|
+
/// `ResultStream`/`NdjsonStream`) first. Returns `false` when discovery
|
|
824
|
+
/// should stop: the consumer is gone, or every worker has already exited.
|
|
825
|
+
///
|
|
826
|
+
/// Without the `out.closed()` arm, an abandoned result kept this producer
|
|
827
|
+
/// issuing `FetchResults` and queueing links for the whole statement, and --
|
|
828
|
+
/// once the workers had noticed the closed channel and exited -- blocked
|
|
829
|
+
/// forever on a full link queue, so `drive_thrift_fetch_loop` never reached
|
|
830
|
+
/// `CloseOperation`/`CloseSession` and its task leaked.
|
|
831
|
+
async fn queue_link(
|
|
832
|
+
link_tx: &mpsc::Sender<ThriftLinkWork>,
|
|
833
|
+
work: ThriftLinkWork,
|
|
834
|
+
out: &mpsc::Sender<Result<ChunkItem, ApiError>>,
|
|
835
|
+
) -> bool {
|
|
836
|
+
tokio::select! {
|
|
837
|
+
biased;
|
|
838
|
+
() = out.closed() => false,
|
|
839
|
+
sent = link_tx.send(work) => sent.is_ok(),
|
|
840
|
+
}
|
|
841
|
+
}
|
|
842
|
+
|
|
797
843
|
/// A panicked download worker has already pulled its `ThriftLinkWork` off
|
|
798
844
|
/// the queue and sends nothing on `out_tx` -- silently discarding that (a
|
|
799
845
|
/// bare `let _ = handle.await`) would mean the channel closing normally
|
|
@@ -867,6 +913,33 @@ fn build_inline_blob(
|
|
|
867
913
|
mod tests {
|
|
868
914
|
use super::*;
|
|
869
915
|
|
|
916
|
+
#[test]
|
|
917
|
+
fn build_inline_blob_decodes_every_lz4_frame_including_an_empty_one() {
|
|
918
|
+
use std::io::Write;
|
|
919
|
+
|
|
920
|
+
fn frame(data: &[u8]) -> bytes::Bytes {
|
|
921
|
+
let mut encoder = lz4_flex::frame::FrameEncoder::new(Vec::new());
|
|
922
|
+
encoder.write_all(data).unwrap();
|
|
923
|
+
bytes::Bytes::from(encoder.finish().unwrap())
|
|
924
|
+
}
|
|
925
|
+
let mut one_batch = frame(b"AAAA").to_vec();
|
|
926
|
+
one_batch.extend(frame(b"")); // a valid frame with no content must not end decoding
|
|
927
|
+
one_batch.extend(frame(b"BBBB"));
|
|
928
|
+
let batches = vec![
|
|
929
|
+
thrift::ArrowBatch {
|
|
930
|
+
batch: bytes::Bytes::from(one_batch),
|
|
931
|
+
row_count: 3,
|
|
932
|
+
},
|
|
933
|
+
thrift::ArrowBatch {
|
|
934
|
+
batch: frame(b"CC"),
|
|
935
|
+
row_count: 2,
|
|
936
|
+
},
|
|
937
|
+
];
|
|
938
|
+
let (blob, rows) = build_inline_blob(Some(bytes::Bytes::from_static(b"S-")), batches, true).unwrap();
|
|
939
|
+
assert_eq!(&blob[..], b"S-AAAABBBBCC");
|
|
940
|
+
assert_eq!(rows, 5);
|
|
941
|
+
}
|
|
942
|
+
|
|
870
943
|
fn ok_worker() -> tokio::task::JoinHandle<()> {
|
|
871
944
|
tokio::spawn(async {})
|
|
872
945
|
}
|
|
@@ -1755,3 +1755,116 @@ async fn abandoning_the_submit_poll_wait_releases_the_session_reservation() {
|
|
|
1755
1755
|
"every abandoned attempt must release its reservation, so each one can create a session"
|
|
1756
1756
|
);
|
|
1757
1757
|
}
|
|
1758
|
+
|
|
1759
|
+
/// A raw cloud-fetch blob server for `sea_dropping_a_partly_read_result_aborts_in_flight_downloads`:
|
|
1760
|
+
/// `/blob-0` is served in full; any other path gets its headers and first
|
|
1761
|
+
/// few bytes, then the connection is held open until the *client* hangs
|
|
1762
|
+
/// up, which is signalled on `hung_up` (`started` fires once such a stalled
|
|
1763
|
+
/// download is in flight). wiremock can't show a client abandoning a
|
|
1764
|
+
/// response mid-body, hence a bare socket, same as `client/download.rs`'s own
|
|
1765
|
+
/// truncated-response tests.
|
|
1766
|
+
async fn spawn_stalling_blob_server(body: Vec<u8>) -> (String, Arc<tokio::sync::Notify>, Arc<tokio::sync::Notify>) {
|
|
1767
|
+
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
|
1768
|
+
|
|
1769
|
+
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
1770
|
+
let base = format!("http://{}", listener.local_addr().unwrap());
|
|
1771
|
+
let started = Arc::new(tokio::sync::Notify::new());
|
|
1772
|
+
let hung_up = Arc::new(tokio::sync::Notify::new());
|
|
1773
|
+
let (started_srv, hung_up_srv) = (started.clone(), hung_up.clone());
|
|
1774
|
+
tokio::spawn(async move {
|
|
1775
|
+
loop {
|
|
1776
|
+
let Ok((mut socket, _)) = listener.accept().await else {
|
|
1777
|
+
return;
|
|
1778
|
+
};
|
|
1779
|
+
let body = body.clone();
|
|
1780
|
+
let (started, hung_up) = (started_srv.clone(), hung_up_srv.clone());
|
|
1781
|
+
tokio::spawn(async move {
|
|
1782
|
+
let mut request = Vec::new();
|
|
1783
|
+
while !request.ends_with(b"\r\n\r\n") {
|
|
1784
|
+
match socket.read_u8().await {
|
|
1785
|
+
Ok(b) => request.push(b),
|
|
1786
|
+
Err(_) => return,
|
|
1787
|
+
}
|
|
1788
|
+
}
|
|
1789
|
+
let head = format!(
|
|
1790
|
+
"HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
|
|
1791
|
+
body.len()
|
|
1792
|
+
);
|
|
1793
|
+
socket.write_all(head.as_bytes()).await.unwrap();
|
|
1794
|
+
if request.starts_with(b"GET /blob-0 ") {
|
|
1795
|
+
socket.write_all(&body).await.unwrap();
|
|
1796
|
+
let _ = socket.shutdown().await;
|
|
1797
|
+
return;
|
|
1798
|
+
}
|
|
1799
|
+
socket.write_all(&body[..10]).await.unwrap();
|
|
1800
|
+
started.notify_one();
|
|
1801
|
+
let mut buf = [0u8; 64];
|
|
1802
|
+
loop {
|
|
1803
|
+
match socket.read(&mut buf).await {
|
|
1804
|
+
Ok(0) | Err(_) => break,
|
|
1805
|
+
Ok(_) => {}
|
|
1806
|
+
}
|
|
1807
|
+
}
|
|
1808
|
+
hung_up.notify_one();
|
|
1809
|
+
});
|
|
1810
|
+
}
|
|
1811
|
+
});
|
|
1812
|
+
(base, started, hung_up)
|
|
1813
|
+
}
|
|
1814
|
+
|
|
1815
|
+
/// Dropping a partly-read `ResultStream` (a cancelled or abandoned fetch)
|
|
1816
|
+
/// must abort the chunk download already in flight, not let it run to
|
|
1817
|
+
/// completion -- here that would mean waiting out the full `http_timeout`
|
|
1818
|
+
/// on a download that never finishes. Before this was fixed the worker
|
|
1819
|
+
/// only noticed the dropped consumer when it tried to hand over a finished
|
|
1820
|
+
/// chunk.
|
|
1821
|
+
#[tokio::test]
|
|
1822
|
+
async fn sea_dropping_a_partly_read_result_aborts_in_flight_downloads() {
|
|
1823
|
+
let server = MockServer::start().await;
|
|
1824
|
+
mount_warehouse_running(&server).await;
|
|
1825
|
+
let (blob_base, started, hung_up) = spawn_stalling_blob_server(build_chunk_bytes(0, 5)).await;
|
|
1826
|
+
let chunks: Vec<_> = (0..3).map(|i| json!({"chunk_index": i, "row_count": 5})).collect();
|
|
1827
|
+
Mock::given(method("POST"))
|
|
1828
|
+
.and(path("/api/2.0/sql/statements"))
|
|
1829
|
+
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
|
1830
|
+
"statement_id": STATEMENT_ID,
|
|
1831
|
+
"status": {"state": "SUCCEEDED"},
|
|
1832
|
+
"manifest": {
|
|
1833
|
+
"chunks": chunks,
|
|
1834
|
+
"schema": {"columns": [
|
|
1835
|
+
{"name": "id", "type_name": "LONG"},
|
|
1836
|
+
{"name": "label", "type_name": "STRING"},
|
|
1837
|
+
]},
|
|
1838
|
+
},
|
|
1839
|
+
})))
|
|
1840
|
+
.mount(&server)
|
|
1841
|
+
.await;
|
|
1842
|
+
for i in 0..3 {
|
|
1843
|
+
Mock::given(method("GET"))
|
|
1844
|
+
.and(path(format!(
|
|
1845
|
+
"/api/2.0/sql/statements/{STATEMENT_ID}/result/chunks/{i}"
|
|
1846
|
+
)))
|
|
1847
|
+
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
|
1848
|
+
"external_links": [{"external_link": format!("{blob_base}/blob-{i}")}]
|
|
1849
|
+
})))
|
|
1850
|
+
.mount(&server)
|
|
1851
|
+
.await;
|
|
1852
|
+
}
|
|
1853
|
+
|
|
1854
|
+
let client = Arc::new(
|
|
1855
|
+
DbClient::new(&server.uri(), WAREHOUSE_ID, "fake-token")
|
|
1856
|
+
.with_protocol(Protocol::Sea)
|
|
1857
|
+
.with_concurrency(1),
|
|
1858
|
+
);
|
|
1859
|
+
let mut stream = execute_lazy(client, "SELECT * FROM t", None, None, None).await.unwrap();
|
|
1860
|
+
let (batches, _schema) = stream.fetchmany_arrow(1).await.unwrap();
|
|
1861
|
+
assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 1);
|
|
1862
|
+
tokio::time::timeout(std::time::Duration::from_secs(5), started.notified())
|
|
1863
|
+
.await
|
|
1864
|
+
.expect("the next chunk's download must be in flight before the drop");
|
|
1865
|
+
|
|
1866
|
+
drop(stream);
|
|
1867
|
+
tokio::time::timeout(std::time::Duration::from_secs(2), hung_up.notified())
|
|
1868
|
+
.await
|
|
1869
|
+
.expect("dropping the result must abort the in-flight download promptly");
|
|
1870
|
+
}
|
|
@@ -2068,3 +2068,117 @@ async fn thrift_abandoning_the_submit_poll_wait_closes_its_session() {
|
|
|
2068
2068
|
wait_for_calls(&close_calls, 1).await;
|
|
2069
2069
|
assert_eq!(close_calls.load(Ordering::SeqCst), 1);
|
|
2070
2070
|
}
|
|
2071
|
+
|
|
2072
|
+
/// Abandoning a partly-read result -- the consumer drops its `ResultStream`
|
|
2073
|
+
/// after the first chunk (a cancelled `fetchall`, a client that stopped
|
|
2074
|
+
/// reading a stream) -- must stop the background fetch loop and close the
|
|
2075
|
+
/// operation promptly. It used to keep the remaining links queued: once the
|
|
2076
|
+
/// one worker noticed the closed channel and exited, the producer blocked
|
|
2077
|
+
/// forever on a full link queue, so `CloseOperation` never went out (and
|
|
2078
|
+
/// the in-flight download ran to completion first in any case). The slow
|
|
2079
|
+
/// links here take 5s, so a `CloseOperation` within `wait_for_calls`' ~1s
|
|
2080
|
+
/// also proves the in-flight download was aborted rather than finished.
|
|
2081
|
+
#[tokio::test]
|
|
2082
|
+
async fn thrift_dropping_a_partly_read_result_aborts_downloads_and_closes_the_operation() {
|
|
2083
|
+
let server = MockServer::start().await;
|
|
2084
|
+
mount_open_session_always(&server, b"sess-abandon").await;
|
|
2085
|
+
Mock::given(method("POST"))
|
|
2086
|
+
.and(path(thrift_path()))
|
|
2087
|
+
.and(IsThriftRpc("ExecuteStatement"))
|
|
2088
|
+
.respond_with(ResponseTemplate::new(200).set_body_raw(
|
|
2089
|
+
build_execute_statement_resp(b"op-abandon", b"opsecret-abandon", None),
|
|
2090
|
+
"application/x-thrift",
|
|
2091
|
+
))
|
|
2092
|
+
.mount(&server)
|
|
2093
|
+
.await;
|
|
2094
|
+
Mock::given(method("POST"))
|
|
2095
|
+
.and(path(thrift_path()))
|
|
2096
|
+
.and(IsThriftRpc("GetOperationStatus"))
|
|
2097
|
+
.respond_with(ResponseTemplate::new(200).set_body_raw(
|
|
2098
|
+
build_get_operation_status_resp(operation_state::FINISHED, None),
|
|
2099
|
+
"application/x-thrift",
|
|
2100
|
+
))
|
|
2101
|
+
.mount(&server)
|
|
2102
|
+
.await;
|
|
2103
|
+
let cancel_calls = mount_cancel_operation_ok(&server).await;
|
|
2104
|
+
let close_calls = Arc::new(AtomicUsize::new(0));
|
|
2105
|
+
let close_calls_for_mock = close_calls.clone();
|
|
2106
|
+
Mock::given(method("POST"))
|
|
2107
|
+
.and(path(thrift_path()))
|
|
2108
|
+
.and(IsThriftRpc("CloseOperation"))
|
|
2109
|
+
.respond_with(move |_req: &Request| {
|
|
2110
|
+
close_calls_for_mock.fetch_add(1, Ordering::SeqCst);
|
|
2111
|
+
ResponseTemplate::new(200).set_body_raw(build_close_operation_resp(), "application/x-thrift")
|
|
2112
|
+
})
|
|
2113
|
+
.mount(&server)
|
|
2114
|
+
.await;
|
|
2115
|
+
|
|
2116
|
+
// More links than one worker plus its one-slot queue can hold.
|
|
2117
|
+
const N_LINKS: i64 = 8;
|
|
2118
|
+
const ROWS_PER_CHUNK: i64 = 5;
|
|
2119
|
+
let uri = server.uri();
|
|
2120
|
+
let links: Vec<(String, i64)> = (0..N_LINKS)
|
|
2121
|
+
.map(|i| (format!("{uri}/_data/abandon-{i}"), ROWS_PER_CHUNK))
|
|
2122
|
+
.collect();
|
|
2123
|
+
Mock::given(method("POST"))
|
|
2124
|
+
.and(path(thrift_path()))
|
|
2125
|
+
.and(IsThriftRpc("FetchResults"))
|
|
2126
|
+
.respond_with(ResponseTemplate::new(200).set_body_raw(
|
|
2127
|
+
build_fetch_results_resp(&FetchSpec {
|
|
2128
|
+
has_more_rows: false,
|
|
2129
|
+
result_links: links,
|
|
2130
|
+
metadata: Some((false, None)),
|
|
2131
|
+
..Default::default()
|
|
2132
|
+
}),
|
|
2133
|
+
"application/x-thrift",
|
|
2134
|
+
))
|
|
2135
|
+
.mount(&server)
|
|
2136
|
+
.await;
|
|
2137
|
+
let schema = test_schema();
|
|
2138
|
+
let data_calls = Arc::new(AtomicUsize::new(0));
|
|
2139
|
+
for i in 0..N_LINKS {
|
|
2140
|
+
let bytes = build_full_stream_bytes(&schema, i * ROWS_PER_CHUNK, (i + 1) * ROWS_PER_CHUNK);
|
|
2141
|
+
let delay = std::time::Duration::from_millis(if i == 0 { 0 } else { 5_000 });
|
|
2142
|
+
let data_calls_for_mock = data_calls.clone();
|
|
2143
|
+
Mock::given(method("GET"))
|
|
2144
|
+
.and(path(format!("/_data/abandon-{i}")))
|
|
2145
|
+
.respond_with(move |_req: &Request| {
|
|
2146
|
+
data_calls_for_mock.fetch_add(1, Ordering::SeqCst);
|
|
2147
|
+
ResponseTemplate::new(200)
|
|
2148
|
+
.set_body_raw(bytes.clone(), "application/vnd.apache.arrow.stream")
|
|
2149
|
+
.set_delay(delay)
|
|
2150
|
+
})
|
|
2151
|
+
.mount(&server)
|
|
2152
|
+
.await;
|
|
2153
|
+
}
|
|
2154
|
+
|
|
2155
|
+
let client = Arc::new(
|
|
2156
|
+
DbClient::new(&server.uri(), WAREHOUSE_ID, "fake-token")
|
|
2157
|
+
.with_protocol(Protocol::Thrift)
|
|
2158
|
+
.with_concurrency(1),
|
|
2159
|
+
);
|
|
2160
|
+
let mut stream = execute_lazy_thrift(client, "SELECT * FROM t", None, None, None)
|
|
2161
|
+
.await
|
|
2162
|
+
.unwrap();
|
|
2163
|
+
let (batches, _schema) = stream.fetchmany_arrow(1).await.unwrap();
|
|
2164
|
+
assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 1);
|
|
2165
|
+
wait_for_calls(&data_calls, 2).await; // the second (slow) download is in flight
|
|
2166
|
+
drop(stream);
|
|
2167
|
+
|
|
2168
|
+
wait_for_calls(&close_calls, 1).await;
|
|
2169
|
+
assert_eq!(
|
|
2170
|
+
close_calls.load(Ordering::SeqCst),
|
|
2171
|
+
1,
|
|
2172
|
+
"CloseOperation must go out promptly once the consumer drops a partly-read result"
|
|
2173
|
+
);
|
|
2174
|
+
assert_eq!(
|
|
2175
|
+
data_calls.load(Ordering::SeqCst),
|
|
2176
|
+
2,
|
|
2177
|
+
"no further download may start after the consumer is gone"
|
|
2178
|
+
);
|
|
2179
|
+
assert_eq!(
|
|
2180
|
+
cancel_calls.load(Ordering::SeqCst),
|
|
2181
|
+
0,
|
|
2182
|
+
"a finished operation is closed, not cancelled"
|
|
2183
|
+
);
|
|
2184
|
+
}
|
|
@@ -13,14 +13,11 @@ from __future__ import annotations
|
|
|
13
13
|
|
|
14
14
|
import contextlib
|
|
15
15
|
from collections.abc import AsyncIterator, Awaitable, Iterator
|
|
16
|
-
from typing import
|
|
16
|
+
from typing import Any, BinaryIO, TypeVar, cast
|
|
17
17
|
|
|
18
18
|
from . import _core
|
|
19
19
|
from .client import DatabricksClient
|
|
20
20
|
|
|
21
|
-
if TYPE_CHECKING:
|
|
22
|
-
import asyncio
|
|
23
|
-
|
|
24
21
|
__all__ = [
|
|
25
22
|
"HEARTBEAT",
|
|
26
23
|
"QueryTimeout",
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/proptest-regressions/json_convert.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/tests/wiremock_volume_files.rs
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/tests_py/test_stream_ndjson_lines.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{arrowbricks-5.1.0 → arrowbricks-5.1.2}/rust/arrowbricks_core/tests_py/test_token_provider.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|