arrowbricks 5.0.3__tar.gz → 5.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/PKG-INFO +1 -1
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/pyproject.toml +1 -1
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/Cargo.lock +2 -1
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/Cargo.toml +10 -2
- arrowbricks-5.1.1/rust/arrowbricks_core/src/alloc.rs +278 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/src/client/download.rs +82 -4
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/src/lib.rs +6 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/src/pipeline/ndjson.rs +8 -12
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/src/pipeline/thrift_exec.rs +27 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/src/arrowbricks/_streaming.py +1 -4
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/LICENSE +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/README.md +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/.gitignore +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/README.md +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/examples/duckdb_query.py +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/examples/fastapi_sse.py +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/proptest-regressions/json_convert.txt +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/rustfmt.toml +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/src/arrow_ffi.rs +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/src/client/error.rs +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/src/client/model.rs +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/src/client/sea.rs +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/src/client/thrift_rpc.rs +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/src/client/volume.rs +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/src/client.rs +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/src/heartbeat.rs +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/src/json_convert.rs +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/src/pipeline/reorder.rs +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/src/pipeline/sea.rs +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/src/pipeline/stats.rs +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/src/pipeline/test_support.rs +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/src/pipeline.rs +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/src/thrift.rs +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/tests/common/mod.rs +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/tests/wiremock_pipeline.rs +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/tests/wiremock_thrift.rs +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/tests/wiremock_volume_files.rs +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/tests_py/conftest.py +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/tests_py/test_ipc_stream.py +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/tests_py/test_parameters.py +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/tests_py/test_stream_ndjson_lines.py +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/tests_py/test_streaming.py +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/tests_py/test_thrift.py +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/tests_py/test_token_provider.py +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/tests_py/test_volume_files.py +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/tests_py/thrift_mock.py +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/src/arrowbricks/__init__.py +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/src/arrowbricks/_core.pyi +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/src/arrowbricks/client.py +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/src/arrowbricks/cursor.py +0 -0
- {arrowbricks-5.0.3 → arrowbricks-5.1.1}/src/arrowbricks/py.typed +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "arrowbricks"
|
|
3
|
-
version = "5.
|
|
3
|
+
version = "5.1.1"
|
|
4
4
|
description = "Runs SQL against a Databricks SQL warehouse via the Statement Execution API and hands you the result as Arrow -- a DB-API-ish Cursor (fetchone/fetchmany/fetchall/fetchall_arrow) or NDJSON streaming. Rust/PyO3 core throughout -- zero required runtime dependencies."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "MIT"
|
|
@@ -177,7 +177,7 @@ dependencies = [
|
|
|
177
177
|
|
|
178
178
|
[[package]]
|
|
179
179
|
name = "arrowbricks_core"
|
|
180
|
-
version = "5.
|
|
180
|
+
version = "5.1.1"
|
|
181
181
|
dependencies = [
|
|
182
182
|
"arrow-array",
|
|
183
183
|
"arrow-buffer",
|
|
@@ -188,6 +188,7 @@ dependencies = [
|
|
|
188
188
|
"bytes",
|
|
189
189
|
"chrono",
|
|
190
190
|
"hyper-rustls",
|
|
191
|
+
"libc",
|
|
191
192
|
"lz4_flex",
|
|
192
193
|
"memchr",
|
|
193
194
|
"proptest",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[package]
|
|
2
2
|
name = "arrowbricks_core"
|
|
3
|
-
version = "5.
|
|
3
|
+
version = "5.1.1"
|
|
4
4
|
edition = "2024"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
|
|
@@ -69,14 +69,22 @@ serde = { version = "1.0.229", features = ["derive"] }
|
|
|
69
69
|
serde_json = "1.0.151"
|
|
70
70
|
tokio = { version = "1.53.1", features = ["rt-multi-thread", "macros"] }
|
|
71
71
|
|
|
72
|
+
# Already in the dependency graph (via tokio/mio); src/alloc.rs uses mmap directly.
|
|
73
|
+
[target.'cfg(target_os = "linux")'.dependencies]
|
|
74
|
+
libc = "0.2"
|
|
75
|
+
|
|
72
76
|
[features]
|
|
73
|
-
default = ["extension-module"]
|
|
77
|
+
default = ["extension-module", "large-block-alloc"]
|
|
74
78
|
# `extension-module` skips linking against libpython -- correct for the
|
|
75
79
|
# cdylib maturin actually ships (Python provides those symbols at import
|
|
76
80
|
# time), but it breaks a plain `cargo test`/`cargo bench` binary, which does
|
|
77
81
|
# need to link libpython itself. Tests/benches build with
|
|
78
82
|
# `--no-default-features` to get a normal link.
|
|
79
83
|
extension-module = ["pyo3/extension-module"]
|
|
84
|
+
# Linux/glibc only: returns buffers of 1 MiB or more to the OS when freed (src/alloc.rs).
|
|
85
|
+
# A Rust crate depending on this one with its own global allocator opts out with
|
|
86
|
+
# `default-features = false`.
|
|
87
|
+
large-block-alloc = []
|
|
80
88
|
|
|
81
89
|
[dev-dependencies]
|
|
82
90
|
tokio = { version = "1.53.1", features = ["rt-multi-thread", "macros", "test-util", "net", "io-util"] }
|
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
//! Returns large buffers to the OS as soon as they are freed.
|
|
2
|
+
//!
|
|
3
|
+
//! glibc raises its mmap threshold (up to 32 MiB) whenever a large mapped
|
|
4
|
+
//! block is freed, so later multi-MiB buffers come from the fragmented heap
|
|
5
|
+
//! and are never handed back: repeating one query in a single process grew
|
|
6
|
+
//! idle RSS from 200 MB to 1.4 GB. Blocks of `LARGE` bytes or more bypass
|
|
7
|
+
//! malloc here (`mmap`/`munmap`, `mremap` to grow without copying); smaller
|
|
8
|
+
//! ones use the system allocator. Only Rust allocations in this extension are
|
|
9
|
+
//! affected, not the host process's malloc. Linux with glibc only (the one
|
|
10
|
+
//! configuration this was measured on); see `lib.rs` for the gate.
|
|
11
|
+
//!
|
|
12
|
+
//! Failure mode: `mmap`/`mremap` return null when the address-space limit is
|
|
13
|
+
//! hit (`RLIMIT_AS`, strict overcommit, `vm.max_map_count`), which Rust turns
|
|
14
|
+
//! into an abort like any other failed allocation.
|
|
15
|
+
|
|
16
|
+
#![warn(clippy::undocumented_unsafe_blocks)]
|
|
17
|
+
|
|
18
|
+
use std::alloc::{GlobalAlloc, Layout, System};
|
|
19
|
+
use std::ptr;
|
|
20
|
+
|
|
21
|
+
const LARGE: usize = 1 << 20;
|
|
22
|
+
const PAGE: usize = 4096;
|
|
23
|
+
|
|
24
|
+
pub struct LargeBlockAlloc;
|
|
25
|
+
|
|
26
|
+
/// Must stay a pure function of the `Layout`: `alloc`, `realloc` and `dealloc`
|
|
27
|
+
/// each call it on the same block and must get the same answer, otherwise a
|
|
28
|
+
/// block is freed by the wrong allocator.
|
|
29
|
+
fn is_large(layout: Layout) -> bool {
|
|
30
|
+
layout.size() >= LARGE && layout.align() <= PAGE
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/// The length the kernel maps for `size` bytes. Cannot overflow: a `Layout`
|
|
34
|
+
/// size is at most `isize::MAX`.
|
|
35
|
+
fn mapped_len(size: usize) -> usize {
|
|
36
|
+
size.next_multiple_of(PAGE)
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
// SAFETY: every method routes a block to `mmap` or to `System` based only on
|
|
40
|
+
// `is_large(layout)`, and the caller must pass the block's original layout to
|
|
41
|
+
// `dealloc`/`realloc` (the `GlobalAlloc` contract), so a block is always
|
|
42
|
+
// released by the allocator that produced it. Mappings are page-aligned, which
|
|
43
|
+
// satisfies every `align <= PAGE`; larger alignments go to `System`.
|
|
44
|
+
unsafe impl GlobalAlloc for LargeBlockAlloc {
|
|
45
|
+
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
|
|
46
|
+
if !is_large(layout) {
|
|
47
|
+
// SAFETY: `layout` is forwarded unchanged; the caller upholds `alloc`'s contract.
|
|
48
|
+
return unsafe { System.alloc(layout) };
|
|
49
|
+
}
|
|
50
|
+
// SAFETY: an anonymous private mapping with no address hint; the length is
|
|
51
|
+
// non-zero because `layout.size() >= LARGE`.
|
|
52
|
+
let p = unsafe {
|
|
53
|
+
libc::mmap(
|
|
54
|
+
ptr::null_mut(),
|
|
55
|
+
mapped_len(layout.size()),
|
|
56
|
+
libc::PROT_READ | libc::PROT_WRITE,
|
|
57
|
+
libc::MAP_PRIVATE | libc::MAP_ANONYMOUS,
|
|
58
|
+
-1,
|
|
59
|
+
0,
|
|
60
|
+
)
|
|
61
|
+
};
|
|
62
|
+
if p == libc::MAP_FAILED {
|
|
63
|
+
ptr::null_mut()
|
|
64
|
+
} else {
|
|
65
|
+
p.cast()
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// Fresh anonymous mappings are already zeroed.
|
|
70
|
+
unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
|
|
71
|
+
if is_large(layout) {
|
|
72
|
+
// SAFETY: same contract as `alloc`, which returns zeroed pages for large layouts.
|
|
73
|
+
unsafe { self.alloc(layout) }
|
|
74
|
+
} else {
|
|
75
|
+
// SAFETY: `layout` is forwarded unchanged to the system allocator.
|
|
76
|
+
unsafe { System.alloc_zeroed(layout) }
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
unsafe fn dealloc(&self, p: *mut u8, layout: Layout) {
|
|
81
|
+
if is_large(layout) {
|
|
82
|
+
// SAFETY: `p` and `layout` came from this allocator, so a large layout means `p`
|
|
83
|
+
// is the start of a mapping of exactly `mapped_len(layout.size())` bytes.
|
|
84
|
+
let rc = unsafe { libc::munmap(p.cast(), mapped_len(layout.size())) };
|
|
85
|
+
debug_assert_eq!(rc, 0, "munmap failed: the layout does not match the block");
|
|
86
|
+
} else {
|
|
87
|
+
// SAFETY: a small layout means `p` came from `System` with this same layout.
|
|
88
|
+
unsafe { System.dealloc(p, layout) };
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
unsafe fn realloc(&self, p: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
|
|
93
|
+
// SAFETY: the caller guarantees `new_size`, rounded up to `layout.align()`, does not
|
|
94
|
+
// overflow `isize::MAX`, and `layout.align()` is already a valid alignment.
|
|
95
|
+
let new_layout = unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) };
|
|
96
|
+
match (is_large(layout), is_large(new_layout)) {
|
|
97
|
+
// SAFETY: both layouts are small, so `p` is a `System` block of `layout`.
|
|
98
|
+
(false, false) => unsafe { System.realloc(p, layout, new_size) },
|
|
99
|
+
(true, true) => {
|
|
100
|
+
let (old_len, new_len) = (mapped_len(layout.size()), mapped_len(new_size));
|
|
101
|
+
if old_len == new_len {
|
|
102
|
+
return p;
|
|
103
|
+
}
|
|
104
|
+
// SAFETY: `p` is the start of a mapping of `old_len` bytes; on failure
|
|
105
|
+
// `mremap` leaves that mapping intact and the null result tells the caller.
|
|
106
|
+
let q = unsafe { libc::mremap(p.cast(), old_len, new_len, libc::MREMAP_MAYMOVE) };
|
|
107
|
+
if q == libc::MAP_FAILED {
|
|
108
|
+
ptr::null_mut()
|
|
109
|
+
} else {
|
|
110
|
+
q.cast()
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
_ => {
|
|
114
|
+
// Crossing the threshold: new block from the matching allocator, copy, free old.
|
|
115
|
+
// SAFETY: `new_layout` is valid (see above) and non-zero sized.
|
|
116
|
+
let q = unsafe { self.alloc(new_layout) };
|
|
117
|
+
if !q.is_null() {
|
|
118
|
+
// SAFETY: `p` is valid for `layout.size()` bytes and `q` for `new_size`, so
|
|
119
|
+
// both are valid for the smaller of the two; they are distinct blocks.
|
|
120
|
+
unsafe {
|
|
121
|
+
ptr::copy_nonoverlapping(p, q, layout.size().min(new_size));
|
|
122
|
+
self.dealloc(p, layout);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
q
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
#[cfg(test)]
|
|
132
|
+
// The tests drive the allocator's raw interface directly, so each block would repeat the same note.
|
|
133
|
+
#[allow(clippy::undocumented_unsafe_blocks)]
|
|
134
|
+
mod tests {
|
|
135
|
+
use super::*;
|
|
136
|
+
|
|
137
|
+
fn fill(p: *mut u8, n: usize, seed: u8) {
|
|
138
|
+
for i in 0..n {
|
|
139
|
+
unsafe { *p.add(i) = seed.wrapping_add(i as u8) };
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
fn check(p: *const u8, n: usize, seed: u8) {
|
|
144
|
+
for i in 0..n {
|
|
145
|
+
assert_eq!(unsafe { *p.add(i) }, seed.wrapping_add(i as u8), "byte {i}");
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/// Fails if the `#[global_allocator]` static in `lib.rs` is removed or mis-gated: only
|
|
150
|
+
/// this allocator hands out page-aligned multi-MiB blocks (glibc's mmap path returns
|
|
151
|
+
/// a 16-byte header offset).
|
|
152
|
+
#[test]
|
|
153
|
+
fn global_allocator_serves_large_vecs_from_page_aligned_mappings() {
|
|
154
|
+
let blocks: Vec<Vec<u8>> = (0..8).map(|_| Vec::with_capacity(2 * LARGE)).collect();
|
|
155
|
+
for b in &blocks {
|
|
156
|
+
assert_eq!(
|
|
157
|
+
b.as_ptr() as usize % PAGE,
|
|
158
|
+
0,
|
|
159
|
+
"LargeBlockAlloc is not the global allocator"
|
|
160
|
+
);
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
#[test]
|
|
165
|
+
fn realloc_keeps_contents_across_the_large_threshold() {
|
|
166
|
+
let a = LargeBlockAlloc;
|
|
167
|
+
let sizes = [
|
|
168
|
+
100,
|
|
169
|
+
4096,
|
|
170
|
+
LARGE - 1,
|
|
171
|
+
LARGE,
|
|
172
|
+
LARGE + 1,
|
|
173
|
+
3 * LARGE + 17,
|
|
174
|
+
40 * LARGE,
|
|
175
|
+
LARGE + 5,
|
|
176
|
+
64,
|
|
177
|
+
2 * LARGE,
|
|
178
|
+
];
|
|
179
|
+
unsafe {
|
|
180
|
+
let mut layout = Layout::from_size_align(sizes[0], 8).unwrap();
|
|
181
|
+
let mut p = a.alloc(layout);
|
|
182
|
+
assert!(!p.is_null());
|
|
183
|
+
fill(p, layout.size(), 7);
|
|
184
|
+
for &next in &sizes[1..] {
|
|
185
|
+
let keep = layout.size().min(next);
|
|
186
|
+
let q = a.realloc(p, layout, next);
|
|
187
|
+
assert!(!q.is_null());
|
|
188
|
+
check(q, keep, 7);
|
|
189
|
+
layout = Layout::from_size_align(next, 8).unwrap();
|
|
190
|
+
fill(q, next, 7);
|
|
191
|
+
p = q;
|
|
192
|
+
}
|
|
193
|
+
a.dealloc(p, layout);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
#[test]
|
|
198
|
+
fn large_blocks_are_zeroed_and_writable_to_the_last_byte() {
|
|
199
|
+
let a = LargeBlockAlloc;
|
|
200
|
+
let layout = Layout::from_size_align(5 * LARGE + 123, 64).unwrap();
|
|
201
|
+
unsafe {
|
|
202
|
+
let p = a.alloc_zeroed(layout);
|
|
203
|
+
assert!(!p.is_null());
|
|
204
|
+
assert_eq!(p as usize % 64, 0);
|
|
205
|
+
assert!((0..layout.size()).step_by(4093).all(|i| *p.add(i) == 0));
|
|
206
|
+
*p.add(layout.size() - 1) = 9;
|
|
207
|
+
a.dealloc(p, layout);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
#[test]
|
|
212
|
+
fn over_aligned_large_requests_fall_back_to_the_system_allocator() {
|
|
213
|
+
let a = LargeBlockAlloc;
|
|
214
|
+
let layout = Layout::from_size_align(2 * LARGE, 1 << 16).unwrap();
|
|
215
|
+
unsafe {
|
|
216
|
+
let p = a.alloc(layout);
|
|
217
|
+
assert!(!p.is_null());
|
|
218
|
+
assert_eq!(p as usize % (1 << 16), 0);
|
|
219
|
+
a.dealloc(p, layout);
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
#[test]
|
|
224
|
+
fn concurrent_random_alloc_realloc_dealloc_keeps_every_block_intact() {
|
|
225
|
+
let handles: Vec<_> = (0..8u64)
|
|
226
|
+
.map(|t| {
|
|
227
|
+
std::thread::spawn(move || {
|
|
228
|
+
let a = LargeBlockAlloc;
|
|
229
|
+
let mut state = 0x9E37_79B9_7F4A_7C15u64 ^ t;
|
|
230
|
+
let mut next = move || {
|
|
231
|
+
state ^= state << 13;
|
|
232
|
+
state ^= state >> 7;
|
|
233
|
+
state ^= state << 17;
|
|
234
|
+
state
|
|
235
|
+
};
|
|
236
|
+
let mut live: Vec<(*mut u8, Layout, u8)> = Vec::new();
|
|
237
|
+
for _ in 0..1500 {
|
|
238
|
+
let r = next();
|
|
239
|
+
let size = match r % 4 {
|
|
240
|
+
0 => (r >> 8) as usize % 5000 + 1,
|
|
241
|
+
1 => LARGE - 3 + (r >> 8) as usize % 7,
|
|
242
|
+
2 => LARGE + (r >> 8) as usize % (3 * LARGE),
|
|
243
|
+
_ => (r >> 8) as usize % (300 * 1024) + 1,
|
|
244
|
+
};
|
|
245
|
+
let seed = (r >> 32) as u8;
|
|
246
|
+
if live.len() < 24 && r % 3 != 0 {
|
|
247
|
+
let layout = Layout::from_size_align(size, 8).unwrap();
|
|
248
|
+
let p = unsafe { a.alloc(layout) };
|
|
249
|
+
assert!(!p.is_null());
|
|
250
|
+
fill(p, size.min(8192), seed);
|
|
251
|
+
live.push((p, layout, seed));
|
|
252
|
+
} else if !live.is_empty() {
|
|
253
|
+
let i = (r >> 16) as usize % live.len();
|
|
254
|
+
let (p, layout, seed) = live.swap_remove(i);
|
|
255
|
+
check(p, layout.size().min(8192), seed);
|
|
256
|
+
if r % 2 == 0 {
|
|
257
|
+
unsafe { a.dealloc(p, layout) };
|
|
258
|
+
} else {
|
|
259
|
+
let q = unsafe { a.realloc(p, layout, size) };
|
|
260
|
+
assert!(!q.is_null());
|
|
261
|
+
check(q, layout.size().min(size).min(8192), seed);
|
|
262
|
+
fill(q, size.min(8192), seed);
|
|
263
|
+
live.push((q, Layout::from_size_align(size, 8).unwrap(), seed));
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
for (p, layout, seed) in live {
|
|
268
|
+
check(p, layout.size().min(8192), seed);
|
|
269
|
+
unsafe { a.dealloc(p, layout) };
|
|
270
|
+
}
|
|
271
|
+
})
|
|
272
|
+
})
|
|
273
|
+
.collect();
|
|
274
|
+
for h in handles {
|
|
275
|
+
h.join().unwrap();
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
}
|
|
@@ -74,15 +74,25 @@ pub(crate) fn lz4_frame_decode_into(src: &[u8], out: &mut Vec<u8>) -> Result<(),
|
|
|
74
74
|
Ok(())
|
|
75
75
|
}
|
|
76
76
|
|
|
77
|
+
/// Most a download reserves up front from `Content-Length`; a longer body
|
|
78
|
+
/// simply grows the buffer.
|
|
79
|
+
const MAX_PREALLOC: usize = 256 << 20;
|
|
80
|
+
|
|
77
81
|
/// Reads the body into one buffer sized from `Content-Length`. Frames are
|
|
78
82
|
/// freed as they are consumed, so a chunk peaks at about its own size instead
|
|
79
83
|
/// of the ~2x `Response::bytes()` reaches while joining collected frames.
|
|
80
84
|
async fn read_body(mut resp: reqwest::Response) -> Result<Bytes, reqwest::Error> {
|
|
81
|
-
//
|
|
82
|
-
|
|
85
|
+
// The header is untrusted: reserve at most `MAX_PREALLOC`, and if that
|
|
86
|
+
// reservation fails (address-space limits) or there is no length, fall
|
|
87
|
+
// back to `bytes()` instead of aborting the process on allocation failure.
|
|
88
|
+
let mut buf = Vec::new();
|
|
89
|
+
let reserved = resp
|
|
90
|
+
.content_length()
|
|
91
|
+
.map(|len| usize::try_from(len).unwrap_or(usize::MAX).min(MAX_PREALLOC))
|
|
92
|
+
.is_some_and(|len| buf.try_reserve_exact(len).is_ok());
|
|
93
|
+
if !reserved {
|
|
83
94
|
return resp.bytes().await;
|
|
84
|
-
}
|
|
85
|
-
let mut buf = Vec::with_capacity(len);
|
|
95
|
+
}
|
|
86
96
|
while let Some(frame) = resp.chunk().await? {
|
|
87
97
|
buf.extend_from_slice(&frame);
|
|
88
98
|
}
|
|
@@ -622,3 +632,71 @@ mod proptests {
|
|
|
622
632
|
}
|
|
623
633
|
}
|
|
624
634
|
}
|
|
635
|
+
|
|
636
|
+
#[cfg(test)]
|
|
637
|
+
mod read_body_tests {
|
|
638
|
+
use std::io::{Read, Write};
|
|
639
|
+
use std::net::TcpListener;
|
|
640
|
+
|
|
641
|
+
use super::*;
|
|
642
|
+
|
|
643
|
+
/// The production HTTP client (`DbClient::new` installs the rustls crypto provider).
|
|
644
|
+
async fn get(url: String) -> reqwest::Response {
|
|
645
|
+
DbClient::new("http://unused.invalid", "wh", "token")
|
|
646
|
+
.http
|
|
647
|
+
.get(url)
|
|
648
|
+
.send()
|
|
649
|
+
.await
|
|
650
|
+
.unwrap()
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
/// Serves `response` verbatim to the first connection, then closes it.
|
|
654
|
+
fn serve_once(response: &'static [u8]) -> String {
|
|
655
|
+
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
|
656
|
+
let url = format!("http://{}/", listener.local_addr().unwrap());
|
|
657
|
+
std::thread::spawn(move || {
|
|
658
|
+
let (mut stream, _) = listener.accept().unwrap();
|
|
659
|
+
let mut request = [0u8; 1024];
|
|
660
|
+
let _ = stream.read(&mut request);
|
|
661
|
+
stream.write_all(response).unwrap();
|
|
662
|
+
});
|
|
663
|
+
url
|
|
664
|
+
}
|
|
665
|
+
|
|
666
|
+
#[tokio::test]
|
|
667
|
+
async fn read_body_reads_a_content_length_body() {
|
|
668
|
+
let url = serve_once(b"HTTP/1.1 200 OK\r\nContent-Length: 11\r\n\r\nhello world");
|
|
669
|
+
let resp = get(url).await;
|
|
670
|
+
assert_eq!(read_body(resp).await.unwrap(), Bytes::from_static(b"hello world"));
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
#[tokio::test]
|
|
674
|
+
async fn read_body_falls_back_when_there_is_no_content_length() {
|
|
675
|
+
let url =
|
|
676
|
+
serve_once(b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nhello\r\n6\r\n world\r\n0\r\n\r\n");
|
|
677
|
+
let resp = get(url).await;
|
|
678
|
+
assert_eq!(read_body(resp).await.unwrap(), Bytes::from_static(b"hello world"));
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
#[tokio::test]
|
|
682
|
+
async fn read_body_errors_on_a_body_shorter_than_content_length() {
|
|
683
|
+
let url = serve_once(b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\nabc");
|
|
684
|
+
let resp = get(url).await;
|
|
685
|
+
assert!(read_body(resp).await.is_err());
|
|
686
|
+
}
|
|
687
|
+
|
|
688
|
+
#[test]
|
|
689
|
+
fn lz4_frame_decode_into_appends_every_frame_including_an_empty_one() {
|
|
690
|
+
fn frame(data: &[u8]) -> Vec<u8> {
|
|
691
|
+
let mut encoder = lz4_flex::frame::FrameEncoder::new(Vec::new());
|
|
692
|
+
encoder.write_all(data).unwrap();
|
|
693
|
+
encoder.finish().unwrap()
|
|
694
|
+
}
|
|
695
|
+
let mut src = frame(b"abc");
|
|
696
|
+
src.extend(frame(b""));
|
|
697
|
+
src.extend(frame(b"def"));
|
|
698
|
+
let mut out = b"prefix-".to_vec();
|
|
699
|
+
lz4_frame_decode_into(&src, &mut out).unwrap();
|
|
700
|
+
assert_eq!(out, b"prefix-abcdef");
|
|
701
|
+
}
|
|
702
|
+
}
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
#[cfg(all(target_os = "linux", target_env = "gnu", feature = "large-block-alloc"))]
|
|
2
|
+
mod alloc;
|
|
1
3
|
mod arrow_ffi;
|
|
2
4
|
pub mod client;
|
|
3
5
|
pub mod heartbeat;
|
|
@@ -24,6 +26,10 @@ use client::{
|
|
|
24
26
|
use heartbeat::{HeartbeatStream, HeartbeatWait, Tick};
|
|
25
27
|
use pipeline::{NdjsonStream, ResultStream};
|
|
26
28
|
|
|
29
|
+
#[cfg(all(target_os = "linux", target_env = "gnu", feature = "large-block-alloc"))]
|
|
30
|
+
#[global_allocator]
|
|
31
|
+
static GLOBAL: alloc::LargeBlockAlloc = alloc::LargeBlockAlloc;
|
|
32
|
+
|
|
27
33
|
/// Writes any object implementing `__arrow_c_stream__` (a `Table`/
|
|
28
34
|
/// `RecordBatchReader` from this crate, arro3, pyarrow, or anything else
|
|
29
35
|
/// Arrow-C-Data-Interface-compatible) as Arrow-IPC stream bytes to `buf` (a
|
|
@@ -54,6 +54,10 @@ impl Write for LineCollector {
|
|
|
54
54
|
}
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
+
fn encode_err(e: impl std::fmt::Display) -> ApiError {
|
|
58
|
+
ApiError::permanent(format!("NDJSON encode error: {e}"))
|
|
59
|
+
}
|
|
60
|
+
|
|
57
61
|
/// Converts one chunk's decoded batches into NDJSON lines, one per row, in
|
|
58
62
|
/// arro3-`write_ndjson(explicit_nulls=True)`-compatible format: null-valued
|
|
59
63
|
/// keys stay present as JSON `null` rather than being omitted, and a
|
|
@@ -83,12 +87,8 @@ fn encode_ndjson_lines(batches: &[RecordBatch], non_finite_as_string: bool) -> R
|
|
|
83
87
|
let builder = arrow_json::WriterBuilder::new().with_explicit_nulls(true);
|
|
84
88
|
let mut writer = builder.build::<_, arrow_json::writer::LineDelimited>(&mut output);
|
|
85
89
|
let refs: Vec<&RecordBatch> = batches.iter().collect();
|
|
86
|
-
writer
|
|
87
|
-
|
|
88
|
-
.map_err(|e| ApiError::permanent(format!("NDJSON encode error: {e}")))?;
|
|
89
|
-
writer
|
|
90
|
-
.finish()
|
|
91
|
-
.map_err(|e| ApiError::permanent(format!("NDJSON encode error: {e}")))?;
|
|
90
|
+
writer.write_batches(&refs).map_err(encode_err)?;
|
|
91
|
+
writer.finish().map_err(encode_err)?;
|
|
92
92
|
}
|
|
93
93
|
if !output.pending.is_empty() {
|
|
94
94
|
return Err(ApiError::permanent("NDJSON encode produced an unterminated row"));
|
|
@@ -403,12 +403,8 @@ mod tests {
|
|
|
403
403
|
let builder = arrow_json::WriterBuilder::new().with_explicit_nulls(true);
|
|
404
404
|
let mut writer = builder.build::<_, arrow_json::writer::LineDelimited>(&mut buf);
|
|
405
405
|
let refs: Vec<&RecordBatch> = batches.iter().collect();
|
|
406
|
-
writer
|
|
407
|
-
|
|
408
|
-
.map_err(|e| ApiError::permanent(format!("NDJSON encode error: {e}")))?;
|
|
409
|
-
writer
|
|
410
|
-
.finish()
|
|
411
|
-
.map_err(|e| ApiError::permanent(format!("NDJSON encode error: {e}")))?;
|
|
406
|
+
writer.write_batches(&refs).map_err(encode_err)?;
|
|
407
|
+
writer.finish().map_err(encode_err)?;
|
|
412
408
|
}
|
|
413
409
|
let mut lines: Vec<String> = String::from_utf8(buf)
|
|
414
410
|
.map_err(|e| ApiError::permanent(format!("NDJSON encode produced invalid UTF-8: {e}")))?
|
|
@@ -867,6 +867,33 @@ fn build_inline_blob(
|
|
|
867
867
|
mod tests {
|
|
868
868
|
use super::*;
|
|
869
869
|
|
|
870
|
+
#[test]
|
|
871
|
+
fn build_inline_blob_decodes_every_lz4_frame_including_an_empty_one() {
|
|
872
|
+
use std::io::Write;
|
|
873
|
+
|
|
874
|
+
fn frame(data: &[u8]) -> bytes::Bytes {
|
|
875
|
+
let mut encoder = lz4_flex::frame::FrameEncoder::new(Vec::new());
|
|
876
|
+
encoder.write_all(data).unwrap();
|
|
877
|
+
bytes::Bytes::from(encoder.finish().unwrap())
|
|
878
|
+
}
|
|
879
|
+
let mut one_batch = frame(b"AAAA").to_vec();
|
|
880
|
+
one_batch.extend(frame(b"")); // a valid frame with no content must not end decoding
|
|
881
|
+
one_batch.extend(frame(b"BBBB"));
|
|
882
|
+
let batches = vec![
|
|
883
|
+
thrift::ArrowBatch {
|
|
884
|
+
batch: bytes::Bytes::from(one_batch),
|
|
885
|
+
row_count: 3,
|
|
886
|
+
},
|
|
887
|
+
thrift::ArrowBatch {
|
|
888
|
+
batch: frame(b"CC"),
|
|
889
|
+
row_count: 2,
|
|
890
|
+
},
|
|
891
|
+
];
|
|
892
|
+
let (blob, rows) = build_inline_blob(Some(bytes::Bytes::from_static(b"S-")), batches, true).unwrap();
|
|
893
|
+
assert_eq!(&blob[..], b"S-AAAABBBBCC");
|
|
894
|
+
assert_eq!(rows, 5);
|
|
895
|
+
}
|
|
896
|
+
|
|
870
897
|
fn ok_worker() -> tokio::task::JoinHandle<()> {
|
|
871
898
|
tokio::spawn(async {})
|
|
872
899
|
}
|
|
@@ -13,14 +13,11 @@ from __future__ import annotations
|
|
|
13
13
|
|
|
14
14
|
import contextlib
|
|
15
15
|
from collections.abc import AsyncIterator, Awaitable, Iterator
|
|
16
|
-
from typing import
|
|
16
|
+
from typing import Any, BinaryIO, TypeVar, cast
|
|
17
17
|
|
|
18
18
|
from . import _core
|
|
19
19
|
from .client import DatabricksClient
|
|
20
20
|
|
|
21
|
-
if TYPE_CHECKING:
|
|
22
|
-
import asyncio
|
|
23
|
-
|
|
24
21
|
__all__ = [
|
|
25
22
|
"HEARTBEAT",
|
|
26
23
|
"QueryTimeout",
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/proptest-regressions/json_convert.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/tests/wiremock_volume_files.rs
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/tests_py/test_stream_ndjson_lines.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{arrowbricks-5.0.3 → arrowbricks-5.1.1}/rust/arrowbricks_core/tests_py/test_token_provider.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|