arrowbricks 5.0.3__tar.gz → 5.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/PKG-INFO +1 -1
  2. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/pyproject.toml +1 -1
  3. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/Cargo.lock +2 -1
  4. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/Cargo.toml +5 -1
  5. arrowbricks-5.1.0/rust/arrowbricks_core/src/alloc.rs +226 -0
  6. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/src/lib.rs +6 -0
  7. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/LICENSE +0 -0
  8. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/README.md +0 -0
  9. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/.gitignore +0 -0
  10. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/README.md +0 -0
  11. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/examples/duckdb_query.py +0 -0
  12. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/examples/fastapi_sse.py +0 -0
  13. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/proptest-regressions/json_convert.txt +0 -0
  14. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/rustfmt.toml +0 -0
  15. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/src/arrow_ffi.rs +0 -0
  16. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/src/client/download.rs +0 -0
  17. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/src/client/error.rs +0 -0
  18. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/src/client/model.rs +0 -0
  19. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/src/client/sea.rs +0 -0
  20. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/src/client/thrift_rpc.rs +0 -0
  21. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/src/client/volume.rs +0 -0
  22. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/src/client.rs +0 -0
  23. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/src/heartbeat.rs +0 -0
  24. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/src/json_convert.rs +0 -0
  25. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/src/pipeline/ndjson.rs +0 -0
  26. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/src/pipeline/reorder.rs +0 -0
  27. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/src/pipeline/sea.rs +0 -0
  28. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/src/pipeline/stats.rs +0 -0
  29. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/src/pipeline/test_support.rs +0 -0
  30. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/src/pipeline/thrift_exec.rs +0 -0
  31. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/src/pipeline.rs +0 -0
  32. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/src/thrift.rs +0 -0
  33. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/tests/common/mod.rs +0 -0
  34. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/tests/wiremock_pipeline.rs +0 -0
  35. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/tests/wiremock_thrift.rs +0 -0
  36. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/tests/wiremock_volume_files.rs +0 -0
  37. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/tests_py/conftest.py +0 -0
  38. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/tests_py/test_ipc_stream.py +0 -0
  39. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/tests_py/test_parameters.py +0 -0
  40. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/tests_py/test_stream_ndjson_lines.py +0 -0
  41. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/tests_py/test_streaming.py +0 -0
  42. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/tests_py/test_thrift.py +0 -0
  43. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/tests_py/test_token_provider.py +0 -0
  44. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/tests_py/test_volume_files.py +0 -0
  45. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/rust/arrowbricks_core/tests_py/thrift_mock.py +0 -0
  46. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/src/arrowbricks/__init__.py +0 -0
  47. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/src/arrowbricks/_core.pyi +0 -0
  48. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/src/arrowbricks/_streaming.py +0 -0
  49. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/src/arrowbricks/client.py +0 -0
  50. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/src/arrowbricks/cursor.py +0 -0
  51. {arrowbricks-5.0.3 → arrowbricks-5.1.0}/src/arrowbricks/py.typed +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: arrowbricks
3
- Version: 5.0.3
3
+ Version: 5.1.0
4
4
  Requires-Dist: arro3-core>=0.8 ; extra == 'arro3'
5
5
  Provides-Extra: arro3
6
6
  License-File: LICENSE
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "arrowbricks"
3
- version = "5.0.3"
3
+ version = "5.1.0"
4
4
  description = "Runs SQL against a Databricks SQL warehouse via the Statement Execution API and hands you the result as Arrow -- a DB-API-ish Cursor (fetchone/fetchmany/fetchall/fetchall_arrow) or NDJSON streaming. Rust/PyO3 core throughout -- zero required runtime dependencies."
5
5
  readme = "README.md"
6
6
  license = "MIT"
@@ -177,7 +177,7 @@ dependencies = [
177
177
 
178
178
  [[package]]
179
179
  name = "arrowbricks_core"
180
- version = "5.0.3"
180
+ version = "5.1.0"
181
181
  dependencies = [
182
182
  "arrow-array",
183
183
  "arrow-buffer",
@@ -188,6 +188,7 @@ dependencies = [
188
188
  "bytes",
189
189
  "chrono",
190
190
  "hyper-rustls",
191
+ "libc",
191
192
  "lz4_flex",
192
193
  "memchr",
193
194
  "proptest",
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "arrowbricks_core"
3
- version = "5.0.3"
3
+ version = "5.1.0"
4
4
  edition = "2024"
5
5
  readme = "README.md"
6
6
 
@@ -69,6 +69,10 @@ serde = { version = "1.0.229", features = ["derive"] }
69
69
  serde_json = "1.0.151"
70
70
  tokio = { version = "1.53.1", features = ["rt-multi-thread", "macros"] }
71
71
 
72
+ # Already in the dependency graph (via tokio/mio); src/alloc.rs uses mmap directly.
73
+ [target.'cfg(target_os = "linux")'.dependencies]
74
+ libc = "0.2"
75
+
72
76
  [features]
73
77
  default = ["extension-module"]
74
78
  # `extension-module` skips linking against libpython -- correct for the
@@ -0,0 +1,226 @@
1
+ //! Returns large buffers to the OS as soon as they are freed.
2
+ //!
3
+ //! glibc raises its mmap threshold (up to 32 MiB) whenever a large mapped
4
+ //! block is freed, so later multi-MiB buffers come from the fragmented heap
5
+ //! and are never handed back: repeating one query in a single process grew
6
+ //! idle RSS from 200 MB to 1.4 GB. Blocks of `LARGE` bytes or more bypass
7
+ //! malloc here (`mmap`/`munmap`, `mremap` to grow without copying); smaller
8
+ //! ones use the system allocator. Only Rust allocations in this extension are
9
+ //! affected, not the host process's malloc. Linux only.
10
+
11
+ use std::alloc::{GlobalAlloc, Layout, System};
12
+ use std::ptr;
13
+
14
+ const LARGE: usize = 1 << 20;
15
+ const PAGE: usize = 4096;
16
+
17
+ pub struct LargeBlockAlloc;
18
+
19
+ fn is_large(layout: Layout) -> bool {
20
+ layout.size() >= LARGE && layout.align() <= PAGE
21
+ }
22
+
23
+ fn mapped_len(size: usize) -> usize {
24
+ size.next_multiple_of(PAGE)
25
+ }
26
+
27
+ unsafe impl GlobalAlloc for LargeBlockAlloc {
28
+ unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
29
+ if !is_large(layout) {
30
+ return unsafe { System.alloc(layout) };
31
+ }
32
+ let p = unsafe {
33
+ libc::mmap(
34
+ ptr::null_mut(),
35
+ mapped_len(layout.size()),
36
+ libc::PROT_READ | libc::PROT_WRITE,
37
+ libc::MAP_PRIVATE | libc::MAP_ANONYMOUS,
38
+ -1,
39
+ 0,
40
+ )
41
+ };
42
+ if p == libc::MAP_FAILED {
43
+ ptr::null_mut()
44
+ } else {
45
+ p.cast()
46
+ }
47
+ }
48
+
49
+ // Fresh anonymous mappings are already zeroed.
50
+ unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
51
+ if is_large(layout) {
52
+ unsafe { self.alloc(layout) }
53
+ } else {
54
+ unsafe { System.alloc_zeroed(layout) }
55
+ }
56
+ }
57
+
58
+ unsafe fn dealloc(&self, p: *mut u8, layout: Layout) {
59
+ if is_large(layout) {
60
+ unsafe { libc::munmap(p.cast(), mapped_len(layout.size())) };
61
+ } else {
62
+ unsafe { System.dealloc(p, layout) };
63
+ }
64
+ }
65
+
66
+ unsafe fn realloc(&self, p: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
67
+ let new_layout = unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) };
68
+ match (is_large(layout), is_large(new_layout)) {
69
+ (false, false) => unsafe { System.realloc(p, layout, new_size) },
70
+ (true, true) => {
71
+ let (old_len, new_len) = (mapped_len(layout.size()), mapped_len(new_size));
72
+ if old_len == new_len {
73
+ return p;
74
+ }
75
+ let q = unsafe { libc::mremap(p.cast(), old_len, new_len, libc::MREMAP_MAYMOVE) };
76
+ if q == libc::MAP_FAILED {
77
+ ptr::null_mut()
78
+ } else {
79
+ q.cast()
80
+ }
81
+ }
82
+ _ => {
83
+ let q = unsafe { self.alloc(new_layout) };
84
+ if !q.is_null() {
85
+ unsafe {
86
+ ptr::copy_nonoverlapping(p, q, layout.size().min(new_size));
87
+ self.dealloc(p, layout);
88
+ }
89
+ }
90
+ q
91
+ }
92
+ }
93
+ }
94
+ }
95
+
96
+ #[cfg(test)]
97
+ mod tests {
98
+ use super::*;
99
+
100
+ fn fill(p: *mut u8, n: usize, seed: u8) {
101
+ for i in 0..n {
102
+ unsafe { *p.add(i) = seed.wrapping_add(i as u8) };
103
+ }
104
+ }
105
+
106
+ fn check(p: *const u8, n: usize, seed: u8) {
107
+ for i in 0..n {
108
+ assert_eq!(unsafe { *p.add(i) }, seed.wrapping_add(i as u8), "byte {i}");
109
+ }
110
+ }
111
+
112
+ #[test]
113
+ fn realloc_keeps_contents_across_the_large_threshold() {
114
+ let a = LargeBlockAlloc;
115
+ let sizes = [
116
+ 100,
117
+ 4096,
118
+ LARGE - 1,
119
+ LARGE,
120
+ LARGE + 1,
121
+ 3 * LARGE + 17,
122
+ 40 * LARGE,
123
+ LARGE + 5,
124
+ 64,
125
+ 2 * LARGE,
126
+ ];
127
+ unsafe {
128
+ let mut layout = Layout::from_size_align(sizes[0], 8).unwrap();
129
+ let mut p = a.alloc(layout);
130
+ assert!(!p.is_null());
131
+ fill(p, layout.size(), 7);
132
+ for &next in &sizes[1..] {
133
+ let keep = layout.size().min(next);
134
+ let q = a.realloc(p, layout, next);
135
+ assert!(!q.is_null());
136
+ check(q, keep, 7);
137
+ layout = Layout::from_size_align(next, 8).unwrap();
138
+ fill(q, next, 7);
139
+ p = q;
140
+ }
141
+ a.dealloc(p, layout);
142
+ }
143
+ }
144
+
145
+ #[test]
146
+ fn large_blocks_are_zeroed_and_writable_to_the_last_byte() {
147
+ let a = LargeBlockAlloc;
148
+ let layout = Layout::from_size_align(5 * LARGE + 123, 64).unwrap();
149
+ unsafe {
150
+ let p = a.alloc_zeroed(layout);
151
+ assert!(!p.is_null());
152
+ assert_eq!(p as usize % 64, 0);
153
+ assert!((0..layout.size()).step_by(4093).all(|i| *p.add(i) == 0));
154
+ *p.add(layout.size() - 1) = 9;
155
+ a.dealloc(p, layout);
156
+ }
157
+ }
158
+
159
+ #[test]
160
+ fn over_aligned_large_requests_fall_back_to_the_system_allocator() {
161
+ let a = LargeBlockAlloc;
162
+ let layout = Layout::from_size_align(2 * LARGE, 1 << 16).unwrap();
163
+ unsafe {
164
+ let p = a.alloc(layout);
165
+ assert!(!p.is_null());
166
+ assert_eq!(p as usize % (1 << 16), 0);
167
+ a.dealloc(p, layout);
168
+ }
169
+ }
170
+
171
+ #[test]
172
+ fn concurrent_random_alloc_realloc_dealloc_keeps_every_block_intact() {
173
+ let handles: Vec<_> = (0..8u64)
174
+ .map(|t| {
175
+ std::thread::spawn(move || {
176
+ let a = LargeBlockAlloc;
177
+ let mut state = 0x9E37_79B9_7F4A_7C15u64 ^ t;
178
+ let mut next = move || {
179
+ state ^= state << 13;
180
+ state ^= state >> 7;
181
+ state ^= state << 17;
182
+ state
183
+ };
184
+ let mut live: Vec<(*mut u8, Layout, u8)> = Vec::new();
185
+ for _ in 0..1500 {
186
+ let r = next();
187
+ let size = match r % 4 {
188
+ 0 => (r >> 8) as usize % 5000 + 1,
189
+ 1 => LARGE - 3 + (r >> 8) as usize % 7,
190
+ 2 => LARGE + (r >> 8) as usize % (3 * LARGE),
191
+ _ => (r >> 8) as usize % (300 * 1024) + 1,
192
+ };
193
+ let seed = (r >> 32) as u8;
194
+ if live.len() < 24 && r % 3 != 0 {
195
+ let layout = Layout::from_size_align(size, 8).unwrap();
196
+ let p = unsafe { a.alloc(layout) };
197
+ assert!(!p.is_null());
198
+ fill(p, size.min(8192), seed);
199
+ live.push((p, layout, seed));
200
+ } else if !live.is_empty() {
201
+ let i = (r >> 16) as usize % live.len();
202
+ let (p, layout, seed) = live.swap_remove(i);
203
+ check(p, layout.size().min(8192), seed);
204
+ if r % 2 == 0 {
205
+ unsafe { a.dealloc(p, layout) };
206
+ } else {
207
+ let q = unsafe { a.realloc(p, layout, size) };
208
+ assert!(!q.is_null());
209
+ check(q, layout.size().min(size).min(8192), seed);
210
+ fill(q, size.min(8192), seed);
211
+ live.push((q, Layout::from_size_align(size, 8).unwrap(), seed));
212
+ }
213
+ }
214
+ }
215
+ for (p, layout, seed) in live {
216
+ check(p, layout.size().min(8192), seed);
217
+ unsafe { a.dealloc(p, layout) };
218
+ }
219
+ })
220
+ })
221
+ .collect();
222
+ for h in handles {
223
+ h.join().unwrap();
224
+ }
225
+ }
226
+ }
@@ -1,3 +1,5 @@
1
+ #[cfg(target_os = "linux")]
2
+ mod alloc;
1
3
  mod arrow_ffi;
2
4
  pub mod client;
3
5
  pub mod heartbeat;
@@ -7,6 +9,10 @@ pub mod thrift;
7
9
 
8
10
  use std::sync::{Arc, Mutex};
9
11
 
12
+ #[cfg(target_os = "linux")]
13
+ #[global_allocator]
14
+ static GLOBAL: alloc::LargeBlockAlloc = alloc::LargeBlockAlloc;
15
+
10
16
  use arrow_array::RecordBatch;
11
17
  use arrow_schema::{Schema, SchemaRef};
12
18
  use pyo3::exceptions::{PyRuntimeError, PyStopAsyncIteration, PyValueError};
File without changes
File without changes