arrowbricks 6.2.0__tar.gz → 6.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/PKG-INFO +10 -2
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/README.md +9 -1
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/pyproject.toml +2 -1
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/Cargo.lock +1 -1
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/Cargo.toml +1 -1
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/client/download.rs +152 -11
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/client/volume.rs +30 -1
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/lib.rs +39 -3
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests/wiremock_thrift.rs +85 -8
- arrowbricks-6.3.0/rust/arrowbricks_core/tests/wiremock_volume_files.rs +213 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/src/arrowbricks/_core.pyi +5 -2
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/src/arrowbricks/_streaming.py +11 -7
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/src/arrowbricks/client.py +19 -3
- arrowbricks-6.2.0/rust/arrowbricks_core/tests/wiremock_volume_files.rs +0 -87
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/LICENSE +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/.gitignore +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/README.md +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/examples/agent_render_bench.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/examples/duckdb_query.py +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/examples/fastapi_sse.py +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/proptest-regressions/json_convert.txt +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/rustfmt.toml +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/agent/diagnostics.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/agent/mod.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/agent/render.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/alloc.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/arrow_ffi.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/client/call_context.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/client/error.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/client/model.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/client/sea.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/client/thrift_rpc.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/client.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/heartbeat.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/json_convert.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/pipeline/ndjson.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/pipeline/reorder.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/pipeline/sea.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/pipeline/stats.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/pipeline/test_support.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/pipeline/thrift_exec.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/pipeline.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/thrift.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests/common/mod.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests/wiremock_agent.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests/wiremock_pipeline.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests_py/conftest.py +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests_py/test_ipc_stream.py +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests_py/test_parameters.py +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests_py/test_stream_ndjson_lines.py +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests_py/test_streaming.py +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests_py/test_thrift.py +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests_py/test_token_provider.py +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests_py/test_volume_files.py +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests_py/thrift_mock.py +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/src/arrowbricks/__init__.py +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/src/arrowbricks/__main__.py +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/src/arrowbricks/_agent.py +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/src/arrowbricks/_sql.py +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/src/arrowbricks/cli.py +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/src/arrowbricks/cursor.py +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.3.0}/src/arrowbricks/py.typed +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: arrowbricks
|
|
3
|
-
Version: 6.
|
|
3
|
+
Version: 6.3.0
|
|
4
4
|
Requires-Dist: arro3-core>=0.8 ; extra == 'arro3'
|
|
5
5
|
Requires-Dist: sqlglot>=27 ; extra == 'sqlglot'
|
|
6
6
|
Provides-Extra: arro3
|
|
@@ -20,7 +20,7 @@ Project-URL: Repository, https://github.com/bmsuisse/arrowbricks
|
|
|
20
20
|
<p align="center">Fast Databricks SQL for Python applications and AI agents.</p>
|
|
21
21
|
|
|
22
22
|
- **Zero required Python dependencies.** One precompiled wheel. No Rust toolchain or cloud SDK needed.
|
|
23
|
-
- **Fast Rust core.** Concurrent chunk downloads, lazy Arrow decoding, and streaming NDJSON, with result order preserved.
|
|
23
|
+
- **Fast Rust core.** Concurrent chunk downloads, lazy Arrow decoding, and streaming NDJSON without materializing the full result, with result order preserved.
|
|
24
24
|
- **Agent mode, new in 6.0.** A CLI and Python API that return compact typed TSV (or a Markdown table), explicit truncation, and actionable JSON errors. Formatting stays in Rust.
|
|
25
25
|
|
|
26
26
|
Use Thrift by default or the Statement Execution API with `protocol="sea"`.
|
|
@@ -118,6 +118,14 @@ a 100-row preview used **44% fewer bytes and 24% fewer tokens** than NDJSON.
|
|
|
118
118
|
See [agent measurements and methodology](docs/research/agent-rendering.md) and
|
|
119
119
|
[client benchmarks](docs/benchmarks.md) for workloads, comparisons, and limits.
|
|
120
120
|
|
|
121
|
+
### Performance tips
|
|
122
|
+
|
|
123
|
+
Numbers are measurements from one ~30 MB/s link to one warehouse, not guarantees.
|
|
124
|
+
|
|
125
|
+
- **Select only the columns you need.** Selecting 3 columns instead of `SELECT *` was 1.5-6x faster and moved 1.9-15.8x fewer bytes across 5 tables.
|
|
126
|
+
- **Keep `compress_results` at its default (`True`)** unless the link is very fast. Here it was 1.8-4.7x faster with 2-5.5x fewer bytes. Turning it off on a very fast link was not tested.
|
|
127
|
+
- **Keep the Thrift default for small lookups.** It was about 0.15 s faster per tiny statement than `protocol="sea"` (a constant gap in these runs).
|
|
128
|
+
|
|
121
129
|
## Learn more
|
|
122
130
|
|
|
123
131
|
- [Usage and API reference](docs/usage.md): authentication, errors, cancellation, observability, and Arrow interop.
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
<p align="center">Fast Databricks SQL for Python applications and AI agents.</p>
|
|
7
7
|
|
|
8
8
|
- **Zero required Python dependencies.** One precompiled wheel. No Rust toolchain or cloud SDK needed.
|
|
9
|
-
- **Fast Rust core.** Concurrent chunk downloads, lazy Arrow decoding, and streaming NDJSON, with result order preserved.
|
|
9
|
+
- **Fast Rust core.** Concurrent chunk downloads, lazy Arrow decoding, and streaming NDJSON without materializing the full result, with result order preserved.
|
|
10
10
|
- **Agent mode, new in 6.0.** A CLI and Python API that return compact typed TSV (or a Markdown table), explicit truncation, and actionable JSON errors. Formatting stays in Rust.
|
|
11
11
|
|
|
12
12
|
Use Thrift by default or the Statement Execution API with `protocol="sea"`.
|
|
@@ -104,6 +104,14 @@ a 100-row preview used **44% fewer bytes and 24% fewer tokens** than NDJSON.
|
|
|
104
104
|
See [agent measurements and methodology](docs/research/agent-rendering.md) and
|
|
105
105
|
[client benchmarks](docs/benchmarks.md) for workloads, comparisons, and limits.
|
|
106
106
|
|
|
107
|
+
### Performance tips
|
|
108
|
+
|
|
109
|
+
Numbers are measurements from one ~30 MB/s link to one warehouse, not guarantees.
|
|
110
|
+
|
|
111
|
+
- **Select only the columns you need.** Selecting 3 columns instead of `SELECT *` was 1.5-6x faster and moved 1.9-15.8x fewer bytes across 5 tables.
|
|
112
|
+
- **Keep `compress_results` at its default (`True`)** unless the link is very fast. Here it was 1.8-4.7x faster with 2-5.5x fewer bytes. Turning it off on a very fast link was not tested.
|
|
113
|
+
- **Keep the Thrift default for small lookups.** It was about 0.15 s faster per tiny statement than `protocol="sea"` (a constant gap in these runs).
|
|
114
|
+
|
|
107
115
|
## Learn more
|
|
108
116
|
|
|
109
117
|
- [Usage and API reference](docs/usage.md): authentication, errors, cancellation, observability, and Arrow interop.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "arrowbricks"
|
|
3
|
-
version = "6.
|
|
3
|
+
version = "6.3.0"
|
|
4
4
|
description = "Rust-powered Databricks SQL client and agent CLI with Arrow/NDJSON streaming, bounded Markdown previews, actionable JSON diagnostics, and zero required Python dependencies."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "MIT"
|
|
@@ -105,3 +105,4 @@ select = ["E", "F", "I", "UP", "S", "B"]
|
|
|
105
105
|
"tests/arro3_free/*" = ["S101", "S106"]
|
|
106
106
|
# One-off benchmark harness: terse scripts, seeded (non-crypto) randomness, asserts as checks.
|
|
107
107
|
"benchmarks/agent-output-format/*" = ["B905", "E501", "E731", "E741", "S101", "S311"]
|
|
108
|
+
"benchmarks/benchmark_duckdb.py" = ["E501", "S608"]
|
|
@@ -16,6 +16,21 @@ use super::MAX_SPLIT_PARTS;
|
|
|
16
16
|
use super::error::{ApiError, join_error};
|
|
17
17
|
use super::model::QueryStatsAccumulator;
|
|
18
18
|
|
|
19
|
+
/// Parses `Content-Range: bytes <start>-<end>/<total>` (the only form a 206
|
|
20
|
+
/// response to a single-range request may use). `*` totals and unsatisfied-range
|
|
21
|
+
/// forms yield `None`.
|
|
22
|
+
fn parse_content_range(headers: &reqwest::header::HeaderMap) -> Option<(u64, u64, u64)> {
|
|
23
|
+
let value = headers.get(reqwest::header::CONTENT_RANGE)?.to_str().ok()?;
|
|
24
|
+
let rest = value.trim().strip_prefix("bytes")?.trim_start();
|
|
25
|
+
let (range, total) = rest.split_once('/')?;
|
|
26
|
+
let (start, end) = range.trim().split_once('-')?;
|
|
27
|
+
Some((
|
|
28
|
+
start.trim().parse().ok()?,
|
|
29
|
+
end.trim().parse().ok()?,
|
|
30
|
+
total.trim().parse().ok()?,
|
|
31
|
+
))
|
|
32
|
+
}
|
|
33
|
+
|
|
19
34
|
/// Unwraps one downloaded chunk file's outer LZ4 Frame compression --
|
|
20
35
|
/// server-side, this is a whole-file wrapper applied because we asked for
|
|
21
36
|
/// `result_compression: "LZ4_FRAME"`, not Arrow-IPC's own (unrelated,
|
|
@@ -226,16 +241,27 @@ impl DbClient {
|
|
|
226
241
|
return Err(ApiError::from_status(status, &text, true));
|
|
227
242
|
}
|
|
228
243
|
let ranged = status == reqwest::StatusCode::PARTIAL_CONTENT;
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
244
|
+
// A 206 must describe exactly the head we asked for: starting
|
|
245
|
+
// at 0 and ending at `part_size - 1` (or the last byte of a
|
|
246
|
+
// smaller file). Anything else would be spliced in as wrong
|
|
247
|
+
// bytes with the right length.
|
|
248
|
+
let mut total: Option<u64> = None;
|
|
249
|
+
let mut head_len: Option<u64> = None;
|
|
250
|
+
if ranged {
|
|
251
|
+
let Some((first, last, size)) = parse_content_range(resp.headers()) else {
|
|
252
|
+
return Err(ApiError::permanent(
|
|
253
|
+
"cloud-fetch link answered a Range request with 206 Partial Content but an \
|
|
254
|
+
unparseable Content-Range header -- refusing to silently return a truncated file",
|
|
255
|
+
));
|
|
256
|
+
};
|
|
257
|
+
if first != 0 || size == 0 || last != (part_size - 1).min(size - 1) {
|
|
258
|
+
return Err(ApiError::permanent(format!(
|
|
259
|
+
"cloud-fetch link answered bytes=0-{} with Content-Range bytes {first}-{last}/{size}",
|
|
260
|
+
part_size - 1
|
|
261
|
+
)));
|
|
262
|
+
}
|
|
263
|
+
total = Some(size);
|
|
264
|
+
head_len = Some(last + 1);
|
|
239
265
|
}
|
|
240
266
|
|
|
241
267
|
let mut downloads = tokio::task::JoinSet::new();
|
|
@@ -248,7 +274,8 @@ impl DbClient {
|
|
|
248
274
|
if start >= total {
|
|
249
275
|
break;
|
|
250
276
|
}
|
|
251
|
-
|
|
277
|
+
// `total` is untrusted: saturate rather than overflow.
|
|
278
|
+
let end = start.saturating_add(rest_size - 1).min(total - 1);
|
|
252
279
|
let this = self.clone();
|
|
253
280
|
let url = url.to_string();
|
|
254
281
|
let part_stats = stats.clone();
|
|
@@ -268,9 +295,36 @@ impl DbClient {
|
|
|
268
295
|
let text = resp.text().await.unwrap_or_default();
|
|
269
296
|
return Err(ApiError::from_status(status, &text, true));
|
|
270
297
|
}
|
|
298
|
+
// The probe proved the server honours ranges, so
|
|
299
|
+
// each part must be a 206 for exactly this range
|
|
300
|
+
// of the same file; a 200 or another interval is
|
|
301
|
+
// wrong data, not a transient failure.
|
|
302
|
+
let got = (status == reqwest::StatusCode::PARTIAL_CONTENT)
|
|
303
|
+
.then(|| parse_content_range(resp.headers()))
|
|
304
|
+
.flatten();
|
|
305
|
+
if got != Some((start, end, total)) {
|
|
306
|
+
return Err(ApiError::permanent(format!(
|
|
307
|
+
"split download part bytes={start}-{end}/{total} was answered with status {status} \
|
|
308
|
+
and Content-Range {got:?}"
|
|
309
|
+
)));
|
|
310
|
+
}
|
|
271
311
|
resp.bytes().await.map_err(|e| ApiError::from_reqwest(e, true))
|
|
272
312
|
})
|
|
273
313
|
.await;
|
|
314
|
+
// A part must be exactly the range asked for: the
|
|
315
|
+
// final sum check alone would accept a short part
|
|
316
|
+
// offset by a long one (wrong bytes, right total).
|
|
317
|
+
let bytes = bytes.and_then(|b| {
|
|
318
|
+
let expected = end - start + 1;
|
|
319
|
+
if b.len() as u64 == expected {
|
|
320
|
+
Ok(b)
|
|
321
|
+
} else {
|
|
322
|
+
Err(ApiError::permanent(format!(
|
|
323
|
+
"split download part bytes={start}-{end} returned {} bytes, expected {expected}",
|
|
324
|
+
b.len()
|
|
325
|
+
)))
|
|
326
|
+
}
|
|
327
|
+
});
|
|
274
328
|
if let Ok(b) = &bytes {
|
|
275
329
|
part_stats.bytes_downloaded.fetch_add(b.len() as u64, Ordering::Relaxed);
|
|
276
330
|
}
|
|
@@ -288,6 +342,15 @@ impl DbClient {
|
|
|
288
342
|
return Err(ApiError::from_reqwest(error, true));
|
|
289
343
|
}
|
|
290
344
|
};
|
|
345
|
+
if let Some(expected) = head_len
|
|
346
|
+
&& head.len() as u64 != expected
|
|
347
|
+
{
|
|
348
|
+
downloads.shutdown().await;
|
|
349
|
+
return Err(ApiError::permanent(format!(
|
|
350
|
+
"split download probe returned {} bytes, expected {expected}",
|
|
351
|
+
head.len()
|
|
352
|
+
)));
|
|
353
|
+
}
|
|
291
354
|
Ok((total, head, downloads))
|
|
292
355
|
})
|
|
293
356
|
.await?;
|
|
@@ -448,6 +511,84 @@ mod tests {
|
|
|
448
511
|
);
|
|
449
512
|
}
|
|
450
513
|
|
|
514
|
+
/// Serves a 12-byte file `abcdefghijkl` that honours `Range`, except where
|
|
515
|
+
/// `fault` says to lie: the failure modes a length check alone cannot see.
|
|
516
|
+
async fn serve_range_file(fault: &'static str) -> std::net::SocketAddr {
|
|
517
|
+
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
|
518
|
+
use tokio::net::TcpListener;
|
|
519
|
+
|
|
520
|
+
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
521
|
+
let address = listener.local_addr().unwrap();
|
|
522
|
+
tokio::spawn(async move {
|
|
523
|
+
loop {
|
|
524
|
+
let (mut socket, _) = listener.accept().await.unwrap();
|
|
525
|
+
tokio::spawn(async move {
|
|
526
|
+
let mut request = Vec::new();
|
|
527
|
+
while !request.ends_with(b"\r\n\r\n") {
|
|
528
|
+
if socket.read_u8().await.map(|b| request.push(b)).is_err() {
|
|
529
|
+
return;
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
let request = String::from_utf8(request).unwrap().to_ascii_lowercase();
|
|
533
|
+
let range = request.lines().find(|s| s.starts_with("range:")).unwrap();
|
|
534
|
+
let (start, end) = range.split_once("bytes=").unwrap().1.split_once('-').unwrap();
|
|
535
|
+
let (start, end): (usize, usize) = (start.parse().unwrap(), end.parse().unwrap());
|
|
536
|
+
let body = &b"abcdefghijkl"[start..=end];
|
|
537
|
+
let (status, content_range) = match (fault, start) {
|
|
538
|
+
("wrong_offset", s) if s > 0 => {
|
|
539
|
+
("206 Partial Content", format!("bytes {}-{}/12", s - 1, end - 1))
|
|
540
|
+
}
|
|
541
|
+
("probe_wrong_offset", 0) => ("206 Partial Content", format!("bytes 1-{}/12", end + 1)),
|
|
542
|
+
("part_ignores_range", s) if s > 0 => ("200 OK", String::new()),
|
|
543
|
+
_ => ("206 Partial Content", format!("bytes {start}-{end}/12")),
|
|
544
|
+
};
|
|
545
|
+
let range_header = if content_range.is_empty() {
|
|
546
|
+
String::new()
|
|
547
|
+
} else {
|
|
548
|
+
format!("Content-Range: {content_range}\r\n")
|
|
549
|
+
};
|
|
550
|
+
let head = format!(
|
|
551
|
+
"HTTP/1.1 {status}\r\nContent-Length: {}\r\n{range_header}Connection: close\r\n\r\n",
|
|
552
|
+
body.len()
|
|
553
|
+
);
|
|
554
|
+
let _ = socket.write_all(head.as_bytes()).await;
|
|
555
|
+
let _ = socket.write_all(body).await;
|
|
556
|
+
});
|
|
557
|
+
}
|
|
558
|
+
});
|
|
559
|
+
address
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
/// Right length from the wrong place passes a byte-count check; only the
|
|
563
|
+
/// status and `Content-Range` of every part can catch it.
|
|
564
|
+
#[tokio::test]
|
|
565
|
+
async fn split_download_rejects_parts_that_are_not_the_requested_range() {
|
|
566
|
+
for fault in ["wrong_offset", "probe_wrong_offset", "part_ignores_range"] {
|
|
567
|
+
let address = serve_range_file(fault).await;
|
|
568
|
+
let client = Arc::new(
|
|
569
|
+
DbClient::new(&format!("http://{address}"), "wh", "token")
|
|
570
|
+
.with_retry_attempts(1)
|
|
571
|
+
.with_retry_max_wait_s(0.0),
|
|
572
|
+
);
|
|
573
|
+
let stats = Arc::new(QueryStatsAccumulator::default());
|
|
574
|
+
let result = client
|
|
575
|
+
.fetch_link_bytes_split(&format!("http://{address}/blob"), false, 4, 3, &stats)
|
|
576
|
+
.await;
|
|
577
|
+
assert!(
|
|
578
|
+
result.is_err(),
|
|
579
|
+
"{fault}: must not assemble wrong bytes, got {result:?}"
|
|
580
|
+
);
|
|
581
|
+
}
|
|
582
|
+
let address = serve_range_file("none").await;
|
|
583
|
+
let client = Arc::new(DbClient::new(&format!("http://{address}"), "wh", "token"));
|
|
584
|
+
let stats = Arc::new(QueryStatsAccumulator::default());
|
|
585
|
+
let bytes = client
|
|
586
|
+
.fetch_link_bytes_split(&format!("http://{address}/blob"), false, 4, 3, &stats)
|
|
587
|
+
.await
|
|
588
|
+
.unwrap();
|
|
589
|
+
assert_eq!(&bytes[..], b"abcdefghijkl");
|
|
590
|
+
}
|
|
591
|
+
|
|
451
592
|
#[tokio::test]
|
|
452
593
|
async fn split_download_starts_tail_requests_before_the_probe_body_finishes() {
|
|
453
594
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
|
@@ -2,12 +2,26 @@
|
|
|
2
2
|
//! (`/api/2.0/fs/files{volume_path}`) -- unrelated to either SQL backend,
|
|
3
3
|
//! just another authenticated HTTP operation this client exposes.
|
|
4
4
|
|
|
5
|
+
use std::time::Duration;
|
|
6
|
+
|
|
5
7
|
use bytes::Bytes;
|
|
6
8
|
use reqwest::StatusCode;
|
|
7
9
|
|
|
8
10
|
use super::DbClient;
|
|
9
11
|
use super::error::ApiError;
|
|
10
12
|
|
|
13
|
+
/// Slowest sustained upload rate the PUT deadline tolerates (1 MiB/s). The
|
|
14
|
+
/// per-request deadline is a total one covering the body, so it scales with
|
|
15
|
+
/// the payload: `max(http_timeout, len / MIN_UPLOAD_BYTES_PER_SEC)`. A slow
|
|
16
|
+
/// but progressing link is not killed at `http_timeout`; a stuck one still
|
|
17
|
+
/// fails once the scaled deadline passes.
|
|
18
|
+
pub const MIN_UPLOAD_BYTES_PER_SEC: u64 = 1 << 20;
|
|
19
|
+
|
|
20
|
+
/// Total deadline for one upload PUT of `len` bytes.
|
|
21
|
+
pub fn upload_deadline(http_timeout: Duration, len: usize) -> Duration {
|
|
22
|
+
http_timeout.max(Duration::from_secs((len as u64).div_ceil(MIN_UPLOAD_BYTES_PER_SEC)))
|
|
23
|
+
}
|
|
24
|
+
|
|
11
25
|
impl DbClient {
|
|
12
26
|
/// Uploads `data` to a Unity Catalog volume path via the Files API,
|
|
13
27
|
/// overwriting anything already there. `volume_path` is caller-supplied
|
|
@@ -17,6 +31,7 @@ impl DbClient {
|
|
|
17
31
|
/// cheap refcount clone, not a real copy.
|
|
18
32
|
pub async fn upload_volume_file(&self, volume_path: &str, data: Bytes) -> Result<(), ApiError> {
|
|
19
33
|
let url = format!("{}/api/2.0/fs/files{}", self.host, volume_path);
|
|
34
|
+
let timeout = upload_deadline(self.http_timeout, data.len());
|
|
20
35
|
self.retry_call(|| async {
|
|
21
36
|
let token = self.token_provider.get_token().await?;
|
|
22
37
|
let resp = self
|
|
@@ -26,7 +41,7 @@ impl DbClient {
|
|
|
26
41
|
.bearer_auth(&token)
|
|
27
42
|
.header("Content-Type", "application/octet-stream")
|
|
28
43
|
.body(data.clone())
|
|
29
|
-
.timeout(
|
|
44
|
+
.timeout(timeout)
|
|
30
45
|
.send()
|
|
31
46
|
.await
|
|
32
47
|
// `overwrite=true` means retrying this exact PUT is safe --
|
|
@@ -67,3 +82,17 @@ impl DbClient {
|
|
|
67
82
|
.await
|
|
68
83
|
}
|
|
69
84
|
}
|
|
85
|
+
|
|
86
|
+
#[cfg(test)]
|
|
87
|
+
mod tests {
|
|
88
|
+
use super::*;
|
|
89
|
+
|
|
90
|
+
#[test]
|
|
91
|
+
fn upload_deadline_scales_with_size_but_never_below_http_timeout() {
|
|
92
|
+
let base = Duration::from_secs(60);
|
|
93
|
+
assert_eq!(upload_deadline(base, 0), base);
|
|
94
|
+
assert_eq!(upload_deadline(base, 10 << 20), base);
|
|
95
|
+
assert_eq!(upload_deadline(base, 600 << 20), Duration::from_secs(600));
|
|
96
|
+
assert_eq!(upload_deadline(base, (600 << 20) + 1), Duration::from_secs(601));
|
|
97
|
+
}
|
|
98
|
+
}
|
|
@@ -130,6 +130,40 @@ fn read_ipc_stream(data: Bound<'_, PyBytes>) -> PyResult<PyTable> {
|
|
|
130
130
|
PyTable::try_new(batches, schema)
|
|
131
131
|
}
|
|
132
132
|
|
|
133
|
+
/// Turns the Python `data` argument of `upload_volume_file` into `Bytes`
|
|
134
|
+
/// without the per-element `Vec<u8>` extraction (20x slower) and, for the
|
|
135
|
+
/// common `bytes` case, without any copy.
|
|
136
|
+
fn upload_body(data: &Bound<'_, PyAny>) -> PyResult<bytes::Bytes> {
|
|
137
|
+
if let Ok(b) = data.cast::<PyBytes>() {
|
|
138
|
+
// Immutable, so the Python object can own the memory for the whole
|
|
139
|
+
// upload (and every retry) with no copy.
|
|
140
|
+
return Ok(bytes::Bytes::from_owner(pyo3::pybacked::PyBackedBytes::from(b.clone())));
|
|
141
|
+
}
|
|
142
|
+
// bytearray/memoryview/numpy/pyarrow are mutable or not owned by us: take
|
|
143
|
+
// one contiguous copy now so later caller mutation cannot change the
|
|
144
|
+
// upload mid-flight.
|
|
145
|
+
let py = data.py();
|
|
146
|
+
if let Ok(buffer) = pyo3::buffer::PyBuffer::<u8>::get(data) {
|
|
147
|
+
return Ok(bytes::Bytes::from(buffer.to_vec(py)?));
|
|
148
|
+
}
|
|
149
|
+
// PyArrow buffers (and `array('b')`) export the signed-byte format `b`,
|
|
150
|
+
// which `PyBuffer<u8>` refuses; the raw bits are the same bytes.
|
|
151
|
+
if let Ok(buffer) = pyo3::buffer::PyBuffer::<i8>::get(data) {
|
|
152
|
+
let signed = buffer.to_vec(py)?;
|
|
153
|
+
return Ok(bytes::Bytes::from(
|
|
154
|
+
signed.into_iter().map(|b| b as u8).collect::<Vec<u8>>(),
|
|
155
|
+
));
|
|
156
|
+
}
|
|
157
|
+
// Last resort keeps the pre-6.3 behaviour: any sequence of ints in 0..=255
|
|
158
|
+
// (a `list[int]`, a numpy int64 array, ...). Slow, but it used to work.
|
|
159
|
+
data.extract::<Vec<u8>>().map(bytes::Bytes::from).map_err(|_| {
|
|
160
|
+
pyo3::exceptions::PyTypeError::new_err(
|
|
161
|
+
"upload_volume_file data must be bytes, a byte buffer (bytearray, memoryview, numpy, pyarrow, ...) \
|
|
162
|
+
or a sequence of ints in 0..=255",
|
|
163
|
+
)
|
|
164
|
+
})
|
|
165
|
+
}
|
|
166
|
+
|
|
133
167
|
/// Wraps a `PyErr` raised by the caller's own `token_provider` callable (its
|
|
134
168
|
/// call itself, awaiting its coroutine, or extracting a non-`str` return
|
|
135
169
|
/// value) into an `ApiError`. This function's *only* caller is
|
|
@@ -869,16 +903,18 @@ impl PyDbClient {
|
|
|
869
903
|
}
|
|
870
904
|
|
|
871
905
|
/// Uploads `data` to a Unity Catalog volume path via the Files API,
|
|
872
|
-
/// overwriting anything already there.
|
|
906
|
+
/// overwriting anything already there. `bytes` is borrowed zero-copy;
|
|
907
|
+
/// `bytearray` and other contiguous byte buffers (memoryview, numpy
|
|
908
|
+
/// `uint8`, pyarrow `Buffer`) are copied once, off the per-element path.
|
|
873
909
|
#[pyo3(signature = (volume_path, data))]
|
|
874
910
|
fn upload_volume_file<'py>(
|
|
875
911
|
&self,
|
|
876
912
|
py: Python<'py>,
|
|
877
913
|
volume_path: String,
|
|
878
|
-
data:
|
|
914
|
+
data: &Bound<'py, PyAny>,
|
|
879
915
|
) -> PyResult<Bound<'py, PyAny>> {
|
|
880
916
|
let client = self.inner.clone();
|
|
881
|
-
let data =
|
|
917
|
+
let data = upload_body(data)?;
|
|
882
918
|
future_into_py_in_call(py, async move {
|
|
883
919
|
client
|
|
884
920
|
.upload_volume_file(&volume_path, data)
|
|
@@ -1702,18 +1702,18 @@ async fn thrift_split_download_fails_on_assembled_length_mismatch() {
|
|
|
1702
1702
|
let server = MockServer::start().await;
|
|
1703
1703
|
mount_single_link_statement(&server, "/_data/short-parts", 60_000).await;
|
|
1704
1704
|
|
|
1705
|
-
// Every part -- including the probe -- honestly declares
|
|
1706
|
-
// via Content-Range, but actually returns far
|
|
1707
|
-
//
|
|
1708
|
-
// guarantees more than one part gets requested; every part coming
|
|
1709
|
-
// short must be caught by the
|
|
1710
|
-
//
|
|
1705
|
+
// Every part -- including the probe -- honestly declares the range it was
|
|
1706
|
+
// asked for of a 5MiB total via Content-Range, but actually returns far
|
|
1707
|
+
// fewer bytes than that range covers (a misbehaving server). The declared
|
|
1708
|
+
// total guarantees more than one part gets requested; every part coming
|
|
1709
|
+
// up short must be caught by the length checks, not silently accepted as
|
|
1710
|
+
// a truncated-but-otherwise-fine result.
|
|
1711
1711
|
const DECLARED_TOTAL: u64 = 5 * (1 << 20);
|
|
1712
1712
|
Mock::given(method("GET"))
|
|
1713
1713
|
.and(path("/_data/short-parts"))
|
|
1714
1714
|
.respond_with(move |req: &Request| {
|
|
1715
|
-
let (start,
|
|
1716
|
-
let end =
|
|
1715
|
+
let (start, end) = parse_range(req).expect("client always sends Range on this path");
|
|
1716
|
+
let end = end.min(DECLARED_TOTAL - 1);
|
|
1717
1717
|
ResponseTemplate::new(206)
|
|
1718
1718
|
.insert_header("Content-Range", format!("bytes {start}-{end}/{DECLARED_TOTAL}"))
|
|
1719
1719
|
.set_body_raw(vec![0u8; 10], "application/octet-stream")
|
|
@@ -1736,6 +1736,83 @@ async fn thrift_split_download_fails_on_assembled_length_mismatch() {
|
|
|
1736
1736
|
);
|
|
1737
1737
|
}
|
|
1738
1738
|
|
|
1739
|
+
#[tokio::test]
|
|
1740
|
+
async fn thrift_split_download_absurd_content_range_total_fails_without_aborting() {
|
|
1741
|
+
let server = MockServer::start().await;
|
|
1742
|
+
mount_single_link_statement(&server, "/_data/absurd-total", 60_000).await;
|
|
1743
|
+
|
|
1744
|
+
// The Content-Range total is untrusted: u64::MAX bytes must never size an
|
|
1745
|
+
// allocation (an allocation failure aborts the whole process, not just
|
|
1746
|
+
// the query), only turn into a normal error.
|
|
1747
|
+
const ABSURD_TOTAL: u64 = u64::MAX;
|
|
1748
|
+
Mock::given(method("GET"))
|
|
1749
|
+
.and(path("/_data/absurd-total"))
|
|
1750
|
+
.respond_with(move |req: &Request| {
|
|
1751
|
+
let (start, end) = parse_range(req).expect("client always sends Range on this path");
|
|
1752
|
+
ResponseTemplate::new(206)
|
|
1753
|
+
.insert_header("Content-Range", format!("bytes {start}-{end}/{ABSURD_TOTAL}"))
|
|
1754
|
+
.set_body_raw(vec![0u8; 1 << 20], "application/octet-stream")
|
|
1755
|
+
})
|
|
1756
|
+
.mount(&server)
|
|
1757
|
+
.await;
|
|
1758
|
+
|
|
1759
|
+
let client = thrift_client(&server);
|
|
1760
|
+
let mut stream = execute_lazy_thrift(client, "SELECT * FROM t", None, None, None)
|
|
1761
|
+
.await
|
|
1762
|
+
.unwrap();
|
|
1763
|
+
let err = stream
|
|
1764
|
+
.fetchall_arrow()
|
|
1765
|
+
.await
|
|
1766
|
+
.expect_err("an absurd declared total must fail loudly, not abort or succeed");
|
|
1767
|
+
assert!(
|
|
1768
|
+
err.message.contains("expected"),
|
|
1769
|
+
"got a different error than expected: {}",
|
|
1770
|
+
err.message
|
|
1771
|
+
);
|
|
1772
|
+
}
|
|
1773
|
+
|
|
1774
|
+
#[tokio::test]
|
|
1775
|
+
async fn thrift_split_download_rejects_short_part_offset_by_long_part() {
|
|
1776
|
+
let server = MockServer::start().await;
|
|
1777
|
+
mount_single_link_statement(&server, "/_data/offset-parts", 60_000).await;
|
|
1778
|
+
|
|
1779
|
+
// 3 MiB file, three 1 MiB ranges. The middle part returns 10 bytes too
|
|
1780
|
+
// few and the last 10 bytes too many, so the assembled length equals the
|
|
1781
|
+
// declared total and only a per-part length check can catch it.
|
|
1782
|
+
const MIB: u64 = 1 << 20;
|
|
1783
|
+
const TOTAL: u64 = 3 * MIB;
|
|
1784
|
+
Mock::given(method("GET"))
|
|
1785
|
+
.and(path("/_data/offset-parts"))
|
|
1786
|
+
.respond_with(move |req: &Request| {
|
|
1787
|
+
let (start, end) = parse_range(req).expect("client always sends Range on this path");
|
|
1788
|
+
let end = end.min(TOTAL - 1);
|
|
1789
|
+
let len = match start {
|
|
1790
|
+
MIB => MIB - 10,
|
|
1791
|
+
s if s == 2 * MIB => MIB + 10,
|
|
1792
|
+
_ => end - start + 1,
|
|
1793
|
+
};
|
|
1794
|
+
ResponseTemplate::new(206)
|
|
1795
|
+
.insert_header("Content-Range", format!("bytes {start}-{end}/{TOTAL}"))
|
|
1796
|
+
.set_body_raw(vec![0u8; len as usize], "application/octet-stream")
|
|
1797
|
+
})
|
|
1798
|
+
.mount(&server)
|
|
1799
|
+
.await;
|
|
1800
|
+
|
|
1801
|
+
let client = thrift_client(&server);
|
|
1802
|
+
let mut stream = execute_lazy_thrift(client, "SELECT * FROM t", None, None, None)
|
|
1803
|
+
.await
|
|
1804
|
+
.unwrap();
|
|
1805
|
+
let err = stream
|
|
1806
|
+
.fetchall_arrow()
|
|
1807
|
+
.await
|
|
1808
|
+
.expect_err("a wrong-length part must fail loudly even when the total adds up");
|
|
1809
|
+
assert!(
|
|
1810
|
+
err.message.contains("split download part"),
|
|
1811
|
+
"got a different error than expected: {}",
|
|
1812
|
+
err.message
|
|
1813
|
+
);
|
|
1814
|
+
}
|
|
1815
|
+
|
|
1739
1816
|
#[tokio::test]
|
|
1740
1817
|
async fn thrift_concurrent_single_link_queries_stay_correct_under_split_contention() {
|
|
1741
1818
|
let server = MockServer::start().await;
|
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
//! Files API (upload_volume_file/delete_volume_file) -- kept separate from
|
|
2
|
+
//! wiremock_pipeline.rs since it's a different API surface entirely (no
|
|
3
|
+
//! statement/manifest/chunk shape to it), mirroring tests/conftest.py's own
|
|
4
|
+
//! `mock_volume_files` fixture being kept apart from `mock_warehouse`.
|
|
5
|
+
|
|
6
|
+
use std::sync::Arc;
|
|
7
|
+
|
|
8
|
+
use arrowbricks_core::client::DbClient;
|
|
9
|
+
use bytes::Bytes;
|
|
10
|
+
use wiremock::matchers::{method, path_regex, query_param};
|
|
11
|
+
use wiremock::{Mock, MockServer, ResponseTemplate};
|
|
12
|
+
|
|
13
|
+
const WAREHOUSE_ID: &str = "wh-test-123";
|
|
14
|
+
|
|
15
|
+
#[tokio::test]
|
|
16
|
+
async fn upload_volume_file_sends_overwrite_and_content_type() {
|
|
17
|
+
let server = MockServer::start().await;
|
|
18
|
+
Mock::given(method("PUT"))
|
|
19
|
+
.and(path_regex(r"^/api/2\.0/fs/files/.*"))
|
|
20
|
+
.and(query_param("overwrite", "true"))
|
|
21
|
+
.respond_with(ResponseTemplate::new(204))
|
|
22
|
+
.mount(&server)
|
|
23
|
+
.await;
|
|
24
|
+
|
|
25
|
+
let client = Arc::new(DbClient::new(&server.uri(), WAREHOUSE_ID, "fake-token"));
|
|
26
|
+
client
|
|
27
|
+
.upload_volume_file("/Volumes/cat/schema/vol/file.bin", Bytes::from_static(b"hello"))
|
|
28
|
+
.await
|
|
29
|
+
.unwrap();
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
#[tokio::test]
|
|
33
|
+
async fn delete_volume_file_succeeds_on_204() {
|
|
34
|
+
let server = MockServer::start().await;
|
|
35
|
+
Mock::given(method("DELETE"))
|
|
36
|
+
.and(path_regex(r"^/api/2\.0/fs/files/.*"))
|
|
37
|
+
.respond_with(ResponseTemplate::new(204))
|
|
38
|
+
.mount(&server)
|
|
39
|
+
.await;
|
|
40
|
+
|
|
41
|
+
let client = Arc::new(DbClient::new(&server.uri(), WAREHOUSE_ID, "fake-token"));
|
|
42
|
+
client
|
|
43
|
+
.delete_volume_file("/Volumes/cat/schema/vol/file.bin")
|
|
44
|
+
.await
|
|
45
|
+
.unwrap();
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
#[tokio::test]
|
|
49
|
+
async fn delete_volume_file_treats_404_as_success() {
|
|
50
|
+
let server = MockServer::start().await;
|
|
51
|
+
Mock::given(method("DELETE"))
|
|
52
|
+
.and(path_regex(r"^/api/2\.0/fs/files/.*"))
|
|
53
|
+
.respond_with(ResponseTemplate::new(404))
|
|
54
|
+
.mount(&server)
|
|
55
|
+
.await;
|
|
56
|
+
|
|
57
|
+
let client = Arc::new(DbClient::new(&server.uri(), WAREHOUSE_ID, "fake-token"));
|
|
58
|
+
client
|
|
59
|
+
.delete_volume_file("/Volumes/cat/schema/vol/already-gone.bin")
|
|
60
|
+
.await
|
|
61
|
+
.expect("404 must be treated as success (idempotent delete), not an error");
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
#[tokio::test]
|
|
65
|
+
async fn delete_volume_file_surfaces_other_errors() {
|
|
66
|
+
// 400, not 500/429/etc -- those are classified transient and would
|
|
67
|
+
// trigger the full 6-attempt retry/backoff cycle (tens of seconds) for
|
|
68
|
+
// no benefit here; that classification logic already has its own
|
|
69
|
+
// coverage. This only needs a status delete_volume_file does NOT treat
|
|
70
|
+
// as "already gone", proving the error still propagates.
|
|
71
|
+
let server = MockServer::start().await;
|
|
72
|
+
Mock::given(method("DELETE"))
|
|
73
|
+
.and(path_regex(r"^/api/2\.0/fs/files/.*"))
|
|
74
|
+
.respond_with(ResponseTemplate::new(400))
|
|
75
|
+
.mount(&server)
|
|
76
|
+
.await;
|
|
77
|
+
|
|
78
|
+
let client = Arc::new(DbClient::new(&server.uri(), WAREHOUSE_ID, "fake-token"));
|
|
79
|
+
let err = client
|
|
80
|
+
.delete_volume_file("/Volumes/cat/schema/vol/file.bin")
|
|
81
|
+
.await
|
|
82
|
+
.unwrap_err();
|
|
83
|
+
assert!(
|
|
84
|
+
!err.transient,
|
|
85
|
+
"400 is not transient, should surface immediately without retrying"
|
|
86
|
+
);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// ---------------------------------------------------------------------------
|
|
90
|
+
// Upload deadline scaling and retry body reuse. wiremock cannot throttle how
|
|
91
|
+
// fast the *server reads* a request body, so these use a tiny raw TCP server.
|
|
92
|
+
// ---------------------------------------------------------------------------
|
|
93
|
+
|
|
94
|
+
use std::time::{Duration, Instant};
|
|
95
|
+
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
|
96
|
+
use tokio::net::TcpListener;
|
|
97
|
+
|
|
98
|
+
const MIB: usize = 1 << 20;
|
|
99
|
+
|
|
100
|
+
/// Raw HTTP/1.1 server answering every PUT with `204` once it has read the
|
|
101
|
+
/// whole body, consuming it at roughly `bytes_per_sec`. `stuck` instead never
|
|
102
|
+
/// reads or answers at all.
|
|
103
|
+
async fn spawn_throttled_server(bytes_per_sec: usize, stuck: bool) -> String {
|
|
104
|
+
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
105
|
+
let addr = listener.local_addr().unwrap();
|
|
106
|
+
tokio::spawn(async move {
|
|
107
|
+
loop {
|
|
108
|
+
let Ok((mut sock, _)) = listener.accept().await else {
|
|
109
|
+
return;
|
|
110
|
+
};
|
|
111
|
+
tokio::spawn(async move {
|
|
112
|
+
if stuck {
|
|
113
|
+
tokio::time::sleep(Duration::from_secs(120)).await;
|
|
114
|
+
return;
|
|
115
|
+
}
|
|
116
|
+
let mut buf = vec![0u8; 64 * 1024];
|
|
117
|
+
let mut head = Vec::new();
|
|
118
|
+
let (mut body_len, mut got_body, mut header_end) = (None::<usize>, 0usize, None::<usize>);
|
|
119
|
+
loop {
|
|
120
|
+
let Ok(n) = sock.read(&mut buf).await else { return };
|
|
121
|
+
if n == 0 {
|
|
122
|
+
return;
|
|
123
|
+
}
|
|
124
|
+
if header_end.is_none() {
|
|
125
|
+
head.extend_from_slice(&buf[..n]);
|
|
126
|
+
if let Some(pos) = head.windows(4).position(|w| w == b"\r\n\r\n") {
|
|
127
|
+
let text = String::from_utf8_lossy(&head[..pos]).to_lowercase();
|
|
128
|
+
body_len = text
|
|
129
|
+
.lines()
|
|
130
|
+
.find_map(|l| l.strip_prefix("content-length:").map(|v| v.trim().parse().unwrap()));
|
|
131
|
+
header_end = Some(pos + 4);
|
|
132
|
+
got_body = head.len() - (pos + 4);
|
|
133
|
+
}
|
|
134
|
+
} else {
|
|
135
|
+
got_body += n;
|
|
136
|
+
}
|
|
137
|
+
if header_end.is_some() {
|
|
138
|
+
if got_body >= body_len.expect("fixed-length body") {
|
|
139
|
+
break;
|
|
140
|
+
}
|
|
141
|
+
tokio::time::sleep(Duration::from_secs_f64(n as f64 / bytes_per_sec as f64)).await;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
let _ = sock
|
|
145
|
+
.write_all(b"HTTP/1.1 204 No Content\r\nConnection: close\r\n\r\n")
|
|
146
|
+
.await;
|
|
147
|
+
});
|
|
148
|
+
}
|
|
149
|
+
});
|
|
150
|
+
format!("http://{addr}")
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
#[tokio::test]
|
|
154
|
+
async fn upload_slow_but_progressing_succeeds_past_http_timeout() {
|
|
155
|
+
// 3 MiB read at ~1.5 MiB/s takes ~2 s: over the 1 s http_timeout, under
|
|
156
|
+
// the scaled deadline (3 MiB / 1 MiB/s = 3 s).
|
|
157
|
+
let uri = spawn_throttled_server(MIB + MIB / 2, false).await;
|
|
158
|
+
let client = Arc::new(DbClient::new(&uri, WAREHOUSE_ID, "fake-token").with_http_timeout(1.0));
|
|
159
|
+
let start = Instant::now();
|
|
160
|
+
client
|
|
161
|
+
.upload_volume_file("/Volumes/c/s/v/big.bin", Bytes::from(vec![7u8; 3 * MIB]))
|
|
162
|
+
.await
|
|
163
|
+
.expect("a progressing upload must not be killed at http_timeout");
|
|
164
|
+
assert!(
|
|
165
|
+
start.elapsed() > Duration::from_secs(1),
|
|
166
|
+
"test must exceed http_timeout to prove anything, took {:?}",
|
|
167
|
+
start.elapsed()
|
|
168
|
+
);
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
#[tokio::test]
|
|
172
|
+
async fn upload_stuck_server_still_times_out_at_scaled_deadline() {
|
|
173
|
+
let uri = spawn_throttled_server(1, true).await;
|
|
174
|
+
let client = Arc::new(DbClient::new(&uri, WAREHOUSE_ID, "fake-token").with_http_timeout(1.0));
|
|
175
|
+
let start = Instant::now();
|
|
176
|
+
let err = client
|
|
177
|
+
.upload_volume_file("/Volumes/c/s/v/big.bin", Bytes::from(vec![7u8; 2 * MIB]))
|
|
178
|
+
.await
|
|
179
|
+
.expect_err("a stuck server must still fail");
|
|
180
|
+
let took = start.elapsed();
|
|
181
|
+
assert!(!err.transient, "timeouts stay non-transient (not retried): {err:?}");
|
|
182
|
+
assert!(
|
|
183
|
+
took >= Duration::from_secs(2) && took < Duration::from_secs(10),
|
|
184
|
+
"deadline should be the scaled ~2 s, took {took:?}"
|
|
185
|
+
);
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
#[tokio::test]
|
|
189
|
+
async fn upload_retry_after_503_resends_full_identical_body() {
|
|
190
|
+
use wiremock::matchers::body_bytes;
|
|
191
|
+
let server = MockServer::start().await;
|
|
192
|
+
let payload: Vec<u8> = (0..300_000u32).map(|i| (i % 251) as u8).collect();
|
|
193
|
+
Mock::given(method("PUT"))
|
|
194
|
+
.and(path_regex(r"^/api/2\.0/fs/files/.*"))
|
|
195
|
+
.respond_with(ResponseTemplate::new(503))
|
|
196
|
+
.up_to_n_times(1)
|
|
197
|
+
.expect(1)
|
|
198
|
+
.mount(&server)
|
|
199
|
+
.await;
|
|
200
|
+
Mock::given(method("PUT"))
|
|
201
|
+
.and(path_regex(r"^/api/2\.0/fs/files/.*"))
|
|
202
|
+
.and(body_bytes(payload.clone()))
|
|
203
|
+
.respond_with(ResponseTemplate::new(204))
|
|
204
|
+
.expect(1)
|
|
205
|
+
.mount(&server)
|
|
206
|
+
.await;
|
|
207
|
+
|
|
208
|
+
let client = Arc::new(DbClient::new(&server.uri(), WAREHOUSE_ID, "fake-token"));
|
|
209
|
+
client
|
|
210
|
+
.upload_volume_file("/Volumes/c/s/v/retry.bin", Bytes::from(payload))
|
|
211
|
+
.await
|
|
212
|
+
.expect("second attempt must carry the complete, identical body");
|
|
213
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
|
|
1
|
+
import array
|
|
2
|
+
from collections.abc import Awaitable, Callable, Sequence
|
|
2
3
|
from typing import Any, BinaryIO, Literal
|
|
3
4
|
|
|
4
5
|
def _validate_timeout(total_timeout_s: float | None) -> None: ...
|
|
@@ -114,7 +115,9 @@ class Client:
|
|
|
114
115
|
schema: str | None = None,
|
|
115
116
|
parameters: list[dict[str, Any]] | None = None,
|
|
116
117
|
) -> str: ...
|
|
117
|
-
async def upload_volume_file(
|
|
118
|
+
async def upload_volume_file(
|
|
119
|
+
self, volume_path: str, data: bytes | bytearray | memoryview | array.array | Sequence[int]
|
|
120
|
+
) -> None: ...
|
|
118
121
|
async def delete_volume_file(self, volume_path: str) -> None: ...
|
|
119
122
|
async def close_sessions(self) -> None: ...
|
|
120
123
|
def stream_ndjson_lines(
|
|
@@ -207,13 +207,17 @@ async def stream_query_json(
|
|
|
207
207
|
mid-download), then each result row as a ready-to-send JSON string --
|
|
208
208
|
one per SSE frame, say.
|
|
209
209
|
|
|
210
|
-
Unlike cursor.py's fetch methods, this
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
210
|
+
Unlike cursor.py's fetch methods, this does not buffer the full result
|
|
211
|
+
first: it emits rows via `._core.Client.stream_ndjson_lines` as chunks
|
|
212
|
+
are decoded, in order (chunks can complete out of order over the
|
|
213
|
+
network; order is restored there). The first row is available once the
|
|
214
|
+
first chunk has been downloaded and decoded. For a result spread over
|
|
215
|
+
many chunks on a bandwidth-limited link that can be most of the total
|
|
216
|
+
download time (measured on one ~30 MB/s link: the first row arrived at
|
|
217
|
+
84-97% of the stream total), so do not count on early rows for large
|
|
218
|
+
results. What streaming does give you: heartbeats while waiting, bounded
|
|
219
|
+
memory, and no full materialization. Decode and NDJSON encoding both
|
|
220
|
+
happen in Rust -- this function only forwards already-formatted lines.
|
|
217
221
|
|
|
218
222
|
`non_finite_floats` -- `"null"` (default) or `"string"`. A NaN/Infinity/
|
|
219
223
|
-Infinity float column comes back from Databricks like any other value,
|
|
@@ -7,7 +7,8 @@ reorder, Arrow/JSON/NDJSON decode)."""
|
|
|
7
7
|
|
|
8
8
|
from __future__ import annotations
|
|
9
9
|
|
|
10
|
-
|
|
10
|
+
import array
|
|
11
|
+
from collections.abc import AsyncIterator, Awaitable, Callable, Sequence
|
|
11
12
|
from typing import Any
|
|
12
13
|
|
|
13
14
|
from . import _core
|
|
@@ -133,12 +134,27 @@ class DatabricksClient:
|
|
|
133
134
|
async def __aexit__(self, *exc_info: object) -> None:
|
|
134
135
|
await self.aclose()
|
|
135
136
|
|
|
136
|
-
async def upload_volume_file(
|
|
137
|
+
async def upload_volume_file(
|
|
138
|
+
self, volume_path: str, data: bytes | bytearray | memoryview | array.array | Sequence[int]
|
|
139
|
+
) -> None:
|
|
137
140
|
"""Uploads `data` to a Unity Catalog volume path via the Files API,
|
|
138
141
|
overwriting anything already there. `volume_path` is caller-supplied
|
|
139
142
|
in full (e.g. `/Volumes/my_catalog/my_schema/my_volume/some/file.parquet`)
|
|
140
143
|
-- this package has no knowledge of any specific catalog/schema/volume.
|
|
141
|
-
|
|
144
|
+
|
|
145
|
+
`data` may be `bytes` (shared with the upload, not copied -- the
|
|
146
|
+
cheapest and recommended form) or any byte buffer (`bytearray`,
|
|
147
|
+
`memoryview`, `array.array`, a numpy array, a pyarrow `Buffer`; signed
|
|
148
|
+
or unsigned bytes), which is copied once up front. A plain sequence of
|
|
149
|
+
ints in 0..=255 (e.g. a `list[int]`) still works but is slow; prefer
|
|
150
|
+
`bytes(data)`. Anything else raises `TypeError`.
|
|
151
|
+
|
|
152
|
+
The PUT's total deadline is `max(http_timeout, len(data) / 1 MiB/s)`,
|
|
153
|
+
so `http_timeout` is a floor, not a ceiling: a slow-but-progressing
|
|
154
|
+
large upload is not cut off at `http_timeout`, and a stalled 1 GiB
|
|
155
|
+
upload can take about 17 minutes to fail. A timeout is not retried;
|
|
156
|
+
transient HTTP errors are. Raises `RuntimeError` (message only) on
|
|
157
|
+
failure."""
|
|
142
158
|
await self._core_client.upload_volume_file(volume_path, data)
|
|
143
159
|
|
|
144
160
|
async def delete_volume_file(self, volume_path: str) -> None:
|
|
@@ -1,87 +0,0 @@
|
|
|
1
|
-
//! Files API (upload_volume_file/delete_volume_file) -- kept separate from
|
|
2
|
-
//! wiremock_pipeline.rs since it's a different API surface entirely (no
|
|
3
|
-
//! statement/manifest/chunk shape to it), mirroring tests/conftest.py's own
|
|
4
|
-
//! `mock_volume_files` fixture being kept apart from `mock_warehouse`.
|
|
5
|
-
|
|
6
|
-
use std::sync::Arc;
|
|
7
|
-
|
|
8
|
-
use arrowbricks_core::client::DbClient;
|
|
9
|
-
use bytes::Bytes;
|
|
10
|
-
use wiremock::matchers::{method, path_regex, query_param};
|
|
11
|
-
use wiremock::{Mock, MockServer, ResponseTemplate};
|
|
12
|
-
|
|
13
|
-
const WAREHOUSE_ID: &str = "wh-test-123";
|
|
14
|
-
|
|
15
|
-
#[tokio::test]
|
|
16
|
-
async fn upload_volume_file_sends_overwrite_and_content_type() {
|
|
17
|
-
let server = MockServer::start().await;
|
|
18
|
-
Mock::given(method("PUT"))
|
|
19
|
-
.and(path_regex(r"^/api/2\.0/fs/files/.*"))
|
|
20
|
-
.and(query_param("overwrite", "true"))
|
|
21
|
-
.respond_with(ResponseTemplate::new(204))
|
|
22
|
-
.mount(&server)
|
|
23
|
-
.await;
|
|
24
|
-
|
|
25
|
-
let client = Arc::new(DbClient::new(&server.uri(), WAREHOUSE_ID, "fake-token"));
|
|
26
|
-
client
|
|
27
|
-
.upload_volume_file("/Volumes/cat/schema/vol/file.bin", Bytes::from_static(b"hello"))
|
|
28
|
-
.await
|
|
29
|
-
.unwrap();
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
#[tokio::test]
|
|
33
|
-
async fn delete_volume_file_succeeds_on_204() {
|
|
34
|
-
let server = MockServer::start().await;
|
|
35
|
-
Mock::given(method("DELETE"))
|
|
36
|
-
.and(path_regex(r"^/api/2\.0/fs/files/.*"))
|
|
37
|
-
.respond_with(ResponseTemplate::new(204))
|
|
38
|
-
.mount(&server)
|
|
39
|
-
.await;
|
|
40
|
-
|
|
41
|
-
let client = Arc::new(DbClient::new(&server.uri(), WAREHOUSE_ID, "fake-token"));
|
|
42
|
-
client
|
|
43
|
-
.delete_volume_file("/Volumes/cat/schema/vol/file.bin")
|
|
44
|
-
.await
|
|
45
|
-
.unwrap();
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
#[tokio::test]
|
|
49
|
-
async fn delete_volume_file_treats_404_as_success() {
|
|
50
|
-
let server = MockServer::start().await;
|
|
51
|
-
Mock::given(method("DELETE"))
|
|
52
|
-
.and(path_regex(r"^/api/2\.0/fs/files/.*"))
|
|
53
|
-
.respond_with(ResponseTemplate::new(404))
|
|
54
|
-
.mount(&server)
|
|
55
|
-
.await;
|
|
56
|
-
|
|
57
|
-
let client = Arc::new(DbClient::new(&server.uri(), WAREHOUSE_ID, "fake-token"));
|
|
58
|
-
client
|
|
59
|
-
.delete_volume_file("/Volumes/cat/schema/vol/already-gone.bin")
|
|
60
|
-
.await
|
|
61
|
-
.expect("404 must be treated as success (idempotent delete), not an error");
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
#[tokio::test]
|
|
65
|
-
async fn delete_volume_file_surfaces_other_errors() {
|
|
66
|
-
// 400, not 500/429/etc -- those are classified transient and would
|
|
67
|
-
// trigger the full 6-attempt retry/backoff cycle (tens of seconds) for
|
|
68
|
-
// no benefit here; that classification logic already has its own
|
|
69
|
-
// coverage. This only needs a status delete_volume_file does NOT treat
|
|
70
|
-
// as "already gone", proving the error still propagates.
|
|
71
|
-
let server = MockServer::start().await;
|
|
72
|
-
Mock::given(method("DELETE"))
|
|
73
|
-
.and(path_regex(r"^/api/2\.0/fs/files/.*"))
|
|
74
|
-
.respond_with(ResponseTemplate::new(400))
|
|
75
|
-
.mount(&server)
|
|
76
|
-
.await;
|
|
77
|
-
|
|
78
|
-
let client = Arc::new(DbClient::new(&server.uri(), WAREHOUSE_ID, "fake-token"));
|
|
79
|
-
let err = client
|
|
80
|
-
.delete_volume_file("/Volumes/cat/schema/vol/file.bin")
|
|
81
|
-
.await
|
|
82
|
-
.unwrap_err();
|
|
83
|
-
assert!(
|
|
84
|
-
!err.transient,
|
|
85
|
-
"400 is not transient, should surface immediately without retrying"
|
|
86
|
-
);
|
|
87
|
-
}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/examples/agent_render_bench.rs
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/proptest-regressions/json_convert.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests_py/test_stream_ndjson_lines.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{arrowbricks-6.2.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests_py/test_token_provider.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|