arrowbricks 6.2.0__tar.gz → 6.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/PKG-INFO +11 -5
  2. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/README.md +10 -4
  3. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/pyproject.toml +2 -1
  4. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/Cargo.lock +1 -1
  5. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/Cargo.toml +1 -1
  6. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/README.md +1 -1
  7. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/client/download.rs +152 -11
  8. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/client/volume.rs +58 -1
  9. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/lib.rs +51 -3
  10. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/tests/wiremock_thrift.rs +85 -8
  11. arrowbricks-6.4.0/rust/arrowbricks_core/tests/wiremock_volume_files.rs +250 -0
  12. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/src/arrowbricks/_core.pyi +6 -2
  13. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/src/arrowbricks/_streaming.py +11 -7
  14. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/src/arrowbricks/client.py +24 -3
  15. arrowbricks-6.2.0/rust/arrowbricks_core/tests/wiremock_volume_files.rs +0 -87
  16. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/LICENSE +0 -0
  17. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/.gitignore +0 -0
  18. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/examples/agent_render_bench.rs +0 -0
  19. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/examples/duckdb_query.py +0 -0
  20. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/examples/fastapi_sse.py +0 -0
  21. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/proptest-regressions/json_convert.txt +0 -0
  22. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/rustfmt.toml +0 -0
  23. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/agent/diagnostics.rs +0 -0
  24. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/agent/mod.rs +0 -0
  25. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/agent/render.rs +0 -0
  26. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/alloc.rs +0 -0
  27. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/arrow_ffi.rs +0 -0
  28. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/client/call_context.rs +0 -0
  29. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/client/error.rs +0 -0
  30. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/client/model.rs +0 -0
  31. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/client/sea.rs +0 -0
  32. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/client/thrift_rpc.rs +0 -0
  33. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/client.rs +0 -0
  34. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/heartbeat.rs +0 -0
  35. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/json_convert.rs +0 -0
  36. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/pipeline/ndjson.rs +0 -0
  37. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/pipeline/reorder.rs +0 -0
  38. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/pipeline/sea.rs +0 -0
  39. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/pipeline/stats.rs +0 -0
  40. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/pipeline/test_support.rs +0 -0
  41. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/pipeline/thrift_exec.rs +0 -0
  42. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/pipeline.rs +0 -0
  43. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/thrift.rs +0 -0
  44. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/tests/common/mod.rs +0 -0
  45. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/tests/wiremock_agent.rs +0 -0
  46. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/tests/wiremock_pipeline.rs +0 -0
  47. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/tests_py/conftest.py +0 -0
  48. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/tests_py/test_ipc_stream.py +0 -0
  49. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/tests_py/test_parameters.py +0 -0
  50. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/tests_py/test_stream_ndjson_lines.py +0 -0
  51. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/tests_py/test_streaming.py +0 -0
  52. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/tests_py/test_thrift.py +0 -0
  53. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/tests_py/test_token_provider.py +0 -0
  54. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/tests_py/test_volume_files.py +0 -0
  55. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/tests_py/thrift_mock.py +0 -0
  56. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/src/arrowbricks/__init__.py +0 -0
  57. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/src/arrowbricks/__main__.py +0 -0
  58. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/src/arrowbricks/_agent.py +0 -0
  59. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/src/arrowbricks/_sql.py +0 -0
  60. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/src/arrowbricks/cli.py +0 -0
  61. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/src/arrowbricks/cursor.py +0 -0
  62. {arrowbricks-6.2.0 → arrowbricks-6.4.0}/src/arrowbricks/py.typed +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: arrowbricks
3
- Version: 6.2.0
3
+ Version: 6.4.0
4
4
  Requires-Dist: arro3-core>=0.8 ; extra == 'arro3'
5
5
  Requires-Dist: sqlglot>=27 ; extra == 'sqlglot'
6
6
  Provides-Extra: arro3
@@ -13,14 +13,12 @@ Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
13
13
  Project-URL: Repository, https://github.com/bmsuisse/arrowbricks
14
14
 
15
15
  <p align="center">
16
- <img src="assets/logo.svg" width="72" height="72" alt="arrowbricks logo">
16
+ <img src="assets/logo.svg" height="120" alt="arrowbricks">
17
17
  </p>
18
-
19
- <h1 align="center">arrowbricks</h1>
20
18
  <p align="center">Fast Databricks SQL for Python applications and AI agents.</p>
21
19
 
22
20
  - **Zero required Python dependencies.** One precompiled wheel. No Rust toolchain or cloud SDK needed.
23
- - **Fast Rust core.** Concurrent chunk downloads, lazy Arrow decoding, and streaming NDJSON, with result order preserved.
21
+ - **Fast Rust core.** Concurrent chunk downloads, lazy Arrow decoding, and streaming NDJSON without materializing the full result, with result order preserved.
24
22
  - **Agent mode, new in 6.0.** A CLI and Python API that return compact typed TSV (or a Markdown table), explicit truncation, and actionable JSON errors. Formatting stays in Rust.
25
23
 
26
24
  Use Thrift by default or the Statement Execution API with `protocol="sea"`.
@@ -118,6 +116,14 @@ a 100-row preview used **44% fewer bytes and 24% fewer tokens** than NDJSON.
118
116
  See [agent measurements and methodology](docs/research/agent-rendering.md) and
119
117
  [client benchmarks](docs/benchmarks.md) for workloads, comparisons, and limits.
120
118
 
119
+ ### Performance tips
120
+
121
+ Numbers are measurements from one ~30 MB/s link to one warehouse, not guarantees.
122
+
123
+ - **Select only the columns you need.** Selecting 3 columns instead of `SELECT *` was 1.5-6x faster and moved 1.9-15.8x fewer bytes across 5 tables.
124
+ - **Keep `compress_results` at its default (`True`)** unless the link is very fast. Here it was 1.8-4.7x faster with 2-5.5x fewer bytes. Turning it off on a very fast link was not tested.
125
+ - **Keep the Thrift default for small lookups.** It was about 0.15 s faster per tiny statement than `protocol="sea"` (a constant gap in these runs).
126
+
121
127
  ## Learn more
122
128
 
123
129
  - [Usage and API reference](docs/usage.md): authentication, errors, cancellation, observability, and Arrow interop.
@@ -1,12 +1,10 @@
1
1
  <p align="center">
2
- <img src="assets/logo.svg" width="72" height="72" alt="arrowbricks logo">
2
+ <img src="assets/logo.svg" height="120" alt="arrowbricks">
3
3
  </p>
4
-
5
- <h1 align="center">arrowbricks</h1>
6
4
  <p align="center">Fast Databricks SQL for Python applications and AI agents.</p>
7
5
 
8
6
  - **Zero required Python dependencies.** One precompiled wheel. No Rust toolchain or cloud SDK needed.
9
- - **Fast Rust core.** Concurrent chunk downloads, lazy Arrow decoding, and streaming NDJSON, with result order preserved.
7
+ - **Fast Rust core.** Concurrent chunk downloads, lazy Arrow decoding, and streaming NDJSON without materializing the full result, with result order preserved.
10
8
  - **Agent mode, new in 6.0.** A CLI and Python API that return compact typed TSV (or a Markdown table), explicit truncation, and actionable JSON errors. Formatting stays in Rust.
11
9
 
12
10
  Use Thrift by default or the Statement Execution API with `protocol="sea"`.
@@ -104,6 +102,14 @@ a 100-row preview used **44% fewer bytes and 24% fewer tokens** than NDJSON.
104
102
  See [agent measurements and methodology](docs/research/agent-rendering.md) and
105
103
  [client benchmarks](docs/benchmarks.md) for workloads, comparisons, and limits.
106
104
 
105
+ ### Performance tips
106
+
107
+ Numbers are measurements from one ~30 MB/s link to one warehouse, not guarantees.
108
+
109
+ - **Select only the columns you need.** Selecting 3 columns instead of `SELECT *` was 1.5-6x faster and moved 1.9-15.8x fewer bytes across 5 tables.
110
+ - **Keep `compress_results` at its default (`True`)** unless the link is very fast. Here it was 1.8-4.7x faster with 2-5.5x fewer bytes. Turning it off on a very fast link was not tested.
111
+ - **Keep the Thrift default for small lookups.** It was about 0.15 s faster per tiny statement than `protocol="sea"` (a constant gap in these runs).
112
+
107
113
  ## Learn more
108
114
 
109
115
  - [Usage and API reference](docs/usage.md): authentication, errors, cancellation, observability, and Arrow interop.
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "arrowbricks"
3
- version = "6.2.0"
3
+ version = "6.4.0"
4
4
  description = "Rust-powered Databricks SQL client and agent CLI with Arrow/NDJSON streaming, bounded Markdown previews, actionable JSON diagnostics, and zero required Python dependencies."
5
5
  readme = "README.md"
6
6
  license = "MIT"
@@ -105,3 +105,4 @@ select = ["E", "F", "I", "UP", "S", "B"]
105
105
  "tests/arro3_free/*" = ["S101", "S106"]
106
106
  # One-off benchmark harness: terse scripts, seeded (non-crypto) randomness, asserts as checks.
107
107
  "benchmarks/agent-output-format/*" = ["B905", "E501", "E731", "E741", "S101", "S311"]
108
+ "benchmarks/benchmark_duckdb.py" = ["E501", "S608"]
@@ -177,7 +177,7 @@ dependencies = [
177
177
 
178
178
  [[package]]
179
179
  name = "arrowbricks_core"
180
- version = "6.2.0"
180
+ version = "6.4.0"
181
181
  dependencies = [
182
182
  "arrow-array",
183
183
  "arrow-buffer",
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "arrowbricks_core"
3
- version = "6.2.0"
3
+ version = "6.4.0"
4
4
  edition = "2024"
5
5
  readme = "README.md"
6
6
 
@@ -79,7 +79,7 @@ Arrow library first (e.g. `arro3.core.Table.from_arrow(table)`).
79
79
  - `ResultSet.schema() -> list[tuple[str, str]] | None` -- the real decoded Arrow schema as `(name, type_name)` string pairs, once known (after >=1 fetch); `None` before that. Computed directly in Rust -- unlike a `Table`'s own `.schema`, this needs no arro3 install.
80
80
  - `ResultSet.statement_id`, `ResultSet.num_chunks`, `ResultSet.columns` (manifest-based pre-fetch schema estimate, also arro3-free).
81
81
  - `Client.stream_ndjson_lines(statement, *, catalog=None, schema=None, parameters=None, total_timeout_s=None)` -- chunk-at-a-time: an async iterator yielding the `HEARTBEAT` singleton while waiting on the statement or any individual chunk, then a `list[str]` of NDJSON lines (one per row, explicit nulls, ISO-8601 timestamps) per chunk in logical order. Decode and JSON encoding both happen in Rust -- backs `stream_query_json` end to end.
82
- - `Client.upload_volume_file(volume_path, data: bytes)` / `Client.delete_volume_file(volume_path)` -- Unity Catalog volume files via the Files API. Delete treats a 404 as success (idempotent). Both raise a plain `RuntimeError` (message only) on failure.
82
+ - `Client.upload_volume_file(volume_path, data: bytes)` / `Client.download_volume_file(volume_path) -> bytes` / `Client.delete_volume_file(volume_path)` -- Unity Catalog volume files via the Files API. Delete treats a 404 as success (idempotent). Both raise a plain `RuntimeError` (message only) on failure.
83
83
  - `write_ipc_stream(stream, buf)` -- free function; writes any object implementing `__arrow_c_stream__` (a `Table` from this crate, arro3, pyarrow, ...) as uncompressed Arrow-IPC stream bytes to a Python file-like object. No dependency needed regardless of the input's origin.
84
84
  - `read_ipc_stream(data: bytes) -> Table` -- free function; the exact inverse of `write_ipc_stream`, parsing raw Arrow-IPC stream bytes back into a `Table`. No dependency needed regardless of where the bytes came from -- backs `arrowbricks.ReplayableArrowChunk`, which needs to re-parse the same cached bytes on every `__arrow_c_stream__` call. Decoding releases the Python interpreter lock so independent replay calls can run concurrently; decoded arrays retain ownership of the immutable input bytes.
85
85
  - `HEARTBEAT` -- module-level singleton; compare with `is`, e.g. `if item is _core.HEARTBEAT: ...`.
@@ -16,6 +16,21 @@ use super::MAX_SPLIT_PARTS;
16
16
  use super::error::{ApiError, join_error};
17
17
  use super::model::QueryStatsAccumulator;
18
18
 
19
+ /// Parses `Content-Range: bytes <start>-<end>/<total>` (the only form a 206
20
+ /// response to a single-range request may use). `*` totals and unsatisfied-range
21
+ /// forms yield `None`.
22
+ fn parse_content_range(headers: &reqwest::header::HeaderMap) -> Option<(u64, u64, u64)> {
23
+ let value = headers.get(reqwest::header::CONTENT_RANGE)?.to_str().ok()?;
24
+ let rest = value.trim().strip_prefix("bytes")?.trim_start();
25
+ let (range, total) = rest.split_once('/')?;
26
+ let (start, end) = range.trim().split_once('-')?;
27
+ Some((
28
+ start.trim().parse().ok()?,
29
+ end.trim().parse().ok()?,
30
+ total.trim().parse().ok()?,
31
+ ))
32
+ }
33
+
19
34
  /// Unwraps one downloaded chunk file's outer LZ4 Frame compression --
20
35
  /// server-side, this is a whole-file wrapper applied because we asked for
21
36
  /// `result_compression: "LZ4_FRAME"`, not Arrow-IPC's own (unrelated,
@@ -226,16 +241,27 @@ impl DbClient {
226
241
  return Err(ApiError::from_status(status, &text, true));
227
242
  }
228
243
  let ranged = status == reqwest::StatusCode::PARTIAL_CONTENT;
229
- let total: Option<u64> = resp
230
- .headers()
231
- .get(reqwest::header::CONTENT_RANGE)
232
- .and_then(|v| v.to_str().ok())
233
- .and_then(|v| v.rsplit('/').next().and_then(|t| t.parse().ok()));
234
- if ranged && total.is_none() {
235
- return Err(ApiError::permanent(
236
- "cloud-fetch link answered a Range request with 206 Partial Content but an \
237
- unparseable Content-Range header -- refusing to silently return a truncated file",
238
- ));
244
+ // A 206 must describe exactly the head we asked for: starting
245
+ // at 0 and ending at `part_size - 1` (or the last byte of a
246
+ // smaller file). Anything else would be spliced in as wrong
247
+ // bytes with the right length.
248
+ let mut total: Option<u64> = None;
249
+ let mut head_len: Option<u64> = None;
250
+ if ranged {
251
+ let Some((first, last, size)) = parse_content_range(resp.headers()) else {
252
+ return Err(ApiError::permanent(
253
+ "cloud-fetch link answered a Range request with 206 Partial Content but an \
254
+ unparseable Content-Range header -- refusing to silently return a truncated file",
255
+ ));
256
+ };
257
+ if first != 0 || size == 0 || last != (part_size - 1).min(size - 1) {
258
+ return Err(ApiError::permanent(format!(
259
+ "cloud-fetch link answered bytes=0-{} with Content-Range bytes {first}-{last}/{size}",
260
+ part_size - 1
261
+ )));
262
+ }
263
+ total = Some(size);
264
+ head_len = Some(last + 1);
239
265
  }
240
266
 
241
267
  let mut downloads = tokio::task::JoinSet::new();
@@ -248,7 +274,8 @@ impl DbClient {
248
274
  if start >= total {
249
275
  break;
250
276
  }
251
- let end = (start + rest_size - 1).min(total - 1);
277
+ // `total` is untrusted: saturate rather than overflow.
278
+ let end = start.saturating_add(rest_size - 1).min(total - 1);
252
279
  let this = self.clone();
253
280
  let url = url.to_string();
254
281
  let part_stats = stats.clone();
@@ -268,9 +295,36 @@ impl DbClient {
268
295
  let text = resp.text().await.unwrap_or_default();
269
296
  return Err(ApiError::from_status(status, &text, true));
270
297
  }
298
+ // The probe proved the server honours ranges, so
299
+ // each part must be a 206 for exactly this range
300
+ // of the same file; a 200 or another interval is
301
+ // wrong data, not a transient failure.
302
+ let got = (status == reqwest::StatusCode::PARTIAL_CONTENT)
303
+ .then(|| parse_content_range(resp.headers()))
304
+ .flatten();
305
+ if got != Some((start, end, total)) {
306
+ return Err(ApiError::permanent(format!(
307
+ "split download part bytes={start}-{end}/{total} was answered with status {status} \
308
+ and Content-Range {got:?}"
309
+ )));
310
+ }
271
311
  resp.bytes().await.map_err(|e| ApiError::from_reqwest(e, true))
272
312
  })
273
313
  .await;
314
+ // A part must be exactly the range asked for: the
315
+ // final sum check alone would accept a short part
316
+ // offset by a long one (wrong bytes, right total).
317
+ let bytes = bytes.and_then(|b| {
318
+ let expected = end - start + 1;
319
+ if b.len() as u64 == expected {
320
+ Ok(b)
321
+ } else {
322
+ Err(ApiError::permanent(format!(
323
+ "split download part bytes={start}-{end} returned {} bytes, expected {expected}",
324
+ b.len()
325
+ )))
326
+ }
327
+ });
274
328
  if let Ok(b) = &bytes {
275
329
  part_stats.bytes_downloaded.fetch_add(b.len() as u64, Ordering::Relaxed);
276
330
  }
@@ -288,6 +342,15 @@ impl DbClient {
288
342
  return Err(ApiError::from_reqwest(error, true));
289
343
  }
290
344
  };
345
+ if let Some(expected) = head_len
346
+ && head.len() as u64 != expected
347
+ {
348
+ downloads.shutdown().await;
349
+ return Err(ApiError::permanent(format!(
350
+ "split download probe returned {} bytes, expected {expected}",
351
+ head.len()
352
+ )));
353
+ }
291
354
  Ok((total, head, downloads))
292
355
  })
293
356
  .await?;
@@ -448,6 +511,84 @@ mod tests {
448
511
  );
449
512
  }
450
513
 
514
+ /// Serves a 12-byte file `abcdefghijkl` that honours `Range`, except where
515
+ /// `fault` says to lie: the failure modes a length check alone cannot see.
516
+ async fn serve_range_file(fault: &'static str) -> std::net::SocketAddr {
517
+ use tokio::io::{AsyncReadExt, AsyncWriteExt};
518
+ use tokio::net::TcpListener;
519
+
520
+ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
521
+ let address = listener.local_addr().unwrap();
522
+ tokio::spawn(async move {
523
+ loop {
524
+ let (mut socket, _) = listener.accept().await.unwrap();
525
+ tokio::spawn(async move {
526
+ let mut request = Vec::new();
527
+ while !request.ends_with(b"\r\n\r\n") {
528
+ if socket.read_u8().await.map(|b| request.push(b)).is_err() {
529
+ return;
530
+ }
531
+ }
532
+ let request = String::from_utf8(request).unwrap().to_ascii_lowercase();
533
+ let range = request.lines().find(|s| s.starts_with("range:")).unwrap();
534
+ let (start, end) = range.split_once("bytes=").unwrap().1.split_once('-').unwrap();
535
+ let (start, end): (usize, usize) = (start.parse().unwrap(), end.parse().unwrap());
536
+ let body = &b"abcdefghijkl"[start..=end];
537
+ let (status, content_range) = match (fault, start) {
538
+ ("wrong_offset", s) if s > 0 => {
539
+ ("206 Partial Content", format!("bytes {}-{}/12", s - 1, end - 1))
540
+ }
541
+ ("probe_wrong_offset", 0) => ("206 Partial Content", format!("bytes 1-{}/12", end + 1)),
542
+ ("part_ignores_range", s) if s > 0 => ("200 OK", String::new()),
543
+ _ => ("206 Partial Content", format!("bytes {start}-{end}/12")),
544
+ };
545
+ let range_header = if content_range.is_empty() {
546
+ String::new()
547
+ } else {
548
+ format!("Content-Range: {content_range}\r\n")
549
+ };
550
+ let head = format!(
551
+ "HTTP/1.1 {status}\r\nContent-Length: {}\r\n{range_header}Connection: close\r\n\r\n",
552
+ body.len()
553
+ );
554
+ let _ = socket.write_all(head.as_bytes()).await;
555
+ let _ = socket.write_all(body).await;
556
+ });
557
+ }
558
+ });
559
+ address
560
+ }
561
+
562
+ /// Right length from the wrong place passes a byte-count check; only the
563
+ /// status and `Content-Range` of every part can catch it.
564
+ #[tokio::test]
565
+ async fn split_download_rejects_parts_that_are_not_the_requested_range() {
566
+ for fault in ["wrong_offset", "probe_wrong_offset", "part_ignores_range"] {
567
+ let address = serve_range_file(fault).await;
568
+ let client = Arc::new(
569
+ DbClient::new(&format!("http://{address}"), "wh", "token")
570
+ .with_retry_attempts(1)
571
+ .with_retry_max_wait_s(0.0),
572
+ );
573
+ let stats = Arc::new(QueryStatsAccumulator::default());
574
+ let result = client
575
+ .fetch_link_bytes_split(&format!("http://{address}/blob"), false, 4, 3, &stats)
576
+ .await;
577
+ assert!(
578
+ result.is_err(),
579
+ "{fault}: must not assemble wrong bytes, got {result:?}"
580
+ );
581
+ }
582
+ let address = serve_range_file("none").await;
583
+ let client = Arc::new(DbClient::new(&format!("http://{address}"), "wh", "token"));
584
+ let stats = Arc::new(QueryStatsAccumulator::default());
585
+ let bytes = client
586
+ .fetch_link_bytes_split(&format!("http://{address}/blob"), false, 4, 3, &stats)
587
+ .await
588
+ .unwrap();
589
+ assert_eq!(&bytes[..], b"abcdefghijkl");
590
+ }
591
+
451
592
  #[tokio::test]
452
593
  async fn split_download_starts_tail_requests_before_the_probe_body_finishes() {
453
594
  use tokio::io::{AsyncReadExt, AsyncWriteExt};
@@ -2,12 +2,26 @@
2
2
  //! (`/api/2.0/fs/files{volume_path}`) -- unrelated to either SQL backend,
3
3
  //! just another authenticated HTTP operation this client exposes.
4
4
 
5
+ use std::time::Duration;
6
+
5
7
  use bytes::Bytes;
6
8
  use reqwest::StatusCode;
7
9
 
8
10
  use super::DbClient;
9
11
  use super::error::ApiError;
10
12
 
13
+ /// Slowest sustained upload rate the PUT deadline tolerates (1 MiB/s). The
14
+ /// per-request deadline is a total one covering the body, so it scales with
15
+ /// the payload: `max(http_timeout, len / MIN_UPLOAD_BYTES_PER_SEC)`. A slow
16
+ /// but progressing link is not killed at `http_timeout`; a stuck one still
17
+ /// fails once the scaled deadline passes.
18
+ pub const MIN_UPLOAD_BYTES_PER_SEC: u64 = 1 << 20;
19
+
20
+ /// Total deadline for one upload PUT of `len` bytes.
21
+ pub fn upload_deadline(http_timeout: Duration, len: usize) -> Duration {
22
+ http_timeout.max(Duration::from_secs((len as u64).div_ceil(MIN_UPLOAD_BYTES_PER_SEC)))
23
+ }
24
+
11
25
  impl DbClient {
12
26
  /// Uploads `data` to a Unity Catalog volume path via the Files API,
13
27
  /// overwriting anything already there. `volume_path` is caller-supplied
@@ -17,6 +31,7 @@ impl DbClient {
17
31
  /// cheap refcount clone, not a real copy.
18
32
  pub async fn upload_volume_file(&self, volume_path: &str, data: Bytes) -> Result<(), ApiError> {
19
33
  let url = format!("{}/api/2.0/fs/files{}", self.host, volume_path);
34
+ let timeout = upload_deadline(self.http_timeout, data.len());
20
35
  self.retry_call(|| async {
21
36
  let token = self.token_provider.get_token().await?;
22
37
  let resp = self
@@ -26,7 +41,7 @@ impl DbClient {
26
41
  .bearer_auth(&token)
27
42
  .header("Content-Type", "application/octet-stream")
28
43
  .body(data.clone())
29
- .timeout(self.http_timeout)
44
+ .timeout(timeout)
30
45
  .send()
31
46
  .await
32
47
  // `overwrite=true` means retrying this exact PUT is safe --
@@ -42,6 +57,34 @@ impl DbClient {
42
57
  .await
43
58
  }
44
59
 
60
+ /// Downloads a Unity Catalog volume file in one GET on the pooled
61
+ /// connection. Single request on purpose: for typical files the server's
62
+ /// time-to-first-byte dominates, so range-splitting only adds round trips.
63
+ /// A non-2xx (incl. 404) is an error, unlike delete.
64
+ pub async fn download_volume_file(&self, volume_path: &str) -> Result<Bytes, ApiError> {
65
+ let url = format!("{}/api/2.0/fs/files{}", self.host, volume_path);
66
+ self.retry_call(|| async {
67
+ let token = self.token_provider.get_token().await?;
68
+ // No total request timeout: headers get `http_timeout`, the body a
69
+ // deadline scaled to Content-Length (same 1 MiB/s floor as upload).
70
+ let resp = tokio::time::timeout(self.http_timeout, self.http.get(&url).bearer_auth(&token).send())
71
+ .await
72
+ .map_err(|_| ApiError::permanent("volume download timed out waiting for response headers"))?
73
+ .map_err(|e| ApiError::from_reqwest(e, true))?;
74
+ let status = resp.status();
75
+ if !status.is_success() {
76
+ let text = resp.text().await.unwrap_or_default();
77
+ return Err(ApiError::from_status(status, &text, true));
78
+ }
79
+ let deadline = upload_deadline(self.http_timeout, resp.content_length().unwrap_or(0) as usize);
80
+ tokio::time::timeout(deadline, resp.bytes())
81
+ .await
82
+ .map_err(|_| ApiError::permanent("volume download timed out reading body"))?
83
+ .map_err(|e| ApiError::from_reqwest(e, true))
84
+ })
85
+ .await
86
+ }
87
+
45
88
  /// Deletes a file at `volume_path` (see `upload_volume_file`). A 404 is
46
89
  /// treated as success -- the file is already gone, which is fine for
47
90
  /// idempotent staging cleanup.
@@ -67,3 +110,17 @@ impl DbClient {
67
110
  .await
68
111
  }
69
112
  }
113
+
114
+ #[cfg(test)]
115
+ mod tests {
116
+ use super::*;
117
+
118
+ #[test]
119
+ fn upload_deadline_scales_with_size_but_never_below_http_timeout() {
120
+ let base = Duration::from_secs(60);
121
+ assert_eq!(upload_deadline(base, 0), base);
122
+ assert_eq!(upload_deadline(base, 10 << 20), base);
123
+ assert_eq!(upload_deadline(base, 600 << 20), Duration::from_secs(600));
124
+ assert_eq!(upload_deadline(base, (600 << 20) + 1), Duration::from_secs(601));
125
+ }
126
+ }
@@ -130,6 +130,40 @@ fn read_ipc_stream(data: Bound<'_, PyBytes>) -> PyResult<PyTable> {
130
130
  PyTable::try_new(batches, schema)
131
131
  }
132
132
 
133
+ /// Turns the Python `data` argument of `upload_volume_file` into `Bytes`
134
+ /// without the per-element `Vec<u8>` extraction (20x slower) and, for the
135
+ /// common `bytes` case, without any copy.
136
+ fn upload_body(data: &Bound<'_, PyAny>) -> PyResult<bytes::Bytes> {
137
+ if let Ok(b) = data.cast::<PyBytes>() {
138
+ // Immutable, so the Python object can own the memory for the whole
139
+ // upload (and every retry) with no copy.
140
+ return Ok(bytes::Bytes::from_owner(pyo3::pybacked::PyBackedBytes::from(b.clone())));
141
+ }
142
+ // bytearray/memoryview/numpy/pyarrow are mutable or not owned by us: take
143
+ // one contiguous copy now so later caller mutation cannot change the
144
+ // upload mid-flight.
145
+ let py = data.py();
146
+ if let Ok(buffer) = pyo3::buffer::PyBuffer::<u8>::get(data) {
147
+ return Ok(bytes::Bytes::from(buffer.to_vec(py)?));
148
+ }
149
+ // PyArrow buffers (and `array('b')`) export the signed-byte format `b`,
150
+ // which `PyBuffer<u8>` refuses; the raw bits are the same bytes.
151
+ if let Ok(buffer) = pyo3::buffer::PyBuffer::<i8>::get(data) {
152
+ let signed = buffer.to_vec(py)?;
153
+ return Ok(bytes::Bytes::from(
154
+ signed.into_iter().map(|b| b as u8).collect::<Vec<u8>>(),
155
+ ));
156
+ }
157
+ // Last resort keeps the pre-6.3 behaviour: any sequence of ints in 0..=255
158
+ // (a `list[int]`, a numpy int64 array, ...). Slow, but it used to work.
159
+ data.extract::<Vec<u8>>().map(bytes::Bytes::from).map_err(|_| {
160
+ pyo3::exceptions::PyTypeError::new_err(
161
+ "upload_volume_file data must be bytes, a byte buffer (bytearray, memoryview, numpy, pyarrow, ...) \
162
+ or a sequence of ints in 0..=255",
163
+ )
164
+ })
165
+ }
166
+
133
167
  /// Wraps a `PyErr` raised by the caller's own `token_provider` callable (its
134
168
  /// call itself, awaiting its coroutine, or extracting a non-`str` return
135
169
  /// value) into an `ApiError`. This function's *only* caller is
@@ -869,16 +903,18 @@ impl PyDbClient {
869
903
  }
870
904
 
871
905
  /// Uploads `data` to a Unity Catalog volume path via the Files API,
872
- /// overwriting anything already there.
906
+ /// overwriting anything already there. `bytes` is borrowed zero-copy;
907
+ /// `bytearray` and other contiguous byte buffers (memoryview, numpy
908
+ /// `uint8`, pyarrow `Buffer`) are copied once, off the per-element path.
873
909
  #[pyo3(signature = (volume_path, data))]
874
910
  fn upload_volume_file<'py>(
875
911
  &self,
876
912
  py: Python<'py>,
877
913
  volume_path: String,
878
- data: Vec<u8>,
914
+ data: &Bound<'py, PyAny>,
879
915
  ) -> PyResult<Bound<'py, PyAny>> {
880
916
  let client = self.inner.clone();
881
- let data = bytes::Bytes::from(data);
917
+ let data = upload_body(data)?;
882
918
  future_into_py_in_call(py, async move {
883
919
  client
884
920
  .upload_volume_file(&volume_path, data)
@@ -887,6 +923,18 @@ impl PyDbClient {
887
923
  })
888
924
  }
889
925
 
926
+ /// Downloads the file at `volume_path` and returns its contents as `bytes`.
927
+ fn download_volume_file<'py>(&self, py: Python<'py>, volume_path: String) -> PyResult<Bound<'py, PyAny>> {
928
+ let client = self.inner.clone();
929
+ future_into_py_in_call(py, async move {
930
+ let data = client
931
+ .download_volume_file(&volume_path)
932
+ .await
933
+ .map_err(api_error_to_pyerr)?;
934
+ Ok(Python::attach(|py| pyo3::types::PyBytes::new(py, &data).unbind()))
935
+ })
936
+ }
937
+
890
938
  /// Deletes a file at `volume_path` (see `upload_volume_file`). A 404 is
891
939
  /// treated as success -- the file is already gone, fine for idempotent
892
940
  /// staging cleanup.
@@ -1702,18 +1702,18 @@ async fn thrift_split_download_fails_on_assembled_length_mismatch() {
1702
1702
  let server = MockServer::start().await;
1703
1703
  mount_single_link_statement(&server, "/_data/short-parts", 60_000).await;
1704
1704
 
1705
- // Every part -- including the probe -- honestly declares a 5MiB total
1706
- // via Content-Range, but actually returns far fewer bytes than its own
1707
- // declared range covers (a misbehaving server). The declared total
1708
- // guarantees more than one part gets requested; every part coming up
1709
- // short must be caught by the final length check, not silently
1710
- // accepted as a truncated-but-otherwise-fine result.
1705
+ // Every part -- including the probe -- honestly declares the range it was
1706
+ // asked for of a 5MiB total via Content-Range, but actually returns far
1707
+ // fewer bytes than that range covers (a misbehaving server). The declared
1708
+ // total guarantees more than one part gets requested; every part coming
1709
+ // up short must be caught by the length checks, not silently accepted as
1710
+ // a truncated-but-otherwise-fine result.
1711
1711
  const DECLARED_TOTAL: u64 = 5 * (1 << 20);
1712
1712
  Mock::given(method("GET"))
1713
1713
  .and(path("/_data/short-parts"))
1714
1714
  .respond_with(move |req: &Request| {
1715
- let (start, _end) = parse_range(req).expect("client always sends Range on this path");
1716
- let end = (start + 9).min(DECLARED_TOTAL - 1);
1715
+ let (start, end) = parse_range(req).expect("client always sends Range on this path");
1716
+ let end = end.min(DECLARED_TOTAL - 1);
1717
1717
  ResponseTemplate::new(206)
1718
1718
  .insert_header("Content-Range", format!("bytes {start}-{end}/{DECLARED_TOTAL}"))
1719
1719
  .set_body_raw(vec![0u8; 10], "application/octet-stream")
@@ -1736,6 +1736,83 @@ async fn thrift_split_download_fails_on_assembled_length_mismatch() {
1736
1736
  );
1737
1737
  }
1738
1738
 
1739
+ #[tokio::test]
1740
+ async fn thrift_split_download_absurd_content_range_total_fails_without_aborting() {
1741
+ let server = MockServer::start().await;
1742
+ mount_single_link_statement(&server, "/_data/absurd-total", 60_000).await;
1743
+
1744
+ // The Content-Range total is untrusted: u64::MAX bytes must never size an
1745
+ // allocation (an allocation failure aborts the whole process, not just
1746
+ // the query), only turn into a normal error.
1747
+ const ABSURD_TOTAL: u64 = u64::MAX;
1748
+ Mock::given(method("GET"))
1749
+ .and(path("/_data/absurd-total"))
1750
+ .respond_with(move |req: &Request| {
1751
+ let (start, end) = parse_range(req).expect("client always sends Range on this path");
1752
+ ResponseTemplate::new(206)
1753
+ .insert_header("Content-Range", format!("bytes {start}-{end}/{ABSURD_TOTAL}"))
1754
+ .set_body_raw(vec![0u8; 1 << 20], "application/octet-stream")
1755
+ })
1756
+ .mount(&server)
1757
+ .await;
1758
+
1759
+ let client = thrift_client(&server);
1760
+ let mut stream = execute_lazy_thrift(client, "SELECT * FROM t", None, None, None)
1761
+ .await
1762
+ .unwrap();
1763
+ let err = stream
1764
+ .fetchall_arrow()
1765
+ .await
1766
+ .expect_err("an absurd declared total must fail loudly, not abort or succeed");
1767
+ assert!(
1768
+ err.message.contains("expected"),
1769
+ "got a different error than expected: {}",
1770
+ err.message
1771
+ );
1772
+ }
1773
+
1774
+ #[tokio::test]
1775
+ async fn thrift_split_download_rejects_short_part_offset_by_long_part() {
1776
+ let server = MockServer::start().await;
1777
+ mount_single_link_statement(&server, "/_data/offset-parts", 60_000).await;
1778
+
1779
+ // 3 MiB file, three 1 MiB ranges. The middle part returns 10 bytes too
1780
+ // few and the last 10 bytes too many, so the assembled length equals the
1781
+ // declared total and only a per-part length check can catch it.
1782
+ const MIB: u64 = 1 << 20;
1783
+ const TOTAL: u64 = 3 * MIB;
1784
+ Mock::given(method("GET"))
1785
+ .and(path("/_data/offset-parts"))
1786
+ .respond_with(move |req: &Request| {
1787
+ let (start, end) = parse_range(req).expect("client always sends Range on this path");
1788
+ let end = end.min(TOTAL - 1);
1789
+ let len = match start {
1790
+ MIB => MIB - 10,
1791
+ s if s == 2 * MIB => MIB + 10,
1792
+ _ => end - start + 1,
1793
+ };
1794
+ ResponseTemplate::new(206)
1795
+ .insert_header("Content-Range", format!("bytes {start}-{end}/{TOTAL}"))
1796
+ .set_body_raw(vec![0u8; len as usize], "application/octet-stream")
1797
+ })
1798
+ .mount(&server)
1799
+ .await;
1800
+
1801
+ let client = thrift_client(&server);
1802
+ let mut stream = execute_lazy_thrift(client, "SELECT * FROM t", None, None, None)
1803
+ .await
1804
+ .unwrap();
1805
+ let err = stream
1806
+ .fetchall_arrow()
1807
+ .await
1808
+ .expect_err("a wrong-length part must fail loudly even when the total adds up");
1809
+ assert!(
1810
+ err.message.contains("split download part"),
1811
+ "got a different error than expected: {}",
1812
+ err.message
1813
+ );
1814
+ }
1815
+
1739
1816
  #[tokio::test]
1740
1817
  async fn thrift_concurrent_single_link_queries_stay_correct_under_split_contention() {
1741
1818
  let server = MockServer::start().await;
@@ -0,0 +1,250 @@
1
+ //! Files API (upload_volume_file/delete_volume_file) -- kept separate from
2
+ //! wiremock_pipeline.rs since it's a different API surface entirely (no
3
+ //! statement/manifest/chunk shape to it), mirroring tests/conftest.py's own
4
+ //! `mock_volume_files` fixture being kept apart from `mock_warehouse`.
5
+
6
+ use std::sync::Arc;
7
+
8
+ use arrowbricks_core::client::DbClient;
9
+ use bytes::Bytes;
10
+ use wiremock::matchers::{method, path_regex, query_param};
11
+ use wiremock::{Mock, MockServer, ResponseTemplate};
12
+
13
+ const WAREHOUSE_ID: &str = "wh-test-123";
14
+
15
+ #[tokio::test]
16
+ async fn upload_volume_file_sends_overwrite_and_content_type() {
17
+ let server = MockServer::start().await;
18
+ Mock::given(method("PUT"))
19
+ .and(path_regex(r"^/api/2\.0/fs/files/.*"))
20
+ .and(query_param("overwrite", "true"))
21
+ .respond_with(ResponseTemplate::new(204))
22
+ .mount(&server)
23
+ .await;
24
+
25
+ let client = Arc::new(DbClient::new(&server.uri(), WAREHOUSE_ID, "fake-token"));
26
+ client
27
+ .upload_volume_file("/Volumes/cat/schema/vol/file.bin", Bytes::from_static(b"hello"))
28
+ .await
29
+ .unwrap();
30
+ }
31
+
32
+ #[tokio::test]
33
+ async fn download_volume_file_returns_exact_body() {
34
+ let server = MockServer::start().await;
35
+ let body = vec![0u8, 255, 1, 254, b'%', b'P', b'D', b'F'];
36
+ Mock::given(method("GET"))
37
+ .and(path_regex(r"^/api/2\.0/fs/files/Volumes/cat/schema/vol/file\.pdf$"))
38
+ .respond_with(ResponseTemplate::new(200).set_body_bytes(body.clone()))
39
+ .mount(&server)
40
+ .await;
41
+
42
+ let client = Arc::new(DbClient::new(&server.uri(), WAREHOUSE_ID, "fake-token"));
43
+ let got = client
44
+ .download_volume_file("/Volumes/cat/schema/vol/file.pdf")
45
+ .await
46
+ .unwrap();
47
+ assert_eq!(got.as_ref(), body.as_slice());
48
+ }
49
+
50
+ #[tokio::test]
51
+ async fn download_volume_file_404_is_an_error() {
52
+ let server = MockServer::start().await;
53
+ Mock::given(method("GET"))
54
+ .respond_with(ResponseTemplate::new(404).set_body_string("not found"))
55
+ .mount(&server)
56
+ .await;
57
+
58
+ let client = Arc::new(DbClient::new(&server.uri(), WAREHOUSE_ID, "fake-token"));
59
+ let err = client
60
+ .download_volume_file("/Volumes/cat/schema/vol/missing.pdf")
61
+ .await
62
+ .unwrap_err();
63
+ assert!(
64
+ err.message.contains("404") || err.message.contains("not found"),
65
+ "{err}"
66
+ );
67
+ }
68
+
69
+ #[tokio::test]
70
+ async fn delete_volume_file_succeeds_on_204() {
71
+ let server = MockServer::start().await;
72
+ Mock::given(method("DELETE"))
73
+ .and(path_regex(r"^/api/2\.0/fs/files/.*"))
74
+ .respond_with(ResponseTemplate::new(204))
75
+ .mount(&server)
76
+ .await;
77
+
78
+ let client = Arc::new(DbClient::new(&server.uri(), WAREHOUSE_ID, "fake-token"));
79
+ client
80
+ .delete_volume_file("/Volumes/cat/schema/vol/file.bin")
81
+ .await
82
+ .unwrap();
83
+ }
84
+
85
+ #[tokio::test]
86
+ async fn delete_volume_file_treats_404_as_success() {
87
+ let server = MockServer::start().await;
88
+ Mock::given(method("DELETE"))
89
+ .and(path_regex(r"^/api/2\.0/fs/files/.*"))
90
+ .respond_with(ResponseTemplate::new(404))
91
+ .mount(&server)
92
+ .await;
93
+
94
+ let client = Arc::new(DbClient::new(&server.uri(), WAREHOUSE_ID, "fake-token"));
95
+ client
96
+ .delete_volume_file("/Volumes/cat/schema/vol/already-gone.bin")
97
+ .await
98
+ .expect("404 must be treated as success (idempotent delete), not an error");
99
+ }
100
+
101
+ #[tokio::test]
102
+ async fn delete_volume_file_surfaces_other_errors() {
103
+ // 400, not 500/429/etc -- those are classified transient and would
104
+ // trigger the full 6-attempt retry/backoff cycle (tens of seconds) for
105
+ // no benefit here; that classification logic already has its own
106
+ // coverage. This only needs a status delete_volume_file does NOT treat
107
+ // as "already gone", proving the error still propagates.
108
+ let server = MockServer::start().await;
109
+ Mock::given(method("DELETE"))
110
+ .and(path_regex(r"^/api/2\.0/fs/files/.*"))
111
+ .respond_with(ResponseTemplate::new(400))
112
+ .mount(&server)
113
+ .await;
114
+
115
+ let client = Arc::new(DbClient::new(&server.uri(), WAREHOUSE_ID, "fake-token"));
116
+ let err = client
117
+ .delete_volume_file("/Volumes/cat/schema/vol/file.bin")
118
+ .await
119
+ .unwrap_err();
120
+ assert!(
121
+ !err.transient,
122
+ "400 is not transient, should surface immediately without retrying"
123
+ );
124
+ }
125
+
126
+ // ---------------------------------------------------------------------------
127
+ // Upload deadline scaling and retry body reuse. wiremock cannot throttle how
128
+ // fast the *server reads* a request body, so these use a tiny raw TCP server.
129
+ // ---------------------------------------------------------------------------
130
+
131
+ use std::time::{Duration, Instant};
132
+ use tokio::io::{AsyncReadExt, AsyncWriteExt};
133
+ use tokio::net::TcpListener;
134
+
135
+ const MIB: usize = 1 << 20;
136
+
137
+ /// Raw HTTP/1.1 server answering every PUT with `204` once it has read the
138
+ /// whole body, consuming it at roughly `bytes_per_sec`. `stuck` instead never
139
+ /// reads or answers at all.
140
+ async fn spawn_throttled_server(bytes_per_sec: usize, stuck: bool) -> String {
141
+ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
142
+ let addr = listener.local_addr().unwrap();
143
+ tokio::spawn(async move {
144
+ loop {
145
+ let Ok((mut sock, _)) = listener.accept().await else {
146
+ return;
147
+ };
148
+ tokio::spawn(async move {
149
+ if stuck {
150
+ tokio::time::sleep(Duration::from_secs(120)).await;
151
+ return;
152
+ }
153
+ let mut buf = vec![0u8; 64 * 1024];
154
+ let mut head = Vec::new();
155
+ let (mut body_len, mut got_body, mut header_end) = (None::<usize>, 0usize, None::<usize>);
156
+ loop {
157
+ let Ok(n) = sock.read(&mut buf).await else { return };
158
+ if n == 0 {
159
+ return;
160
+ }
161
+ if header_end.is_none() {
162
+ head.extend_from_slice(&buf[..n]);
163
+ if let Some(pos) = head.windows(4).position(|w| w == b"\r\n\r\n") {
164
+ let text = String::from_utf8_lossy(&head[..pos]).to_lowercase();
165
+ body_len = text
166
+ .lines()
167
+ .find_map(|l| l.strip_prefix("content-length:").map(|v| v.trim().parse().unwrap()));
168
+ header_end = Some(pos + 4);
169
+ got_body = head.len() - (pos + 4);
170
+ }
171
+ } else {
172
+ got_body += n;
173
+ }
174
+ if header_end.is_some() {
175
+ if got_body >= body_len.expect("fixed-length body") {
176
+ break;
177
+ }
178
+ tokio::time::sleep(Duration::from_secs_f64(n as f64 / bytes_per_sec as f64)).await;
179
+ }
180
+ }
181
+ let _ = sock
182
+ .write_all(b"HTTP/1.1 204 No Content\r\nConnection: close\r\n\r\n")
183
+ .await;
184
+ });
185
+ }
186
+ });
187
+ format!("http://{addr}")
188
+ }
189
+
190
+ #[tokio::test]
191
+ async fn upload_slow_but_progressing_succeeds_past_http_timeout() {
192
+ // 3 MiB read at ~1.5 MiB/s takes ~2 s: over the 1 s http_timeout, under
193
+ // the scaled deadline (3 MiB / 1 MiB/s = 3 s).
194
+ let uri = spawn_throttled_server(MIB + MIB / 2, false).await;
195
+ let client = Arc::new(DbClient::new(&uri, WAREHOUSE_ID, "fake-token").with_http_timeout(1.0));
196
+ let start = Instant::now();
197
+ client
198
+ .upload_volume_file("/Volumes/c/s/v/big.bin", Bytes::from(vec![7u8; 3 * MIB]))
199
+ .await
200
+ .expect("a progressing upload must not be killed at http_timeout");
201
+ assert!(
202
+ start.elapsed() > Duration::from_secs(1),
203
+ "test must exceed http_timeout to prove anything, took {:?}",
204
+ start.elapsed()
205
+ );
206
+ }
207
+
208
+ #[tokio::test]
209
+ async fn upload_stuck_server_still_times_out_at_scaled_deadline() {
210
+ let uri = spawn_throttled_server(1, true).await;
211
+ let client = Arc::new(DbClient::new(&uri, WAREHOUSE_ID, "fake-token").with_http_timeout(1.0));
212
+ let start = Instant::now();
213
+ let err = client
214
+ .upload_volume_file("/Volumes/c/s/v/big.bin", Bytes::from(vec![7u8; 2 * MIB]))
215
+ .await
216
+ .expect_err("a stuck server must still fail");
217
+ let took = start.elapsed();
218
+ assert!(!err.transient, "timeouts stay non-transient (not retried): {err:?}");
219
+ assert!(
220
+ took >= Duration::from_secs(2) && took < Duration::from_secs(10),
221
+ "deadline should be the scaled ~2 s, took {took:?}"
222
+ );
223
+ }
224
+
225
+ #[tokio::test]
226
+ async fn upload_retry_after_503_resends_full_identical_body() {
227
+ use wiremock::matchers::body_bytes;
228
+ let server = MockServer::start().await;
229
+ let payload: Vec<u8> = (0..300_000u32).map(|i| (i % 251) as u8).collect();
230
+ Mock::given(method("PUT"))
231
+ .and(path_regex(r"^/api/2\.0/fs/files/.*"))
232
+ .respond_with(ResponseTemplate::new(503))
233
+ .up_to_n_times(1)
234
+ .expect(1)
235
+ .mount(&server)
236
+ .await;
237
+ Mock::given(method("PUT"))
238
+ .and(path_regex(r"^/api/2\.0/fs/files/.*"))
239
+ .and(body_bytes(payload.clone()))
240
+ .respond_with(ResponseTemplate::new(204))
241
+ .expect(1)
242
+ .mount(&server)
243
+ .await;
244
+
245
+ let client = Arc::new(DbClient::new(&server.uri(), WAREHOUSE_ID, "fake-token"));
246
+ client
247
+ .upload_volume_file("/Volumes/c/s/v/retry.bin", Bytes::from(payload))
248
+ .await
249
+ .expect("second attempt must carry the complete, identical body");
250
+ }
@@ -1,4 +1,5 @@
1
- from collections.abc import Awaitable, Callable
1
+ import array
2
+ from collections.abc import Awaitable, Callable, Sequence
2
3
  from typing import Any, BinaryIO, Literal
3
4
 
4
5
  def _validate_timeout(total_timeout_s: float | None) -> None: ...
@@ -114,7 +115,10 @@ class Client:
114
115
  schema: str | None = None,
115
116
  parameters: list[dict[str, Any]] | None = None,
116
117
  ) -> str: ...
117
- async def upload_volume_file(self, volume_path: str, data: bytes) -> None: ...
118
+ async def upload_volume_file(
119
+ self, volume_path: str, data: bytes | bytearray | memoryview | array.array | Sequence[int]
120
+ ) -> None: ...
121
+ async def download_volume_file(self, volume_path: str) -> bytes: ...
118
122
  async def delete_volume_file(self, volume_path: str) -> None: ...
119
123
  async def close_sessions(self) -> None: ...
120
124
  def stream_ndjson_lines(
@@ -207,13 +207,17 @@ async def stream_query_json(
207
207
  mid-download), then each result row as a ready-to-send JSON string --
208
208
  one per SSE frame, say.
209
209
 
210
- Unlike cursor.py's fetch methods, this registers and emits each Databricks
211
- chunk AS IT ARRIVES rather than buffering the full result first, via
212
- `._core.Client.stream_ndjson_lines` -- the first row reaches the caller
213
- after ~one chunk's fetch/decode/reorder/encode time, not the whole
214
- statement's; chunk order is restored there too (chunks can complete out
215
- of order over the network). Decode and NDJSON encoding both happen in
216
- Rust -- this function only forwards already-formatted lines.
210
+ Unlike cursor.py's fetch methods, this does not buffer the full result
211
+ first: it emits rows via `._core.Client.stream_ndjson_lines` as chunks
212
+ are decoded, in order (chunks can complete out of order over the
213
+ network; order is restored there). The first row is available once the
214
+ first chunk has been downloaded and decoded. For a result spread over
215
+ many chunks on a bandwidth-limited link that can be most of the total
216
+ download time (measured on one ~30 MB/s link: the first row arrived at
217
+ 84-97% of the stream total), so do not count on early rows for large
218
+ results. What streaming does give you: heartbeats while waiting, bounded
219
+ memory, and no full materialization. Decode and NDJSON encoding both
220
+ happen in Rust -- this function only forwards already-formatted lines.
217
221
 
218
222
  `non_finite_floats` -- `"null"` (default) or `"string"`. A NaN/Infinity/
219
223
  -Infinity float column comes back from Databricks like any other value,
@@ -7,7 +7,8 @@ reorder, Arrow/JSON/NDJSON decode)."""
7
7
 
8
8
  from __future__ import annotations
9
9
 
10
- from collections.abc import AsyncIterator, Awaitable, Callable
10
+ import array
11
+ from collections.abc import AsyncIterator, Awaitable, Callable, Sequence
11
12
  from typing import Any
12
13
 
13
14
  from . import _core
@@ -133,14 +134,34 @@ class DatabricksClient:
133
134
  async def __aexit__(self, *exc_info: object) -> None:
134
135
  await self.aclose()
135
136
 
136
- async def upload_volume_file(self, volume_path: str, data: bytes) -> None:
137
+ async def upload_volume_file(
138
+ self, volume_path: str, data: bytes | bytearray | memoryview | array.array | Sequence[int]
139
+ ) -> None:
137
140
  """Uploads `data` to a Unity Catalog volume path via the Files API,
138
141
  overwriting anything already there. `volume_path` is caller-supplied
139
142
  in full (e.g. `/Volumes/my_catalog/my_schema/my_volume/some/file.parquet`)
140
143
  -- this package has no knowledge of any specific catalog/schema/volume.
141
- Raises `RuntimeError` (message only) on failure."""
144
+
145
+ `data` may be `bytes` (shared with the upload, not copied -- the
146
+ cheapest and recommended form) or any byte buffer (`bytearray`,
147
+ `memoryview`, `array.array`, a numpy array, a pyarrow `Buffer`; signed
148
+ or unsigned bytes), which is copied once up front. A plain sequence of
149
+ ints in 0..=255 (e.g. a `list[int]`) still works but is slow; prefer
150
+ `bytes(data)`. Anything else raises `TypeError`.
151
+
152
+ The PUT's total deadline is `max(http_timeout, len(data) / 1 MiB/s)`,
153
+ so `http_timeout` is a floor, not a ceiling: a slow-but-progressing
154
+ large upload is not cut off at `http_timeout`, and a stalled 1 GiB
155
+ upload can take about 17 minutes to fail. A timeout is not retried;
156
+ transient HTTP errors are. Raises `RuntimeError` (message only) on
157
+ failure."""
142
158
  await self._core_client.upload_volume_file(volume_path, data)
143
159
 
160
+ async def download_volume_file(self, volume_path: str) -> bytes:
161
+ """Downloads the file at `volume_path` (see upload_volume_file) and
162
+ returns its contents. Raises on a missing file (404)."""
163
+ return await self._core_client.download_volume_file(volume_path)
164
+
144
165
  async def delete_volume_file(self, volume_path: str) -> None:
145
166
  """Deletes a file at `volume_path` (see upload_volume_file). A 404 is
146
167
  treated as success -- the file is already gone, which is fine for
@@ -1,87 +0,0 @@
1
- //! Files API (upload_volume_file/delete_volume_file) -- kept separate from
2
- //! wiremock_pipeline.rs since it's a different API surface entirely (no
3
- //! statement/manifest/chunk shape to it), mirroring tests/conftest.py's own
4
- //! `mock_volume_files` fixture being kept apart from `mock_warehouse`.
5
-
6
- use std::sync::Arc;
7
-
8
- use arrowbricks_core::client::DbClient;
9
- use bytes::Bytes;
10
- use wiremock::matchers::{method, path_regex, query_param};
11
- use wiremock::{Mock, MockServer, ResponseTemplate};
12
-
13
- const WAREHOUSE_ID: &str = "wh-test-123";
14
-
15
- #[tokio::test]
16
- async fn upload_volume_file_sends_overwrite_and_content_type() {
17
- let server = MockServer::start().await;
18
- Mock::given(method("PUT"))
19
- .and(path_regex(r"^/api/2\.0/fs/files/.*"))
20
- .and(query_param("overwrite", "true"))
21
- .respond_with(ResponseTemplate::new(204))
22
- .mount(&server)
23
- .await;
24
-
25
- let client = Arc::new(DbClient::new(&server.uri(), WAREHOUSE_ID, "fake-token"));
26
- client
27
- .upload_volume_file("/Volumes/cat/schema/vol/file.bin", Bytes::from_static(b"hello"))
28
- .await
29
- .unwrap();
30
- }
31
-
32
- #[tokio::test]
33
- async fn delete_volume_file_succeeds_on_204() {
34
- let server = MockServer::start().await;
35
- Mock::given(method("DELETE"))
36
- .and(path_regex(r"^/api/2\.0/fs/files/.*"))
37
- .respond_with(ResponseTemplate::new(204))
38
- .mount(&server)
39
- .await;
40
-
41
- let client = Arc::new(DbClient::new(&server.uri(), WAREHOUSE_ID, "fake-token"));
42
- client
43
- .delete_volume_file("/Volumes/cat/schema/vol/file.bin")
44
- .await
45
- .unwrap();
46
- }
47
-
48
- #[tokio::test]
49
- async fn delete_volume_file_treats_404_as_success() {
50
- let server = MockServer::start().await;
51
- Mock::given(method("DELETE"))
52
- .and(path_regex(r"^/api/2\.0/fs/files/.*"))
53
- .respond_with(ResponseTemplate::new(404))
54
- .mount(&server)
55
- .await;
56
-
57
- let client = Arc::new(DbClient::new(&server.uri(), WAREHOUSE_ID, "fake-token"));
58
- client
59
- .delete_volume_file("/Volumes/cat/schema/vol/already-gone.bin")
60
- .await
61
- .expect("404 must be treated as success (idempotent delete), not an error");
62
- }
63
-
64
- #[tokio::test]
65
- async fn delete_volume_file_surfaces_other_errors() {
66
- // 400, not 500/429/etc -- those are classified transient and would
67
- // trigger the full 6-attempt retry/backoff cycle (tens of seconds) for
68
- // no benefit here; that classification logic already has its own
69
- // coverage. This only needs a status delete_volume_file does NOT treat
70
- // as "already gone", proving the error still propagates.
71
- let server = MockServer::start().await;
72
- Mock::given(method("DELETE"))
73
- .and(path_regex(r"^/api/2\.0/fs/files/.*"))
74
- .respond_with(ResponseTemplate::new(400))
75
- .mount(&server)
76
- .await;
77
-
78
- let client = Arc::new(DbClient::new(&server.uri(), WAREHOUSE_ID, "fake-token"));
79
- let err = client
80
- .delete_volume_file("/Volumes/cat/schema/vol/file.bin")
81
- .await
82
- .unwrap_err();
83
- assert!(
84
- !err.transient,
85
- "400 is not transient, should surface immediately without retrying"
86
- );
87
- }
File without changes