arrowbricks 6.2.0__tar.gz → 6.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/PKG-INFO +11 -5
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/README.md +10 -4
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/pyproject.toml +2 -1
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/Cargo.lock +1 -1
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/Cargo.toml +1 -1
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/README.md +1 -1
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/client/download.rs +152 -11
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/client/volume.rs +58 -1
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/lib.rs +51 -3
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/tests/wiremock_thrift.rs +85 -8
- arrowbricks-6.4.0/rust/arrowbricks_core/tests/wiremock_volume_files.rs +250 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/src/arrowbricks/_core.pyi +6 -2
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/src/arrowbricks/_streaming.py +11 -7
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/src/arrowbricks/client.py +24 -3
- arrowbricks-6.2.0/rust/arrowbricks_core/tests/wiremock_volume_files.rs +0 -87
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/LICENSE +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/.gitignore +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/examples/agent_render_bench.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/examples/duckdb_query.py +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/examples/fastapi_sse.py +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/proptest-regressions/json_convert.txt +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/rustfmt.toml +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/agent/diagnostics.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/agent/mod.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/agent/render.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/alloc.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/arrow_ffi.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/client/call_context.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/client/error.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/client/model.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/client/sea.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/client/thrift_rpc.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/client.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/heartbeat.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/json_convert.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/pipeline/ndjson.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/pipeline/reorder.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/pipeline/sea.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/pipeline/stats.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/pipeline/test_support.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/pipeline/thrift_exec.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/pipeline.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/src/thrift.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/tests/common/mod.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/tests/wiremock_agent.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/tests/wiremock_pipeline.rs +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/tests_py/conftest.py +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/tests_py/test_ipc_stream.py +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/tests_py/test_parameters.py +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/tests_py/test_stream_ndjson_lines.py +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/tests_py/test_streaming.py +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/tests_py/test_thrift.py +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/tests_py/test_token_provider.py +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/tests_py/test_volume_files.py +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/tests_py/thrift_mock.py +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/src/arrowbricks/__init__.py +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/src/arrowbricks/__main__.py +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/src/arrowbricks/_agent.py +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/src/arrowbricks/_sql.py +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/src/arrowbricks/cli.py +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/src/arrowbricks/cursor.py +0 -0
- {arrowbricks-6.2.0 → arrowbricks-6.4.0}/src/arrowbricks/py.typed +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: arrowbricks
|
|
3
|
-
Version: 6.
|
|
3
|
+
Version: 6.4.0
|
|
4
4
|
Requires-Dist: arro3-core>=0.8 ; extra == 'arro3'
|
|
5
5
|
Requires-Dist: sqlglot>=27 ; extra == 'sqlglot'
|
|
6
6
|
Provides-Extra: arro3
|
|
@@ -13,14 +13,12 @@ Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
|
|
|
13
13
|
Project-URL: Repository, https://github.com/bmsuisse/arrowbricks
|
|
14
14
|
|
|
15
15
|
<p align="center">
|
|
16
|
-
<img src="assets/logo.svg"
|
|
16
|
+
<img src="assets/logo.svg" height="120" alt="arrowbricks">
|
|
17
17
|
</p>
|
|
18
|
-
|
|
19
|
-
<h1 align="center">arrowbricks</h1>
|
|
20
18
|
<p align="center">Fast Databricks SQL for Python applications and AI agents.</p>
|
|
21
19
|
|
|
22
20
|
- **Zero required Python dependencies.** One precompiled wheel. No Rust toolchain or cloud SDK needed.
|
|
23
|
-
- **Fast Rust core.** Concurrent chunk downloads, lazy Arrow decoding, and streaming NDJSON, with result order preserved.
|
|
21
|
+
- **Fast Rust core.** Concurrent chunk downloads, lazy Arrow decoding, and streaming NDJSON without materializing the full result, with result order preserved.
|
|
24
22
|
- **Agent mode, new in 6.0.** A CLI and Python API that return compact typed TSV (or a Markdown table), explicit truncation, and actionable JSON errors. Formatting stays in Rust.
|
|
25
23
|
|
|
26
24
|
Use Thrift by default or the Statement Execution API with `protocol="sea"`.
|
|
@@ -118,6 +116,14 @@ a 100-row preview used **44% fewer bytes and 24% fewer tokens** than NDJSON.
|
|
|
118
116
|
See [agent measurements and methodology](docs/research/agent-rendering.md) and
|
|
119
117
|
[client benchmarks](docs/benchmarks.md) for workloads, comparisons, and limits.
|
|
120
118
|
|
|
119
|
+
### Performance tips
|
|
120
|
+
|
|
121
|
+
Numbers are measurements from one ~30 MB/s link to one warehouse, not guarantees.
|
|
122
|
+
|
|
123
|
+
- **Select only the columns you need.** Selecting 3 columns instead of `SELECT *` was 1.5-6x faster and moved 1.9-15.8x fewer bytes across 5 tables.
|
|
124
|
+
- **Keep `compress_results` at its default (`True`)** unless the link is very fast. Here it was 1.8-4.7x faster with 2-5.5x fewer bytes. Turning it off on a very fast link was not tested.
|
|
125
|
+
- **Keep the Thrift default for small lookups.** It was about 0.15 s faster per tiny statement than `protocol="sea"` (a constant gap in these runs).
|
|
126
|
+
|
|
121
127
|
## Learn more
|
|
122
128
|
|
|
123
129
|
- [Usage and API reference](docs/usage.md): authentication, errors, cancellation, observability, and Arrow interop.
|
|
@@ -1,12 +1,10 @@
|
|
|
1
1
|
<p align="center">
|
|
2
|
-
<img src="assets/logo.svg"
|
|
2
|
+
<img src="assets/logo.svg" height="120" alt="arrowbricks">
|
|
3
3
|
</p>
|
|
4
|
-
|
|
5
|
-
<h1 align="center">arrowbricks</h1>
|
|
6
4
|
<p align="center">Fast Databricks SQL for Python applications and AI agents.</p>
|
|
7
5
|
|
|
8
6
|
- **Zero required Python dependencies.** One precompiled wheel. No Rust toolchain or cloud SDK needed.
|
|
9
|
-
- **Fast Rust core.** Concurrent chunk downloads, lazy Arrow decoding, and streaming NDJSON, with result order preserved.
|
|
7
|
+
- **Fast Rust core.** Concurrent chunk downloads, lazy Arrow decoding, and streaming NDJSON without materializing the full result, with result order preserved.
|
|
10
8
|
- **Agent mode, new in 6.0.** A CLI and Python API that return compact typed TSV (or a Markdown table), explicit truncation, and actionable JSON errors. Formatting stays in Rust.
|
|
11
9
|
|
|
12
10
|
Use Thrift by default or the Statement Execution API with `protocol="sea"`.
|
|
@@ -104,6 +102,14 @@ a 100-row preview used **44% fewer bytes and 24% fewer tokens** than NDJSON.
|
|
|
104
102
|
See [agent measurements and methodology](docs/research/agent-rendering.md) and
|
|
105
103
|
[client benchmarks](docs/benchmarks.md) for workloads, comparisons, and limits.
|
|
106
104
|
|
|
105
|
+
### Performance tips
|
|
106
|
+
|
|
107
|
+
Numbers are measurements from one ~30 MB/s link to one warehouse, not guarantees.
|
|
108
|
+
|
|
109
|
+
- **Select only the columns you need.** Selecting 3 columns instead of `SELECT *` was 1.5-6x faster and moved 1.9-15.8x fewer bytes across 5 tables.
|
|
110
|
+
- **Keep `compress_results` at its default (`True`)** unless the link is very fast. Here it was 1.8-4.7x faster with 2-5.5x fewer bytes. Turning it off on a very fast link was not tested.
|
|
111
|
+
- **Keep the Thrift default for small lookups.** It was about 0.15 s faster per tiny statement than `protocol="sea"` (a constant gap in these runs).
|
|
112
|
+
|
|
107
113
|
## Learn more
|
|
108
114
|
|
|
109
115
|
- [Usage and API reference](docs/usage.md): authentication, errors, cancellation, observability, and Arrow interop.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "arrowbricks"
|
|
3
|
-
version = "6.
|
|
3
|
+
version = "6.4.0"
|
|
4
4
|
description = "Rust-powered Databricks SQL client and agent CLI with Arrow/NDJSON streaming, bounded Markdown previews, actionable JSON diagnostics, and zero required Python dependencies."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "MIT"
|
|
@@ -105,3 +105,4 @@ select = ["E", "F", "I", "UP", "S", "B"]
|
|
|
105
105
|
"tests/arro3_free/*" = ["S101", "S106"]
|
|
106
106
|
# One-off benchmark harness: terse scripts, seeded (non-crypto) randomness, asserts as checks.
|
|
107
107
|
"benchmarks/agent-output-format/*" = ["B905", "E501", "E731", "E741", "S101", "S311"]
|
|
108
|
+
"benchmarks/benchmark_duckdb.py" = ["E501", "S608"]
|
|
@@ -79,7 +79,7 @@ Arrow library first (e.g. `arro3.core.Table.from_arrow(table)`).
|
|
|
79
79
|
- `ResultSet.schema() -> list[tuple[str, str]] | None` -- the real decoded Arrow schema as `(name, type_name)` string pairs, once known (after >=1 fetch); `None` before that. Computed directly in Rust -- unlike a `Table`'s own `.schema`, this needs no arro3 install.
|
|
80
80
|
- `ResultSet.statement_id`, `ResultSet.num_chunks`, `ResultSet.columns` (manifest-based pre-fetch schema estimate, also arro3-free).
|
|
81
81
|
- `Client.stream_ndjson_lines(statement, *, catalog=None, schema=None, parameters=None, total_timeout_s=None)` -- chunk-at-a-time: an async iterator yielding the `HEARTBEAT` singleton while waiting on the statement or any individual chunk, then a `list[str]` of NDJSON lines (one per row, explicit nulls, ISO-8601 timestamps) per chunk in logical order. Decode and JSON encoding both happen in Rust -- backs `stream_query_json` end to end.
|
|
82
|
-
- `Client.upload_volume_file(volume_path, data: bytes)` / `Client.delete_volume_file(volume_path)` -- Unity Catalog volume files via the Files API. Delete treats a 404 as success (idempotent). Both raise a plain `RuntimeError` (message only) on failure.
|
|
82
|
+
- `Client.upload_volume_file(volume_path, data: bytes)` / `Client.download_volume_file(volume_path) -> bytes` / `Client.delete_volume_file(volume_path)` -- Unity Catalog volume files via the Files API. Delete treats a 404 as success (idempotent). Both raise a plain `RuntimeError` (message only) on failure.
|
|
83
83
|
- `write_ipc_stream(stream, buf)` -- free function; writes any object implementing `__arrow_c_stream__` (a `Table` from this crate, arro3, pyarrow, ...) as uncompressed Arrow-IPC stream bytes to a Python file-like object. No dependency needed regardless of the input's origin.
|
|
84
84
|
- `read_ipc_stream(data: bytes) -> Table` -- free function; the exact inverse of `write_ipc_stream`, parsing raw Arrow-IPC stream bytes back into a `Table`. No dependency needed regardless of where the bytes came from -- backs `arrowbricks.ReplayableArrowChunk`, which needs to re-parse the same cached bytes on every `__arrow_c_stream__` call. Decoding releases the Python interpreter lock so independent replay calls can run concurrently; decoded arrays retain ownership of the immutable input bytes.
|
|
85
85
|
- `HEARTBEAT` -- module-level singleton; compare with `is`, e.g. `if item is _core.HEARTBEAT: ...`.
|
|
@@ -16,6 +16,21 @@ use super::MAX_SPLIT_PARTS;
|
|
|
16
16
|
use super::error::{ApiError, join_error};
|
|
17
17
|
use super::model::QueryStatsAccumulator;
|
|
18
18
|
|
|
19
|
+
/// Parses `Content-Range: bytes <start>-<end>/<total>` (the only form a 206
|
|
20
|
+
/// response to a single-range request may use). `*` totals and unsatisfied-range
|
|
21
|
+
/// forms yield `None`.
|
|
22
|
+
fn parse_content_range(headers: &reqwest::header::HeaderMap) -> Option<(u64, u64, u64)> {
|
|
23
|
+
let value = headers.get(reqwest::header::CONTENT_RANGE)?.to_str().ok()?;
|
|
24
|
+
let rest = value.trim().strip_prefix("bytes")?.trim_start();
|
|
25
|
+
let (range, total) = rest.split_once('/')?;
|
|
26
|
+
let (start, end) = range.trim().split_once('-')?;
|
|
27
|
+
Some((
|
|
28
|
+
start.trim().parse().ok()?,
|
|
29
|
+
end.trim().parse().ok()?,
|
|
30
|
+
total.trim().parse().ok()?,
|
|
31
|
+
))
|
|
32
|
+
}
|
|
33
|
+
|
|
19
34
|
/// Unwraps one downloaded chunk file's outer LZ4 Frame compression --
|
|
20
35
|
/// server-side, this is a whole-file wrapper applied because we asked for
|
|
21
36
|
/// `result_compression: "LZ4_FRAME"`, not Arrow-IPC's own (unrelated,
|
|
@@ -226,16 +241,27 @@ impl DbClient {
|
|
|
226
241
|
return Err(ApiError::from_status(status, &text, true));
|
|
227
242
|
}
|
|
228
243
|
let ranged = status == reqwest::StatusCode::PARTIAL_CONTENT;
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
244
|
+
// A 206 must describe exactly the head we asked for: starting
|
|
245
|
+
// at 0 and ending at `part_size - 1` (or the last byte of a
|
|
246
|
+
// smaller file). Anything else would be spliced in as wrong
|
|
247
|
+
// bytes with the right length.
|
|
248
|
+
let mut total: Option<u64> = None;
|
|
249
|
+
let mut head_len: Option<u64> = None;
|
|
250
|
+
if ranged {
|
|
251
|
+
let Some((first, last, size)) = parse_content_range(resp.headers()) else {
|
|
252
|
+
return Err(ApiError::permanent(
|
|
253
|
+
"cloud-fetch link answered a Range request with 206 Partial Content but an \
|
|
254
|
+
unparseable Content-Range header -- refusing to silently return a truncated file",
|
|
255
|
+
));
|
|
256
|
+
};
|
|
257
|
+
if first != 0 || size == 0 || last != (part_size - 1).min(size - 1) {
|
|
258
|
+
return Err(ApiError::permanent(format!(
|
|
259
|
+
"cloud-fetch link answered bytes=0-{} with Content-Range bytes {first}-{last}/{size}",
|
|
260
|
+
part_size - 1
|
|
261
|
+
)));
|
|
262
|
+
}
|
|
263
|
+
total = Some(size);
|
|
264
|
+
head_len = Some(last + 1);
|
|
239
265
|
}
|
|
240
266
|
|
|
241
267
|
let mut downloads = tokio::task::JoinSet::new();
|
|
@@ -248,7 +274,8 @@ impl DbClient {
|
|
|
248
274
|
if start >= total {
|
|
249
275
|
break;
|
|
250
276
|
}
|
|
251
|
-
|
|
277
|
+
// `total` is untrusted: saturate rather than overflow.
|
|
278
|
+
let end = start.saturating_add(rest_size - 1).min(total - 1);
|
|
252
279
|
let this = self.clone();
|
|
253
280
|
let url = url.to_string();
|
|
254
281
|
let part_stats = stats.clone();
|
|
@@ -268,9 +295,36 @@ impl DbClient {
|
|
|
268
295
|
let text = resp.text().await.unwrap_or_default();
|
|
269
296
|
return Err(ApiError::from_status(status, &text, true));
|
|
270
297
|
}
|
|
298
|
+
// The probe proved the server honours ranges, so
|
|
299
|
+
// each part must be a 206 for exactly this range
|
|
300
|
+
// of the same file; a 200 or another interval is
|
|
301
|
+
// wrong data, not a transient failure.
|
|
302
|
+
let got = (status == reqwest::StatusCode::PARTIAL_CONTENT)
|
|
303
|
+
.then(|| parse_content_range(resp.headers()))
|
|
304
|
+
.flatten();
|
|
305
|
+
if got != Some((start, end, total)) {
|
|
306
|
+
return Err(ApiError::permanent(format!(
|
|
307
|
+
"split download part bytes={start}-{end}/{total} was answered with status {status} \
|
|
308
|
+
and Content-Range {got:?}"
|
|
309
|
+
)));
|
|
310
|
+
}
|
|
271
311
|
resp.bytes().await.map_err(|e| ApiError::from_reqwest(e, true))
|
|
272
312
|
})
|
|
273
313
|
.await;
|
|
314
|
+
// A part must be exactly the range asked for: the
|
|
315
|
+
// final sum check alone would accept a short part
|
|
316
|
+
// offset by a long one (wrong bytes, right total).
|
|
317
|
+
let bytes = bytes.and_then(|b| {
|
|
318
|
+
let expected = end - start + 1;
|
|
319
|
+
if b.len() as u64 == expected {
|
|
320
|
+
Ok(b)
|
|
321
|
+
} else {
|
|
322
|
+
Err(ApiError::permanent(format!(
|
|
323
|
+
"split download part bytes={start}-{end} returned {} bytes, expected {expected}",
|
|
324
|
+
b.len()
|
|
325
|
+
)))
|
|
326
|
+
}
|
|
327
|
+
});
|
|
274
328
|
if let Ok(b) = &bytes {
|
|
275
329
|
part_stats.bytes_downloaded.fetch_add(b.len() as u64, Ordering::Relaxed);
|
|
276
330
|
}
|
|
@@ -288,6 +342,15 @@ impl DbClient {
|
|
|
288
342
|
return Err(ApiError::from_reqwest(error, true));
|
|
289
343
|
}
|
|
290
344
|
};
|
|
345
|
+
if let Some(expected) = head_len
|
|
346
|
+
&& head.len() as u64 != expected
|
|
347
|
+
{
|
|
348
|
+
downloads.shutdown().await;
|
|
349
|
+
return Err(ApiError::permanent(format!(
|
|
350
|
+
"split download probe returned {} bytes, expected {expected}",
|
|
351
|
+
head.len()
|
|
352
|
+
)));
|
|
353
|
+
}
|
|
291
354
|
Ok((total, head, downloads))
|
|
292
355
|
})
|
|
293
356
|
.await?;
|
|
@@ -448,6 +511,84 @@ mod tests {
|
|
|
448
511
|
);
|
|
449
512
|
}
|
|
450
513
|
|
|
514
|
+
/// Serves a 12-byte file `abcdefghijkl` that honours `Range`, except where
|
|
515
|
+
/// `fault` says to lie: the failure modes a length check alone cannot see.
|
|
516
|
+
async fn serve_range_file(fault: &'static str) -> std::net::SocketAddr {
|
|
517
|
+
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
|
518
|
+
use tokio::net::TcpListener;
|
|
519
|
+
|
|
520
|
+
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
521
|
+
let address = listener.local_addr().unwrap();
|
|
522
|
+
tokio::spawn(async move {
|
|
523
|
+
loop {
|
|
524
|
+
let (mut socket, _) = listener.accept().await.unwrap();
|
|
525
|
+
tokio::spawn(async move {
|
|
526
|
+
let mut request = Vec::new();
|
|
527
|
+
while !request.ends_with(b"\r\n\r\n") {
|
|
528
|
+
if socket.read_u8().await.map(|b| request.push(b)).is_err() {
|
|
529
|
+
return;
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
let request = String::from_utf8(request).unwrap().to_ascii_lowercase();
|
|
533
|
+
let range = request.lines().find(|s| s.starts_with("range:")).unwrap();
|
|
534
|
+
let (start, end) = range.split_once("bytes=").unwrap().1.split_once('-').unwrap();
|
|
535
|
+
let (start, end): (usize, usize) = (start.parse().unwrap(), end.parse().unwrap());
|
|
536
|
+
let body = &b"abcdefghijkl"[start..=end];
|
|
537
|
+
let (status, content_range) = match (fault, start) {
|
|
538
|
+
("wrong_offset", s) if s > 0 => {
|
|
539
|
+
("206 Partial Content", format!("bytes {}-{}/12", s - 1, end - 1))
|
|
540
|
+
}
|
|
541
|
+
("probe_wrong_offset", 0) => ("206 Partial Content", format!("bytes 1-{}/12", end + 1)),
|
|
542
|
+
("part_ignores_range", s) if s > 0 => ("200 OK", String::new()),
|
|
543
|
+
_ => ("206 Partial Content", format!("bytes {start}-{end}/12")),
|
|
544
|
+
};
|
|
545
|
+
let range_header = if content_range.is_empty() {
|
|
546
|
+
String::new()
|
|
547
|
+
} else {
|
|
548
|
+
format!("Content-Range: {content_range}\r\n")
|
|
549
|
+
};
|
|
550
|
+
let head = format!(
|
|
551
|
+
"HTTP/1.1 {status}\r\nContent-Length: {}\r\n{range_header}Connection: close\r\n\r\n",
|
|
552
|
+
body.len()
|
|
553
|
+
);
|
|
554
|
+
let _ = socket.write_all(head.as_bytes()).await;
|
|
555
|
+
let _ = socket.write_all(body).await;
|
|
556
|
+
});
|
|
557
|
+
}
|
|
558
|
+
});
|
|
559
|
+
address
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
/// Right length from the wrong place passes a byte-count check; only the
|
|
563
|
+
/// status and `Content-Range` of every part can catch it.
|
|
564
|
+
#[tokio::test]
|
|
565
|
+
async fn split_download_rejects_parts_that_are_not_the_requested_range() {
|
|
566
|
+
for fault in ["wrong_offset", "probe_wrong_offset", "part_ignores_range"] {
|
|
567
|
+
let address = serve_range_file(fault).await;
|
|
568
|
+
let client = Arc::new(
|
|
569
|
+
DbClient::new(&format!("http://{address}"), "wh", "token")
|
|
570
|
+
.with_retry_attempts(1)
|
|
571
|
+
.with_retry_max_wait_s(0.0),
|
|
572
|
+
);
|
|
573
|
+
let stats = Arc::new(QueryStatsAccumulator::default());
|
|
574
|
+
let result = client
|
|
575
|
+
.fetch_link_bytes_split(&format!("http://{address}/blob"), false, 4, 3, &stats)
|
|
576
|
+
.await;
|
|
577
|
+
assert!(
|
|
578
|
+
result.is_err(),
|
|
579
|
+
"{fault}: must not assemble wrong bytes, got {result:?}"
|
|
580
|
+
);
|
|
581
|
+
}
|
|
582
|
+
let address = serve_range_file("none").await;
|
|
583
|
+
let client = Arc::new(DbClient::new(&format!("http://{address}"), "wh", "token"));
|
|
584
|
+
let stats = Arc::new(QueryStatsAccumulator::default());
|
|
585
|
+
let bytes = client
|
|
586
|
+
.fetch_link_bytes_split(&format!("http://{address}/blob"), false, 4, 3, &stats)
|
|
587
|
+
.await
|
|
588
|
+
.unwrap();
|
|
589
|
+
assert_eq!(&bytes[..], b"abcdefghijkl");
|
|
590
|
+
}
|
|
591
|
+
|
|
451
592
|
#[tokio::test]
|
|
452
593
|
async fn split_download_starts_tail_requests_before_the_probe_body_finishes() {
|
|
453
594
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
|
@@ -2,12 +2,26 @@
|
|
|
2
2
|
//! (`/api/2.0/fs/files{volume_path}`) -- unrelated to either SQL backend,
|
|
3
3
|
//! just another authenticated HTTP operation this client exposes.
|
|
4
4
|
|
|
5
|
+
use std::time::Duration;
|
|
6
|
+
|
|
5
7
|
use bytes::Bytes;
|
|
6
8
|
use reqwest::StatusCode;
|
|
7
9
|
|
|
8
10
|
use super::DbClient;
|
|
9
11
|
use super::error::ApiError;
|
|
10
12
|
|
|
13
|
+
/// Slowest sustained upload rate the PUT deadline tolerates (1 MiB/s). The
|
|
14
|
+
/// per-request deadline is a total one covering the body, so it scales with
|
|
15
|
+
/// the payload: `max(http_timeout, len / MIN_UPLOAD_BYTES_PER_SEC)`. A slow
|
|
16
|
+
/// but progressing link is not killed at `http_timeout`; a stuck one still
|
|
17
|
+
/// fails once the scaled deadline passes.
|
|
18
|
+
pub const MIN_UPLOAD_BYTES_PER_SEC: u64 = 1 << 20;
|
|
19
|
+
|
|
20
|
+
/// Total deadline for one upload PUT of `len` bytes.
|
|
21
|
+
pub fn upload_deadline(http_timeout: Duration, len: usize) -> Duration {
|
|
22
|
+
http_timeout.max(Duration::from_secs((len as u64).div_ceil(MIN_UPLOAD_BYTES_PER_SEC)))
|
|
23
|
+
}
|
|
24
|
+
|
|
11
25
|
impl DbClient {
|
|
12
26
|
/// Uploads `data` to a Unity Catalog volume path via the Files API,
|
|
13
27
|
/// overwriting anything already there. `volume_path` is caller-supplied
|
|
@@ -17,6 +31,7 @@ impl DbClient {
|
|
|
17
31
|
/// cheap refcount clone, not a real copy.
|
|
18
32
|
pub async fn upload_volume_file(&self, volume_path: &str, data: Bytes) -> Result<(), ApiError> {
|
|
19
33
|
let url = format!("{}/api/2.0/fs/files{}", self.host, volume_path);
|
|
34
|
+
let timeout = upload_deadline(self.http_timeout, data.len());
|
|
20
35
|
self.retry_call(|| async {
|
|
21
36
|
let token = self.token_provider.get_token().await?;
|
|
22
37
|
let resp = self
|
|
@@ -26,7 +41,7 @@ impl DbClient {
|
|
|
26
41
|
.bearer_auth(&token)
|
|
27
42
|
.header("Content-Type", "application/octet-stream")
|
|
28
43
|
.body(data.clone())
|
|
29
|
-
.timeout(
|
|
44
|
+
.timeout(timeout)
|
|
30
45
|
.send()
|
|
31
46
|
.await
|
|
32
47
|
// `overwrite=true` means retrying this exact PUT is safe --
|
|
@@ -42,6 +57,34 @@ impl DbClient {
|
|
|
42
57
|
.await
|
|
43
58
|
}
|
|
44
59
|
|
|
60
|
+
/// Downloads a Unity Catalog volume file in one GET on the pooled
|
|
61
|
+
/// connection. Single request on purpose: for typical files the server's
|
|
62
|
+
/// time-to-first-byte dominates, so range-splitting only adds round trips.
|
|
63
|
+
/// A non-2xx (incl. 404) is an error, unlike delete.
|
|
64
|
+
pub async fn download_volume_file(&self, volume_path: &str) -> Result<Bytes, ApiError> {
|
|
65
|
+
let url = format!("{}/api/2.0/fs/files{}", self.host, volume_path);
|
|
66
|
+
self.retry_call(|| async {
|
|
67
|
+
let token = self.token_provider.get_token().await?;
|
|
68
|
+
// No total request timeout: headers get `http_timeout`, the body a
|
|
69
|
+
// deadline scaled to Content-Length (same 1 MiB/s floor as upload).
|
|
70
|
+
let resp = tokio::time::timeout(self.http_timeout, self.http.get(&url).bearer_auth(&token).send())
|
|
71
|
+
.await
|
|
72
|
+
.map_err(|_| ApiError::permanent("volume download timed out waiting for response headers"))?
|
|
73
|
+
.map_err(|e| ApiError::from_reqwest(e, true))?;
|
|
74
|
+
let status = resp.status();
|
|
75
|
+
if !status.is_success() {
|
|
76
|
+
let text = resp.text().await.unwrap_or_default();
|
|
77
|
+
return Err(ApiError::from_status(status, &text, true));
|
|
78
|
+
}
|
|
79
|
+
let deadline = upload_deadline(self.http_timeout, resp.content_length().unwrap_or(0) as usize);
|
|
80
|
+
tokio::time::timeout(deadline, resp.bytes())
|
|
81
|
+
.await
|
|
82
|
+
.map_err(|_| ApiError::permanent("volume download timed out reading body"))?
|
|
83
|
+
.map_err(|e| ApiError::from_reqwest(e, true))
|
|
84
|
+
})
|
|
85
|
+
.await
|
|
86
|
+
}
|
|
87
|
+
|
|
45
88
|
/// Deletes a file at `volume_path` (see `upload_volume_file`). A 404 is
|
|
46
89
|
/// treated as success -- the file is already gone, which is fine for
|
|
47
90
|
/// idempotent staging cleanup.
|
|
@@ -67,3 +110,17 @@ impl DbClient {
|
|
|
67
110
|
.await
|
|
68
111
|
}
|
|
69
112
|
}
|
|
113
|
+
|
|
114
|
+
#[cfg(test)]
|
|
115
|
+
mod tests {
|
|
116
|
+
use super::*;
|
|
117
|
+
|
|
118
|
+
#[test]
|
|
119
|
+
fn upload_deadline_scales_with_size_but_never_below_http_timeout() {
|
|
120
|
+
let base = Duration::from_secs(60);
|
|
121
|
+
assert_eq!(upload_deadline(base, 0), base);
|
|
122
|
+
assert_eq!(upload_deadline(base, 10 << 20), base);
|
|
123
|
+
assert_eq!(upload_deadline(base, 600 << 20), Duration::from_secs(600));
|
|
124
|
+
assert_eq!(upload_deadline(base, (600 << 20) + 1), Duration::from_secs(601));
|
|
125
|
+
}
|
|
126
|
+
}
|
|
@@ -130,6 +130,40 @@ fn read_ipc_stream(data: Bound<'_, PyBytes>) -> PyResult<PyTable> {
|
|
|
130
130
|
PyTable::try_new(batches, schema)
|
|
131
131
|
}
|
|
132
132
|
|
|
133
|
+
/// Turns the Python `data` argument of `upload_volume_file` into `Bytes`
|
|
134
|
+
/// without the per-element `Vec<u8>` extraction (20x slower) and, for the
|
|
135
|
+
/// common `bytes` case, without any copy.
|
|
136
|
+
fn upload_body(data: &Bound<'_, PyAny>) -> PyResult<bytes::Bytes> {
|
|
137
|
+
if let Ok(b) = data.cast::<PyBytes>() {
|
|
138
|
+
// Immutable, so the Python object can own the memory for the whole
|
|
139
|
+
// upload (and every retry) with no copy.
|
|
140
|
+
return Ok(bytes::Bytes::from_owner(pyo3::pybacked::PyBackedBytes::from(b.clone())));
|
|
141
|
+
}
|
|
142
|
+
// bytearray/memoryview/numpy/pyarrow are mutable or not owned by us: take
|
|
143
|
+
// one contiguous copy now so later caller mutation cannot change the
|
|
144
|
+
// upload mid-flight.
|
|
145
|
+
let py = data.py();
|
|
146
|
+
if let Ok(buffer) = pyo3::buffer::PyBuffer::<u8>::get(data) {
|
|
147
|
+
return Ok(bytes::Bytes::from(buffer.to_vec(py)?));
|
|
148
|
+
}
|
|
149
|
+
// PyArrow buffers (and `array('b')`) export the signed-byte format `b`,
|
|
150
|
+
// which `PyBuffer<u8>` refuses; the raw bits are the same bytes.
|
|
151
|
+
if let Ok(buffer) = pyo3::buffer::PyBuffer::<i8>::get(data) {
|
|
152
|
+
let signed = buffer.to_vec(py)?;
|
|
153
|
+
return Ok(bytes::Bytes::from(
|
|
154
|
+
signed.into_iter().map(|b| b as u8).collect::<Vec<u8>>(),
|
|
155
|
+
));
|
|
156
|
+
}
|
|
157
|
+
// Last resort keeps the pre-6.3 behaviour: any sequence of ints in 0..=255
|
|
158
|
+
// (a `list[int]`, a numpy int64 array, ...). Slow, but it used to work.
|
|
159
|
+
data.extract::<Vec<u8>>().map(bytes::Bytes::from).map_err(|_| {
|
|
160
|
+
pyo3::exceptions::PyTypeError::new_err(
|
|
161
|
+
"upload_volume_file data must be bytes, a byte buffer (bytearray, memoryview, numpy, pyarrow, ...) \
|
|
162
|
+
or a sequence of ints in 0..=255",
|
|
163
|
+
)
|
|
164
|
+
})
|
|
165
|
+
}
|
|
166
|
+
|
|
133
167
|
/// Wraps a `PyErr` raised by the caller's own `token_provider` callable (its
|
|
134
168
|
/// call itself, awaiting its coroutine, or extracting a non-`str` return
|
|
135
169
|
/// value) into an `ApiError`. This function's *only* caller is
|
|
@@ -869,16 +903,18 @@ impl PyDbClient {
|
|
|
869
903
|
}
|
|
870
904
|
|
|
871
905
|
/// Uploads `data` to a Unity Catalog volume path via the Files API,
|
|
872
|
-
/// overwriting anything already there.
|
|
906
|
+
/// overwriting anything already there. `bytes` is borrowed zero-copy;
|
|
907
|
+
/// `bytearray` and other contiguous byte buffers (memoryview, numpy
|
|
908
|
+
/// `uint8`, pyarrow `Buffer`) are copied once, off the per-element path.
|
|
873
909
|
#[pyo3(signature = (volume_path, data))]
|
|
874
910
|
fn upload_volume_file<'py>(
|
|
875
911
|
&self,
|
|
876
912
|
py: Python<'py>,
|
|
877
913
|
volume_path: String,
|
|
878
|
-
data:
|
|
914
|
+
data: &Bound<'py, PyAny>,
|
|
879
915
|
) -> PyResult<Bound<'py, PyAny>> {
|
|
880
916
|
let client = self.inner.clone();
|
|
881
|
-
let data =
|
|
917
|
+
let data = upload_body(data)?;
|
|
882
918
|
future_into_py_in_call(py, async move {
|
|
883
919
|
client
|
|
884
920
|
.upload_volume_file(&volume_path, data)
|
|
@@ -887,6 +923,18 @@ impl PyDbClient {
|
|
|
887
923
|
})
|
|
888
924
|
}
|
|
889
925
|
|
|
926
|
+
/// Downloads the file at `volume_path` and returns its contents as `bytes`.
|
|
927
|
+
fn download_volume_file<'py>(&self, py: Python<'py>, volume_path: String) -> PyResult<Bound<'py, PyAny>> {
|
|
928
|
+
let client = self.inner.clone();
|
|
929
|
+
future_into_py_in_call(py, async move {
|
|
930
|
+
let data = client
|
|
931
|
+
.download_volume_file(&volume_path)
|
|
932
|
+
.await
|
|
933
|
+
.map_err(api_error_to_pyerr)?;
|
|
934
|
+
Ok(Python::attach(|py| pyo3::types::PyBytes::new(py, &data).unbind()))
|
|
935
|
+
})
|
|
936
|
+
}
|
|
937
|
+
|
|
890
938
|
/// Deletes a file at `volume_path` (see `upload_volume_file`). A 404 is
|
|
891
939
|
/// treated as success -- the file is already gone, fine for idempotent
|
|
892
940
|
/// staging cleanup.
|
|
@@ -1702,18 +1702,18 @@ async fn thrift_split_download_fails_on_assembled_length_mismatch() {
|
|
|
1702
1702
|
let server = MockServer::start().await;
|
|
1703
1703
|
mount_single_link_statement(&server, "/_data/short-parts", 60_000).await;
|
|
1704
1704
|
|
|
1705
|
-
// Every part -- including the probe -- honestly declares
|
|
1706
|
-
// via Content-Range, but actually returns far
|
|
1707
|
-
//
|
|
1708
|
-
// guarantees more than one part gets requested; every part coming
|
|
1709
|
-
// short must be caught by the
|
|
1710
|
-
//
|
|
1705
|
+
// Every part -- including the probe -- honestly declares the range it was
|
|
1706
|
+
// asked for of a 5MiB total via Content-Range, but actually returns far
|
|
1707
|
+
// fewer bytes than that range covers (a misbehaving server). The declared
|
|
1708
|
+
// total guarantees more than one part gets requested; every part coming
|
|
1709
|
+
// up short must be caught by the length checks, not silently accepted as
|
|
1710
|
+
// a truncated-but-otherwise-fine result.
|
|
1711
1711
|
const DECLARED_TOTAL: u64 = 5 * (1 << 20);
|
|
1712
1712
|
Mock::given(method("GET"))
|
|
1713
1713
|
.and(path("/_data/short-parts"))
|
|
1714
1714
|
.respond_with(move |req: &Request| {
|
|
1715
|
-
let (start,
|
|
1716
|
-
let end =
|
|
1715
|
+
let (start, end) = parse_range(req).expect("client always sends Range on this path");
|
|
1716
|
+
let end = end.min(DECLARED_TOTAL - 1);
|
|
1717
1717
|
ResponseTemplate::new(206)
|
|
1718
1718
|
.insert_header("Content-Range", format!("bytes {start}-{end}/{DECLARED_TOTAL}"))
|
|
1719
1719
|
.set_body_raw(vec![0u8; 10], "application/octet-stream")
|
|
@@ -1736,6 +1736,83 @@ async fn thrift_split_download_fails_on_assembled_length_mismatch() {
|
|
|
1736
1736
|
);
|
|
1737
1737
|
}
|
|
1738
1738
|
|
|
1739
|
+
#[tokio::test]
|
|
1740
|
+
async fn thrift_split_download_absurd_content_range_total_fails_without_aborting() {
|
|
1741
|
+
let server = MockServer::start().await;
|
|
1742
|
+
mount_single_link_statement(&server, "/_data/absurd-total", 60_000).await;
|
|
1743
|
+
|
|
1744
|
+
// The Content-Range total is untrusted: u64::MAX bytes must never size an
|
|
1745
|
+
// allocation (an allocation failure aborts the whole process, not just
|
|
1746
|
+
// the query), only turn into a normal error.
|
|
1747
|
+
const ABSURD_TOTAL: u64 = u64::MAX;
|
|
1748
|
+
Mock::given(method("GET"))
|
|
1749
|
+
.and(path("/_data/absurd-total"))
|
|
1750
|
+
.respond_with(move |req: &Request| {
|
|
1751
|
+
let (start, end) = parse_range(req).expect("client always sends Range on this path");
|
|
1752
|
+
ResponseTemplate::new(206)
|
|
1753
|
+
.insert_header("Content-Range", format!("bytes {start}-{end}/{ABSURD_TOTAL}"))
|
|
1754
|
+
.set_body_raw(vec![0u8; 1 << 20], "application/octet-stream")
|
|
1755
|
+
})
|
|
1756
|
+
.mount(&server)
|
|
1757
|
+
.await;
|
|
1758
|
+
|
|
1759
|
+
let client = thrift_client(&server);
|
|
1760
|
+
let mut stream = execute_lazy_thrift(client, "SELECT * FROM t", None, None, None)
|
|
1761
|
+
.await
|
|
1762
|
+
.unwrap();
|
|
1763
|
+
let err = stream
|
|
1764
|
+
.fetchall_arrow()
|
|
1765
|
+
.await
|
|
1766
|
+
.expect_err("an absurd declared total must fail loudly, not abort or succeed");
|
|
1767
|
+
assert!(
|
|
1768
|
+
err.message.contains("expected"),
|
|
1769
|
+
"got a different error than expected: {}",
|
|
1770
|
+
err.message
|
|
1771
|
+
);
|
|
1772
|
+
}
|
|
1773
|
+
|
|
1774
|
+
#[tokio::test]
|
|
1775
|
+
async fn thrift_split_download_rejects_short_part_offset_by_long_part() {
|
|
1776
|
+
let server = MockServer::start().await;
|
|
1777
|
+
mount_single_link_statement(&server, "/_data/offset-parts", 60_000).await;
|
|
1778
|
+
|
|
1779
|
+
// 3 MiB file, three 1 MiB ranges. The middle part returns 10 bytes too
|
|
1780
|
+
// few and the last 10 bytes too many, so the assembled length equals the
|
|
1781
|
+
// declared total and only a per-part length check can catch it.
|
|
1782
|
+
const MIB: u64 = 1 << 20;
|
|
1783
|
+
const TOTAL: u64 = 3 * MIB;
|
|
1784
|
+
Mock::given(method("GET"))
|
|
1785
|
+
.and(path("/_data/offset-parts"))
|
|
1786
|
+
.respond_with(move |req: &Request| {
|
|
1787
|
+
let (start, end) = parse_range(req).expect("client always sends Range on this path");
|
|
1788
|
+
let end = end.min(TOTAL - 1);
|
|
1789
|
+
let len = match start {
|
|
1790
|
+
MIB => MIB - 10,
|
|
1791
|
+
s if s == 2 * MIB => MIB + 10,
|
|
1792
|
+
_ => end - start + 1,
|
|
1793
|
+
};
|
|
1794
|
+
ResponseTemplate::new(206)
|
|
1795
|
+
.insert_header("Content-Range", format!("bytes {start}-{end}/{TOTAL}"))
|
|
1796
|
+
.set_body_raw(vec![0u8; len as usize], "application/octet-stream")
|
|
1797
|
+
})
|
|
1798
|
+
.mount(&server)
|
|
1799
|
+
.await;
|
|
1800
|
+
|
|
1801
|
+
let client = thrift_client(&server);
|
|
1802
|
+
let mut stream = execute_lazy_thrift(client, "SELECT * FROM t", None, None, None)
|
|
1803
|
+
.await
|
|
1804
|
+
.unwrap();
|
|
1805
|
+
let err = stream
|
|
1806
|
+
.fetchall_arrow()
|
|
1807
|
+
.await
|
|
1808
|
+
.expect_err("a wrong-length part must fail loudly even when the total adds up");
|
|
1809
|
+
assert!(
|
|
1810
|
+
err.message.contains("split download part"),
|
|
1811
|
+
"got a different error than expected: {}",
|
|
1812
|
+
err.message
|
|
1813
|
+
);
|
|
1814
|
+
}
|
|
1815
|
+
|
|
1739
1816
|
#[tokio::test]
|
|
1740
1817
|
async fn thrift_concurrent_single_link_queries_stay_correct_under_split_contention() {
|
|
1741
1818
|
let server = MockServer::start().await;
|
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
//! Files API (upload_volume_file/delete_volume_file) -- kept separate from
|
|
2
|
+
//! wiremock_pipeline.rs since it's a different API surface entirely (no
|
|
3
|
+
//! statement/manifest/chunk shape to it), mirroring tests/conftest.py's own
|
|
4
|
+
//! `mock_volume_files` fixture being kept apart from `mock_warehouse`.
|
|
5
|
+
|
|
6
|
+
use std::sync::Arc;
|
|
7
|
+
|
|
8
|
+
use arrowbricks_core::client::DbClient;
|
|
9
|
+
use bytes::Bytes;
|
|
10
|
+
use wiremock::matchers::{method, path_regex, query_param};
|
|
11
|
+
use wiremock::{Mock, MockServer, ResponseTemplate};
|
|
12
|
+
|
|
13
|
+
const WAREHOUSE_ID: &str = "wh-test-123";
|
|
14
|
+
|
|
15
|
+
#[tokio::test]
|
|
16
|
+
async fn upload_volume_file_sends_overwrite_and_content_type() {
|
|
17
|
+
let server = MockServer::start().await;
|
|
18
|
+
Mock::given(method("PUT"))
|
|
19
|
+
.and(path_regex(r"^/api/2\.0/fs/files/.*"))
|
|
20
|
+
.and(query_param("overwrite", "true"))
|
|
21
|
+
.respond_with(ResponseTemplate::new(204))
|
|
22
|
+
.mount(&server)
|
|
23
|
+
.await;
|
|
24
|
+
|
|
25
|
+
let client = Arc::new(DbClient::new(&server.uri(), WAREHOUSE_ID, "fake-token"));
|
|
26
|
+
client
|
|
27
|
+
.upload_volume_file("/Volumes/cat/schema/vol/file.bin", Bytes::from_static(b"hello"))
|
|
28
|
+
.await
|
|
29
|
+
.unwrap();
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
#[tokio::test]
|
|
33
|
+
async fn download_volume_file_returns_exact_body() {
|
|
34
|
+
let server = MockServer::start().await;
|
|
35
|
+
let body = vec![0u8, 255, 1, 254, b'%', b'P', b'D', b'F'];
|
|
36
|
+
Mock::given(method("GET"))
|
|
37
|
+
.and(path_regex(r"^/api/2\.0/fs/files/Volumes/cat/schema/vol/file\.pdf$"))
|
|
38
|
+
.respond_with(ResponseTemplate::new(200).set_body_bytes(body.clone()))
|
|
39
|
+
.mount(&server)
|
|
40
|
+
.await;
|
|
41
|
+
|
|
42
|
+
let client = Arc::new(DbClient::new(&server.uri(), WAREHOUSE_ID, "fake-token"));
|
|
43
|
+
let got = client
|
|
44
|
+
.download_volume_file("/Volumes/cat/schema/vol/file.pdf")
|
|
45
|
+
.await
|
|
46
|
+
.unwrap();
|
|
47
|
+
assert_eq!(got.as_ref(), body.as_slice());
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
#[tokio::test]
|
|
51
|
+
async fn download_volume_file_404_is_an_error() {
|
|
52
|
+
let server = MockServer::start().await;
|
|
53
|
+
Mock::given(method("GET"))
|
|
54
|
+
.respond_with(ResponseTemplate::new(404).set_body_string("not found"))
|
|
55
|
+
.mount(&server)
|
|
56
|
+
.await;
|
|
57
|
+
|
|
58
|
+
let client = Arc::new(DbClient::new(&server.uri(), WAREHOUSE_ID, "fake-token"));
|
|
59
|
+
let err = client
|
|
60
|
+
.download_volume_file("/Volumes/cat/schema/vol/missing.pdf")
|
|
61
|
+
.await
|
|
62
|
+
.unwrap_err();
|
|
63
|
+
assert!(
|
|
64
|
+
err.message.contains("404") || err.message.contains("not found"),
|
|
65
|
+
"{err}"
|
|
66
|
+
);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
#[tokio::test]
|
|
70
|
+
async fn delete_volume_file_succeeds_on_204() {
|
|
71
|
+
let server = MockServer::start().await;
|
|
72
|
+
Mock::given(method("DELETE"))
|
|
73
|
+
.and(path_regex(r"^/api/2\.0/fs/files/.*"))
|
|
74
|
+
.respond_with(ResponseTemplate::new(204))
|
|
75
|
+
.mount(&server)
|
|
76
|
+
.await;
|
|
77
|
+
|
|
78
|
+
let client = Arc::new(DbClient::new(&server.uri(), WAREHOUSE_ID, "fake-token"));
|
|
79
|
+
client
|
|
80
|
+
.delete_volume_file("/Volumes/cat/schema/vol/file.bin")
|
|
81
|
+
.await
|
|
82
|
+
.unwrap();
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
#[tokio::test]
|
|
86
|
+
async fn delete_volume_file_treats_404_as_success() {
|
|
87
|
+
let server = MockServer::start().await;
|
|
88
|
+
Mock::given(method("DELETE"))
|
|
89
|
+
.and(path_regex(r"^/api/2\.0/fs/files/.*"))
|
|
90
|
+
.respond_with(ResponseTemplate::new(404))
|
|
91
|
+
.mount(&server)
|
|
92
|
+
.await;
|
|
93
|
+
|
|
94
|
+
let client = Arc::new(DbClient::new(&server.uri(), WAREHOUSE_ID, "fake-token"));
|
|
95
|
+
client
|
|
96
|
+
.delete_volume_file("/Volumes/cat/schema/vol/already-gone.bin")
|
|
97
|
+
.await
|
|
98
|
+
.expect("404 must be treated as success (idempotent delete), not an error");
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
#[tokio::test]
|
|
102
|
+
async fn delete_volume_file_surfaces_other_errors() {
|
|
103
|
+
// 400, not 500/429/etc -- those are classified transient and would
|
|
104
|
+
// trigger the full 6-attempt retry/backoff cycle (tens of seconds) for
|
|
105
|
+
// no benefit here; that classification logic already has its own
|
|
106
|
+
// coverage. This only needs a status delete_volume_file does NOT treat
|
|
107
|
+
// as "already gone", proving the error still propagates.
|
|
108
|
+
let server = MockServer::start().await;
|
|
109
|
+
Mock::given(method("DELETE"))
|
|
110
|
+
.and(path_regex(r"^/api/2\.0/fs/files/.*"))
|
|
111
|
+
.respond_with(ResponseTemplate::new(400))
|
|
112
|
+
.mount(&server)
|
|
113
|
+
.await;
|
|
114
|
+
|
|
115
|
+
let client = Arc::new(DbClient::new(&server.uri(), WAREHOUSE_ID, "fake-token"));
|
|
116
|
+
let err = client
|
|
117
|
+
.delete_volume_file("/Volumes/cat/schema/vol/file.bin")
|
|
118
|
+
.await
|
|
119
|
+
.unwrap_err();
|
|
120
|
+
assert!(
|
|
121
|
+
!err.transient,
|
|
122
|
+
"400 is not transient, should surface immediately without retrying"
|
|
123
|
+
);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
// ---------------------------------------------------------------------------
|
|
127
|
+
// Upload deadline scaling and retry body reuse. wiremock cannot throttle how
|
|
128
|
+
// fast the *server reads* a request body, so these use a tiny raw TCP server.
|
|
129
|
+
// ---------------------------------------------------------------------------
|
|
130
|
+
|
|
131
|
+
use std::time::{Duration, Instant};
|
|
132
|
+
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
|
133
|
+
use tokio::net::TcpListener;
|
|
134
|
+
|
|
135
|
+
const MIB: usize = 1 << 20;
|
|
136
|
+
|
|
137
|
+
/// Raw HTTP/1.1 server answering every PUT with `204` once it has read the
|
|
138
|
+
/// whole body, consuming it at roughly `bytes_per_sec`. `stuck` instead never
|
|
139
|
+
/// reads or answers at all.
|
|
140
|
+
async fn spawn_throttled_server(bytes_per_sec: usize, stuck: bool) -> String {
|
|
141
|
+
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
142
|
+
let addr = listener.local_addr().unwrap();
|
|
143
|
+
tokio::spawn(async move {
|
|
144
|
+
loop {
|
|
145
|
+
let Ok((mut sock, _)) = listener.accept().await else {
|
|
146
|
+
return;
|
|
147
|
+
};
|
|
148
|
+
tokio::spawn(async move {
|
|
149
|
+
if stuck {
|
|
150
|
+
tokio::time::sleep(Duration::from_secs(120)).await;
|
|
151
|
+
return;
|
|
152
|
+
}
|
|
153
|
+
let mut buf = vec![0u8; 64 * 1024];
|
|
154
|
+
let mut head = Vec::new();
|
|
155
|
+
let (mut body_len, mut got_body, mut header_end) = (None::<usize>, 0usize, None::<usize>);
|
|
156
|
+
loop {
|
|
157
|
+
let Ok(n) = sock.read(&mut buf).await else { return };
|
|
158
|
+
if n == 0 {
|
|
159
|
+
return;
|
|
160
|
+
}
|
|
161
|
+
if header_end.is_none() {
|
|
162
|
+
head.extend_from_slice(&buf[..n]);
|
|
163
|
+
if let Some(pos) = head.windows(4).position(|w| w == b"\r\n\r\n") {
|
|
164
|
+
let text = String::from_utf8_lossy(&head[..pos]).to_lowercase();
|
|
165
|
+
body_len = text
|
|
166
|
+
.lines()
|
|
167
|
+
.find_map(|l| l.strip_prefix("content-length:").map(|v| v.trim().parse().unwrap()));
|
|
168
|
+
header_end = Some(pos + 4);
|
|
169
|
+
got_body = head.len() - (pos + 4);
|
|
170
|
+
}
|
|
171
|
+
} else {
|
|
172
|
+
got_body += n;
|
|
173
|
+
}
|
|
174
|
+
if header_end.is_some() {
|
|
175
|
+
if got_body >= body_len.expect("fixed-length body") {
|
|
176
|
+
break;
|
|
177
|
+
}
|
|
178
|
+
tokio::time::sleep(Duration::from_secs_f64(n as f64 / bytes_per_sec as f64)).await;
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
let _ = sock
|
|
182
|
+
.write_all(b"HTTP/1.1 204 No Content\r\nConnection: close\r\n\r\n")
|
|
183
|
+
.await;
|
|
184
|
+
});
|
|
185
|
+
}
|
|
186
|
+
});
|
|
187
|
+
format!("http://{addr}")
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
#[tokio::test]
|
|
191
|
+
async fn upload_slow_but_progressing_succeeds_past_http_timeout() {
|
|
192
|
+
// 3 MiB read at ~1.5 MiB/s takes ~2 s: over the 1 s http_timeout, under
|
|
193
|
+
// the scaled deadline (3 MiB / 1 MiB/s = 3 s).
|
|
194
|
+
let uri = spawn_throttled_server(MIB + MIB / 2, false).await;
|
|
195
|
+
let client = Arc::new(DbClient::new(&uri, WAREHOUSE_ID, "fake-token").with_http_timeout(1.0));
|
|
196
|
+
let start = Instant::now();
|
|
197
|
+
client
|
|
198
|
+
.upload_volume_file("/Volumes/c/s/v/big.bin", Bytes::from(vec![7u8; 3 * MIB]))
|
|
199
|
+
.await
|
|
200
|
+
.expect("a progressing upload must not be killed at http_timeout");
|
|
201
|
+
assert!(
|
|
202
|
+
start.elapsed() > Duration::from_secs(1),
|
|
203
|
+
"test must exceed http_timeout to prove anything, took {:?}",
|
|
204
|
+
start.elapsed()
|
|
205
|
+
);
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
#[tokio::test]
|
|
209
|
+
async fn upload_stuck_server_still_times_out_at_scaled_deadline() {
|
|
210
|
+
let uri = spawn_throttled_server(1, true).await;
|
|
211
|
+
let client = Arc::new(DbClient::new(&uri, WAREHOUSE_ID, "fake-token").with_http_timeout(1.0));
|
|
212
|
+
let start = Instant::now();
|
|
213
|
+
let err = client
|
|
214
|
+
.upload_volume_file("/Volumes/c/s/v/big.bin", Bytes::from(vec![7u8; 2 * MIB]))
|
|
215
|
+
.await
|
|
216
|
+
.expect_err("a stuck server must still fail");
|
|
217
|
+
let took = start.elapsed();
|
|
218
|
+
assert!(!err.transient, "timeouts stay non-transient (not retried): {err:?}");
|
|
219
|
+
assert!(
|
|
220
|
+
took >= Duration::from_secs(2) && took < Duration::from_secs(10),
|
|
221
|
+
"deadline should be the scaled ~2 s, took {took:?}"
|
|
222
|
+
);
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
#[tokio::test]
|
|
226
|
+
async fn upload_retry_after_503_resends_full_identical_body() {
|
|
227
|
+
use wiremock::matchers::body_bytes;
|
|
228
|
+
let server = MockServer::start().await;
|
|
229
|
+
let payload: Vec<u8> = (0..300_000u32).map(|i| (i % 251) as u8).collect();
|
|
230
|
+
Mock::given(method("PUT"))
|
|
231
|
+
.and(path_regex(r"^/api/2\.0/fs/files/.*"))
|
|
232
|
+
.respond_with(ResponseTemplate::new(503))
|
|
233
|
+
.up_to_n_times(1)
|
|
234
|
+
.expect(1)
|
|
235
|
+
.mount(&server)
|
|
236
|
+
.await;
|
|
237
|
+
Mock::given(method("PUT"))
|
|
238
|
+
.and(path_regex(r"^/api/2\.0/fs/files/.*"))
|
|
239
|
+
.and(body_bytes(payload.clone()))
|
|
240
|
+
.respond_with(ResponseTemplate::new(204))
|
|
241
|
+
.expect(1)
|
|
242
|
+
.mount(&server)
|
|
243
|
+
.await;
|
|
244
|
+
|
|
245
|
+
let client = Arc::new(DbClient::new(&server.uri(), WAREHOUSE_ID, "fake-token"));
|
|
246
|
+
client
|
|
247
|
+
.upload_volume_file("/Volumes/c/s/v/retry.bin", Bytes::from(payload))
|
|
248
|
+
.await
|
|
249
|
+
.expect("second attempt must carry the complete, identical body");
|
|
250
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
|
|
1
|
+
import array
|
|
2
|
+
from collections.abc import Awaitable, Callable, Sequence
|
|
2
3
|
from typing import Any, BinaryIO, Literal
|
|
3
4
|
|
|
4
5
|
def _validate_timeout(total_timeout_s: float | None) -> None: ...
|
|
@@ -114,7 +115,10 @@ class Client:
|
|
|
114
115
|
schema: str | None = None,
|
|
115
116
|
parameters: list[dict[str, Any]] | None = None,
|
|
116
117
|
) -> str: ...
|
|
117
|
-
async def upload_volume_file(
|
|
118
|
+
async def upload_volume_file(
|
|
119
|
+
self, volume_path: str, data: bytes | bytearray | memoryview | array.array | Sequence[int]
|
|
120
|
+
) -> None: ...
|
|
121
|
+
async def download_volume_file(self, volume_path: str) -> bytes: ...
|
|
118
122
|
async def delete_volume_file(self, volume_path: str) -> None: ...
|
|
119
123
|
async def close_sessions(self) -> None: ...
|
|
120
124
|
def stream_ndjson_lines(
|
|
@@ -207,13 +207,17 @@ async def stream_query_json(
|
|
|
207
207
|
mid-download), then each result row as a ready-to-send JSON string --
|
|
208
208
|
one per SSE frame, say.
|
|
209
209
|
|
|
210
|
-
Unlike cursor.py's fetch methods, this
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
210
|
+
Unlike cursor.py's fetch methods, this does not buffer the full result
|
|
211
|
+
first: it emits rows via `._core.Client.stream_ndjson_lines` as chunks
|
|
212
|
+
are decoded, in order (chunks can complete out of order over the
|
|
213
|
+
network; order is restored there). The first row is available once the
|
|
214
|
+
first chunk has been downloaded and decoded. For a result spread over
|
|
215
|
+
many chunks on a bandwidth-limited link that can be most of the total
|
|
216
|
+
download time (measured on one ~30 MB/s link: the first row arrived at
|
|
217
|
+
84-97% of the stream total), so do not count on early rows for large
|
|
218
|
+
results. What streaming does give you: heartbeats while waiting, bounded
|
|
219
|
+
memory, and no full materialization. Decode and NDJSON encoding both
|
|
220
|
+
happen in Rust -- this function only forwards already-formatted lines.
|
|
217
221
|
|
|
218
222
|
`non_finite_floats` -- `"null"` (default) or `"string"`. A NaN/Infinity/
|
|
219
223
|
-Infinity float column comes back from Databricks like any other value,
|
|
@@ -7,7 +7,8 @@ reorder, Arrow/JSON/NDJSON decode)."""
|
|
|
7
7
|
|
|
8
8
|
from __future__ import annotations
|
|
9
9
|
|
|
10
|
-
|
|
10
|
+
import array
|
|
11
|
+
from collections.abc import AsyncIterator, Awaitable, Callable, Sequence
|
|
11
12
|
from typing import Any
|
|
12
13
|
|
|
13
14
|
from . import _core
|
|
@@ -133,14 +134,34 @@ class DatabricksClient:
|
|
|
133
134
|
async def __aexit__(self, *exc_info: object) -> None:
|
|
134
135
|
await self.aclose()
|
|
135
136
|
|
|
136
|
-
async def upload_volume_file(
|
|
137
|
+
async def upload_volume_file(
|
|
138
|
+
self, volume_path: str, data: bytes | bytearray | memoryview | array.array | Sequence[int]
|
|
139
|
+
) -> None:
|
|
137
140
|
"""Uploads `data` to a Unity Catalog volume path via the Files API,
|
|
138
141
|
overwriting anything already there. `volume_path` is caller-supplied
|
|
139
142
|
in full (e.g. `/Volumes/my_catalog/my_schema/my_volume/some/file.parquet`)
|
|
140
143
|
-- this package has no knowledge of any specific catalog/schema/volume.
|
|
141
|
-
|
|
144
|
+
|
|
145
|
+
`data` may be `bytes` (shared with the upload, not copied -- the
|
|
146
|
+
cheapest and recommended form) or any byte buffer (`bytearray`,
|
|
147
|
+
`memoryview`, `array.array`, a numpy array, a pyarrow `Buffer`; signed
|
|
148
|
+
or unsigned bytes), which is copied once up front. A plain sequence of
|
|
149
|
+
ints in 0..=255 (e.g. a `list[int]`) still works but is slow; prefer
|
|
150
|
+
`bytes(data)`. Anything else raises `TypeError`.
|
|
151
|
+
|
|
152
|
+
The PUT's total deadline is `max(http_timeout, len(data) / 1 MiB/s)`,
|
|
153
|
+
so `http_timeout` is a floor, not a ceiling: a slow-but-progressing
|
|
154
|
+
large upload is not cut off at `http_timeout`, and a stalled 1 GiB
|
|
155
|
+
upload can take about 17 minutes to fail. A timeout is not retried;
|
|
156
|
+
transient HTTP errors are. Raises `RuntimeError` (message only) on
|
|
157
|
+
failure."""
|
|
142
158
|
await self._core_client.upload_volume_file(volume_path, data)
|
|
143
159
|
|
|
160
|
+
async def download_volume_file(self, volume_path: str) -> bytes:
|
|
161
|
+
"""Downloads the file at `volume_path` (see upload_volume_file) and
|
|
162
|
+
returns its contents. Raises on a missing file (404)."""
|
|
163
|
+
return await self._core_client.download_volume_file(volume_path)
|
|
164
|
+
|
|
144
165
|
async def delete_volume_file(self, volume_path: str) -> None:
|
|
145
166
|
"""Deletes a file at `volume_path` (see upload_volume_file). A 404 is
|
|
146
167
|
treated as success -- the file is already gone, which is fine for
|
|
@@ -1,87 +0,0 @@
|
|
|
1
|
-
//! Files API (upload_volume_file/delete_volume_file) -- kept separate from
|
|
2
|
-
//! wiremock_pipeline.rs since it's a different API surface entirely (no
|
|
3
|
-
//! statement/manifest/chunk shape to it), mirroring tests/conftest.py's own
|
|
4
|
-
//! `mock_volume_files` fixture being kept apart from `mock_warehouse`.
|
|
5
|
-
|
|
6
|
-
use std::sync::Arc;
|
|
7
|
-
|
|
8
|
-
use arrowbricks_core::client::DbClient;
|
|
9
|
-
use bytes::Bytes;
|
|
10
|
-
use wiremock::matchers::{method, path_regex, query_param};
|
|
11
|
-
use wiremock::{Mock, MockServer, ResponseTemplate};
|
|
12
|
-
|
|
13
|
-
const WAREHOUSE_ID: &str = "wh-test-123";
|
|
14
|
-
|
|
15
|
-
#[tokio::test]
|
|
16
|
-
async fn upload_volume_file_sends_overwrite_and_content_type() {
|
|
17
|
-
let server = MockServer::start().await;
|
|
18
|
-
Mock::given(method("PUT"))
|
|
19
|
-
.and(path_regex(r"^/api/2\.0/fs/files/.*"))
|
|
20
|
-
.and(query_param("overwrite", "true"))
|
|
21
|
-
.respond_with(ResponseTemplate::new(204))
|
|
22
|
-
.mount(&server)
|
|
23
|
-
.await;
|
|
24
|
-
|
|
25
|
-
let client = Arc::new(DbClient::new(&server.uri(), WAREHOUSE_ID, "fake-token"));
|
|
26
|
-
client
|
|
27
|
-
.upload_volume_file("/Volumes/cat/schema/vol/file.bin", Bytes::from_static(b"hello"))
|
|
28
|
-
.await
|
|
29
|
-
.unwrap();
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
#[tokio::test]
|
|
33
|
-
async fn delete_volume_file_succeeds_on_204() {
|
|
34
|
-
let server = MockServer::start().await;
|
|
35
|
-
Mock::given(method("DELETE"))
|
|
36
|
-
.and(path_regex(r"^/api/2\.0/fs/files/.*"))
|
|
37
|
-
.respond_with(ResponseTemplate::new(204))
|
|
38
|
-
.mount(&server)
|
|
39
|
-
.await;
|
|
40
|
-
|
|
41
|
-
let client = Arc::new(DbClient::new(&server.uri(), WAREHOUSE_ID, "fake-token"));
|
|
42
|
-
client
|
|
43
|
-
.delete_volume_file("/Volumes/cat/schema/vol/file.bin")
|
|
44
|
-
.await
|
|
45
|
-
.unwrap();
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
#[tokio::test]
|
|
49
|
-
async fn delete_volume_file_treats_404_as_success() {
|
|
50
|
-
let server = MockServer::start().await;
|
|
51
|
-
Mock::given(method("DELETE"))
|
|
52
|
-
.and(path_regex(r"^/api/2\.0/fs/files/.*"))
|
|
53
|
-
.respond_with(ResponseTemplate::new(404))
|
|
54
|
-
.mount(&server)
|
|
55
|
-
.await;
|
|
56
|
-
|
|
57
|
-
let client = Arc::new(DbClient::new(&server.uri(), WAREHOUSE_ID, "fake-token"));
|
|
58
|
-
client
|
|
59
|
-
.delete_volume_file("/Volumes/cat/schema/vol/already-gone.bin")
|
|
60
|
-
.await
|
|
61
|
-
.expect("404 must be treated as success (idempotent delete), not an error");
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
#[tokio::test]
|
|
65
|
-
async fn delete_volume_file_surfaces_other_errors() {
|
|
66
|
-
// 400, not 500/429/etc -- those are classified transient and would
|
|
67
|
-
// trigger the full 6-attempt retry/backoff cycle (tens of seconds) for
|
|
68
|
-
// no benefit here; that classification logic already has its own
|
|
69
|
-
// coverage. This only needs a status delete_volume_file does NOT treat
|
|
70
|
-
// as "already gone", proving the error still propagates.
|
|
71
|
-
let server = MockServer::start().await;
|
|
72
|
-
Mock::given(method("DELETE"))
|
|
73
|
-
.and(path_regex(r"^/api/2\.0/fs/files/.*"))
|
|
74
|
-
.respond_with(ResponseTemplate::new(400))
|
|
75
|
-
.mount(&server)
|
|
76
|
-
.await;
|
|
77
|
-
|
|
78
|
-
let client = Arc::new(DbClient::new(&server.uri(), WAREHOUSE_ID, "fake-token"));
|
|
79
|
-
let err = client
|
|
80
|
-
.delete_volume_file("/Volumes/cat/schema/vol/file.bin")
|
|
81
|
-
.await
|
|
82
|
-
.unwrap_err();
|
|
83
|
-
assert!(
|
|
84
|
-
!err.transient,
|
|
85
|
-
"400 is not transient, should surface immediately without retrying"
|
|
86
|
-
);
|
|
87
|
-
}
|
|
File without changes
|
|
File without changes
|
{arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/examples/agent_render_bench.rs
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/proptest-regressions/json_convert.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/tests_py/test_stream_ndjson_lines.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{arrowbricks-6.2.0 → arrowbricks-6.4.0}/rust/arrowbricks_core/tests_py/test_token_provider.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|