arrowbricks 3.1.3__tar.gz → 3.1.4__tar.gz

{arrowbricks-3.1.3 → arrowbricks-3.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: arrowbricks
-Version: 3.1.3
+Version: 3.1.4
 Requires-Dist: arro3-core>=0.8 ; extra == 'arro3'
 Provides-Extra: arro3
 License-File: LICENSE
@@ -300,11 +300,19 @@ python examples/benchmark_vs_connector.py
 For comparing arrowbricks versions with identical APIs, use
 [`examples/benchmark_versions.py`](examples/benchmark_versions.py). It keeps
 one connection per version, discards warm-ups, alternates execution order,
-passes concurrency explicitly, and checks row counts. See
+passes concurrency explicitly, and checks row counts. Add `--verify-ipc`
+(requires `arro3-core`) to compare serialized Arrow results in memory after
+timing. Results must have stable values, schema metadata, row order, and batch boundaries;
+verification contributes to peak process memory, and checksums are not printed. See
 [`benchmarks/2026-09-06.md`](benchmarks/2026-09-06.md) for replay/cache measurements
 and [`benchmarks/2026-09-06-downloads.md`](benchmarks/2026-09-06-downloads.md)
 for the subsequent cloud-fetch scheduling measurements and limits.
 
+Follow-up experiments cover [spare request slots](benchmarks/2026-09-06-spare-slots.md),
+[concurrent replay and NDJSON encoding](benchmarks/2026-09-06-lowlevel.md),
+and the rejected [LZ4 capacity](benchmarks/2026-09-06-lz4-capacity.md) and
+[direct-fill buffer](benchmarks/2026-09-06-direct-fill.md) changes.
+
 Cached IPC replay can be measured without a warehouse:
 
 ```bash
@@ -315,6 +323,8 @@ Replays now share immutable input bytes across decoded tables, reducing
 repeated copies and memory use. Arrow may still copy misaligned fixed-width
 buffers or decompress IPC-compressed bodies. Keeping a small slice of a
 decoded array can retain the full source allocation until that slice is released.
+Decoding releases the Python interpreter lock, so independent replay calls
+can run concurrently on separate Python threads.
 
 `BENCHMARK_SQL` overrides the query, `BENCHMARK_RUNS` (default 3) controls how many timed runs to average.
 

{arrowbricks-3.1.3 → arrowbricks-3.1.4}/README.md

@@ -288,11 +288,19 @@ python examples/benchmark_vs_connector.py
 For comparing arrowbricks versions with identical APIs, use
 [`examples/benchmark_versions.py`](examples/benchmark_versions.py). It keeps
 one connection per version, discards warm-ups, alternates execution order,
-passes concurrency explicitly, and checks row counts. See
+passes concurrency explicitly, and checks row counts. Add `--verify-ipc`
+(requires `arro3-core`) to compare serialized Arrow results in memory after
+timing. Results must have stable values, schema metadata, row order, and batch boundaries;
+verification contributes to peak process memory, and checksums are not printed. See
 [`benchmarks/2026-09-06.md`](benchmarks/2026-09-06.md) for replay/cache measurements
 and [`benchmarks/2026-09-06-downloads.md`](benchmarks/2026-09-06-downloads.md)
 for the subsequent cloud-fetch scheduling measurements and limits.
 
+Follow-up experiments cover [spare request slots](benchmarks/2026-09-06-spare-slots.md),
+[concurrent replay and NDJSON encoding](benchmarks/2026-09-06-lowlevel.md),
+and the rejected [LZ4 capacity](benchmarks/2026-09-06-lz4-capacity.md) and
+[direct-fill buffer](benchmarks/2026-09-06-direct-fill.md) changes.
+
 Cached IPC replay can be measured without a warehouse:
 
 ```bash
@@ -303,6 +311,8 @@ Replays now share immutable input bytes across decoded tables, reducing
 repeated copies and memory use. Arrow may still copy misaligned fixed-width
 buffers or decompress IPC-compressed bodies. Keeping a small slice of a
 decoded array can retain the full source allocation until that slice is released.
+Decoding releases the Python interpreter lock, so independent replay calls
+can run concurrently on separate Python threads.
 
 `BENCHMARK_SQL` overrides the query, `BENCHMARK_RUNS` (default 3) controls how many timed runs to average.
 

{arrowbricks-3.1.3 → arrowbricks-3.1.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "arrowbricks"
-version = "3.1.3"
+version = "3.1.4"
 description = "Runs SQL against a Databricks SQL warehouse via the Statement Execution API and hands you the result as Arrow -- a DB-API-ish Cursor (fetchone/fetchmany/fetchall/fetchall_arrow) or NDJSON streaming. Rust/PyO3 core throughout -- zero required runtime dependencies."
 readme = "README.md"
 license = "MIT"

{arrowbricks-3.1.3 → arrowbricks-3.1.4}/rust/arrowbricks_core/Cargo.lock

@@ -178,7 +178,7 @@ dependencies = [
 
 [[package]]
 name = "arrowbricks_core"
-version = "3.1.3"
+version = "3.1.4"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -190,6 +190,7 @@ dependencies = [
  "chrono",
  "hyper-rustls",
  "lz4_flex",
+ "memchr",
  "proptest",
  "pyo3",
  "pyo3-arrow",

{arrowbricks-3.1.3 → arrowbricks-3.1.4}/rust/arrowbricks_core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "arrowbricks_core"
-version = "3.1.3"
+version = "3.1.4"
 edition = "2024"
 readme = "README.md"
 
@@ -17,6 +17,8 @@ arrow-ipc = "59.1.0"
 arrow-schema = "59.1.0"
 arrow-json = "59.1.0"
 bytes = "1.12.1"
+# Already in the dependency graph; vectorized newline search in NDJSON output.
+memchr = "2"
 # Already an unconditional transitive dependency of arrow-cast (pulled in by
 # pyo3-arrow) at this exact version -- declared directly here too so
 # json_convert.rs can decode BINARY columns without hand-rolling base64. No

{arrowbricks-3.1.3 → arrowbricks-3.1.4}/rust/arrowbricks_core/README.md

@@ -73,7 +73,7 @@ unlike the table object itself.
 ## API
 
 - `Client(host, warehouse_id, *, token=None, token_provider=None, chunk_fetch_concurrency=64, http_timeout=60.0, wait_timeout="30s", warehouse_start_timeout=300.0, warehouse_confirmed_running_ttl_s=30.0, compress_results=True, protocol="thrift", retry_attempts=6, retry_max_wait_s=20.0)` -- exactly one of `token`/`token_provider`. `retry_attempts`/`retry_max_wait_s` tune the retry policy (total attempts, exponential-backoff ceiling in seconds) behind every retryable request this client makes; `retry_attempts` must be at least 1. `token_provider` is a callable (sync or async) returning a token string, called fresh on every request, no caching. `compress_results` requests LZ4-compressed cloud-fetch chunks (see above); set `False` to opt out. `protocol="thrift"` (the default, as of this crate's own real-workspace benchmarking -- see `AGENTS.md`'s design-invariant entry) speaks the same HiveServer2-compatible Thrift-over-HTTPS protocol `databricks-sql-connector` uses by default (`thrift.rs`, a hand-rolled `TBinaryProtocol` reader/writer -- no new Cargo dependency) -- measurably faster for small queries (its `ExecuteStatement` RPC can return a small result inline via `getDirectResults`, in the same call that submits the statement) at the cost of `prefer_inline` becoming a silent no-op (Thrift has no INLINE-disposition equivalent, and doesn't need one); never slower than SEA on any query shape tested. `protocol="sea"` instead talks to the REST Statement Execution API -- still fully supported, opt in explicitly if you have a reason to prefer it. On par with SEA for a large, multi-chunk result too -- `run_thrift_fetch_loop` (`pipeline.rs`) fans its chunk downloads out across a `chunk_fetch_concurrency`-sized worker pool spanning the *whole* result (pipelined with the sequential `FetchResults` discovery calls, not serialized behind them), the same concurrency shape as SEA's own `fetch_chunks_with_backpressure` (see `AGENTS.md`'s own entry on this).
-- `Client.execute(statement, *, catalog=None, schema=None, parameters=None, prefer_inline=False) -> ResultSet` -- submits and starts background chunk fetching without pulling anything yet. `parameters` is Databricks' own named-parameter format (`[{"name":..., "value":..., "type":...}]`), passed straight through. `prefer_inline=True` submits with `disposition=INLINE, format=JSON_ARRAY` instead, for a caller who expects a small (well under Databricks' 25 MiB inline cap) result and wants to skip the chunk-fetch round trip -- on a result too big for INLINE, or containing a column type `json_convert.rs` doesn't map (STRUCT/ARRAY-of-STRUCT/MAP/VARIANT), it transparently falls back to a second, normal `execute()` and the query runs twice. See `AGENTS.md`'s "Design invariants" section (in the root package) for the full reasoning and real-workspace verification behind this.
+- `Client.execute(statement, *, catalog=None, schema=None, parameters=None, prefer_inline=False) -> ResultSet` -- submits and starts background chunk fetching without pulling anything yet. `parameters` is Databricks' own named-parameter format (`[{"name":..., "value":..., "type":...}]`), passed straight through. `prefer_inline=True` submits with `disposition=INLINE, format=JSON_ARRAY` instead, for a caller who expects a small (well under Databricks' 25 MiB inline cap) result and wants to skip the chunk-fetch round trip -- the recognized INLINE byte-limit failure falls back to a second, normal `execute()`. If an already-succeeded result cannot be converted (including unsupported empty STRUCT arrays), it raises `ArrowbricksError` without resubmitting the statement. Use the byte-limit retry only with SQL that is safe to execute again. See `AGENTS.md`'s "Design invariants" section (in the root package) for the full reasoning and real-workspace verification behind this.
   - `ResultSet.fetchmany_arrow(n) -> Table` -- pulls/decodes only as many chunks as needed for `n` rows, buffering the rest; may return fewer than `n` once exhausted.
   - `ResultSet.fetchall_arrow() -> Table` -- drains everything remaining.
   - `ResultSet.fetchall_arrow_streamed(*, total_timeout_s=None)` -- same as `fetchall_arrow()`, but an async iterator yielding the `HEARTBEAT` singleton while pulling chunks instead of blocking silently (bridge e.g. an SSE connection through the download), then a `Table` exactly once. Raises if `total_timeout_s` elapses first.
@@ -82,7 +82,7 @@ unlike the table object itself.
 - `Client.stream_ndjson_lines(statement, *, catalog=None, schema=None, parameters=None, total_timeout_s=None)` -- chunk-at-a-time: an async iterator yielding the `HEARTBEAT` singleton while waiting on the statement or any individual chunk, then a `list[str]` of NDJSON lines (one per row, explicit nulls, ISO-8601 timestamps) per chunk in logical order. Decode and JSON encoding both happen in Rust -- backs `stream_query_json` end to end.
 - `Client.upload_volume_file(volume_path, data: bytes)` / `Client.delete_volume_file(volume_path)` -- Unity Catalog volume files via the Files API. Delete treats a 404 as success (idempotent). Both raise a plain `RuntimeError` (message only) on failure.
 - `write_ipc_stream(stream, buf)` -- free function; writes any object implementing `__arrow_c_stream__` (a `Table` from this crate, arro3, pyarrow, ...) as uncompressed Arrow-IPC stream bytes to a Python file-like object. No dependency needed regardless of the input's origin.
-- `read_ipc_stream(data: bytes) -> Table` -- free function; the exact inverse of `write_ipc_stream`, parsing raw Arrow-IPC stream bytes back into a `Table`. No dependency needed regardless of where the bytes came from -- backs `arrowbricks.ReplayableArrowChunk`, which needs to re-parse the same cached bytes on every `__arrow_c_stream__` call.
+- `read_ipc_stream(data: bytes) -> Table` -- free function; the exact inverse of `write_ipc_stream`, parsing raw Arrow-IPC stream bytes back into a `Table`. No dependency needed regardless of where the bytes came from -- backs `arrowbricks.ReplayableArrowChunk`, which needs to re-parse the same cached bytes on every `__arrow_c_stream__` call. Decoding releases the Python interpreter lock so independent replay calls can run concurrently; decoded arrays retain ownership of the immutable input bytes.
 - `HEARTBEAT` -- module-level singleton; compare with `is`, e.g. `if item is _core.HEARTBEAT: ...`.
 
 ## With DuckDB

arrowbricks-3.1.4/rust/arrowbricks_core/proptest-regressions/json_convert.txt

@@ -0,0 +1,7 @@
+# Seeds for failure cases proptest has generated in the past. It is
+# automatically read and these particular cases re-run before any
+# novel cases are generated.
+#
+# It is recommended to check this file in to source control so that
+# everyone who runs the test benefits from these saved cases.
+cc c23ce2dbe22693eff6644d643de77c6c113dc918c876434391fb93a05438d706 # shrinks to type_name = "STRUCT", precision = None, scale = None, type_text = Some(""), values = []

{arrowbricks-3.1.3 → arrowbricks-3.1.4}/rust/arrowbricks_core/src/client/download.rs

@@ -36,18 +36,15 @@ use super::model::QueryStatsAccumulator;
 /// after each `EndMark` and picks up the next concatenated frame on a
 /// subsequent `read_to_end` call against the *same* instance (verified: the
 /// decoder's position in the underlying byte slice carries over across
-/// calls) -- so looping `read_to_end` on one decoder until it stops growing
-/// `out` reads every frame without reconstructing a decoder per frame.
+/// calls) -- so looping `read_to_end` on one decoder until its underlying
+/// reader is exhausted reads every frame without reconstructing a decoder.
 pub(crate) fn decompress_lz4_frame(compressed: &Bytes) -> Result<Bytes, ApiError> {
     use std::io::Read;
-    // `compressed.len()` is a real lower bound, but LZ4 on Arrow-IPC data
-    // (long dictionary/offset-buffer runs, mostly-repeated bytes) typically
-    // compresses several-fold -- estimating just the lower bound means the
-    // real decompressed size almost always blows past initial capacity,
-    // paying for repeated doubling-and-copy growth on every chunk. `* 4` is
-    // a heuristic, not a guarantee (`Vec` still grows normally if it's wrong
-    // either way) -- just a better starting point than the guaranteed-too-
-    // small lower bound.
+    // LZ4 on Arrow-IPC data often compresses several-fold. Starting at the
+    // compressed size can therefore pay for repeated buffer growth. Neither
+    // size is a bound on the other: incompressible input can expand slightly.
+    // `* 4` remains a heuristic; Vec grows normally when it underestimates.
+    // See benchmark_lz4_capacity_hypotheses for the allocation tradeoffs.
     let mut out = Vec::with_capacity(compressed.len() * 4);
     let mut decoder = lz4_flex::frame::FrameDecoder::new(&compressed[..]);
     // Terminate on the *reader* being exhausted, not on "output stopped
@@ -301,6 +298,65 @@ impl DbClient {
 mod tests {
     use super::*;
 
+    #[test]
+    #[ignore = "manual release-mode decompression allocation experiment"]
+    fn benchmark_lz4_capacity_hypotheses() {
+        use std::hint::black_box;
+        use std::io::{Read, Write};
+        use std::time::Instant;
+
+        // Deterministic random bytes mixed with repeated bytes cover different
+        // compression ratios without depending on warehouse data.
+        for random_fraction in [0, 25, 50, 100] {
+            let mut state = 0x12345678_u64;
+            let input: Vec<u8> = (0..8 * 1024 * 1024)
+                .map(|i| {
+                    state ^= state << 13;
+                    state ^= state >> 7;
+                    state ^= state << 17;
+                    if i % 100 < random_fraction { state as u8 } else { 0 }
+                })
+                .collect();
+            let mut compressed = Vec::new();
+            for part in input.chunks(512 * 1024) {
+                let mut encoder = lz4_flex::frame::FrameEncoder::new(Vec::new());
+                encoder.write_all(part).unwrap();
+                compressed.extend(encoder.finish().unwrap());
+            }
+            let variants = [
+                ("compressed_x1", compressed.len()),
+                ("compressed_x2", compressed.len() * 2),
+                ("compressed_x4", compressed.len() * 4),
+                ("exact", input.len()),
+                ("short_hint", input.len() - 64),
+            ];
+            for round in 0..10 {
+                for index in 0..variants.len() {
+                    let (variant, capacity) = variants[(index + round) % variants.len()];
+                    let start = Instant::now();
+                    let mut output = Vec::with_capacity(capacity);
+                    let mut decoder = lz4_flex::frame::FrameDecoder::new(black_box(compressed.as_slice()));
+                    while !decoder.get_ref().is_empty() {
+                        decoder.read_to_end(&mut output).unwrap();
+                    }
+                    let seconds = start.elapsed().as_secs_f64();
+                    assert_eq!(output, input);
+                    if round > 1 {
+                        println!(
+                            "{}",
+                            serde_json::json!({
+                                "random_percent": random_fraction, "variant": variant,
+                                "round": round, "seconds": seconds,
+                                "compressed_bytes": compressed.len(), "decoded_bytes": output.len(),
+                                "retained_capacity": output.capacity()
+                            })
+                        );
+                    }
+                }
+            }
+        }
+    }
+
     #[tokio::test]
     async fn split_download_starts_tail_requests_before_the_probe_body_finishes() {
         use tokio::io::{AsyncReadExt, AsyncWriteExt};

{arrowbricks-3.1.3 → arrowbricks-3.1.4}/rust/arrowbricks_core/src/json_convert.rs

@@ -334,7 +334,8 @@ fn build_column(
 
             let validity: Vec<bool> = rows_parsed.iter().map(Option::is_some).collect();
             let fields = Fields::from(child_fields);
-            let struct_array = StructArray::new(fields.clone(), child_arrays, Some(NullBuffer::from(validity)));
+            let struct_array = StructArray::try_new(fields.clone(), child_arrays, Some(NullBuffer::from(validity)))
+                .map_err(|e| ApiError::permanent(format!("column `{name}`: invalid STRUCT array: {e}")))?;
             Ok((DataType::Struct(fields), Arc::new(struct_array)))
         }
         other => Err(ApiError {
@@ -702,6 +703,15 @@ mod tests {
         assert!(s.is_null(1), "row 1's whole struct must be null");
     }
 
+    #[test]
+    fn empty_struct_returns_a_conversion_error_instead_of_panicking() {
+        let columns = vec![struct_col("s", "STRUCT<>")];
+        for rows in [vec![], vec![vec![Some("{}".to_string())]], vec![vec![None]]] {
+            let err = json_array_to_record_batch(&rows, &columns).unwrap_err();
+            assert!(err.message.contains("invalid STRUCT array"));
+        }
+    }
+
     #[test]
     fn struct_with_a_nested_unsupported_composite_field_errors_instead_of_guessing() {
         let columns = vec![struct_col("s", "STRUCT<a: INT, nested: STRUCT<x: INT>>")];

{arrowbricks-3.1.3 → arrowbricks-3.1.4}/rust/arrowbricks_core/src/lib.rs

@@ -62,11 +62,14 @@ fn write_ipc_stream(py: Python<'_>, stream: Bound<'_, PyAny>, buf: Bound<'_, PyA
 #[pyfunction]
 #[pyo3(signature = (data))]
 fn read_ipc_stream(data: Bound<'_, PyBytes>) -> PyResult<PyTable> {
+    let py = data.py();
     // The immutable Python bytes own the memory for as long as any decoded
     // array needs it, including arrays exported through the C Data Interface.
     // PyBackedBytes provides that ownership without copying or custom unsafe code.
     let blob = bytes::Bytes::from_owner(pyo3::pybacked::PyBackedBytes::from(data));
-    let (batches, schema) = pipeline::decode_ipc_stream(&blob).map_err(|e| PyRuntimeError::new_err(e.message))?;
+    let (batches, schema) = py
+        .detach(|| pipeline::decode_ipc_stream(&blob))
+        .map_err(|e| PyRuntimeError::new_err(e.message))?;
     PyTable::try_new(batches, schema).map_err(|e| PyRuntimeError::new_err(e.to_string()))
 }
 

{arrowbricks-3.1.3 → arrowbricks-3.1.4}/rust/arrowbricks_core/src/pipeline/ndjson.rs

@@ -6,6 +6,7 @@
 //! `Infinity`/`-Infinity`) string-patching arrow-json's own fixed encoding
 //! needs help with.
 
+use std::io::{self, Write};
 use std::sync::Arc;
 
 use arrow_array::Array;
@@ -22,6 +23,37 @@ use super::sea::submit_sea_and_report;
 use super::stats::{ReportOnDrop, StatsReporter};
 use super::thrift_exec::{ThriftSubmitResult, submit_thrift_and_start_fetch};
 
+/// Collect rows as Arrow writes them, avoiding a second, chunk-sized buffer.
+/// Writes may end within a row (or even a UTF-8 character).
+struct LineCollector {
+    lines: Vec<String>,
+    pending: Vec<u8>,
+}
+
+impl Write for LineCollector {
+    fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
+        let mut start = 0;
+        for end in memchr::memchr_iter(b'\n', bytes) {
+            let line = &bytes[start..end];
+            let row = if self.pending.is_empty() {
+                line.to_vec()
+            } else {
+                self.pending.extend_from_slice(line);
+                std::mem::take(&mut self.pending)
+            };
+            self.lines
+                .push(String::from_utf8(row).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?);
+            start = end + 1;
+        }
+        self.pending.extend_from_slice(&bytes[start..]);
+        Ok(bytes.len())
+    }
+
+    fn flush(&mut self) -> io::Result<()> {
+        Ok(())
+    }
+}
+
 /// Converts one chunk's decoded batches into NDJSON lines, one per row, in
 /// arro3-`write_ndjson(explicit_nulls=True)`-compatible format: null-valued
 /// keys stay present as JSON `null` rather than being omitted, and a
@@ -43,10 +75,13 @@ fn encode_ndjson_lines(batches: &[RecordBatch], non_finite_as_string: bool) -> R
     if batches.is_empty() {
         return Ok(Vec::new());
     }
-    let mut buf = Vec::new();
+    let mut output = LineCollector {
+        lines: Vec::with_capacity(batches.iter().map(RecordBatch::num_rows).sum()),
+        pending: Vec::new(),
+    };
     {
         let builder = arrow_json::WriterBuilder::new().with_explicit_nulls(true);
-        let mut writer = builder.build::<_, arrow_json::writer::LineDelimited>(&mut buf);
+        let mut writer = builder.build::<_, arrow_json::writer::LineDelimited>(&mut output);
         let refs: Vec<&RecordBatch> = batches.iter().collect();
         writer.write_batches(&refs).map_err(|e| ApiError {
             message: format!("NDJSON encode error: {e}"),
@@ -59,15 +94,10 @@ fn encode_ndjson_lines(batches: &[RecordBatch], non_finite_as_string: bool) -> R
             kind: ApiErrorKind::Other,
         })?;
     }
-    let mut lines: Vec<String> = String::from_utf8(buf)
-        .map_err(|e| ApiError {
-            message: format!("NDJSON encode produced invalid UTF-8: {e}"),
-            transient: false,
-            kind: ApiErrorKind::Other,
-        })?
-        .lines()
-        .map(|line| line.to_string())
-        .collect();
+    if !output.pending.is_empty() {
+        return Err(ApiError::permanent("NDJSON encode produced an unterminated row"));
+    }
+    let mut lines = output.lines;
 
     if non_finite_as_string {
         patch_non_finite_floats(batches, &mut lines);
@@ -369,6 +399,120 @@ mod tests {
     use super::*;
     use crate::pipeline::test_support::make_batch;
 
+    fn encode_ndjson_reference(batches: &[RecordBatch], non_finite_as_string: bool) -> Result<Vec<String>, ApiError> {
+        if batches.is_empty() {
+            return Ok(Vec::new());
+        }
+        let mut buf = Vec::new();
+        {
+            let builder = arrow_json::WriterBuilder::new().with_explicit_nulls(true);
+            let mut writer = builder.build::<_, arrow_json::writer::LineDelimited>(&mut buf);
+            let refs: Vec<&RecordBatch> = batches.iter().collect();
+            writer.write_batches(&refs).map_err(|e| ApiError {
+                message: format!("NDJSON encode error: {e}"),
+                transient: false,
+                kind: ApiErrorKind::Other,
+            })?;
+            writer.finish().map_err(|e| ApiError {
+                message: format!("NDJSON encode error: {e}"),
+                transient: false,
+                kind: ApiErrorKind::Other,
+            })?;
+        }
+        let mut lines: Vec<String> = String::from_utf8(buf)
+            .map_err(|e| ApiError {
+                message: format!("NDJSON encode produced invalid UTF-8: {e}"),
+                transient: false,
+                kind: ApiErrorKind::Other,
+            })?
+            .lines()
+            .map(|line| line.to_string())
+            .collect();
+
+        if non_finite_as_string {
+            patch_non_finite_floats(batches, &mut lines);
+        }
+        Ok(lines)
+    }
+
+    #[test]
+    fn line_collector_handles_split_unicode_and_rejects_invalid_utf8() {
+        let mut output = LineCollector {
+            lines: Vec::new(),
+            pending: Vec::new(),
+        };
+        for byte in "{\"label\":\"café 🦀\"}\n{}\n".as_bytes() {
+            output.write_all(&[*byte]).unwrap();
+        }
+        assert_eq!(output.lines, ["{\"label\":\"café 🦀\"}", "{}"]);
+        assert!(output.pending.is_empty());
+        assert_eq!(
+            output.write_all(&[0xff, b'\n']).unwrap_err().kind(),
+            io::ErrorKind::InvalidData
+        );
+    }
+
+    fn text_batch(rows: usize, columns: usize, width: usize) -> RecordBatch {
+        use arrow_array::StringArray;
+        use arrow_schema::{Field, Schema};
+        let value = format!("café 🦀 \"quoted\"\n{}", "x".repeat(width));
+        let array = Arc::new(StringArray::from_iter(
+            (0..rows).map(|i| (i % 7 != 0).then_some(value.as_str())),
+        ));
+        let fields = (0..columns)
+            .map(|i| Field::new(format!("c{i}"), DataType::Utf8, true))
+            .collect::<Vec<_>>();
+        RecordBatch::try_new(
+            Arc::new(Schema::new(fields)),
+            (0..columns).map(|_| array.clone() as _).collect(),
+        )
+        .unwrap()
+    }
+
+    #[test]
+    fn collected_ndjson_matches_reference_for_large_unicode_rows_and_multiple_batches() {
+        let batches = vec![
+            text_batch(20, 3, 12000),
+            text_batch(0, 3, 12000),
+            text_batch(17, 3, 12000),
+        ];
+        let expected = encode_ndjson_reference(&batches, false).unwrap();
+        let actual = encode_ndjson_lines(&batches, false).unwrap();
+        assert_eq!(actual.len(), 37);
+        assert_eq!(actual, expected);
+    }
+
+    #[test]
+    #[ignore = "manual release-mode allocation/encoding benchmark"]
+    fn benchmark_ndjson_collector() {
+        use std::hint::black_box;
+        use std::time::Instant;
+        for (name, rows, columns, width) in [("narrow", 100000, 4, 16), ("wide", 20000, 32, 96)] {
+            let batches = vec![text_batch(rows, columns, width)];
+            assert_eq!(
+                encode_ndjson_lines(&batches, false).unwrap(),
+                encode_ndjson_reference(&batches, false).unwrap()
+            );
+            for round in 0..9 {
+                for candidate in if round % 2 == 0 { [false, true] } else { [true, false] } {
+                    let start = Instant::now();
+                    let lines = if candidate {
+                        encode_ndjson_lines(black_box(&batches), false)
+                    } else {
+                        encode_ndjson_reference(black_box(&batches), false)
+                    }
+                    .unwrap();
+                    let elapsed = start.elapsed().as_secs_f64() * 1000.0;
+                    black_box(&lines);
+                    println!(
+                        "{}",
+                        serde_json::json!({"workload":name,"round":round,"candidate":candidate,"ms":elapsed,"rows":lines.len(),"bytes":lines.iter().map(String::len).sum::<usize>()})
+                    );
+                }
+            }
+        }
+    }
+
     #[test]
     fn non_finite_token_covers_nan_and_both_infinities_only() {
         assert_eq!(non_finite_token(f64::NAN), Some("\"NaN\""));

{arrowbricks-3.1.3 → arrowbricks-3.1.4}/rust/arrowbricks_core/src/pipeline/thrift_exec.rs

@@ -686,8 +686,13 @@ async fn run_thrift_fetch_loop(
         // Share the budget across all links already known in this batch.
         // Otherwise the first workers can claim eight slots each while
         // the remaining files wait without even starting a request.
-        let split_limit = (concurrency / row_set.result_links.len().max(1)).max(1);
-        for link in row_set.result_links {
+        let link_count = row_set.result_links.len().max(1);
+        for (link_position, link) in row_set.result_links.into_iter().enumerate() {
+            // Distribute the remainder too: 35 files under a 64-slot budget
+            // get 29 two-slot shares and six one-slot shares, rather than
+            // leaving 29 slots unused. Every file still gets at least one;
+            // the semaphore queues files when there are more than slots.
+            let split_limit = (concurrency / link_count + usize::from(link_position < concurrency % link_count)).max(1);
             let idx = chunk_index;
             chunk_index += 1;
             stats.chunks_seen.fetch_add(1, Ordering::Relaxed);

{arrowbricks-3.1.3 → arrowbricks-3.1.4}/rust/arrowbricks_core/tests/wiremock_thrift.rs

@@ -1456,6 +1456,61 @@ async fn mount_single_link_statement(server: &MockServer, link_path: &'static st
         .await;
 }
 
+#[tokio::test]
+async fn thrift_distributes_spare_slots_across_known_links() {
+    let server = MockServer::start().await;
+    mount_open_session_always(&server, b"sess").await;
+    mount_close_operation_ok(&server).await;
+    let direct = DirectResultsSpec {
+        operation_state: Some(operation_state::FINISHED),
+        metadata: Some((false, None)),
+        fetch: Some(FetchSpec {
+            result_links: vec![
+                (format!("{}/_data/first", server.uri()), 60_000),
+                (format!("{}/_data/second", server.uri()), 60_000),
+            ],
+            ..Default::default()
+        }),
+        ..Default::default()
+    };
+    Mock::given(method("POST"))
+        .and(IsThriftRpc("ExecuteStatement"))
+        .respond_with(ResponseTemplate::new(200).set_body_raw(
+            build_execute_statement_resp(b"op", b"secret", Some(direct)),
+            "application/x-thrift",
+        ))
+        .mount(&server)
+        .await;
+    let first = mount_ranged_blob(
+        &server,
+        "/_data/first",
+        build_full_stream_bytes(&test_schema(), 0, 60_000),
+    )
+    .await;
+    let second = mount_ranged_blob(
+        &server,
+        "/_data/second",
+        build_full_stream_bytes(&test_schema(), 60_000, 120_000),
+    )
+    .await;
+    let client = Arc::new(
+        DbClient::new(&server.uri(), WAREHOUSE_ID, "fake-token")
+            .with_protocol(Protocol::Thrift)
+            .with_concurrency(3),
+    );
+    let mut stream = execute_lazy_thrift(client, "SELECT * FROM t", None, None, None)
+        .await
+        .unwrap();
+    let (batches, _) = stream.fetchall_arrow().await.unwrap();
+    assert_ids_in_order(&batches, 120_000);
+    assert_eq!(
+        first.load(Ordering::SeqCst),
+        2,
+        "the spare slot should split the first file"
+    );
+    assert_eq!(second.load(Ordering::SeqCst), 1, "the second file keeps its own slot");
+}
+
 #[tokio::test]
 async fn thrift_single_link_downloads_via_parallel_range_requests() {
     let server = MockServer::start().await;

{arrowbricks-3.1.3 → arrowbricks-3.1.4}/rust/arrowbricks_core/tests_py/test_ipc_stream.py

@@ -127,3 +127,40 @@ def test_read_ipc_stream_result_is_callable_more_than_once():
     second = arrowbricks_core.read_ipc_stream(data)
     assert first.num_rows == 3
     assert second.num_rows == 3
+
+
+def test_read_ipc_stream_allows_another_python_thread_to_run():
+    import threading
+
+    table = core.Table.from_pydict(
+        {"label": core.Array(["synthetic text " * 16] * 100_000, type=core.DataType.string())}
+    )
+    buf = io.BytesIO()
+    arrowbricks_core.write_ipc_stream(table, buf)
+    data = buf.getvalue()
+    ready = threading.Event()
+    go = threading.Event()
+    progressed = threading.Event()
+
+    def other_thread():
+        ready.set()
+        go.wait()
+        progressed.set()
+
+    thread = threading.Thread(target=other_thread)
+    thread.start()
+    ready.wait()
+    previous = sys.getswitchinterval()
+    try:
+        # Prevent a periodic Python bytecode switch from satisfying the test.
+        # The decoder itself must release the interpreter for the worker.
+        sys.setswitchinterval(10)
+        go.set()
+        result = arrowbricks_core.read_ipc_stream(data)
+        ran_during_decode = progressed.is_set()
+    finally:
+        sys.setswitchinterval(previous)
+        go.set()
+        thread.join(timeout=5)
+    assert result.num_rows == 100_000
+    assert ran_during_decode, "IPC decoding must release the interpreter lock"