arrowbricks 3.0.3__tar.gz → 3.0.4__tar.gz

{arrowbricks-3.0.3 → arrowbricks-3.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: arrowbricks
-Version: 3.0.3
+Version: 3.0.4
 Requires-Dist: arro3-core>=0.8 ; extra == 'arro3'
 Provides-Extra: arro3
 License-File: LICENSE

{arrowbricks-3.0.3 → arrowbricks-3.0.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "arrowbricks"
-version = "3.0.3"
+version = "3.0.4"
 description = "Runs SQL against a Databricks SQL warehouse via the Statement Execution API and hands you the result as Arrow -- a DB-API-ish Cursor (fetchone/fetchmany/fetchall/fetchall_arrow) or NDJSON streaming. Rust/PyO3 core throughout -- zero required runtime dependencies."
 readme = "README.md"
 license = "MIT"

{arrowbricks-3.0.3 → arrowbricks-3.0.4}/rust/arrowbricks_core/Cargo.lock

@@ -241,7 +241,7 @@ dependencies = [
 
 [[package]]
 name = "arrowbricks_core"
-version = "3.0.3"
+version = "3.0.4"
 dependencies = [
  "arrow",
  "arrow-json",

{arrowbricks-3.0.3 → arrowbricks-3.0.4}/rust/arrowbricks_core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "arrowbricks_core"
-version = "3.0.3"
+version = "3.0.4"
 edition = "2024"
 readme = "README.md"
 

{arrowbricks-3.0.3 → arrowbricks-3.0.4}/rust/arrowbricks_core/src/client.rs

@@ -738,20 +738,41 @@ impl DbClient {
             // Python's DatabricksClient defaults to 6, tuned for asyncio+GIL
             // where higher concurrency stops paying off past single digits
             // (see its own comment). This Rust core's real OS-thread
-            // parallelism keeps paying off well past that -- measured against
-            // a real 400-chunk/5.6M-row/120-column table, repeated runs (full
-            // download time, same query, same warehouse): 16=140s,
-            // 32=~113s (avg of 3), 64=114s, 96=~102s (avg of 2), 128=122s.
-            // 16 is clearly worse and 128 clearly regresses (concurrency
-            // outrunning what the warehouse/network can actually keep fed);
-            // 32-96 all land in roughly the same band with 96 nominally
-            // fastest on this table/warehouse, but the gap over 32 is modest
-            // (~10%) and the exact peak is workload/warehouse-shaped, not a
-            // fixed constant -- 64 is picked as a safe middle-ground default
-            // headroom above the old value with no observed downside, not
-            // a claim that 64 is the true optimum. A caller with a very
-            // large chunk count or a fast/low-latency link to the warehouse
-            // may still want to raise it further.
+            // parallelism keeps paying off well past that -- measured
+            // against a 400-chunk/5.6M-row/120-column table (16=140s,
+            // 32=~113s avg of 3, 64=114s, 96=~102s avg of 2, 128=122s). 64
+            // was picked as a safe middle-ground bump with no observed
+            // downside on real data, not a claim that it's the true optimum.
+            //
+            // Re-attempted 2026-08-09 after switching away from http2/
+            // aws-lc-rs to ring, on the theory that either transport change
+            // could have moved the optimum: a first pass reported 96 as
+            // ~13-16% faster than 64 on both this table and a second,
+            // larger one (fact_sales_order_invoiced, 20M rows/295 cols) --
+            // **found on review to be a false positive**. The benchmark
+            // script called `connect()`/`arrowbricks.connect()` without
+            // ever passing `chunk_fetch_concurrency=` explicitly, so every
+            // "level" it claimed to test actually ran at whatever
+            // `PyDbClient::new`'s own `#[pyo3(signature = ...)]` default
+            // was (unchanged at 64 throughout, since only this Rust-side
+            // constant had been edited, and it's overridden unconditionally
+            // by `PyDbClient::new`'s explicit `.with_concurrency(...)` call
+            // for every real Python caller regardless of this constant's
+            // value) -- i.e. every "64 vs 96 vs 128" comparison was actually
+            // 64 vs 64 vs 64, and the reported gap was warehouse/network
+            // run-to-run noise, not a code effect. Caught by re-running a
+            // controlled, interleaved A/B (`chunk_fetch_concurrency=`
+            // passed explicitly each time, no rebuild needed) on
+            // `dim_article`: 64/96/64/96 measured 125.60s/129.92s/137.50s/
+            // 134.30s -- no consistent winner, well within run-to-run
+            // noise. Reverted to 64. If re-attempting this again, always
+            // pass `chunk_fetch_concurrency=` explicitly in the benchmark
+            // script itself rather than relying on rebuilding with a
+            // different default -- this exact mistake is easy to repeat
+            // otherwise, since the four independent places this default is
+            // hardcoded (see AGENTS.md) make "I changed the constant" and "a
+            // real Python caller now uses that constant" two different,
+            // easily-conflated claims.
             chunk_fetch_concurrency: DEFAULT_CHUNK_FETCH_CONCURRENCY,
             warehouse_start_timeout: Duration::from_secs(300),
             warehouse_confirmed_running_ttl: Duration::from_secs(30),

{arrowbricks-3.0.3 → arrowbricks-3.0.4}/rust/arrowbricks_core/src/pipeline.rs

@@ -1008,7 +1008,24 @@ fn build_inline_blob(
     batches: Vec<thrift::ArrowBatch>,
     lz4_compressed: bool,
 ) -> Result<(bytes::Bytes, i64), ApiError> {
-    let mut out = Vec::new();
+    // Pre-size the output buffer to avoid repeated allocations. Schema is
+    // typically a few KB; batches are typically 10s-100s of KB uncompressed.
+    // For compressed data, use the same * 4 heuristic as decompress_lz4_frame
+    // (Arrow IPC data typically compresses several-fold); for uncompressed,
+    // the exact sizes are known.
+    let schema_size = schema_bytes.as_ref().map(|s| s.len()).unwrap_or(0);
+    let batches_size: usize = batches
+        .iter()
+        .map(|b| {
+            if lz4_compressed {
+                b.batch.len() * 4
+            } else {
+                b.batch.len()
+            }
+        })
+        .sum();
+    let mut out = Vec::with_capacity(schema_size + batches_size);
+
     if let Some(s) = &schema_bytes {
         out.extend_from_slice(s);
     }
@@ -1016,8 +1033,16 @@ fn build_inline_blob(
     for b in batches {
         row_count += b.row_count;
         if lz4_compressed {
-            let decoded = crate::client::decompress_lz4_frame(&b.batch)?;
-            out.extend_from_slice(&decoded);
+            // Decompress directly into the output buffer using FrameDecoder,
+            // avoiding a separate intermediate allocation and copy. The
+            // FrameDecoder's read_to_end appends to the existing buffer.
+            use std::io::Read;
+            let mut decoder = lz4_flex::frame::FrameDecoder::new(&b.batch[..]);
+            while !decoder.get_ref().is_empty() {
+                decoder
+                    .read_to_end(&mut out)
+                    .map_err(|e| ApiError::permanent(format!("LZ4 frame decompress failed: {e}")))?;
+            }
         } else {
             out.extend_from_slice(&b.batch);
         }

{arrowbricks-3.0.3 → arrowbricks-3.0.4}/src/arrowbricks/client.py

@@ -50,7 +50,9 @@ class DatabricksClient:
         # chunk's raw bytes in memory. 64 -- tuned for the Rust core's real
         # OS-thread parallelism, which scales well past what asyncio+GIL
         # concurrency used to buy; measured against a real 400-chunk/
-        # 5.6M-row table (see client.rs's own comment for the numbers).
+        # 5.6M-row table (see client.rs's own comment for the numbers, and
+        # for a since-reverted attempt to bump this further that turned out
+        # to be a benchmarking false positive).
         # compress_results: requests LZ4-compressed cloud-fetch chunks
         # (matches databricks-sql-connector's own
         # enable_query_result_lz4_compression default) -- measured ~2x