arrowbricks 1.3.2__tar.gz → 1.4.0__tar.gz

{arrowbricks-1.3.2 → arrowbricks-1.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: arrowbricks
-Version: 1.3.2
+Version: 1.4.0
 Requires-Dist: arro3-core>=0.8 ; extra == 'arro3'
 Provides-Extra: arro3
 License-File: LICENSE
@@ -139,7 +139,7 @@ conn = connect(host=..., warehouse_id=..., token_provider=my_token_provider)
 - `connect(host, warehouse_id, *, token=None, token_provider=None, ...) -> Connection`
 - `Connection.cursor() -> Cursor`
 - `Connection.client -> DatabricksClient` -- the same client `cursor()` uses, for lower-level access (e.g. `stream_query_json`, `upload_volume_file`).
-- `Cursor.execute(sql, parameters=None, *, row_limit=None, offset=None, catalog=None, schema=None, total_timeout_s=None) -> Cursor` -- submits and waits for the statement, like a real DB-API cursor. `parameters`, if given, is Databricks' own named-parameter format -- `[{"name": ..., "value": ..., "type": ...}]` bound against `:name` markers in `sql`.
+- `Cursor.execute(sql, parameters=None, *, row_limit=None, offset=None, catalog=None, schema=None, total_timeout_s=None, prefer_inline=False) -> Cursor` -- submits and waits for the statement, like a real DB-API cursor. `parameters`, if given, is Databricks' own named-parameter format -- `[{"name": ..., "value": ..., "type": ...}]` bound against `:name` markers in `sql`. `prefer_inline=True` tries fetching a small result (well under Databricks' 25 MiB inline cap) in the same round trip as the submission itself, skipping the chunk-fetch entirely -- if the result turns out too big, or has a column type this can't convert (nested ARRAY/MAP/STRUCT, VARIANT), it transparently re-runs the query the normal way, so a caller who sets this without actually expecting a small result pays for the query twice. Leave it off unless you know the result is small.
 - `Cursor.execute_streamed(...)` -- same args, but an async generator yielding `HEARTBEAT` while waiting on a slow cold start, then the ready `Cursor` -- for bridging e.g. an SSE connection. Its timeout/heartbeats stop the moment the statement is ready, *before* any chunk has been downloaded -- see `fetchall_streamed` below for the download phase itself.
 - `Cursor.fetchone() -> tuple | None`, `Cursor.fetchmany(size) -> list[tuple]`, `Cursor.fetchall() -> list[tuple]`, and iterating a `Cursor` directly -- row tuples; needs the `arro3` extra.
 - `Cursor.fetchmany_arrow(size) -> Table`, `Cursor.fetchall_arrow() -> Table` -- an Arrow table (implements `__arrow_c_stream__`, so arro3/pyarrow/DuckDB can all consume it directly, zero-copy).
@@ -191,7 +191,7 @@ con.register("my_table", table)
 con.sql("SELECT count(*) FROM my_table").show()
 ```
 
-`ReplayableArrowChunk` works the same way for Arrow-IPC bytes you fetched and stored earlier (e.g. `client.execute_arrow_statement(...)`'s raw chunk bytes, cached in Redis/a file/wherever) -- DuckDB's registration path calls `__arrow_c_stream__` twice (a schema peek, then the actual scan), which is exactly what `ReplayableArrowChunk` exists to support:
+`ReplayableArrowChunk` works the same way for Arrow-IPC bytes you fetched and stored earlier (e.g. a raw chunk's bytes, cached in Redis/a file/wherever) -- DuckDB's registration path calls `__arrow_c_stream__` twice (a schema peek, then the actual scan), which is exactly what `ReplayableArrowChunk` exists to support:
 
 ```python
 from arrowbricks import ReplayableArrowChunk

{arrowbricks-1.3.2 → arrowbricks-1.4.0}/README.md

@@ -127,7 +127,7 @@ conn = connect(host=..., warehouse_id=..., token_provider=my_token_provider)
 - `connect(host, warehouse_id, *, token=None, token_provider=None, ...) -> Connection`
 - `Connection.cursor() -> Cursor`
 - `Connection.client -> DatabricksClient` -- the same client `cursor()` uses, for lower-level access (e.g. `stream_query_json`, `upload_volume_file`).
-- `Cursor.execute(sql, parameters=None, *, row_limit=None, offset=None, catalog=None, schema=None, total_timeout_s=None) -> Cursor` -- submits and waits for the statement, like a real DB-API cursor. `parameters`, if given, is Databricks' own named-parameter format -- `[{"name": ..., "value": ..., "type": ...}]` bound against `:name` markers in `sql`.
+- `Cursor.execute(sql, parameters=None, *, row_limit=None, offset=None, catalog=None, schema=None, total_timeout_s=None, prefer_inline=False) -> Cursor` -- submits and waits for the statement, like a real DB-API cursor. `parameters`, if given, is Databricks' own named-parameter format -- `[{"name": ..., "value": ..., "type": ...}]` bound against `:name` markers in `sql`. `prefer_inline=True` tries fetching a small result (well under Databricks' 25 MiB inline cap) in the same round trip as the submission itself, skipping the chunk-fetch entirely -- if the result turns out too big, or has a column type this can't convert (nested ARRAY/MAP/STRUCT, VARIANT), it transparently re-runs the query the normal way, so a caller who sets this without actually expecting a small result pays for the query twice. Leave it off unless you know the result is small.
 - `Cursor.execute_streamed(...)` -- same args, but an async generator yielding `HEARTBEAT` while waiting on a slow cold start, then the ready `Cursor` -- for bridging e.g. an SSE connection. Its timeout/heartbeats stop the moment the statement is ready, *before* any chunk has been downloaded -- see `fetchall_streamed` below for the download phase itself.
 - `Cursor.fetchone() -> tuple | None`, `Cursor.fetchmany(size) -> list[tuple]`, `Cursor.fetchall() -> list[tuple]`, and iterating a `Cursor` directly -- row tuples; needs the `arro3` extra.
 - `Cursor.fetchmany_arrow(size) -> Table`, `Cursor.fetchall_arrow() -> Table` -- an Arrow table (implements `__arrow_c_stream__`, so arro3/pyarrow/DuckDB can all consume it directly, zero-copy).
@@ -179,7 +179,7 @@ con.register("my_table", table)
 con.sql("SELECT count(*) FROM my_table").show()
 ```
 
-`ReplayableArrowChunk` works the same way for Arrow-IPC bytes you fetched and stored earlier (e.g. `client.execute_arrow_statement(...)`'s raw chunk bytes, cached in Redis/a file/wherever) -- DuckDB's registration path calls `__arrow_c_stream__` twice (a schema peek, then the actual scan), which is exactly what `ReplayableArrowChunk` exists to support:
+`ReplayableArrowChunk` works the same way for Arrow-IPC bytes you fetched and stored earlier (e.g. a raw chunk's bytes, cached in Redis/a file/wherever) -- DuckDB's registration path calls `__arrow_c_stream__` twice (a schema peek, then the actual scan), which is exactly what `ReplayableArrowChunk` exists to support:
 
 ```python
 from arrowbricks import ReplayableArrowChunk

{arrowbricks-1.3.2 → arrowbricks-1.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "arrowbricks"
-version = "1.3.2"
+version = "1.4.0"
 description = "Runs SQL against a Databricks SQL warehouse via the Statement Execution API and hands you the result as Arrow -- a DB-API-ish Cursor (fetchone/fetchmany/fetchall/fetchall_arrow) or NDJSON streaming. Rust/PyO3 core throughout -- zero required runtime dependencies."
 readme = "README.md"
 license = "MIT"
@@ -10,8 +10,8 @@ requires-python = ">=3.11"
 # (see rust/arrowbricks_core's own reqwest client + Arrow-IPC/NDJSON
 # writer). arro3-core is optional (see below): only Cursor.fetchone/
 # fetchmany/fetchall's row materialization needs it -- fetchall_arrow,
-# execute_arrow, stream_query_json, upload_volume_file/delete_volume_file,
-# and Cursor.description all work with zero dependencies installed.
+# stream_query_json, upload_volume_file/delete_volume_file, and
+# Cursor.description all work with zero dependencies installed.
 dependencies = []
 
 [project.optional-dependencies]

{arrowbricks-1.3.2 → arrowbricks-1.4.0}/rust/arrowbricks_core/Cargo.lock

@@ -240,11 +240,12 @@ dependencies = [
 
 [[package]]
 name = "arrowbricks_core"
-version = "1.3.2"
+version = "1.4.0"
 dependencies = [
  "arrow",
  "arrow-json",
  "bytes",
+ "chrono",
  "lz4_flex",
  "pyo3",
  "pyo3-arrow",

{arrowbricks-1.3.2 → arrowbricks-1.4.0}/rust/arrowbricks_core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "arrowbricks_core"
-version = "1.3.2"
+version = "1.4.0"
 edition = "2024"
 readme = "README.md"
 
@@ -15,6 +15,11 @@ crate-type = ["cdylib", "rlib"]
 arrow = { version = "59.1.0", default-features = false, features = ["ipc"] }
 arrow-json = "59.1.0"
 bytes = "1.12.1"
+# Already an unconditional transitive dependency of pyo3-arrow (see its own
+# comment below) -- declared directly here too so json_convert.rs can parse
+# DATE/TIMESTAMP/TIMESTAMP_NTZ strings without hand-rolling calendar
+# arithmetic. No new code ships that wasn't already linked in.
+chrono = { version = "0.4", default-features = false, features = ["std"] }
 # Decompresses cloud-fetch chunk bytes when the server honors our
 # `result_compression: LZ4_FRAME` request (see client.rs's execute_statement)
 # -- the same LZ4 Frame format databricks-sql-python's `lz4.frame` decodes.

{arrowbricks-1.3.2 → arrowbricks-1.4.0}/rust/arrowbricks_core/README.md

@@ -48,7 +48,8 @@ async def main():
         warehouse_id="abcd1234efgh5678",
         token="dapi...",
     )
-    table = await client.execute_arrow("SELECT * FROM my_catalog.my_schema.my_table LIMIT 100")
+    result = await client.execute("SELECT * FROM my_catalog.my_schema.my_table LIMIT 100")
+    table = await result.fetchall_arrow()
     print(table.num_rows)
 
 
@@ -66,17 +67,14 @@ unlike the table object itself.
 ## API
 
 - `Client(host, warehouse_id, *, token=None, token_provider=None, chunk_fetch_concurrency=64, http_timeout=60.0, wait_timeout="30s", warehouse_start_timeout=300.0, warehouse_confirmed_running_ttl_s=30.0, compress_results=True)` -- exactly one of `token`/`token_provider`. `token_provider` is a callable (sync or async) returning a token string, called fresh on every request, no caching. `compress_results` requests LZ4-compressed cloud-fetch chunks (see above); set `False` to opt out.
-- `Client.execute_arrow(statement, *, catalog=None, schema=None, parameters=None) -> Table` -- eager: fetches and assembles the whole result before returning. `parameters` is Databricks' own named-parameter format (`[{"name":..., "value":..., "type":...}]`), passed straight through.
-- `Client.execute(statement, *, catalog=None, schema=None, parameters=None) -> ResultSet` -- submits and starts background chunk fetching without pulling anything yet.
+- `Client.execute(statement, *, catalog=None, schema=None, parameters=None, prefer_inline=False) -> ResultSet` -- submits and starts background chunk fetching without pulling anything yet. `parameters` is Databricks' own named-parameter format (`[{"name":..., "value":..., "type":...}]`), passed straight through. `prefer_inline=True` submits with `disposition=INLINE, format=JSON_ARRAY` instead, for a caller who expects a small (well under Databricks' 25 MiB inline cap) result and wants to skip the chunk-fetch round trip -- on a result too big for INLINE, or containing a column type `json_convert.rs` doesn't map (STRUCT/ARRAY-of-STRUCT/MAP/VARIANT), it transparently falls back to a second, normal `execute()` and the query runs twice. See `AGENTS.md`'s "Design invariants" section (in the root package) for the full reasoning and real-workspace verification behind this.
   - `ResultSet.fetchmany_arrow(n) -> Table` -- pulls/decodes only as many chunks as needed for `n` rows, buffering the rest; may return fewer than `n` once exhausted.
   - `ResultSet.fetchall_arrow() -> Table` -- drains everything remaining.
+  - `ResultSet.fetchall_arrow_streamed(*, total_timeout_s=None)` -- same as `fetchall_arrow()`, but an async iterator yielding the `HEARTBEAT` singleton while pulling chunks instead of blocking silently (bridge e.g. an SSE connection through the download), then a `Table` exactly once. Raises if `total_timeout_s` elapses first.
   - `ResultSet.schema() -> list[tuple[str, str]] | None` -- the real decoded Arrow schema as `(name, type_name)` string pairs, once known (after >=1 fetch); `None` before that. Computed directly in Rust -- unlike a `Table`'s own `.schema`, this needs no arro3 install.
   - `ResultSet.statement_id`, `ResultSet.num_chunks`, `ResultSet.columns` (manifest-based pre-fetch schema estimate, also arro3-free).
-- `Client.execute_json(statement, *, catalog=None, schema=None, parameters=None) -> list[list[str | None]]` -- JSON_ARRAY format, no Arrow parse. Every non-null value comes back as a string regardless of real column type (Databricks' own contract) -- cast by the manifest's column type yourself if you want native Python types.
 - `Client.stream_ndjson_lines(statement, *, catalog=None, schema=None, parameters=None, total_timeout_s=None)` -- chunk-at-a-time: an async iterator yielding the `HEARTBEAT` singleton while waiting on the statement or any individual chunk, then a `list[str]` of NDJSON lines (one per row, explicit nulls, ISO-8601 timestamps) per chunk in logical order. Decode and JSON encoding both happen in Rust -- backs `stream_query_json` end to end.
 - `Client.upload_volume_file(volume_path, data: bytes)` / `Client.delete_volume_file(volume_path)` -- Unity Catalog volume files via the Files API. Delete treats a 404 as success (idempotent). Both raise a plain `RuntimeError` (message only) on failure.
-- `Client.execute_streamed(statement, *, catalog=None, schema=None, parameters=None, total_timeout_s=None)` -- like `execute()`, but an async iterator yielding the `HEARTBEAT` singleton while waiting on Databricks instead of blocking silently (bridge e.g. an SSE connection through a slow warehouse cold start), then a `ResultSet` exactly once. Raises if `total_timeout_s` elapses first.
-  - `ResultSet.fetchall_arrow_streamed(*, total_timeout_s=None)` -- same idea for the chunk-download phase: yields `HEARTBEAT` while pulling chunks, then a `Table` exactly once.
 - `write_ipc_stream(stream, buf)` -- free function; writes any object implementing `__arrow_c_stream__` (a `Table` from this crate, arro3, pyarrow, ...) as uncompressed Arrow-IPC stream bytes to a Python file-like object. No dependency needed regardless of the input's origin.
 - `read_ipc_stream(data: bytes) -> Table` -- free function; the exact inverse of `write_ipc_stream`, parsing raw Arrow-IPC stream bytes back into a `Table`. No dependency needed regardless of where the bytes came from -- backs `arrowbricks.ReplayableArrowChunk`, which needs to re-parse the same cached bytes on every `__arrow_c_stream__` call.
 - `HEARTBEAT` -- module-level singleton; compare with `is`, e.g. `if item is _core.HEARTBEAT: ...`.
@@ -90,7 +88,8 @@ directly, no pandas/pyarrow conversion step, no extra dependency:
 ```python
 import duckdb
 
-result = await client.execute_arrow("SELECT * FROM my_catalog.my_schema.my_table")
+result_set = await client.execute("SELECT * FROM my_catalog.my_schema.my_table")
+result = await result_set.fetchall_arrow()
 print(duckdb.sql("SELECT count(*) FROM result").fetchall())
 ```
 

{arrowbricks-1.3.2 → arrowbricks-1.4.0}/rust/arrowbricks_core/examples/duckdb_query.py

@@ -27,7 +27,8 @@ async def main() -> None:
         warehouse_id=os.environ["DATABRICKS_WAREHOUSE_ID"],
         token=os.environ["DATABRICKS_TOKEN"],
     )
-    result = await client.execute_arrow("SELECT * FROM my_catalog.my_schema.my_table LIMIT 1000000")  # noqa: F841 -- read by DuckDB's replacement scan via local variable name, not a literal Python reference
+    result_set = await client.execute("SELECT * FROM my_catalog.my_schema.my_table LIMIT 1000000")
+    result = await result_set.fetchall_arrow()  # noqa: F841 -- read by DuckDB's replacement scan via local variable name, not a literal Python reference
 
     # `result` is queryable by variable name -- DuckDB imports it zero-copy,
     # no pandas/pyarrow conversion step in between.

{arrowbricks-1.3.2 → arrowbricks-1.4.0}/rust/arrowbricks_core/src/client.rs

@@ -56,6 +56,15 @@ pub struct ColumnDescription {
     pub name: String,
     #[serde(default)]
     pub type_name: Option<String>,
+    /// Only present for `type_name == "DECIMAL"` -- needed by
+    /// `json_convert`'s INLINE/JSON_ARRAY-to-Arrow conversion to build a
+    /// correctly-scaled `Decimal128Array`. Absent (and unused) for every
+    /// other type, including on the Arrow-IPC path, which gets its decimal
+    /// precision/scale from the IPC schema itself, not from here.
+    #[serde(default)]
+    pub type_precision: Option<u8>,
+    #[serde(default)]
+    pub type_scale: Option<i8>,
 }
 
 #[derive(Deserialize, Default)]
@@ -119,6 +128,14 @@ struct ResultLinkBody {
 struct ResultBody {
     #[serde(default)]
     external_links: Vec<ResultLinkBody>,
+    /// Only present for `disposition: INLINE` + `format: JSON_ARRAY` -- each
+    /// row a `Vec<Option<String>>` (every non-null value a string,
+    /// Databricks' own JSON_ARRAY contract, same as `execute_json_statement`'s
+    /// normal `EXTERNAL_LINKS`+`JSON_ARRAY` chunks). Consumed by
+    /// `json_convert::json_array_to_record_batch` on the `prefer_inline`
+    /// fast path.
+    #[serde(default)]
+    data_array: Option<Vec<Vec<Option<String>>>>,
 }
 
 #[derive(Deserialize)]
@@ -197,7 +214,18 @@ impl ApiError {
     /// (excluding `is_connect()`/`is_timeout()`, already handled) also covers
     /// hyper's own "connection closed before message completed" pooled-
     /// connection-reuse race, which fires before any body is even read and
-    /// is equally safe to retry on an idempotent request.
+    /// is equally safe to retry on an idempotent request. Unlike `is_decode()`
+    /// (hit for real on a 400-chunk fetch and reproduced with a directed
+    /// test), this specific race is reasoned from reqwest/hyper's own source,
+    /// not independently reproduced -- a directed attempt to force it
+    /// self-healed 20/20 times (hyper's own idle-connection health check
+    /// evidently detects a closed peer and opens a fresh connection before
+    /// ever handing the dead one out for reuse, at least under a simple,
+    /// low-concurrency test). Kept anyway since it's safe regardless (scoped
+    /// to idempotent requests only) and the window may still be reachable
+    /// under real production concurrency even though a quick local test
+    /// couldn't force it -- treat as a plausible defensive measure, not a
+    /// confirmed fix for an observed failure.
     fn from_reqwest(e: reqwest::Error, idempotent: bool) -> Self {
         // `is_decode()` (reqwest's `Kind::Decode`) isn't only content-decoding
         // -- `Response::bytes()`/`.text()`/`.json()` also wrap a body read
@@ -219,8 +247,18 @@ impl ApiError {
         }
     }
 
-    fn from_status(status: StatusCode, body: &str) -> Self {
-        let transient = matches!(status.as_u16(), 401 | 403 | 408 | 429) || status.is_server_error();
+    /// `idempotent` only gates the 5xx case -- 401/403/408/429 mean the
+    /// request was rejected before any processing started (auth failure,
+    /// rate limit, client-side timeout), safe to retry regardless of method.
+    /// A 5xx is murkier: it usually means the same, but it can also mean the
+    /// backend already accepted and started the statement before some later
+    /// failure (e.g. a gateway timeout) produced the 5xx anyway -- found in
+    /// code review that this call was unconditionally transient even for the
+    /// statement-submit POST, bypassing the exact idempotency reasoning
+    /// `from_reqwest`'s `idempotent` param exists for (retrying that POST
+    /// risks a second, duplicate execution of arbitrary caller SQL).
+    fn from_status(status: StatusCode, body: &str, idempotent: bool) -> Self {
+        let transient = matches!(status.as_u16(), 401 | 403 | 408 | 429) || (idempotent && status.is_server_error());
         Self {
             message: format!("HTTP {status}: {body}"),
             transient,
@@ -271,6 +309,23 @@ pub struct StatementSubmitResult {
     pub compressed: bool,
 }
 
+/// What `execute_arrow_statement_prefer_inline` gets you -- either the whole
+/// result inline as raw JSON_ARRAY rows (every non-null value a string,
+/// same contract as `execute_json_statement`; converting these into a real
+/// `RecordBatch` is `pipeline.rs`'s job via `json_convert`, not this
+/// Arrow-agnostic module's -- see this file's own module doc comment), or a
+/// normal `StatementSubmitResult` to fetch chunks for exactly as if
+/// `prefer_inline` had never been asked for. See that function's own doc
+/// comment for when each happens.
+pub enum InlineOrExternal {
+    Inline {
+        statement_id: String,
+        rows: Vec<Vec<Option<String>>>,
+        columns: Vec<ColumnDescription>,
+    },
+    External(StatementSubmitResult),
+}
+
 #[derive(Debug)]
 pub struct ChunkItem {
     /// `Bytes` (not `Vec<u8>`) so the bytes reqwest already received off the
@@ -316,14 +371,22 @@ fn decompress_lz4_frame(compressed: &Bytes) -> Result<Bytes, ApiError> {
     // small lower bound.
     let mut out = Vec::with_capacity(compressed.len() * 4);
     let mut decoder = lz4_flex::frame::FrameDecoder::new(&compressed[..]);
-    loop {
-        let before = out.len();
+    // Terminate on the *reader* being exhausted, not on "output stopped
+    // growing" -- found in code review that a frame which happens to decode
+    // to zero bytes (a real, valid LZ4 Frame shape: header + immediate
+    // EndMark) makes one `read_to_end` call return `Ok(0)` for that frame
+    // without erroring and without necessarily advancing into the next
+    // frame yet, which the old `out.len() == before` check read as "no more
+    // frames" -- silently dropping every subsequent concatenated frame with
+    // no error at all. Same silent-truncation shape as the original
+    // multi-frame bug this loop exists to fix. Looping while the reader
+    // still has bytes left (regardless of whether the last call grew `out`)
+    // is the correct fix -- verified against a zero-content frame sandwiched
+    // between two real ones, see `decompress_lz4_frame_survives_a_zero_content_frame_in_the_middle`.
+    while !decoder.get_ref().is_empty() {
         decoder
             .read_to_end(&mut out)
             .map_err(|e| ApiError::permanent(format!("LZ4 frame decompress failed: {e}")))?;
-        if out.len() == before {
-            break;
-        }
     }
     Ok(Bytes::from(out))
 }
@@ -499,7 +562,7 @@ impl DbClient {
             let status = resp.status();
             let text = resp.text().await.map_err(|e| ApiError::from_reqwest(e, idempotent))?;
             if !status.is_success() {
-                return Err(ApiError::from_status(status, &text));
+                return Err(ApiError::from_status(status, &text, idempotent));
             }
             serde_json::from_str::<T>(&text).map_err(|e| ApiError::permanent(format!("bad JSON body: {e}")))
         })
@@ -530,7 +593,7 @@ impl DbClient {
             let status = resp.status();
             if !status.is_success() {
                 let text = resp.text().await.unwrap_or_default();
-                return Err(ApiError::from_status(status, &text));
+                return Err(ApiError::from_status(status, &text, true));
             }
             let bytes = resp.bytes().await.map_err(|e| ApiError::from_reqwest(e, true))?;
             if !compressed {
@@ -595,6 +658,101 @@ impl DbClient {
             .await
     }
 
+    /// Tries `disposition: INLINE` + `format: JSON_ARRAY` first -- for a
+    /// small result, Databricks embeds the whole result directly in this
+    /// same submit/poll response (`result.data_array`), skipping the
+    /// separate chunk-resolution-and-blob-fetch round trip the normal
+    /// `EXTERNAL_LINKS` path always needs. Confirmed against a real
+    /// workspace: exceeding INLINE's byte limit (26,214,400 bytes / 25MiB)
+    /// fails the statement cleanly with a specific, matchable error
+    /// message -- never silent truncation -- and `INLINE_OR_EXTERNAL_LINKS`
+    /// ("HYBRID", which would let the server choose per-query) returned a
+    /// clean "not a supported disposition" 400 on that same workspace, so
+    /// isn't used here. `format` must be `JSON_ARRAY` for `INLINE` --
+    /// Databricks rejects `INLINE`+`ARROW_STREAM` outright (also confirmed).
+    /// This module stays Arrow-agnostic on purpose (see its own module doc
+    /// comment) -- `InlineOrExternal::Inline` carries the raw JSON_ARRAY
+    /// rows straight through; converting them into a `RecordBatch` (and
+    /// falling back to a fresh `EXTERNAL_LINKS` submission if that
+    /// conversion hits a column type it doesn't handle) is `pipeline.rs`'s
+    /// job via `json_convert`, same division of responsibility as every
+    /// other decode step in this crate.
+    ///
+    /// Falls back to a **fresh, independent** `execute_arrow_statement` call
+    /// (not a retry of this same submission) whenever INLINE doesn't pan
+    /// out at the HTTP/statement level: the byte limit was exceeded, or
+    /// `data_array` is unexpectedly absent despite SUCCEEDED. This is a
+    /// second, distinct statement execution, not a retry of a possibly-
+    /// already-run one, so it carries none of the double-execution risk
+    /// that made POST retries unsafe elsewhere in this file -- the first
+    /// attempt's outcome is fully known (it reached a terminal state)
+    /// before the second one is ever submitted. A genuine query error (bad
+    /// SQL, permission denied) propagates immediately instead, same as the
+    /// normal path -- no reason to mask it behind a pointless second
+    /// attempt.
+    ///
+    /// Not the default -- opt-in only (`prefer_inline` on the Python-facing
+    /// `execute()`/`Cursor.execute()`), since a caller who doesn't expect a
+    /// small result pays for two full statement executions on the (common,
+    /// for them) fallback path instead of one.
+    pub async fn execute_arrow_statement_prefer_inline(
+        &self,
+        statement: &str,
+        catalog: Option<&str>,
+        schema: Option<&str>,
+        parameters: Option<Value>,
+    ) -> Result<InlineOrExternal, ApiError> {
+        let mut body = json!({
+            "warehouse_id": self.warehouse_id,
+            "statement": statement,
+            "disposition": "INLINE",
+            "format": "JSON_ARRAY",
+            "wait_timeout": self.wait_timeout,
+            "on_wait_timeout": "CONTINUE",
+        });
+        if let Some(c) = catalog {
+            body["catalog"] = json!(c);
+        }
+        if let Some(s) = schema {
+            body["schema"] = json!(s);
+        }
+        if let Some(p) = parameters.clone() {
+            body["parameters"] = p;
+        }
+
+        let outcome = self.submit_and_poll(body).await;
+        let data = match outcome {
+            Ok(d) => d,
+            Err(e) if e.message.contains("Inline byte limit exceeded") => {
+                return self
+                    .execute_arrow_statement(statement, catalog, schema, parameters)
+                    .await
+                    .map(InlineOrExternal::External);
+            }
+            Err(e) => return Err(e),
+        };
+
+        let manifest = data.manifest.unwrap_or_default();
+        let columns = manifest.schema.map(|s| s.columns).unwrap_or_default();
+        let data_array = data.result.and_then(|r| r.data_array);
+        match data_array {
+            Some(rows) => Ok(InlineOrExternal::Inline {
+                statement_id: data.statement_id,
+                rows,
+                columns,
+            }),
+            // No data_array despite SUCCEEDED -- shouldn't happen given a
+            // non-error status, but this crate never guesses at a missing
+            // field; a fresh EXTERNAL_LINKS submission is exactly as safe as
+            // the byte-limit fallback above (a distinct statement, not a
+            // retry of this one).
+            None => self
+                .execute_arrow_statement(statement, catalog, schema, parameters)
+                .await
+                .map(InlineOrExternal::External),
+        }
+    }
+
     /// Like `execute_arrow_statement`, fixed to JSON_ARRAY -- each fetched
     /// chunk's bytes are then a JSON array of rows, each row itself an array
     /// of values where every non-null value is a *string* regardless of its
@@ -613,6 +771,41 @@ impl DbClient {
             .await
     }
 
+    /// Shared by `execute_statement` and `execute_arrow_statement_prefer_inline`:
+    /// POST the statement, poll until a terminal state, and turn FAILED/
+    /// CANCELED into an `Err` -- everything both callers need before they
+    /// diverge on how to interpret a SUCCEEDED response's `result`/`manifest`.
+    async fn submit_and_poll(&self, body: Value) -> Result<StatementResponseBody, ApiError> {
+        self.ensure_warehouse_running().await?;
+        let url = format!("{}/api/2.0/sql/statements", self.host);
+        let mut data: StatementResponseBody = self.authed_json(reqwest::Method::POST, &url, Some(&body)).await?;
+
+        while !matches!(
+            data.status.state.as_str(),
+            "SUCCEEDED" | "FAILED" | "CANCELED" | "CLOSED"
+        ) {
+            tokio::time::sleep(POLL_INTERVAL).await;
+            let poll_url = format!("{}/api/2.0/sql/statements/{}", self.host, data.statement_id);
+            data = self.authed_json(reqwest::Method::GET, &poll_url, None).await?;
+        }
+
+        match data.status.state.as_str() {
+            "FAILED" => {
+                let err = data.status.error.unwrap_or(StatementErrorBody {
+                    error_code: None,
+                    message: None,
+                });
+                Err(ApiError::permanent(format!(
+                    "Databricks statement failed [{}]: {}",
+                    err.error_code.as_deref().unwrap_or(""),
+                    err.message.as_deref().unwrap_or(""),
+                )))
+            }
+            "CANCELED" => Err(ApiError::permanent("Databricks statement was canceled")),
+            _ => Ok(data),
+        }
+    }
+
     async fn execute_statement(
         &self,
         statement: &str,
@@ -649,35 +842,7 @@ impl DbClient {
             body["parameters"] = p;
         }
 
-        self.ensure_warehouse_running().await?;
-        let url = format!("{}/api/2.0/sql/statements", self.host);
-        let mut data: StatementResponseBody = self.authed_json(reqwest::Method::POST, &url, Some(&body)).await?;
-
-        while !matches!(
-            data.status.state.as_str(),
-            "SUCCEEDED" | "FAILED" | "CANCELED" | "CLOSED"
-        ) {
-            tokio::time::sleep(POLL_INTERVAL).await;
-            let poll_url = format!("{}/api/2.0/sql/statements/{}", self.host, data.statement_id);
-            data = self.authed_json(reqwest::Method::GET, &poll_url, None).await?;
-        }
-
-        match data.status.state.as_str() {
-            "FAILED" => {
-                let err = data.status.error.unwrap_or(StatementErrorBody {
-                    error_code: None,
-                    message: None,
-                });
-                return Err(ApiError::permanent(format!(
-                    "Databricks statement failed [{}]: {}",
-                    err.error_code.as_deref().unwrap_or(""),
-                    err.message.as_deref().unwrap_or(""),
-                )));
-            }
-            "CANCELED" => return Err(ApiError::permanent("Databricks statement was canceled")),
-            _ => {}
-        }
-
+        let data = self.submit_and_poll(body).await?;
         let manifest = data.manifest.unwrap_or_default();
         let compressed = manifest.result_compression.as_deref() == Some("LZ4_FRAME");
         // `Vec` per index, not a plain map entry -- see `ChunkMeta::pre_resolved_links`'s
@@ -837,7 +1002,7 @@ impl DbClient {
             let status = resp.status();
             if !status.is_success() {
                 let text = resp.text().await.unwrap_or_default();
-                return Err(ApiError::from_status(status, &text));
+                return Err(ApiError::from_status(status, &text, true));
             }
             Ok(())
         })
@@ -866,7 +1031,7 @@ impl DbClient {
                 return Ok(());
             }
             let text = resp.text().await.unwrap_or_default();
-            Err(ApiError::from_status(status, &text))
+            Err(ApiError::from_status(status, &text, true))
         })
         .await
     }
@@ -932,6 +1097,44 @@ mod tests {
         assert_eq!(decompressed, Bytes::from(expected));
     }
 
+    /// Regression test for a bug found in code review: a real, valid LZ4
+    /// Frame that happens to decode to zero bytes (a header immediately
+    /// followed by an EndMark -- a legal frame shape, not malformed input)
+    /// makes `read_to_end` return `Ok(0)` for that frame without erroring.
+    /// The old loop read "output didn't grow" as "no more frames" and
+    /// stopped there, silently dropping every frame concatenated after the
+    /// empty one. `decompress_lz4_frame` must keep going as long as the
+    /// underlying reader still has bytes left, not just as long as output
+    /// keeps growing.
+    #[test]
+    fn decompress_lz4_frame_survives_a_zero_content_frame_in_the_middle() {
+        use std::io::Write;
+
+        fn compress_one_frame(data: &[u8]) -> Vec<u8> {
+            let mut encoder = lz4_flex::frame::FrameEncoder::new(Vec::new());
+            encoder.write_all(data).unwrap();
+            encoder.finish().unwrap()
+        }
+
+        let part_a = b"the quick brown fox jumps over the lazy dog ".repeat(50);
+        let part_b = b"pack my box with five dozen liquor jugs ".repeat(50);
+        let mut concatenated_frames = Vec::new();
+        concatenated_frames.extend(compress_one_frame(&part_a));
+        concatenated_frames.extend(compress_one_frame(b"")); // real frame, zero content
+        concatenated_frames.extend(compress_one_frame(&part_b));
+
+        let decompressed = decompress_lz4_frame(&Bytes::from(concatenated_frames)).unwrap();
+
+        let mut expected = Vec::new();
+        expected.extend_from_slice(&part_a);
+        expected.extend_from_slice(&part_b);
+        assert_eq!(
+            decompressed,
+            Bytes::from(expected),
+            "the frame after the zero-content one must not be silently dropped"
+        );
+    }
+
     fn ok_task() -> tokio::task::JoinHandle<Result<(), ApiError>> {
         tokio::spawn(async { Ok(()) })
     }