arrowbricks 5.2.1__tar.gz → 6.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. arrowbricks-5.2.1/README.md → arrowbricks-6.0.0/PKG-INFO +146 -2
  2. arrowbricks-5.2.1/PKG-INFO → arrowbricks-6.0.0/README.md +133 -15
  3. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/pyproject.toml +5 -2
  4. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/Cargo.lock +2 -1
  5. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/Cargo.toml +2 -1
  6. arrowbricks-6.0.0/rust/arrowbricks_core/examples/agent_render_bench.rs +156 -0
  7. arrowbricks-6.0.0/rust/arrowbricks_core/src/agent/diagnostics.rs +188 -0
  8. arrowbricks-6.0.0/rust/arrowbricks_core/src/agent/mod.rs +355 -0
  9. arrowbricks-6.0.0/rust/arrowbricks_core/src/agent/render.rs +646 -0
  10. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/client/error.rs +9 -0
  11. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/client/sea.rs +13 -1
  12. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/client/thrift_rpc.rs +26 -3
  13. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/client.rs +11 -0
  14. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/lib.rs +38 -0
  15. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/pipeline/ndjson.rs +1 -0
  16. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/pipeline/sea.rs +22 -2
  17. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/pipeline/stats.rs +2 -1
  18. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/pipeline/thrift_exec.rs +109 -71
  19. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/pipeline.rs +2 -0
  20. arrowbricks-6.0.0/rust/arrowbricks_core/tests/wiremock_agent.rs +317 -0
  21. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/tests/wiremock_thrift.rs +101 -1
  22. arrowbricks-6.0.0/src/arrowbricks/__main__.py +5 -0
  23. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/src/arrowbricks/_core.pyi +12 -1
  24. arrowbricks-6.0.0/src/arrowbricks/cli.py +118 -0
  25. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/src/arrowbricks/client.py +32 -0
  26. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/LICENSE +0 -0
  27. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/.gitignore +0 -0
  28. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/README.md +0 -0
  29. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/examples/duckdb_query.py +0 -0
  30. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/examples/fastapi_sse.py +0 -0
  31. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/proptest-regressions/json_convert.txt +0 -0
  32. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/rustfmt.toml +0 -0
  33. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/alloc.rs +0 -0
  34. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/arrow_ffi.rs +0 -0
  35. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/client/call_context.rs +0 -0
  36. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/client/download.rs +0 -0
  37. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/client/model.rs +0 -0
  38. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/client/volume.rs +0 -0
  39. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/heartbeat.rs +0 -0
  40. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/json_convert.rs +0 -0
  41. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/pipeline/reorder.rs +0 -0
  42. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/pipeline/test_support.rs +0 -0
  43. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/thrift.rs +0 -0
  44. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/tests/common/mod.rs +0 -0
  45. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/tests/wiremock_pipeline.rs +0 -0
  46. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/tests/wiremock_volume_files.rs +0 -0
  47. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/tests_py/conftest.py +0 -0
  48. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/tests_py/test_ipc_stream.py +0 -0
  49. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/tests_py/test_parameters.py +0 -0
  50. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/tests_py/test_stream_ndjson_lines.py +0 -0
  51. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/tests_py/test_streaming.py +0 -0
  52. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/tests_py/test_thrift.py +0 -0
  53. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/tests_py/test_token_provider.py +0 -0
  54. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/tests_py/test_volume_files.py +0 -0
  55. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/tests_py/thrift_mock.py +0 -0
  56. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/src/arrowbricks/__init__.py +0 -0
  57. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/src/arrowbricks/_streaming.py +0 -0
  58. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/src/arrowbricks/cursor.py +0 -0
  59. {arrowbricks-5.2.1 → arrowbricks-6.0.0}/src/arrowbricks/py.typed +0 -0
@@ -1,13 +1,28 @@
1
+ Metadata-Version: 2.4
2
+ Name: arrowbricks
3
+ Version: 6.0.0
4
+ Requires-Dist: arro3-core>=0.8 ; extra == 'arro3'
5
+ Provides-Extra: arro3
6
+ License-File: LICENSE
7
+ Summary: Rust-powered Databricks SQL client and agent CLI with Arrow/NDJSON streaming, bounded Markdown previews, actionable JSON diagnostics, and zero required Python dependencies.
8
+ License-Expression: MIT
9
+ Requires-Python: >=3.11
10
+ Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
11
+ Project-URL: Repository, https://github.com/bmsuisse/arrowbricks
12
+
1
13
  <p align="center">
2
14
  <img src="assets/logo.svg" width="72" height="72" alt="arrowbricks logo">
3
15
  </p>
4
16
 
5
17
  <h1 align="center">arrowbricks</h1>
6
- <p align="center">Databricks SQL to Arrow, with a Rust core.</p>
18
+ <p align="center">Rust-powered Databricks SQL for applications and agents: Arrow streaming, compact Markdown, and actionable diagnostics.</p>
19
+
20
+ Arrowbricks is a Databricks SQL client and agent toolkit with a Rust execution and rendering core. Applications can consume lazy Arrow results or streaming NDJSON; agents can use the CLI or `query_agent` for compact typed Markdown and actionable JSON diagnostics. Version 6.0 adds strict preview budgets, explicit completeness metadata, total deadlines, cancellation, and a reusable agent skill, with zero required Python dependencies.
7
21
 
8
- Runs SQL against a Databricks SQL warehouse and hands you the result as Arrow -- a `Cursor` shaped like [`databricks-sql-python`](https://github.com/databricks/databricks-sql-python)'s (`execute`, `fetchone`/`fetchmany`/`fetchall`, `fetchall_arrow`/`fetchmany_arrow`), or `stream_query_json` for streaming NDJSON. Talks to Databricks over Thrift by default (`protocol="thrift"`), or the REST Statement Execution API if you prefer (`protocol="sea"`) -- see below.
22
+ The Python `Cursor` also offers familiar [`databricks-sql-python`](https://github.com/databricks/databricks-sql-python)-style methods (`execute`, `fetchone`/`fetchmany`/`fetchall`, `fetchall_arrow`/`fetchmany_arrow`). Connect over Thrift by default (`protocol="thrift"`) or the REST Statement Execution API (`protocol="sea"`).
9
23
 
10
24
  - **Rust core.** Statement submit/poll, bounded-concurrency chunk fetch, the reorder buffer, and Arrow-IPC decode all run in a [PyO3](https://pyo3.rs)/[arrow-rs](https://github.com/apache/arrow-rs) extension bundled in this same package -- 1.6x-2.5x faster than a pure-Python/asyncio client on a multi-chunk result, scaling further with chunk count and concurrency where asyncio+GIL plateaus.
25
+ - **Agent CLI and Python API.** Render directly from Arrow to bounded typed Markdown in Rust. Row, output-byte, cell and deadline budgets make previews predictable; the footer identifies incomplete or clipped results, and JSON errors retain the database's suggestions with a recovery hint.
11
26
  - **Thrift by default, SEA/REST as a fully-supported alternative** (`protocol="sea"` on `connect()`/`DatabricksClient(...)`). Thrift speaks the same HiveServer2-compatible protocol `databricks-sql-connector` uses by default -- measurably faster for *small* results (a statement's data can come back inline in the very same call that submits it, instead of SEA's separate poll-then-fetch round trip), and on par with SEA for a large, multi-chunk result too (its chunk downloads fan out concurrently across the whole result, same as SEA's own bounded-concurrency fetch, not serialized batch by batch) -- never slower than SEA on any query shape tested. Confirmed against a real production warehouse; see `AGENTS.md`'s design-invariant entry for the benchmarking history behind the switch.
12
27
  - **Compressed cloud-fetch transport by default.** Every statement requests LZ4-compressed chunk downloads (same default as the official `databricks-sql-connector`) and decompresses them in Rust before you ever see the bytes -- less data over the wire, which matters more than local decode speed for a large result. Measured ~2x faster chunk-fetch time against a real 120-column/100k-row table. Disable per-client with `compress_results=False` (`connect()`/`DatabricksClient(...)`) if your link to the warehouse is fast enough that decompression CPU time stops paying for itself.
13
28
  - **Zero required dependencies.** `pip install arrowbricks` and go.
@@ -24,6 +39,78 @@ pip install arrowbricks
24
39
 
25
40
  Ships as precompiled platform wheels (Linux/macOS/Windows) -- no Rust toolchain needed, and nothing else to install for most of the API. Row-tuple fetches (`fetchone`/`fetchmany`/`fetchall`) need one optional extra: `pip install arrowbricks[arro3]` -- see [Arrow vs. row-tuple fetches](#arrow-vs-row-tuple-fetches) below.
26
41
 
42
+ ## CLI quickstart (6.0)
43
+
44
+ ```bash
45
+ pip install 'arrowbricks>=6.0'
46
+ export DATABRICKS_HOST='your-workspace.azuredatabricks.net'
47
+ export DATABRICKS_WAREHOUSE_ID='your-warehouse-id'
48
+ export DATABRICKS_TOKEN='your-access-token'
49
+
50
+ arrowbricks -c 'SELECT 1 AS answer'
51
+ # | answer (Int32) |
52
+ # | --- |
53
+ # | 1 |
54
+ #
55
+ # rows_shown=1; complete=true; total_rows=1
56
+
57
+ arrowbricks --agent -c 'SELECT * FROM your_catalog.your_schema.your_table' --max-rows 20 --max-bytes 4000
58
+ arrowbricks -agent --file query.sql
59
+ printf 'SELECT 1 AS answer' | arrowbricks
60
+ python -m arrowbricks --protocol sea --command 'SELECT 1 AS answer'
61
+ ```
62
+
63
+ Agent output is the default; `--agent` and `-agent` are optional aliases.
64
+ `-c`/`--sql`/`--command` sends SQL unchanged. `-f`/`--file` reads a UTF-8 file
65
+ (or `-` for stdin); without either option, piped stdin supplies SQL. Interactive
66
+ invocations without SQL return a configuration error. Flags `--host`,
67
+ `--warehouse-id` and `--token` override the corresponding environment variables;
68
+ `--catalog` and `--schema` set the query namespace.
69
+
70
+ The default limits are `--max-rows 1000`, `--max-bytes 10000`,
71
+ `--max-cell-chars 500` and `--total-timeout-s 60` (`--timeout` is an alias).
72
+ Clipped previews have `complete=false`; they remain successful queries with
73
+ exit code 0. These limits bound the returned preview and stop further fetching
74
+ once full; they do not limit the warehouse's scan cost. See [Agent results](#agent-results) for completeness and cancellation.
75
+
76
+ Success writes only Markdown to stdout, so it can be piped directly into an
77
+ agent. Query failures write JSON to stderr and exit 1; invalid configuration
78
+ writes JSON to stderr and exits 2. Help uses the usual `--help` output and exits
79
+ 0. Ctrl-C interrupts the query, writes JSON to stderr and exits 130.
80
+ Credentials are never included in help output. For example:
81
+
82
+ ```bash
83
+ arrowbricks -c 'SELEC 1' 2>error.json
84
+ # exit 1; error.json contains category, original message and next_step
85
+ ```
86
+
87
+ Existing Python cursor and streaming APIs remain available in 6.0. `query_agent`
88
+ is an explicit Python API; it does not change the output of existing methods.
89
+
90
+ ## Agent skill
91
+
92
+ The reusable [Arrowbricks skill](skills/arrowbricks/SKILL.md) teaches agents the
93
+ CLI and Python API, secure environment authentication, preview completeness,
94
+ JSON recovery and the limits of scan/cost information. The skill is distributed
95
+ in this repository; installing the Python package supplies the CLI and API.
96
+ It targets 6.0+: for an unreleased checkout, install its built wheel; the
97
+ `pip install` examples apply once a compatible release is published.
98
+
99
+ From a repository checkout, copy the skill folder into your agent's skill
100
+ directory. For project discovery with Codex:
101
+
102
+ ```bash
103
+ mkdir -p .agents/skills
104
+ cp -R skills/arrowbricks .agents/skills/arrowbricks
105
+ ```
106
+
107
+ For Claude Code, use `.claude/skills/arrowbricks` instead; for a personal
108
+ installation, use your agent's corresponding user skill directory. Keep
109
+ `SKILL.md` inside the `arrowbricks` folder. After your agent reloads its skills,
110
+ invoke `$arrowbricks` in Codex or `/arrowbricks` in Claude Code, or ask it to use
111
+ Arrowbricks for a Databricks SQL preview. Credentials remain in your environment
112
+ or credential provider, separate from the skill.
113
+
27
114
  ## Quickstart
28
115
 
29
116
  ```python
@@ -74,6 +161,62 @@ client-credentials flow instead -- like `databricks-sql-connector`'s
74
161
  `auth_type="databricks-oauth"`, but stdlib-only (`urllib.request`), no
75
162
  extra dependency.
76
163
 
164
+ ## Agent results
165
+
166
+ `query_agent` returns one compact string: a typed Markdown table on success,
167
+ or JSON with `category`, `message` and `next_step` on database or transport
168
+ failure. It formats Arrow directly in Rust and needs no optional Python Arrow
169
+ packages. Both `thrift` (default) and `sea` are supported.
170
+
171
+ ```python
172
+ from arrowbricks import DatabricksClient
173
+
174
+ async with DatabricksClient(host=..., warehouse_id=..., token=...) as client:
175
+ result = await client.query_agent(
176
+ "SELECT id, name FROM my_catalog.my_schema.my_table ORDER BY id",
177
+ max_rows=1000,
178
+ max_bytes=10000,
179
+ max_cell_chars=500,
180
+ total_timeout_s=60.0,
181
+ # catalog=None, schema=None, params=None
182
+ )
183
+ print(result)
184
+ ```
185
+
186
+ Column headers include Arrow types. Strings are quoted, so SQL `NULL` differs
187
+ from the string `"NULL"`; Markdown, HTML, pipes and line breaks are escaped.
188
+ Clipped cells carry `…[truncated]`. The footer gives `rows_shown`, `complete`,
189
+ and an exact `total_rows` only when the result is exhausted. Row, byte or cell
190
+ clipping makes `complete=false`; an unexhausted result reports only a fetched
191
+ lower bound. Every returned string, including its header and footer, fits
192
+ `max_bytes` in UTF-8. Row and cell limits must be positive, with `max_bytes >= 256`; the timeout
193
+ must be finite and non-negative. Invalid budgets raise before SQL submission.
194
+
195
+ The single timeout covers submission, polling, downloads and formatting.
196
+ Stopping requests best-effort server cancellation; cancelling the Python task
197
+ preserves `asyncio.CancelledError`. A failure after some rows were fetched
198
+ returns an error JSON string instead of claiming a successful partial result.
199
+ Database diagnostics and suggestions are preserved; corrected SQL is never
200
+ executed automatically. Pass named parameters with `params`, using the same
201
+ list of `{"name": ..., "value": ..., "type": ...}` objects as the other APIs.
202
+
203
+ These budgets bound the returned preview, not the warehouse scan or its cost.
204
+ Pre-execution cost estimates and percentage progress are unavailable. This
205
+ method does not hash the full result, fetch its last rows or offer persistent
206
+ result paging. Use the cursor API when you need to consume a larger result.
207
+
208
+ [`examples/agent_mode.py`](examples/agent_mode.py) is runnable with
209
+ `DATABRICKS_HOST`, `DATABRICKS_WAREHOUSE_ID` and `DATABRICKS_TOKEN`:
210
+
211
+ ```bash
212
+ python examples/agent_mode.py --protocol sea --sql 'SELECT 1 AS answer'
213
+ ```
214
+
215
+ The example delegates to the installed CLI: Markdown goes to stdout and error
216
+ JSON goes to stderr. Its exit codes are 0 for Markdown, 1 for a query error and
217
+ 2 for invalid configuration. A clipped Markdown preview is still a successful query; inspect
218
+ its completeness footer before using it as a complete dataset.
219
+
77
220
  ## FastAPI SSE example
78
221
 
79
222
  `stream_query_json` is the full-speed way to serve a query over HTTP: the first row reaches the client after roughly one chunk's fetch/decode time, not the whole query's -- the Rust core is fetching, decoding, and reordering chunks concurrently the entire time, and the `HEARTBEAT`s keep the connection alive through a slow cold warehouse start instead of the client just seeing dead air:
@@ -366,3 +509,4 @@ real data (`--mode rows` or `--mode ipc`), to distinguish CPU work from network
366
509
  ## License
367
510
 
368
511
  MIT
512
+
@@ -1,25 +1,16 @@
1
- Metadata-Version: 2.4
2
- Name: arrowbricks
3
- Version: 5.2.1
4
- Requires-Dist: arro3-core>=0.8 ; extra == 'arro3'
5
- Provides-Extra: arro3
6
- License-File: LICENSE
7
- Summary: Runs SQL against a Databricks SQL warehouse via the Statement Execution API and hands you the result as Arrow -- a DB-API-ish Cursor (fetchone/fetchmany/fetchall/fetchall_arrow) or NDJSON streaming. Rust/PyO3 core throughout -- zero required runtime dependencies.
8
- License-Expression: MIT
9
- Requires-Python: >=3.11
10
- Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
11
- Project-URL: Repository, https://github.com/bmsuisse/arrowbricks
12
-
13
1
  <p align="center">
14
2
  <img src="assets/logo.svg" width="72" height="72" alt="arrowbricks logo">
15
3
  </p>
16
4
 
17
5
  <h1 align="center">arrowbricks</h1>
18
- <p align="center">Databricks SQL to Arrow, with a Rust core.</p>
6
+ <p align="center">Rust-powered Databricks SQL for applications and agents: Arrow streaming, compact Markdown, and actionable diagnostics.</p>
7
+
8
+ Arrowbricks is a Databricks SQL client and agent toolkit with a Rust execution and rendering core. Applications can consume lazy Arrow results or streaming NDJSON; agents can use the CLI or `query_agent` for compact typed Markdown and actionable JSON diagnostics. Version 6.0 adds strict preview budgets, explicit completeness metadata, total deadlines, cancellation, and a reusable agent skill, with zero required Python dependencies.
19
9
 
20
- Runs SQL against a Databricks SQL warehouse and hands you the result as Arrow -- a `Cursor` shaped like [`databricks-sql-python`](https://github.com/databricks/databricks-sql-python)'s (`execute`, `fetchone`/`fetchmany`/`fetchall`, `fetchall_arrow`/`fetchmany_arrow`), or `stream_query_json` for streaming NDJSON. Talks to Databricks over Thrift by default (`protocol="thrift"`), or the REST Statement Execution API if you prefer (`protocol="sea"`) -- see below.
10
+ The Python `Cursor` also offers familiar [`databricks-sql-python`](https://github.com/databricks/databricks-sql-python)-style methods (`execute`, `fetchone`/`fetchmany`/`fetchall`, `fetchall_arrow`/`fetchmany_arrow`). Connect over Thrift by default (`protocol="thrift"`) or the REST Statement Execution API (`protocol="sea"`).
21
11
 
22
12
  - **Rust core.** Statement submit/poll, bounded-concurrency chunk fetch, the reorder buffer, and Arrow-IPC decode all run in a [PyO3](https://pyo3.rs)/[arrow-rs](https://github.com/apache/arrow-rs) extension bundled in this same package -- 1.6x-2.5x faster than a pure-Python/asyncio client on a multi-chunk result, scaling further with chunk count and concurrency where asyncio+GIL plateaus.
13
+ - **Agent CLI and Python API.** Render directly from Arrow to bounded typed Markdown in Rust. Row, output-byte, cell and deadline budgets make previews predictable; the footer identifies incomplete or clipped results, and JSON errors retain the database's suggestions with a recovery hint.
23
14
  - **Thrift by default, SEA/REST as a fully-supported alternative** (`protocol="sea"` on `connect()`/`DatabricksClient(...)`). Thrift speaks the same HiveServer2-compatible protocol `databricks-sql-connector` uses by default -- measurably faster for *small* results (a statement's data can come back inline in the very same call that submits it, instead of SEA's separate poll-then-fetch round trip), and on par with SEA for a large, multi-chunk result too (its chunk downloads fan out concurrently across the whole result, same as SEA's own bounded-concurrency fetch, not serialized batch by batch) -- never slower than SEA on any query shape tested. Confirmed against a real production warehouse; see `AGENTS.md`'s design-invariant entry for the benchmarking history behind the switch.
24
15
  - **Compressed cloud-fetch transport by default.** Every statement requests LZ4-compressed chunk downloads (same default as the official `databricks-sql-connector`) and decompresses them in Rust before you ever see the bytes -- less data over the wire, which matters more than local decode speed for a large result. Measured ~2x faster chunk-fetch time against a real 120-column/100k-row table. Disable per-client with `compress_results=False` (`connect()`/`DatabricksClient(...)`) if your link to the warehouse is fast enough that decompression CPU time stops paying for itself.
25
16
  - **Zero required dependencies.** `pip install arrowbricks` and go.
@@ -36,6 +27,78 @@ pip install arrowbricks
36
27
 
37
28
  Ships as precompiled platform wheels (Linux/macOS/Windows) -- no Rust toolchain needed, and nothing else to install for most of the API. Row-tuple fetches (`fetchone`/`fetchmany`/`fetchall`) need one optional extra: `pip install arrowbricks[arro3]` -- see [Arrow vs. row-tuple fetches](#arrow-vs-row-tuple-fetches) below.
38
29
 
30
+ ## CLI quickstart (6.0)
31
+
32
+ ```bash
33
+ pip install 'arrowbricks>=6.0'
34
+ export DATABRICKS_HOST='your-workspace.azuredatabricks.net'
35
+ export DATABRICKS_WAREHOUSE_ID='your-warehouse-id'
36
+ export DATABRICKS_TOKEN='your-access-token'
37
+
38
+ arrowbricks -c 'SELECT 1 AS answer'
39
+ # | answer (Int32) |
40
+ # | --- |
41
+ # | 1 |
42
+ #
43
+ # rows_shown=1; complete=true; total_rows=1
44
+
45
+ arrowbricks --agent -c 'SELECT * FROM your_catalog.your_schema.your_table' --max-rows 20 --max-bytes 4000
46
+ arrowbricks -agent --file query.sql
47
+ printf 'SELECT 1 AS answer' | arrowbricks
48
+ python -m arrowbricks --protocol sea --command 'SELECT 1 AS answer'
49
+ ```
50
+
51
+ Agent output is the default; `--agent` and `-agent` are optional aliases.
52
+ `-c`/`--sql`/`--command` sends SQL unchanged. `-f`/`--file` reads a UTF-8 file
53
+ (or `-` for stdin); without either option, piped stdin supplies SQL. Interactive
54
+ invocations without SQL return a configuration error. Flags `--host`,
55
+ `--warehouse-id` and `--token` override the corresponding environment variables;
56
+ `--catalog` and `--schema` set the query namespace.
57
+
58
+ The default limits are `--max-rows 1000`, `--max-bytes 10000`,
59
+ `--max-cell-chars 500` and `--total-timeout-s 60` (`--timeout` is an alias).
60
+ Clipped previews have `complete=false`; they remain successful queries with
61
+ exit code 0. These limits bound the returned preview and stop further fetching
62
+ once full; they do not limit the warehouse's scan cost. See [Agent results](#agent-results) for completeness and cancellation.
63
+
64
+ Success writes only Markdown to stdout, so it can be piped directly into an
65
+ agent. Query failures write JSON to stderr and exit 1; invalid configuration
66
+ writes JSON to stderr and exits 2. Help uses the usual `--help` output and exits
67
+ 0. Ctrl-C interrupts the query, writes JSON to stderr and exits 130.
68
+ Credentials are never included in help output. For example:
69
+
70
+ ```bash
71
+ arrowbricks -c 'SELEC 1' 2>error.json
72
+ # exit 1; error.json contains category, original message and next_step
73
+ ```
74
+
75
+ Existing Python cursor and streaming APIs remain available in 6.0. `query_agent`
76
+ is an explicit Python API; it does not change the output of existing methods.
77
+
78
+ ## Agent skill
79
+
80
+ The reusable [Arrowbricks skill](skills/arrowbricks/SKILL.md) teaches agents the
81
+ CLI and Python API, secure environment authentication, preview completeness,
82
+ JSON recovery and the limits of scan/cost information. The skill is distributed
83
+ in this repository; installing the Python package supplies the CLI and API.
84
+ It targets 6.0+: for an unreleased checkout, install its built wheel; the
85
+ `pip install` examples apply once a compatible release is published.
86
+
87
+ From a repository checkout, copy the skill folder into your agent's skill
88
+ directory. For project discovery with Codex:
89
+
90
+ ```bash
91
+ mkdir -p .agents/skills
92
+ cp -R skills/arrowbricks .agents/skills/arrowbricks
93
+ ```
94
+
95
+ For Claude Code, use `.claude/skills/arrowbricks` instead; for a personal
96
+ installation, use your agent's corresponding user skill directory. Keep
97
+ `SKILL.md` inside the `arrowbricks` folder. After your agent reloads its skills,
98
+ invoke `$arrowbricks` in Codex or `/arrowbricks` in Claude Code, or ask it to use
99
+ Arrowbricks for a Databricks SQL preview. Credentials remain in your environment
100
+ or credential provider, separate from the skill.
101
+
39
102
  ## Quickstart
40
103
 
41
104
  ```python
@@ -86,6 +149,62 @@ client-credentials flow instead -- like `databricks-sql-connector`'s
86
149
  `auth_type="databricks-oauth"`, but stdlib-only (`urllib.request`), no
87
150
  extra dependency.
88
151
 
152
+ ## Agent results
153
+
154
+ `query_agent` returns one compact string: a typed Markdown table on success,
155
+ or JSON with `category`, `message` and `next_step` on database or transport
156
+ failure. It formats Arrow directly in Rust and needs no optional Python Arrow
157
+ packages. Both `thrift` (default) and `sea` are supported.
158
+
159
+ ```python
160
+ from arrowbricks import DatabricksClient
161
+
162
+ async with DatabricksClient(host=..., warehouse_id=..., token=...) as client:
163
+ result = await client.query_agent(
164
+ "SELECT id, name FROM my_catalog.my_schema.my_table ORDER BY id",
165
+ max_rows=1000,
166
+ max_bytes=10000,
167
+ max_cell_chars=500,
168
+ total_timeout_s=60.0,
169
+ # catalog=None, schema=None, params=None
170
+ )
171
+ print(result)
172
+ ```
173
+
174
+ Column headers include Arrow types. Strings are quoted, so SQL `NULL` differs
175
+ from the string `"NULL"`; Markdown, HTML, pipes and line breaks are escaped.
176
+ Clipped cells carry `…[truncated]`. The footer gives `rows_shown`, `complete`,
177
+ and an exact `total_rows` only when the result is exhausted. Row, byte or cell
178
+ clipping makes `complete=false`; an unexhausted result reports only a fetched
179
+ lower bound. Every returned string, including its header and footer, fits
180
+ `max_bytes` in UTF-8. Row and cell limits must be positive, with `max_bytes >= 256`; the timeout
181
+ must be finite and non-negative. Invalid budgets raise before SQL submission.
182
+
183
+ The single timeout covers submission, polling, downloads and formatting.
184
+ Stopping requests best-effort server cancellation; cancelling the Python task
185
+ preserves `asyncio.CancelledError`. A failure after some rows were fetched
186
+ returns an error JSON string instead of claiming a successful partial result.
187
+ Database diagnostics and suggestions are preserved; corrected SQL is never
188
+ executed automatically. Pass named parameters with `params`, using the same
189
+ list of `{"name": ..., "value": ..., "type": ...}` objects as the other APIs.
190
+
191
+ These budgets bound the returned preview, not the warehouse scan or its cost.
192
+ Pre-execution cost estimates and percentage progress are unavailable. This
193
+ method does not hash the full result, fetch its last rows or offer persistent
194
+ result paging. Use the cursor API when you need to consume a larger result.
195
+
196
+ [`examples/agent_mode.py`](examples/agent_mode.py) is runnable with
197
+ `DATABRICKS_HOST`, `DATABRICKS_WAREHOUSE_ID` and `DATABRICKS_TOKEN`:
198
+
199
+ ```bash
200
+ python examples/agent_mode.py --protocol sea --sql 'SELECT 1 AS answer'
201
+ ```
202
+
203
+ The example delegates to the installed CLI: Markdown goes to stdout and error
204
+ JSON goes to stderr. Its exit codes are 0 for Markdown, 1 for a query error and
205
+ 2 for invalid configuration. A clipped Markdown preview is still a successful query; inspect
206
+ its completeness footer before using it as a complete dataset.
207
+
89
208
  ## FastAPI SSE example
90
209
 
91
210
  `stream_query_json` is the full-speed way to serve a query over HTTP: the first row reaches the client after roughly one chunk's fetch/decode time, not the whole query's -- the Rust core is fetching, decoding, and reordering chunks concurrently the entire time, and the `HEARTBEAT`s keep the connection alive through a slow cold warehouse start instead of the client just seeing dead air:
@@ -378,4 +497,3 @@ real data (`--mode rows` or `--mode ipc`), to distinguish CPU work from network
378
497
  ## License
379
498
 
380
499
  MIT
381
-
@@ -1,7 +1,7 @@
1
1
  [project]
2
2
  name = "arrowbricks"
3
- version = "5.2.1"
4
- description = "Runs SQL against a Databricks SQL warehouse via the Statement Execution API and hands you the result as Arrow -- a DB-API-ish Cursor (fetchone/fetchmany/fetchall/fetchall_arrow) or NDJSON streaming. Rust/PyO3 core throughout -- zero required runtime dependencies."
3
+ version = "6.0.0"
4
+ description = "Rust-powered Databricks SQL client and agent CLI with Arrow/NDJSON streaming, bounded Markdown previews, actionable JSON diagnostics, and zero required Python dependencies."
5
5
  readme = "README.md"
6
6
  license = "MIT"
7
7
  license-files = ["LICENSE"]
@@ -14,6 +14,9 @@ requires-python = ">=3.11"
14
14
  # Cursor.description all work with zero dependencies installed.
15
15
  dependencies = []
16
16
 
17
+ [project.scripts]
18
+ arrowbricks = "arrowbricks.cli:main"
19
+
17
20
  [project.optional-dependencies]
18
21
  # Row materialization converts our capsule table through arro3-core.
19
22
  # Arrow fetches and streaming do not need this optional dependency.
@@ -177,10 +177,11 @@ dependencies = [
177
177
 
178
178
  [[package]]
179
179
  name = "arrowbricks_core"
180
- version = "5.2.1"
180
+ version = "6.0.0"
181
181
  dependencies = [
182
182
  "arrow-array",
183
183
  "arrow-buffer",
184
+ "arrow-cast",
184
185
  "arrow-ipc",
185
186
  "arrow-json",
186
187
  "arrow-schema",
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "arrowbricks_core"
3
- version = "5.2.1"
3
+ version = "6.0.0"
4
4
  edition = "2024"
5
5
  readme = "README.md"
6
6
 
@@ -19,6 +19,7 @@ arrow-buffer = "59.1.0"
19
19
  arrow-ipc = "59.1.0"
20
20
  arrow-schema = "59.1.0"
21
21
  arrow-json = "59.1.0"
22
+ arrow-cast = "59.1.0"
22
23
  bytes = "1.12.1"
23
24
  # Already in the dependency graph; vectorized newline search in NDJSON output.
24
25
  memchr = "2"
@@ -0,0 +1,156 @@
1
+ use arrow_array::types::Int32Type;
2
+ use arrow_array::{ArrayRef, Float64Array, Int64Array, ListArray, RecordBatch, StringArray};
3
+ use arrow_schema::{Field, Schema};
4
+ use arrowbricks_core::agent::render::MarkdownRenderer;
5
+ use std::{hint::black_box, sync::Arc, time::Instant};
6
+
7
+ fn batch(columns: Vec<ArrayRef>) -> RecordBatch {
8
+ let fields: Vec<_> = columns
9
+ .iter()
10
+ .enumerate()
11
+ .map(|(i, a)| Field::new(format!("c{i}"), a.data_type().clone(), true))
12
+ .collect();
13
+ RecordBatch::try_new(Arc::new(Schema::new(fields)), columns).unwrap()
14
+ }
15
+ fn direct(b: &RecordBatch, bytes: usize, interrupts: bool) -> String {
16
+ let mut renderer = if interrupts {
17
+ MarkdownRenderer::new_with_interrupt(
18
+ b.schema(),
19
+ 1000,
20
+ bytes,
21
+ 500,
22
+ Instant::now() + std::time::Duration::from_secs(60),
23
+ Arc::new(std::sync::atomic::AtomicBool::new(false)),
24
+ )
25
+ } else {
26
+ MarkdownRenderer::new(b.schema(), 1000, bytes, 500)
27
+ }
28
+ .unwrap();
29
+ renderer.push_batch(b).unwrap();
30
+ renderer.finish(true)
31
+ }
32
+ fn json(b: &RecordBatch) -> Vec<serde_json::Value> {
33
+ let mut writer = arrow_json::ArrayWriter::new(Vec::new());
34
+ writer.write_batches(&[b]).unwrap();
35
+ writer.finish().unwrap();
36
+ serde_json::from_slice(&writer.into_inner()).unwrap()
37
+ }
38
+ fn json_markdown(b: &RecordBatch, bytes: usize) -> String {
39
+ let values = json(b);
40
+ let mut output = String::new();
41
+ for row in &values {
42
+ let mut line = String::from("|");
43
+ for field in b.schema().fields() {
44
+ let value = &row[field.name()];
45
+ let owned;
46
+ let text = if let Some(value) = value.as_str() {
47
+ value
48
+ } else {
49
+ owned = value.to_string();
50
+ &owned
51
+ };
52
+ line.push(' ');
53
+ if value.is_string() {
54
+ line.push('"');
55
+ }
56
+ for ch in text.chars().take(500) {
57
+ match ch {
58
+ '&' => line.push_str("&amp;"),
59
+ '<' => line.push_str("&lt;"),
60
+ '>' => line.push_str("&gt;"),
61
+ '\n' => line.push_str("\\n"),
62
+ '\r' => line.push_str("\\r"),
63
+ '\t' => line.push_str("\\t"),
64
+ '\\' | '|' | '`' | '*' | '_' | '[' | ']' | '~' | '"' => {
65
+ line.push('\\');
66
+ line.push(ch);
67
+ }
68
+ c => line.push(c),
69
+ }
70
+ }
71
+ if value.is_string() {
72
+ line.push('"');
73
+ }
74
+ line.push_str(" |");
75
+ }
76
+ line.push('\n');
77
+ if output.len() + line.len() > bytes.saturating_sub(160) {
78
+ break;
79
+ }
80
+ output.push_str(&line);
81
+ }
82
+ output
83
+ }
84
+ fn time(mut f: impl FnMut(), loops: usize) -> f64 {
85
+ let start = Instant::now();
86
+ for _ in 0..loops {
87
+ f();
88
+ }
89
+ start.elapsed().as_secs_f64() * 1e6 / loops as f64
90
+ }
91
+ fn main() {
92
+ let agent_only = std::env::args().any(|arg| arg == "--agent-only");
93
+ let n = 1000;
94
+ let numeric = batch(vec![
95
+ Arc::new(Int64Array::from_iter_values(0..n)),
96
+ Arc::new(Float64Array::from_iter_values((0..n).map(|i| i as f64 * 0.1))),
97
+ ]);
98
+ let unicode = batch(vec![
99
+ Arc::new(StringArray::from(vec!["é🦀|<>&\\".repeat(100); n as usize])),
100
+ Arc::new(ListArray::from_iter_primitive::<Int32Type, _, _>(
101
+ (0..n).map(|_| Some(vec![Some(1), None, Some(3)])),
102
+ )),
103
+ ]);
104
+ let ascii = batch(vec![Arc::new(StringArray::from(vec![
105
+ "ordinary query results without markup ".repeat(20); n as usize
106
+ ]))]);
107
+ let controls = batch(vec![Arc::new(StringArray::from(vec![
108
+ "a\0\u{0001}b\t\n".repeat(30);
109
+ n as usize
110
+ ]))]);
111
+ let huge = batch(vec![Arc::new(StringArray::from(vec!["🦀".repeat(1_000_000)]))]);
112
+ let mut builder = arrow_array::builder::ListBuilder::new(arrow_array::builder::StringBuilder::new());
113
+ builder.values().append_value("🦀".repeat(2_500_000));
114
+ builder.append(true);
115
+ let nested_huge = batch(vec![Arc::new(builder.finish())]);
116
+ for (name, b, loops) in [
117
+ ("numeric", numeric, 1000),
118
+ ("unicode_nested", unicode, 100),
119
+ ("ascii_strings", ascii, 100),
120
+ ("controls", controls, 100),
121
+ ("4MB_string", huge, 20),
122
+ ("10MB_nested_string", nested_huge, 10),
123
+ ] {
124
+ let bounded = time(
125
+ || {
126
+ black_box(direct(&b, 10_000, agent_only));
127
+ },
128
+ if agent_only { 10000 } else { loops },
129
+ );
130
+ let full = time(
131
+ || {
132
+ black_box(direct(&b, 2_000_000, agent_only));
133
+ },
134
+ loops,
135
+ );
136
+ if agent_only {
137
+ println!("{name}: direct_10KB={bounded:.2}us direct_2MB={full:.2}us");
138
+ continue;
139
+ }
140
+ let json_render = time(
141
+ || {
142
+ black_box(json_markdown(&b, 10_000));
143
+ },
144
+ loops,
145
+ );
146
+ let via_json = time(
147
+ || {
148
+ black_box(json(&b));
149
+ },
150
+ loops,
151
+ );
152
+ println!(
153
+ "{name}: direct_10KB={bounded:.2}us direct_2MB={full:.2}us JSON_materialize={via_json:.2}us JSON_then_Markdown_10KB={json_render:.2}us"
154
+ );
155
+ }
156
+ }