arrowbricks 5.2.1__tar.gz → 6.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- arrowbricks-5.2.1/README.md → arrowbricks-6.0.0/PKG-INFO +146 -2
- arrowbricks-5.2.1/PKG-INFO → arrowbricks-6.0.0/README.md +133 -15
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/pyproject.toml +5 -2
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/Cargo.lock +2 -1
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/Cargo.toml +2 -1
- arrowbricks-6.0.0/rust/arrowbricks_core/examples/agent_render_bench.rs +156 -0
- arrowbricks-6.0.0/rust/arrowbricks_core/src/agent/diagnostics.rs +188 -0
- arrowbricks-6.0.0/rust/arrowbricks_core/src/agent/mod.rs +355 -0
- arrowbricks-6.0.0/rust/arrowbricks_core/src/agent/render.rs +646 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/client/error.rs +9 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/client/sea.rs +13 -1
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/client/thrift_rpc.rs +26 -3
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/client.rs +11 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/lib.rs +38 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/pipeline/ndjson.rs +1 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/pipeline/sea.rs +22 -2
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/pipeline/stats.rs +2 -1
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/pipeline/thrift_exec.rs +109 -71
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/pipeline.rs +2 -0
- arrowbricks-6.0.0/rust/arrowbricks_core/tests/wiremock_agent.rs +317 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/tests/wiremock_thrift.rs +101 -1
- arrowbricks-6.0.0/src/arrowbricks/__main__.py +5 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/src/arrowbricks/_core.pyi +12 -1
- arrowbricks-6.0.0/src/arrowbricks/cli.py +118 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/src/arrowbricks/client.py +32 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/LICENSE +0 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/.gitignore +0 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/README.md +0 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/examples/duckdb_query.py +0 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/examples/fastapi_sse.py +0 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/proptest-regressions/json_convert.txt +0 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/rustfmt.toml +0 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/alloc.rs +0 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/arrow_ffi.rs +0 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/client/call_context.rs +0 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/client/download.rs +0 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/client/model.rs +0 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/client/volume.rs +0 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/heartbeat.rs +0 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/json_convert.rs +0 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/pipeline/reorder.rs +0 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/pipeline/test_support.rs +0 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/src/thrift.rs +0 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/tests/common/mod.rs +0 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/tests/wiremock_pipeline.rs +0 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/tests/wiremock_volume_files.rs +0 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/tests_py/conftest.py +0 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/tests_py/test_ipc_stream.py +0 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/tests_py/test_parameters.py +0 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/tests_py/test_stream_ndjson_lines.py +0 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/tests_py/test_streaming.py +0 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/tests_py/test_thrift.py +0 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/tests_py/test_token_provider.py +0 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/tests_py/test_volume_files.py +0 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/rust/arrowbricks_core/tests_py/thrift_mock.py +0 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/src/arrowbricks/__init__.py +0 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/src/arrowbricks/_streaming.py +0 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/src/arrowbricks/cursor.py +0 -0
- {arrowbricks-5.2.1 → arrowbricks-6.0.0}/src/arrowbricks/py.typed +0 -0
|
@@ -1,13 +1,28 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: arrowbricks
|
|
3
|
+
Version: 6.0.0
|
|
4
|
+
Requires-Dist: arro3-core>=0.8 ; extra == 'arro3'
|
|
5
|
+
Provides-Extra: arro3
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
Summary: Rust-powered Databricks SQL client and agent CLI with Arrow/NDJSON streaming, bounded Markdown previews, actionable JSON diagnostics, and zero required Python dependencies.
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
Requires-Python: >=3.11
|
|
10
|
+
Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
|
|
11
|
+
Project-URL: Repository, https://github.com/bmsuisse/arrowbricks
|
|
12
|
+
|
|
1
13
|
<p align="center">
|
|
2
14
|
<img src="assets/logo.svg" width="72" height="72" alt="arrowbricks logo">
|
|
3
15
|
</p>
|
|
4
16
|
|
|
5
17
|
<h1 align="center">arrowbricks</h1>
|
|
6
|
-
<p align="center">Databricks SQL
|
|
18
|
+
<p align="center">Rust-powered Databricks SQL for applications and agents: Arrow streaming, compact Markdown, and actionable diagnostics.</p>
|
|
19
|
+
|
|
20
|
+
Arrowbricks is a Databricks SQL client and agent toolkit with a Rust execution and rendering core. Applications can consume lazy Arrow results or streaming NDJSON; agents can use the CLI or `query_agent` for compact typed Markdown and actionable JSON diagnostics. Version 6.0 adds strict preview budgets, explicit completeness metadata, total deadlines, cancellation, and a reusable agent skill, with zero required Python dependencies.
|
|
7
21
|
|
|
8
|
-
|
|
22
|
+
The Python `Cursor` also offers familiar [`databricks-sql-python`](https://github.com/databricks/databricks-sql-python)-style methods (`execute`, `fetchone`/`fetchmany`/`fetchall`, `fetchall_arrow`/`fetchmany_arrow`). Connect over Thrift by default (`protocol="thrift"`) or the REST Statement Execution API (`protocol="sea"`).
|
|
9
23
|
|
|
10
24
|
- **Rust core.** Statement submit/poll, bounded-concurrency chunk fetch, the reorder buffer, and Arrow-IPC decode all run in a [PyO3](https://pyo3.rs)/[arrow-rs](https://github.com/apache/arrow-rs) extension bundled in this same package -- 1.6x-2.5x faster than a pure-Python/asyncio client on a multi-chunk result, scaling further with chunk count and concurrency where asyncio+GIL plateaus.
|
|
25
|
+
- **Agent CLI and Python API.** Render directly from Arrow to bounded typed Markdown in Rust. Row, output-byte, cell and deadline budgets make previews predictable; the footer identifies incomplete or clipped results, and JSON errors retain the database's suggestions with a recovery hint.
|
|
11
26
|
- **Thrift by default, SEA/REST as a fully-supported alternative** (`protocol="sea"` on `connect()`/`DatabricksClient(...)`). Thrift speaks the same HiveServer2-compatible protocol `databricks-sql-connector` uses by default -- measurably faster for *small* results (a statement's data can come back inline in the very same call that submits it, instead of SEA's separate poll-then-fetch round trip), and on par with SEA for a large, multi-chunk result too (its chunk downloads fan out concurrently across the whole result, same as SEA's own bounded-concurrency fetch, not serialized batch by batch) -- never slower than SEA on any query shape tested. Confirmed against a real production warehouse; see `AGENTS.md`'s design-invariant entry for the benchmarking history behind the switch.
|
|
12
27
|
- **Compressed cloud-fetch transport by default.** Every statement requests LZ4-compressed chunk downloads (same default as the official `databricks-sql-connector`) and decompresses them in Rust before you ever see the bytes -- less data over the wire, which matters more than local decode speed for a large result. Measured ~2x faster chunk-fetch time against a real 120-column/100k-row table. Disable per-client with `compress_results=False` (`connect()`/`DatabricksClient(...)`) if your link to the warehouse is fast enough that decompression CPU time stops paying for itself.
|
|
13
28
|
- **Zero required dependencies.** `pip install arrowbricks` and go.
|
|
@@ -24,6 +39,78 @@ pip install arrowbricks
|
|
|
24
39
|
|
|
25
40
|
Ships as precompiled platform wheels (Linux/macOS/Windows) -- no Rust toolchain needed, and nothing else to install for most of the API. Row-tuple fetches (`fetchone`/`fetchmany`/`fetchall`) need one optional extra: `pip install arrowbricks[arro3]` -- see [Arrow vs. row-tuple fetches](#arrow-vs-row-tuple-fetches) below.
|
|
26
41
|
|
|
42
|
+
## CLI quickstart (6.0)
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
pip install 'arrowbricks>=6.0'
|
|
46
|
+
export DATABRICKS_HOST='your-workspace.azuredatabricks.net'
|
|
47
|
+
export DATABRICKS_WAREHOUSE_ID='your-warehouse-id'
|
|
48
|
+
export DATABRICKS_TOKEN='your-access-token'
|
|
49
|
+
|
|
50
|
+
arrowbricks -c 'SELECT 1 AS answer'
|
|
51
|
+
# | answer (Int32) |
|
|
52
|
+
# | --- |
|
|
53
|
+
# | 1 |
|
|
54
|
+
#
|
|
55
|
+
# rows_shown=1; complete=true; total_rows=1
|
|
56
|
+
|
|
57
|
+
arrowbricks --agent -c 'SELECT * FROM your_catalog.your_schema.your_table' --max-rows 20 --max-bytes 4000
|
|
58
|
+
arrowbricks -agent --file query.sql
|
|
59
|
+
printf 'SELECT 1 AS answer' | arrowbricks
|
|
60
|
+
python -m arrowbricks --protocol sea --command 'SELECT 1 AS answer'
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
Agent output is the default; `--agent` and `-agent` are optional aliases.
|
|
64
|
+
`-c`/`--sql`/`--command` sends SQL unchanged. `-f`/`--file` reads a UTF-8 file
|
|
65
|
+
(or `-` for stdin); without either option, piped stdin supplies SQL. Interactive
|
|
66
|
+
invocations without SQL return a configuration error. Flags `--host`,
|
|
67
|
+
`--warehouse-id` and `--token` override the corresponding environment variables;
|
|
68
|
+
`--catalog` and `--schema` set the query namespace.
|
|
69
|
+
|
|
70
|
+
The default limits are `--max-rows 1000`, `--max-bytes 10000`,
|
|
71
|
+
`--max-cell-chars 500` and `--total-timeout-s 60` (`--timeout` is an alias).
|
|
72
|
+
Clipped previews have `complete=false`; they remain successful queries with
|
|
73
|
+
exit code 0. These limits bound the returned preview and stop further fetching
|
|
74
|
+
once full; they do not limit the warehouse's scan cost. See [Agent results](#agent-results) for completeness and cancellation.
|
|
75
|
+
|
|
76
|
+
Success writes only Markdown to stdout, so it can be piped directly into an
|
|
77
|
+
agent. Query failures write JSON to stderr and exit 1; invalid configuration
|
|
78
|
+
writes JSON to stderr and exits 2. Help uses the usual `--help` output and exits
|
|
79
|
+
0. Ctrl-C interrupts the query, writes JSON to stderr and exits 130.
|
|
80
|
+
Credentials are never included in help output. For example:
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
arrowbricks -c 'SELEC 1' 2>error.json
|
|
84
|
+
# exit 1; error.json contains category, original message and next_step
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
Existing Python cursor and streaming APIs remain available in 6.0. `query_agent`
|
|
88
|
+
is an explicit Python API; it does not change the output of existing methods.
|
|
89
|
+
|
|
90
|
+
## Agent skill
|
|
91
|
+
|
|
92
|
+
The reusable [Arrowbricks skill](skills/arrowbricks/SKILL.md) teaches agents the
|
|
93
|
+
CLI and Python API, secure environment authentication, preview completeness,
|
|
94
|
+
JSON recovery and the limits of scan/cost information. The skill is distributed
|
|
95
|
+
in this repository; installing the Python package supplies the CLI and API.
|
|
96
|
+
It targets 6.0+: for an unreleased checkout, install its built wheel; the
|
|
97
|
+
`pip install` examples apply once a compatible release is published.
|
|
98
|
+
|
|
99
|
+
From a repository checkout, copy the skill folder into your agent's skill
|
|
100
|
+
directory. For project discovery with Codex:
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
mkdir -p .agents/skills
|
|
104
|
+
cp -R skills/arrowbricks .agents/skills/arrowbricks
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
For Claude Code, use `.claude/skills/arrowbricks` instead; for a personal
|
|
108
|
+
installation, use your agent's corresponding user skill directory. Keep
|
|
109
|
+
`SKILL.md` inside the `arrowbricks` folder. After your agent reloads its skills,
|
|
110
|
+
invoke `$arrowbricks` in Codex or `/arrowbricks` in Claude Code, or ask it to use
|
|
111
|
+
Arrowbricks for a Databricks SQL preview. Credentials remain in your environment
|
|
112
|
+
or credential provider, separate from the skill.
|
|
113
|
+
|
|
27
114
|
## Quickstart
|
|
28
115
|
|
|
29
116
|
```python
|
|
@@ -74,6 +161,62 @@ client-credentials flow instead -- like `databricks-sql-connector`'s
|
|
|
74
161
|
`auth_type="databricks-oauth"`, but stdlib-only (`urllib.request`), no
|
|
75
162
|
extra dependency.
|
|
76
163
|
|
|
164
|
+
## Agent results
|
|
165
|
+
|
|
166
|
+
`query_agent` returns one compact string: a typed Markdown table on success,
|
|
167
|
+
or JSON with `category`, `message` and `next_step` on database or transport
|
|
168
|
+
failure. It formats Arrow directly in Rust and needs no optional Python Arrow
|
|
169
|
+
packages. Both `thrift` (default) and `sea` are supported.
|
|
170
|
+
|
|
171
|
+
```python
|
|
172
|
+
from arrowbricks import DatabricksClient
|
|
173
|
+
|
|
174
|
+
async with DatabricksClient(host=..., warehouse_id=..., token=...) as client:
|
|
175
|
+
result = await client.query_agent(
|
|
176
|
+
"SELECT id, name FROM my_catalog.my_schema.my_table ORDER BY id",
|
|
177
|
+
max_rows=1000,
|
|
178
|
+
max_bytes=10000,
|
|
179
|
+
max_cell_chars=500,
|
|
180
|
+
total_timeout_s=60.0,
|
|
181
|
+
# catalog=None, schema=None, params=None
|
|
182
|
+
)
|
|
183
|
+
print(result)
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
Column headers include Arrow types. Strings are quoted, so SQL `NULL` differs
|
|
187
|
+
from the string `"NULL"`; Markdown, HTML, pipes and line breaks are escaped.
|
|
188
|
+
Clipped cells carry `…[truncated]`. The footer gives `rows_shown`, `complete`,
|
|
189
|
+
and an exact `total_rows` only when the result is exhausted. Row, byte or cell
|
|
190
|
+
clipping makes `complete=false`; an unexhausted result reports only a fetched
|
|
191
|
+
lower bound. Every returned string, including its header and footer, fits
|
|
192
|
+
`max_bytes` in UTF-8. Row and cell limits must be positive, with `max_bytes >= 256`; the timeout
|
|
193
|
+
must be finite and non-negative. Invalid budgets raise before SQL submission.
|
|
194
|
+
|
|
195
|
+
The single timeout covers submission, polling, downloads and formatting.
|
|
196
|
+
Stopping requests best-effort server cancellation; cancelling the Python task
|
|
197
|
+
preserves `asyncio.CancelledError`. A failure after some rows were fetched
|
|
198
|
+
returns an error JSON string instead of claiming a successful partial result.
|
|
199
|
+
Database diagnostics and suggestions are preserved; corrected SQL is never
|
|
200
|
+
executed automatically. Pass named parameters with `params`, using the same
|
|
201
|
+
list of `{"name": ..., "value": ..., "type": ...}` objects as the other APIs.
|
|
202
|
+
|
|
203
|
+
These budgets bound the returned preview, not the warehouse scan or its cost.
|
|
204
|
+
Pre-execution cost estimates and percentage progress are unavailable. This
|
|
205
|
+
method does not hash the full result, fetch its last rows or offer persistent
|
|
206
|
+
result paging. Use the cursor API when you need to consume a larger result.
|
|
207
|
+
|
|
208
|
+
[`examples/agent_mode.py`](examples/agent_mode.py) is runnable with
|
|
209
|
+
`DATABRICKS_HOST`, `DATABRICKS_WAREHOUSE_ID` and `DATABRICKS_TOKEN`:
|
|
210
|
+
|
|
211
|
+
```bash
|
|
212
|
+
python examples/agent_mode.py --protocol sea --sql 'SELECT 1 AS answer'
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
The example delegates to the installed CLI: Markdown goes to stdout and error
|
|
216
|
+
JSON goes to stderr. Its exit codes are 0 for Markdown, 1 for a query error and
|
|
217
|
+
2 for invalid configuration. A clipped Markdown preview is still a successful query; inspect
|
|
218
|
+
its completeness footer before using it as a complete dataset.
|
|
219
|
+
|
|
77
220
|
## FastAPI SSE example
|
|
78
221
|
|
|
79
222
|
`stream_query_json` is the full-speed way to serve a query over HTTP: the first row reaches the client after roughly one chunk's fetch/decode time, not the whole query's -- the Rust core is fetching, decoding, and reordering chunks concurrently the entire time, and the `HEARTBEAT`s keep the connection alive through a slow cold warehouse start instead of the client just seeing dead air:
|
|
@@ -366,3 +509,4 @@ real data (`--mode rows` or `--mode ipc`), to distinguish CPU work from network
|
|
|
366
509
|
## License
|
|
367
510
|
|
|
368
511
|
MIT
|
|
512
|
+
|
|
@@ -1,25 +1,16 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: arrowbricks
|
|
3
|
-
Version: 5.2.1
|
|
4
|
-
Requires-Dist: arro3-core>=0.8 ; extra == 'arro3'
|
|
5
|
-
Provides-Extra: arro3
|
|
6
|
-
License-File: LICENSE
|
|
7
|
-
Summary: Runs SQL against a Databricks SQL warehouse via the Statement Execution API and hands you the result as Arrow -- a DB-API-ish Cursor (fetchone/fetchmany/fetchall/fetchall_arrow) or NDJSON streaming. Rust/PyO3 core throughout -- zero required runtime dependencies.
|
|
8
|
-
License-Expression: MIT
|
|
9
|
-
Requires-Python: >=3.11
|
|
10
|
-
Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
|
|
11
|
-
Project-URL: Repository, https://github.com/bmsuisse/arrowbricks
|
|
12
|
-
|
|
13
1
|
<p align="center">
|
|
14
2
|
<img src="assets/logo.svg" width="72" height="72" alt="arrowbricks logo">
|
|
15
3
|
</p>
|
|
16
4
|
|
|
17
5
|
<h1 align="center">arrowbricks</h1>
|
|
18
|
-
<p align="center">Databricks SQL
|
|
6
|
+
<p align="center">Rust-powered Databricks SQL for applications and agents: Arrow streaming, compact Markdown, and actionable diagnostics.</p>
|
|
7
|
+
|
|
8
|
+
Arrowbricks is a Databricks SQL client and agent toolkit with a Rust execution and rendering core. Applications can consume lazy Arrow results or streaming NDJSON; agents can use the CLI or `query_agent` for compact typed Markdown and actionable JSON diagnostics. Version 6.0 adds strict preview budgets, explicit completeness metadata, total deadlines, cancellation, and a reusable agent skill, with zero required Python dependencies.
|
|
19
9
|
|
|
20
|
-
|
|
10
|
+
The Python `Cursor` also offers familiar [`databricks-sql-python`](https://github.com/databricks/databricks-sql-python)-style methods (`execute`, `fetchone`/`fetchmany`/`fetchall`, `fetchall_arrow`/`fetchmany_arrow`). Connect over Thrift by default (`protocol="thrift"`) or the REST Statement Execution API (`protocol="sea"`).
|
|
21
11
|
|
|
22
12
|
- **Rust core.** Statement submit/poll, bounded-concurrency chunk fetch, the reorder buffer, and Arrow-IPC decode all run in a [PyO3](https://pyo3.rs)/[arrow-rs](https://github.com/apache/arrow-rs) extension bundled in this same package -- 1.6x-2.5x faster than a pure-Python/asyncio client on a multi-chunk result, scaling further with chunk count and concurrency where asyncio+GIL plateaus.
|
|
13
|
+
- **Agent CLI and Python API.** Render directly from Arrow to bounded typed Markdown in Rust. Row, output-byte, cell and deadline budgets make previews predictable; the footer identifies incomplete or clipped results, and JSON errors retain the database's suggestions with a recovery hint.
|
|
23
14
|
- **Thrift by default, SEA/REST as a fully-supported alternative** (`protocol="sea"` on `connect()`/`DatabricksClient(...)`). Thrift speaks the same HiveServer2-compatible protocol `databricks-sql-connector` uses by default -- measurably faster for *small* results (a statement's data can come back inline in the very same call that submits it, instead of SEA's separate poll-then-fetch round trip), and on par with SEA for a large, multi-chunk result too (its chunk downloads fan out concurrently across the whole result, same as SEA's own bounded-concurrency fetch, not serialized batch by batch) -- never slower than SEA on any query shape tested. Confirmed against a real production warehouse; see `AGENTS.md`'s design-invariant entry for the benchmarking history behind the switch.
|
|
24
15
|
- **Compressed cloud-fetch transport by default.** Every statement requests LZ4-compressed chunk downloads (same default as the official `databricks-sql-connector`) and decompresses them in Rust before you ever see the bytes -- less data over the wire, which matters more than local decode speed for a large result. Measured ~2x faster chunk-fetch time against a real 120-column/100k-row table. Disable per-client with `compress_results=False` (`connect()`/`DatabricksClient(...)`) if your link to the warehouse is fast enough that decompression CPU time stops paying for itself.
|
|
25
16
|
- **Zero required dependencies.** `pip install arrowbricks` and go.
|
|
@@ -36,6 +27,78 @@ pip install arrowbricks
|
|
|
36
27
|
|
|
37
28
|
Ships as precompiled platform wheels (Linux/macOS/Windows) -- no Rust toolchain needed, and nothing else to install for most of the API. Row-tuple fetches (`fetchone`/`fetchmany`/`fetchall`) need one optional extra: `pip install arrowbricks[arro3]` -- see [Arrow vs. row-tuple fetches](#arrow-vs-row-tuple-fetches) below.
|
|
38
29
|
|
|
30
|
+
## CLI quickstart (6.0)
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
pip install 'arrowbricks>=6.0'
|
|
34
|
+
export DATABRICKS_HOST='your-workspace.azuredatabricks.net'
|
|
35
|
+
export DATABRICKS_WAREHOUSE_ID='your-warehouse-id'
|
|
36
|
+
export DATABRICKS_TOKEN='your-access-token'
|
|
37
|
+
|
|
38
|
+
arrowbricks -c 'SELECT 1 AS answer'
|
|
39
|
+
# | answer (Int32) |
|
|
40
|
+
# | --- |
|
|
41
|
+
# | 1 |
|
|
42
|
+
#
|
|
43
|
+
# rows_shown=1; complete=true; total_rows=1
|
|
44
|
+
|
|
45
|
+
arrowbricks --agent -c 'SELECT * FROM your_catalog.your_schema.your_table' --max-rows 20 --max-bytes 4000
|
|
46
|
+
arrowbricks -agent --file query.sql
|
|
47
|
+
printf 'SELECT 1 AS answer' | arrowbricks
|
|
48
|
+
python -m arrowbricks --protocol sea --command 'SELECT 1 AS answer'
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
Agent output is the default; `--agent` and `-agent` are optional aliases.
|
|
52
|
+
`-c`/`--sql`/`--command` sends SQL unchanged. `-f`/`--file` reads a UTF-8 file
|
|
53
|
+
(or `-` for stdin); without either option, piped stdin supplies SQL. Interactive
|
|
54
|
+
invocations without SQL return a configuration error. Flags `--host`,
|
|
55
|
+
`--warehouse-id` and `--token` override the corresponding environment variables;
|
|
56
|
+
`--catalog` and `--schema` set the query namespace.
|
|
57
|
+
|
|
58
|
+
The default limits are `--max-rows 1000`, `--max-bytes 10000`,
|
|
59
|
+
`--max-cell-chars 500` and `--total-timeout-s 60` (`--timeout` is an alias).
|
|
60
|
+
Clipped previews have `complete=false`; they remain successful queries with
|
|
61
|
+
exit code 0. These limits bound the returned preview and stop further fetching
|
|
62
|
+
once full; they do not limit the warehouse's scan cost. See [Agent results](#agent-results) for completeness and cancellation.
|
|
63
|
+
|
|
64
|
+
Success writes only Markdown to stdout, so it can be piped directly into an
|
|
65
|
+
agent. Query failures write JSON to stderr and exit 1; invalid configuration
|
|
66
|
+
writes JSON to stderr and exits 2. Help uses the usual `--help` output and exits
|
|
67
|
+
0. Ctrl-C interrupts the query, writes JSON to stderr and exits 130.
|
|
68
|
+
Credentials are never included in help output. For example:
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
arrowbricks -c 'SELEC 1' 2>error.json
|
|
72
|
+
# exit 1; error.json contains category, original message and next_step
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
Existing Python cursor and streaming APIs remain available in 6.0. `query_agent`
|
|
76
|
+
is an explicit Python API; it does not change the output of existing methods.
|
|
77
|
+
|
|
78
|
+
## Agent skill
|
|
79
|
+
|
|
80
|
+
The reusable [Arrowbricks skill](skills/arrowbricks/SKILL.md) teaches agents the
|
|
81
|
+
CLI and Python API, secure environment authentication, preview completeness,
|
|
82
|
+
JSON recovery and the limits of scan/cost information. The skill is distributed
|
|
83
|
+
in this repository; installing the Python package supplies the CLI and API.
|
|
84
|
+
It targets 6.0+: for an unreleased checkout, install its built wheel; the
|
|
85
|
+
`pip install` examples apply once a compatible release is published.
|
|
86
|
+
|
|
87
|
+
From a repository checkout, copy the skill folder into your agent's skill
|
|
88
|
+
directory. For project discovery with Codex:
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
mkdir -p .agents/skills
|
|
92
|
+
cp -R skills/arrowbricks .agents/skills/arrowbricks
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
For Claude Code, use `.claude/skills/arrowbricks` instead; for a personal
|
|
96
|
+
installation, use your agent's corresponding user skill directory. Keep
|
|
97
|
+
`SKILL.md` inside the `arrowbricks` folder. After your agent reloads its skills,
|
|
98
|
+
invoke `$arrowbricks` in Codex or `/arrowbricks` in Claude Code, or ask it to use
|
|
99
|
+
Arrowbricks for a Databricks SQL preview. Credentials remain in your environment
|
|
100
|
+
or credential provider, separate from the skill.
|
|
101
|
+
|
|
39
102
|
## Quickstart
|
|
40
103
|
|
|
41
104
|
```python
|
|
@@ -86,6 +149,62 @@ client-credentials flow instead -- like `databricks-sql-connector`'s
|
|
|
86
149
|
`auth_type="databricks-oauth"`, but stdlib-only (`urllib.request`), no
|
|
87
150
|
extra dependency.
|
|
88
151
|
|
|
152
|
+
## Agent results
|
|
153
|
+
|
|
154
|
+
`query_agent` returns one compact string: a typed Markdown table on success,
|
|
155
|
+
or JSON with `category`, `message` and `next_step` on database or transport
|
|
156
|
+
failure. It formats Arrow directly in Rust and needs no optional Python Arrow
|
|
157
|
+
packages. Both `thrift` (default) and `sea` are supported.
|
|
158
|
+
|
|
159
|
+
```python
|
|
160
|
+
from arrowbricks import DatabricksClient
|
|
161
|
+
|
|
162
|
+
async with DatabricksClient(host=..., warehouse_id=..., token=...) as client:
|
|
163
|
+
result = await client.query_agent(
|
|
164
|
+
"SELECT id, name FROM my_catalog.my_schema.my_table ORDER BY id",
|
|
165
|
+
max_rows=1000,
|
|
166
|
+
max_bytes=10000,
|
|
167
|
+
max_cell_chars=500,
|
|
168
|
+
total_timeout_s=60.0,
|
|
169
|
+
# catalog=None, schema=None, params=None
|
|
170
|
+
)
|
|
171
|
+
print(result)
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
Column headers include Arrow types. Strings are quoted, so SQL `NULL` differs
|
|
175
|
+
from the string `"NULL"`; Markdown, HTML, pipes and line breaks are escaped.
|
|
176
|
+
Clipped cells carry `…[truncated]`. The footer gives `rows_shown`, `complete`,
|
|
177
|
+
and an exact `total_rows` only when the result is exhausted. Row, byte or cell
|
|
178
|
+
clipping makes `complete=false`; an unexhausted result reports only a fetched
|
|
179
|
+
lower bound. Every returned string, including its header and footer, fits
|
|
180
|
+
`max_bytes` in UTF-8. Row and cell limits must be positive, with `max_bytes >= 256`; the timeout
|
|
181
|
+
must be finite and non-negative. Invalid budgets raise before SQL submission.
|
|
182
|
+
|
|
183
|
+
The single timeout covers submission, polling, downloads and formatting.
|
|
184
|
+
Stopping requests best-effort server cancellation; cancelling the Python task
|
|
185
|
+
preserves `asyncio.CancelledError`. A failure after some rows were fetched
|
|
186
|
+
returns an error JSON string instead of claiming a successful partial result.
|
|
187
|
+
Database diagnostics and suggestions are preserved; corrected SQL is never
|
|
188
|
+
executed automatically. Pass named parameters with `params`, using the same
|
|
189
|
+
list of `{"name": ..., "value": ..., "type": ...}` objects as the other APIs.
|
|
190
|
+
|
|
191
|
+
These budgets bound the returned preview, not the warehouse scan or its cost.
|
|
192
|
+
Pre-execution cost estimates and percentage progress are unavailable. This
|
|
193
|
+
method does not hash the full result, fetch its last rows or offer persistent
|
|
194
|
+
result paging. Use the cursor API when you need to consume a larger result.
|
|
195
|
+
|
|
196
|
+
[`examples/agent_mode.py`](examples/agent_mode.py) is runnable with
|
|
197
|
+
`DATABRICKS_HOST`, `DATABRICKS_WAREHOUSE_ID` and `DATABRICKS_TOKEN`:
|
|
198
|
+
|
|
199
|
+
```bash
|
|
200
|
+
python examples/agent_mode.py --protocol sea --sql 'SELECT 1 AS answer'
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
The example delegates to the installed CLI: Markdown goes to stdout and error
|
|
204
|
+
JSON goes to stderr. Its exit codes are 0 for Markdown, 1 for a query error and
|
|
205
|
+
2 for invalid configuration. A clipped Markdown preview is still a successful query; inspect
|
|
206
|
+
its completeness footer before using it as a complete dataset.
|
|
207
|
+
|
|
89
208
|
## FastAPI SSE example
|
|
90
209
|
|
|
91
210
|
`stream_query_json` is the full-speed way to serve a query over HTTP: the first row reaches the client after roughly one chunk's fetch/decode time, not the whole query's -- the Rust core is fetching, decoding, and reordering chunks concurrently the entire time, and the `HEARTBEAT`s keep the connection alive through a slow cold warehouse start instead of the client just seeing dead air:
|
|
@@ -378,4 +497,3 @@ real data (`--mode rows` or `--mode ipc`), to distinguish CPU work from network
|
|
|
378
497
|
## License
|
|
379
498
|
|
|
380
499
|
MIT
|
|
381
|
-
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "arrowbricks"
|
|
3
|
-
version = "
|
|
4
|
-
description = "
|
|
3
|
+
version = "6.0.0"
|
|
4
|
+
description = "Rust-powered Databricks SQL client and agent CLI with Arrow/NDJSON streaming, bounded Markdown previews, actionable JSON diagnostics, and zero required Python dependencies."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "MIT"
|
|
7
7
|
license-files = ["LICENSE"]
|
|
@@ -14,6 +14,9 @@ requires-python = ">=3.11"
|
|
|
14
14
|
# Cursor.description all work with zero dependencies installed.
|
|
15
15
|
dependencies = []
|
|
16
16
|
|
|
17
|
+
[project.scripts]
|
|
18
|
+
arrowbricks = "arrowbricks.cli:main"
|
|
19
|
+
|
|
17
20
|
[project.optional-dependencies]
|
|
18
21
|
# Row materialization converts our capsule table through arro3-core.
|
|
19
22
|
# Arrow fetches and streaming do not need this optional dependency.
|
|
@@ -177,10 +177,11 @@ dependencies = [
|
|
|
177
177
|
|
|
178
178
|
[[package]]
|
|
179
179
|
name = "arrowbricks_core"
|
|
180
|
-
version = "
|
|
180
|
+
version = "6.0.0"
|
|
181
181
|
dependencies = [
|
|
182
182
|
"arrow-array",
|
|
183
183
|
"arrow-buffer",
|
|
184
|
+
"arrow-cast",
|
|
184
185
|
"arrow-ipc",
|
|
185
186
|
"arrow-json",
|
|
186
187
|
"arrow-schema",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[package]
|
|
2
2
|
name = "arrowbricks_core"
|
|
3
|
-
version = "
|
|
3
|
+
version = "6.0.0"
|
|
4
4
|
edition = "2024"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
|
|
@@ -19,6 +19,7 @@ arrow-buffer = "59.1.0"
|
|
|
19
19
|
arrow-ipc = "59.1.0"
|
|
20
20
|
arrow-schema = "59.1.0"
|
|
21
21
|
arrow-json = "59.1.0"
|
|
22
|
+
arrow-cast = "59.1.0"
|
|
22
23
|
bytes = "1.12.1"
|
|
23
24
|
# Already in the dependency graph; vectorized newline search in NDJSON output.
|
|
24
25
|
memchr = "2"
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
use arrow_array::types::Int32Type;
|
|
2
|
+
use arrow_array::{ArrayRef, Float64Array, Int64Array, ListArray, RecordBatch, StringArray};
|
|
3
|
+
use arrow_schema::{Field, Schema};
|
|
4
|
+
use arrowbricks_core::agent::render::MarkdownRenderer;
|
|
5
|
+
use std::{hint::black_box, sync::Arc, time::Instant};
|
|
6
|
+
|
|
7
|
+
fn batch(columns: Vec<ArrayRef>) -> RecordBatch {
|
|
8
|
+
let fields: Vec<_> = columns
|
|
9
|
+
.iter()
|
|
10
|
+
.enumerate()
|
|
11
|
+
.map(|(i, a)| Field::new(format!("c{i}"), a.data_type().clone(), true))
|
|
12
|
+
.collect();
|
|
13
|
+
RecordBatch::try_new(Arc::new(Schema::new(fields)), columns).unwrap()
|
|
14
|
+
}
|
|
15
|
+
fn direct(b: &RecordBatch, bytes: usize, interrupts: bool) -> String {
|
|
16
|
+
let mut renderer = if interrupts {
|
|
17
|
+
MarkdownRenderer::new_with_interrupt(
|
|
18
|
+
b.schema(),
|
|
19
|
+
1000,
|
|
20
|
+
bytes,
|
|
21
|
+
500,
|
|
22
|
+
Instant::now() + std::time::Duration::from_secs(60),
|
|
23
|
+
Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
|
24
|
+
)
|
|
25
|
+
} else {
|
|
26
|
+
MarkdownRenderer::new(b.schema(), 1000, bytes, 500)
|
|
27
|
+
}
|
|
28
|
+
.unwrap();
|
|
29
|
+
renderer.push_batch(b).unwrap();
|
|
30
|
+
renderer.finish(true)
|
|
31
|
+
}
|
|
32
|
+
fn json(b: &RecordBatch) -> Vec<serde_json::Value> {
|
|
33
|
+
let mut writer = arrow_json::ArrayWriter::new(Vec::new());
|
|
34
|
+
writer.write_batches(&[b]).unwrap();
|
|
35
|
+
writer.finish().unwrap();
|
|
36
|
+
serde_json::from_slice(&writer.into_inner()).unwrap()
|
|
37
|
+
}
|
|
38
|
+
fn json_markdown(b: &RecordBatch, bytes: usize) -> String {
|
|
39
|
+
let values = json(b);
|
|
40
|
+
let mut output = String::new();
|
|
41
|
+
for row in &values {
|
|
42
|
+
let mut line = String::from("|");
|
|
43
|
+
for field in b.schema().fields() {
|
|
44
|
+
let value = &row[field.name()];
|
|
45
|
+
let owned;
|
|
46
|
+
let text = if let Some(value) = value.as_str() {
|
|
47
|
+
value
|
|
48
|
+
} else {
|
|
49
|
+
owned = value.to_string();
|
|
50
|
+
&owned
|
|
51
|
+
};
|
|
52
|
+
line.push(' ');
|
|
53
|
+
if value.is_string() {
|
|
54
|
+
line.push('"');
|
|
55
|
+
}
|
|
56
|
+
for ch in text.chars().take(500) {
|
|
57
|
+
match ch {
|
|
58
|
+
'&' => line.push_str("&"),
|
|
59
|
+
'<' => line.push_str("<"),
|
|
60
|
+
'>' => line.push_str(">"),
|
|
61
|
+
'\n' => line.push_str("\\n"),
|
|
62
|
+
'\r' => line.push_str("\\r"),
|
|
63
|
+
'\t' => line.push_str("\\t"),
|
|
64
|
+
'\\' | '|' | '`' | '*' | '_' | '[' | ']' | '~' | '"' => {
|
|
65
|
+
line.push('\\');
|
|
66
|
+
line.push(ch);
|
|
67
|
+
}
|
|
68
|
+
c => line.push(c),
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
if value.is_string() {
|
|
72
|
+
line.push('"');
|
|
73
|
+
}
|
|
74
|
+
line.push_str(" |");
|
|
75
|
+
}
|
|
76
|
+
line.push('\n');
|
|
77
|
+
if output.len() + line.len() > bytes.saturating_sub(160) {
|
|
78
|
+
break;
|
|
79
|
+
}
|
|
80
|
+
output.push_str(&line);
|
|
81
|
+
}
|
|
82
|
+
output
|
|
83
|
+
}
|
|
84
|
+
fn time(mut f: impl FnMut(), loops: usize) -> f64 {
|
|
85
|
+
let start = Instant::now();
|
|
86
|
+
for _ in 0..loops {
|
|
87
|
+
f();
|
|
88
|
+
}
|
|
89
|
+
start.elapsed().as_secs_f64() * 1e6 / loops as f64
|
|
90
|
+
}
|
|
91
|
+
fn main() {
|
|
92
|
+
let agent_only = std::env::args().any(|arg| arg == "--agent-only");
|
|
93
|
+
let n = 1000;
|
|
94
|
+
let numeric = batch(vec![
|
|
95
|
+
Arc::new(Int64Array::from_iter_values(0..n)),
|
|
96
|
+
Arc::new(Float64Array::from_iter_values((0..n).map(|i| i as f64 * 0.1))),
|
|
97
|
+
]);
|
|
98
|
+
let unicode = batch(vec![
|
|
99
|
+
Arc::new(StringArray::from(vec!["é🦀|<>&\\".repeat(100); n as usize])),
|
|
100
|
+
Arc::new(ListArray::from_iter_primitive::<Int32Type, _, _>(
|
|
101
|
+
(0..n).map(|_| Some(vec![Some(1), None, Some(3)])),
|
|
102
|
+
)),
|
|
103
|
+
]);
|
|
104
|
+
let ascii = batch(vec![Arc::new(StringArray::from(vec![
|
|
105
|
+
"ordinary query results without markup ".repeat(20); n as usize
|
|
106
|
+
]))]);
|
|
107
|
+
let controls = batch(vec![Arc::new(StringArray::from(vec![
|
|
108
|
+
"a\0\u{0001}b\t\n".repeat(30);
|
|
109
|
+
n as usize
|
|
110
|
+
]))]);
|
|
111
|
+
let huge = batch(vec![Arc::new(StringArray::from(vec!["🦀".repeat(1_000_000)]))]);
|
|
112
|
+
let mut builder = arrow_array::builder::ListBuilder::new(arrow_array::builder::StringBuilder::new());
|
|
113
|
+
builder.values().append_value("🦀".repeat(2_500_000));
|
|
114
|
+
builder.append(true);
|
|
115
|
+
let nested_huge = batch(vec![Arc::new(builder.finish())]);
|
|
116
|
+
for (name, b, loops) in [
|
|
117
|
+
("numeric", numeric, 1000),
|
|
118
|
+
("unicode_nested", unicode, 100),
|
|
119
|
+
("ascii_strings", ascii, 100),
|
|
120
|
+
("controls", controls, 100),
|
|
121
|
+
("4MB_string", huge, 20),
|
|
122
|
+
("10MB_nested_string", nested_huge, 10),
|
|
123
|
+
] {
|
|
124
|
+
let bounded = time(
|
|
125
|
+
|| {
|
|
126
|
+
black_box(direct(&b, 10_000, agent_only));
|
|
127
|
+
},
|
|
128
|
+
if agent_only { 10000 } else { loops },
|
|
129
|
+
);
|
|
130
|
+
let full = time(
|
|
131
|
+
|| {
|
|
132
|
+
black_box(direct(&b, 2_000_000, agent_only));
|
|
133
|
+
},
|
|
134
|
+
loops,
|
|
135
|
+
);
|
|
136
|
+
if agent_only {
|
|
137
|
+
println!("{name}: direct_10KB={bounded:.2}us direct_2MB={full:.2}us");
|
|
138
|
+
continue;
|
|
139
|
+
}
|
|
140
|
+
let json_render = time(
|
|
141
|
+
|| {
|
|
142
|
+
black_box(json_markdown(&b, 10_000));
|
|
143
|
+
},
|
|
144
|
+
loops,
|
|
145
|
+
);
|
|
146
|
+
let via_json = time(
|
|
147
|
+
|| {
|
|
148
|
+
black_box(json(&b));
|
|
149
|
+
},
|
|
150
|
+
loops,
|
|
151
|
+
);
|
|
152
|
+
println!(
|
|
153
|
+
"{name}: direct_10KB={bounded:.2}us direct_2MB={full:.2}us JSON_materialize={via_json:.2}us JSON_then_Markdown_10KB={json_render:.2}us"
|
|
154
|
+
);
|
|
155
|
+
}
|
|
156
|
+
}
|