arrowbricks 6.1.0__tar.gz → 6.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/PKG-INFO +19 -8
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/README.md +18 -7
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/pyproject.toml +4 -1
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/Cargo.lock +1 -1
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/Cargo.toml +1 -1
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/examples/agent_render_bench.rs +4 -3
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/agent/mod.rs +7 -3
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/agent/render.rs +304 -31
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/client/download.rs +152 -11
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/client/volume.rs +30 -1
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/lib.rs +44 -5
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests/wiremock_thrift.rs +85 -8
- arrowbricks-6.3.0/rust/arrowbricks_core/tests/wiremock_volume_files.rs +213 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/src/arrowbricks/__init__.py +2 -0
- arrowbricks-6.3.0/src/arrowbricks/_agent.py +11 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/src/arrowbricks/_core.pyi +6 -2
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/src/arrowbricks/_streaming.py +11 -7
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/src/arrowbricks/cli.py +26 -4
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/src/arrowbricks/client.py +31 -4
- arrowbricks-6.1.0/rust/arrowbricks_core/tests/wiremock_volume_files.rs +0 -87
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/LICENSE +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/.gitignore +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/README.md +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/examples/duckdb_query.py +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/examples/fastapi_sse.py +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/proptest-regressions/json_convert.txt +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/rustfmt.toml +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/agent/diagnostics.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/alloc.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/arrow_ffi.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/client/call_context.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/client/error.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/client/model.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/client/sea.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/client/thrift_rpc.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/client.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/heartbeat.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/json_convert.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/pipeline/ndjson.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/pipeline/reorder.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/pipeline/sea.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/pipeline/stats.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/pipeline/test_support.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/pipeline/thrift_exec.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/pipeline.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/thrift.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests/common/mod.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests/wiremock_agent.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests/wiremock_pipeline.rs +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests_py/conftest.py +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests_py/test_ipc_stream.py +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests_py/test_parameters.py +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests_py/test_stream_ndjson_lines.py +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests_py/test_streaming.py +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests_py/test_thrift.py +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests_py/test_token_provider.py +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests_py/test_volume_files.py +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests_py/thrift_mock.py +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/src/arrowbricks/__main__.py +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/src/arrowbricks/_sql.py +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/src/arrowbricks/cursor.py +0 -0
- {arrowbricks-6.1.0 → arrowbricks-6.3.0}/src/arrowbricks/py.typed +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: arrowbricks
|
|
3
|
-
Version: 6.
|
|
3
|
+
Version: 6.3.0
|
|
4
4
|
Requires-Dist: arro3-core>=0.8 ; extra == 'arro3'
|
|
5
5
|
Requires-Dist: sqlglot>=27 ; extra == 'sqlglot'
|
|
6
6
|
Provides-Extra: arro3
|
|
@@ -20,8 +20,8 @@ Project-URL: Repository, https://github.com/bmsuisse/arrowbricks
|
|
|
20
20
|
<p align="center">Fast Databricks SQL for Python applications and AI agents.</p>
|
|
21
21
|
|
|
22
22
|
- **Zero required Python dependencies.** One precompiled wheel. No Rust toolchain or cloud SDK needed.
|
|
23
|
-
- **Fast Rust core.** Concurrent chunk downloads, lazy Arrow decoding, and streaming NDJSON, with result order preserved.
|
|
24
|
-
- **Agent mode, new in 6.0.** A CLI and Python API that return compact typed Markdown, explicit truncation, and actionable JSON errors. Formatting stays in Rust.
|
|
23
|
+
- **Fast Rust core.** Concurrent chunk downloads, lazy Arrow decoding, and streaming NDJSON without materializing the full result, with result order preserved.
|
|
24
|
+
- **Agent mode, new in 6.0.** A CLI and Python API that return compact typed TSV (or a Markdown table), explicit truncation, and actionable JSON errors. Formatting stays in Rust.
|
|
25
25
|
|
|
26
26
|
Use Thrift by default or the Statement Execution API with `protocol="sea"`.
|
|
27
27
|
Bring a token or your own token-refresh callable.
|
|
@@ -48,14 +48,17 @@ export DATABRICKS_TOKEN='your-access-token'
|
|
|
48
48
|
arrowbricks -c 'SELECT 1 AS answer'
|
|
49
49
|
```
|
|
50
50
|
|
|
51
|
-
```
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
| 1 |
|
|
51
|
+
```text
|
|
52
|
+
answer:Int32
|
|
53
|
+
1
|
|
55
54
|
|
|
56
55
|
rows_shown=1; complete=true; total_rows=1
|
|
57
56
|
```
|
|
58
57
|
|
|
58
|
+
Output is tab-separated by default (cheapest to read; tabs, newlines and
|
|
59
|
+
backslashes in values are escaped, NULL is `\N`). `--format markdown` /
|
|
60
|
+
`output_format="markdown"` gives a pipe table instead.
|
|
61
|
+
|
|
59
62
|
Agent mode is the CLI default. SQL files and piped stdin work too:
|
|
60
63
|
|
|
61
64
|
```bash
|
|
@@ -75,7 +78,7 @@ from arrowbricks import DatabricksClient
|
|
|
75
78
|
|
|
76
79
|
async with DatabricksClient(host=..., warehouse_id=..., token=...) as client:
|
|
77
80
|
text = await client.query_agent("SELECT 1 AS answer", max_bytes=4000)
|
|
78
|
-
print(text) #
|
|
81
|
+
print(text) # TSV on success; JSON on query failure
|
|
79
82
|
```
|
|
80
83
|
|
|
81
84
|
Give your agent the reusable [Arrowbricks skill](skills/arrowbricks/SKILL.md).
|
|
@@ -115,6 +118,14 @@ a 100-row preview used **44% fewer bytes and 24% fewer tokens** than NDJSON.
|
|
|
115
118
|
See [agent measurements and methodology](docs/research/agent-rendering.md) and
|
|
116
119
|
[client benchmarks](docs/benchmarks.md) for workloads, comparisons, and limits.
|
|
117
120
|
|
|
121
|
+
### Performance tips
|
|
122
|
+
|
|
123
|
+
Numbers are measurements from one ~30 MB/s link to one warehouse, not guarantees.
|
|
124
|
+
|
|
125
|
+
- **Select only the columns you need.** Selecting 3 columns instead of `SELECT *` was 1.5-6x faster and moved 1.9-15.8x fewer bytes across 5 tables.
|
|
126
|
+
- **Keep `compress_results` at its default (`True`)** unless the link is very fast. Here it was 1.8-4.7x faster with 2-5.5x fewer bytes. Turning it off on a very fast link was not tested.
|
|
127
|
+
- **Keep the Thrift default for small lookups.** It was about 0.15 s faster per tiny statement than `protocol="sea"` (a constant gap in these runs).
|
|
128
|
+
|
|
118
129
|
## Learn more
|
|
119
130
|
|
|
120
131
|
- [Usage and API reference](docs/usage.md): authentication, errors, cancellation, observability, and Arrow interop.
|
|
@@ -6,8 +6,8 @@
|
|
|
6
6
|
<p align="center">Fast Databricks SQL for Python applications and AI agents.</p>
|
|
7
7
|
|
|
8
8
|
- **Zero required Python dependencies.** One precompiled wheel. No Rust toolchain or cloud SDK needed.
|
|
9
|
-
- **Fast Rust core.** Concurrent chunk downloads, lazy Arrow decoding, and streaming NDJSON, with result order preserved.
|
|
10
|
-
- **Agent mode, new in 6.0.** A CLI and Python API that return compact typed Markdown, explicit truncation, and actionable JSON errors. Formatting stays in Rust.
|
|
9
|
+
- **Fast Rust core.** Concurrent chunk downloads, lazy Arrow decoding, and streaming NDJSON without materializing the full result, with result order preserved.
|
|
10
|
+
- **Agent mode, new in 6.0.** A CLI and Python API that return compact typed TSV (or a Markdown table), explicit truncation, and actionable JSON errors. Formatting stays in Rust.
|
|
11
11
|
|
|
12
12
|
Use Thrift by default or the Statement Execution API with `protocol="sea"`.
|
|
13
13
|
Bring a token or your own token-refresh callable.
|
|
@@ -34,14 +34,17 @@ export DATABRICKS_TOKEN='your-access-token'
|
|
|
34
34
|
arrowbricks -c 'SELECT 1 AS answer'
|
|
35
35
|
```
|
|
36
36
|
|
|
37
|
-
```
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
| 1 |
|
|
37
|
+
```text
|
|
38
|
+
answer:Int32
|
|
39
|
+
1
|
|
41
40
|
|
|
42
41
|
rows_shown=1; complete=true; total_rows=1
|
|
43
42
|
```
|
|
44
43
|
|
|
44
|
+
Output is tab-separated by default (cheapest to read; tabs, newlines and
|
|
45
|
+
backslashes in values are escaped, NULL is `\N`). `--format markdown` /
|
|
46
|
+
`output_format="markdown"` gives a pipe table instead.
|
|
47
|
+
|
|
45
48
|
Agent mode is the CLI default. SQL files and piped stdin work too:
|
|
46
49
|
|
|
47
50
|
```bash
|
|
@@ -61,7 +64,7 @@ from arrowbricks import DatabricksClient
|
|
|
61
64
|
|
|
62
65
|
async with DatabricksClient(host=..., warehouse_id=..., token=...) as client:
|
|
63
66
|
text = await client.query_agent("SELECT 1 AS answer", max_bytes=4000)
|
|
64
|
-
print(text) #
|
|
67
|
+
print(text) # TSV on success; JSON on query failure
|
|
65
68
|
```
|
|
66
69
|
|
|
67
70
|
Give your agent the reusable [Arrowbricks skill](skills/arrowbricks/SKILL.md).
|
|
@@ -101,6 +104,14 @@ a 100-row preview used **44% fewer bytes and 24% fewer tokens** than NDJSON.
|
|
|
101
104
|
See [agent measurements and methodology](docs/research/agent-rendering.md) and
|
|
102
105
|
[client benchmarks](docs/benchmarks.md) for workloads, comparisons, and limits.
|
|
103
106
|
|
|
107
|
+
### Performance tips
|
|
108
|
+
|
|
109
|
+
Numbers are measurements from one ~30 MB/s link to one warehouse, not guarantees.
|
|
110
|
+
|
|
111
|
+
- **Select only the columns you need.** Selecting 3 columns instead of `SELECT *` was 1.5-6x faster and moved 1.9-15.8x fewer bytes across 5 tables.
|
|
112
|
+
- **Keep `compress_results` at its default (`True`)** unless the link is very fast. Here it was 1.8-4.7x faster with 2-5.5x fewer bytes. Turning it off on a very fast link was not tested.
|
|
113
|
+
- **Keep the Thrift default for small lookups.** It was about 0.15 s faster per tiny statement than `protocol="sea"` (a constant gap in these runs).
|
|
114
|
+
|
|
104
115
|
## Learn more
|
|
105
116
|
|
|
106
117
|
- [Usage and API reference](docs/usage.md): authentication, errors, cancellation, observability, and Arrow interop.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "arrowbricks"
|
|
3
|
-
version = "6.
|
|
3
|
+
version = "6.3.0"
|
|
4
4
|
description = "Rust-powered Databricks SQL client and agent CLI with Arrow/NDJSON streaming, bounded Markdown previews, actionable JSON diagnostics, and zero required Python dependencies."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "MIT"
|
|
@@ -103,3 +103,6 @@ select = ["E", "F", "I", "UP", "S", "B"]
|
|
|
103
103
|
"tests/*" = ["S101", "S106"]
|
|
104
104
|
"rust/arrowbricks_core/tests_py/*" = ["S101", "S106"]
|
|
105
105
|
"tests/arro3_free/*" = ["S101", "S106"]
|
|
106
|
+
# One-off benchmark harness: terse scripts, seeded (non-crypto) randomness, asserts as checks.
|
|
107
|
+
"benchmarks/agent-output-format/*" = ["B905", "E501", "E731", "E741", "S101", "S311"]
|
|
108
|
+
"benchmarks/benchmark_duckdb.py" = ["E501", "S608"]
|
{arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/examples/agent_render_bench.rs
RENAMED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
use arrow_array::types::Int32Type;
|
|
2
2
|
use arrow_array::{ArrayRef, Float64Array, Int64Array, ListArray, RecordBatch, StringArray};
|
|
3
3
|
use arrow_schema::{Field, Schema};
|
|
4
|
-
use arrowbricks_core::agent::render::
|
|
4
|
+
use arrowbricks_core::agent::render::{AgentRenderer, OutputFormat};
|
|
5
5
|
use std::{hint::black_box, sync::Arc, time::Instant};
|
|
6
6
|
|
|
7
7
|
fn batch(columns: Vec<ArrayRef>) -> RecordBatch {
|
|
@@ -14,16 +14,17 @@ fn batch(columns: Vec<ArrayRef>) -> RecordBatch {
|
|
|
14
14
|
}
|
|
15
15
|
fn direct(b: &RecordBatch, bytes: usize, interrupts: bool) -> String {
|
|
16
16
|
let mut renderer = if interrupts {
|
|
17
|
-
|
|
17
|
+
AgentRenderer::new_with_interrupt(
|
|
18
18
|
b.schema(),
|
|
19
19
|
1000,
|
|
20
20
|
bytes,
|
|
21
21
|
500,
|
|
22
|
+
OutputFormat::Markdown,
|
|
22
23
|
Instant::now() + std::time::Duration::from_secs(60),
|
|
23
24
|
Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
|
24
25
|
)
|
|
25
26
|
} else {
|
|
26
|
-
|
|
27
|
+
AgentRenderer::new(b.schema(), 1000, bytes, 500, OutputFormat::Markdown)
|
|
27
28
|
}
|
|
28
29
|
.unwrap();
|
|
29
30
|
renderer.push_batch(b).unwrap();
|
|
@@ -10,7 +10,8 @@ use serde_json::Value;
|
|
|
10
10
|
|
|
11
11
|
use crate::client::{ApiError, DbClient, Protocol};
|
|
12
12
|
use crate::pipeline;
|
|
13
|
-
use render::
|
|
13
|
+
use render::AgentRenderer;
|
|
14
|
+
pub use render::OutputFormat;
|
|
14
15
|
|
|
15
16
|
#[derive(Clone, Copy)]
|
|
16
17
|
pub struct AgentOptions {
|
|
@@ -18,6 +19,7 @@ pub struct AgentOptions {
|
|
|
18
19
|
pub max_bytes: usize,
|
|
19
20
|
pub max_cell_chars: usize,
|
|
20
21
|
pub total_timeout_s: f64,
|
|
22
|
+
pub format: OutputFormat,
|
|
21
23
|
}
|
|
22
24
|
|
|
23
25
|
impl Default for AgentOptions {
|
|
@@ -27,6 +29,7 @@ impl Default for AgentOptions {
|
|
|
27
29
|
max_bytes: 10_000,
|
|
28
30
|
max_cell_chars: 500,
|
|
29
31
|
total_timeout_s: 60.0,
|
|
32
|
+
format: OutputFormat::default(),
|
|
30
33
|
}
|
|
31
34
|
}
|
|
32
35
|
}
|
|
@@ -158,7 +161,7 @@ pub async fn query(
|
|
|
158
161
|
while !finished {
|
|
159
162
|
let want = renderer
|
|
160
163
|
.as_ref()
|
|
161
|
-
.map_or(128, |r: &
|
|
164
|
+
.map_or(128, |r: &AgentRenderer| {
|
|
162
165
|
options.max_rows.saturating_sub(r.rows()).min(128)
|
|
163
166
|
})
|
|
164
167
|
.min(options.max_rows);
|
|
@@ -167,11 +170,12 @@ pub async fn query(
|
|
|
167
170
|
let mut render_task = tokio::task::spawn_blocking(move || {
|
|
168
171
|
let mut renderer = match renderer {
|
|
169
172
|
Some(renderer) => renderer,
|
|
170
|
-
None =>
|
|
173
|
+
None => AgentRenderer::new_with_interrupt(
|
|
171
174
|
schema.unwrap_or_else(|| Arc::new(Schema::empty())),
|
|
172
175
|
options.max_rows,
|
|
173
176
|
options.max_bytes,
|
|
174
177
|
options.max_cell_chars,
|
|
178
|
+
options.format,
|
|
175
179
|
deadline.into_std(),
|
|
176
180
|
interrupted,
|
|
177
181
|
)?,
|