arrowbricks 6.1.0__tar.gz → 6.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/PKG-INFO +19 -8
  2. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/README.md +18 -7
  3. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/pyproject.toml +4 -1
  4. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/Cargo.lock +1 -1
  5. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/Cargo.toml +1 -1
  6. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/examples/agent_render_bench.rs +4 -3
  7. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/agent/mod.rs +7 -3
  8. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/agent/render.rs +304 -31
  9. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/client/download.rs +152 -11
  10. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/client/volume.rs +30 -1
  11. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/lib.rs +44 -5
  12. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests/wiremock_thrift.rs +85 -8
  13. arrowbricks-6.3.0/rust/arrowbricks_core/tests/wiremock_volume_files.rs +213 -0
  14. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/src/arrowbricks/__init__.py +2 -0
  15. arrowbricks-6.3.0/src/arrowbricks/_agent.py +11 -0
  16. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/src/arrowbricks/_core.pyi +6 -2
  17. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/src/arrowbricks/_streaming.py +11 -7
  18. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/src/arrowbricks/cli.py +26 -4
  19. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/src/arrowbricks/client.py +31 -4
  20. arrowbricks-6.1.0/rust/arrowbricks_core/tests/wiremock_volume_files.rs +0 -87
  21. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/LICENSE +0 -0
  22. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/.gitignore +0 -0
  23. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/README.md +0 -0
  24. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/examples/duckdb_query.py +0 -0
  25. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/examples/fastapi_sse.py +0 -0
  26. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/proptest-regressions/json_convert.txt +0 -0
  27. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/rustfmt.toml +0 -0
  28. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/agent/diagnostics.rs +0 -0
  29. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/alloc.rs +0 -0
  30. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/arrow_ffi.rs +0 -0
  31. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/client/call_context.rs +0 -0
  32. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/client/error.rs +0 -0
  33. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/client/model.rs +0 -0
  34. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/client/sea.rs +0 -0
  35. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/client/thrift_rpc.rs +0 -0
  36. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/client.rs +0 -0
  37. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/heartbeat.rs +0 -0
  38. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/json_convert.rs +0 -0
  39. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/pipeline/ndjson.rs +0 -0
  40. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/pipeline/reorder.rs +0 -0
  41. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/pipeline/sea.rs +0 -0
  42. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/pipeline/stats.rs +0 -0
  43. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/pipeline/test_support.rs +0 -0
  44. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/pipeline/thrift_exec.rs +0 -0
  45. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/pipeline.rs +0 -0
  46. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/src/thrift.rs +0 -0
  47. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests/common/mod.rs +0 -0
  48. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests/wiremock_agent.rs +0 -0
  49. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests/wiremock_pipeline.rs +0 -0
  50. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests_py/conftest.py +0 -0
  51. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests_py/test_ipc_stream.py +0 -0
  52. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests_py/test_parameters.py +0 -0
  53. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests_py/test_stream_ndjson_lines.py +0 -0
  54. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests_py/test_streaming.py +0 -0
  55. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests_py/test_thrift.py +0 -0
  56. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests_py/test_token_provider.py +0 -0
  57. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests_py/test_volume_files.py +0 -0
  58. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/rust/arrowbricks_core/tests_py/thrift_mock.py +0 -0
  59. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/src/arrowbricks/__main__.py +0 -0
  60. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/src/arrowbricks/_sql.py +0 -0
  61. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/src/arrowbricks/cursor.py +0 -0
  62. {arrowbricks-6.1.0 → arrowbricks-6.3.0}/src/arrowbricks/py.typed +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: arrowbricks
3
- Version: 6.1.0
3
+ Version: 6.3.0
4
4
  Requires-Dist: arro3-core>=0.8 ; extra == 'arro3'
5
5
  Requires-Dist: sqlglot>=27 ; extra == 'sqlglot'
6
6
  Provides-Extra: arro3
@@ -20,8 +20,8 @@ Project-URL: Repository, https://github.com/bmsuisse/arrowbricks
20
20
  <p align="center">Fast Databricks SQL for Python applications and AI agents.</p>
21
21
 
22
22
  - **Zero required Python dependencies.** One precompiled wheel. No Rust toolchain or cloud SDK needed.
23
- - **Fast Rust core.** Concurrent chunk downloads, lazy Arrow decoding, and streaming NDJSON, with result order preserved.
24
- - **Agent mode, new in 6.0.** A CLI and Python API that return compact typed Markdown, explicit truncation, and actionable JSON errors. Formatting stays in Rust.
23
+ - **Fast Rust core.** Concurrent chunk downloads, lazy Arrow decoding, and streaming NDJSON without materializing the full result, with result order preserved.
24
+ - **Agent mode, new in 6.0.** A CLI and Python API that return compact typed TSV (or a Markdown table), explicit truncation, and actionable JSON errors. Formatting stays in Rust.
25
25
 
26
26
  Use Thrift by default or the Statement Execution API with `protocol="sea"`.
27
27
  Bring a token or your own token-refresh callable.
@@ -48,14 +48,17 @@ export DATABRICKS_TOKEN='your-access-token'
48
48
  arrowbricks -c 'SELECT 1 AS answer'
49
49
  ```
50
50
 
51
- ```markdown
52
- | answer (Int32) |
53
- | --- |
54
- | 1 |
51
+ ```text
52
+ answer:Int32
53
+ 1
55
54
 
56
55
  rows_shown=1; complete=true; total_rows=1
57
56
  ```
58
57
 
58
+ Output is tab-separated by default (cheapest to read; tabs, newlines and
59
+ backslashes in values are escaped, NULL is `\N`). `--format markdown` /
60
+ `output_format="markdown"` gives a pipe table instead.
61
+
59
62
  Agent mode is the CLI default. SQL files and piped stdin work too:
60
63
 
61
64
  ```bash
@@ -75,7 +78,7 @@ from arrowbricks import DatabricksClient
75
78
 
76
79
  async with DatabricksClient(host=..., warehouse_id=..., token=...) as client:
77
80
  text = await client.query_agent("SELECT 1 AS answer", max_bytes=4000)
78
- print(text) # Markdown on success; JSON on query failure
81
+ print(text) # TSV on success; JSON on query failure
79
82
  ```
80
83
 
81
84
  Give your agent the reusable [Arrowbricks skill](skills/arrowbricks/SKILL.md).
@@ -115,6 +118,14 @@ a 100-row preview used **44% fewer bytes and 24% fewer tokens** than NDJSON.
115
118
  See [agent measurements and methodology](docs/research/agent-rendering.md) and
116
119
  [client benchmarks](docs/benchmarks.md) for workloads, comparisons, and limits.
117
120
 
121
+ ### Performance tips
122
+
123
+ Numbers are measurements from one ~30 MB/s link to one warehouse, not guarantees.
124
+
125
+ - **Select only the columns you need.** Selecting 3 columns instead of `SELECT *` was 1.5-6x faster and moved 1.9-15.8x fewer bytes across 5 tables.
126
+ - **Keep `compress_results` at its default (`True`)** unless the link is very fast. Here it was 1.8-4.7x faster with 2-5.5x fewer bytes. Turning it off on a very fast link was not tested.
127
+ - **Keep the Thrift default for small lookups.** It was about 0.15 s faster per tiny statement than `protocol="sea"` (a constant gap in these runs).
128
+
118
129
  ## Learn more
119
130
 
120
131
  - [Usage and API reference](docs/usage.md): authentication, errors, cancellation, observability, and Arrow interop.
@@ -6,8 +6,8 @@
6
6
  <p align="center">Fast Databricks SQL for Python applications and AI agents.</p>
7
7
 
8
8
  - **Zero required Python dependencies.** One precompiled wheel. No Rust toolchain or cloud SDK needed.
9
- - **Fast Rust core.** Concurrent chunk downloads, lazy Arrow decoding, and streaming NDJSON, with result order preserved.
10
- - **Agent mode, new in 6.0.** A CLI and Python API that return compact typed Markdown, explicit truncation, and actionable JSON errors. Formatting stays in Rust.
9
+ - **Fast Rust core.** Concurrent chunk downloads, lazy Arrow decoding, and streaming NDJSON without materializing the full result, with result order preserved.
10
+ - **Agent mode, new in 6.0.** A CLI and Python API that return compact typed TSV (or a Markdown table), explicit truncation, and actionable JSON errors. Formatting stays in Rust.
11
11
 
12
12
  Use Thrift by default or the Statement Execution API with `protocol="sea"`.
13
13
  Bring a token or your own token-refresh callable.
@@ -34,14 +34,17 @@ export DATABRICKS_TOKEN='your-access-token'
34
34
  arrowbricks -c 'SELECT 1 AS answer'
35
35
  ```
36
36
 
37
- ```markdown
38
- | answer (Int32) |
39
- | --- |
40
- | 1 |
37
+ ```text
38
+ answer:Int32
39
+ 1
41
40
 
42
41
  rows_shown=1; complete=true; total_rows=1
43
42
  ```
44
43
 
44
+ Output is tab-separated by default (cheapest to read; tabs, newlines and
45
+ backslashes in values are escaped, NULL is `\N`). `--format markdown` /
46
+ `output_format="markdown"` gives a pipe table instead.
47
+
45
48
  Agent mode is the CLI default. SQL files and piped stdin work too:
46
49
 
47
50
  ```bash
@@ -61,7 +64,7 @@ from arrowbricks import DatabricksClient
61
64
 
62
65
  async with DatabricksClient(host=..., warehouse_id=..., token=...) as client:
63
66
  text = await client.query_agent("SELECT 1 AS answer", max_bytes=4000)
64
- print(text) # Markdown on success; JSON on query failure
67
+ print(text) # TSV on success; JSON on query failure
65
68
  ```
66
69
 
67
70
  Give your agent the reusable [Arrowbricks skill](skills/arrowbricks/SKILL.md).
@@ -101,6 +104,14 @@ a 100-row preview used **44% fewer bytes and 24% fewer tokens** than NDJSON.
101
104
  See [agent measurements and methodology](docs/research/agent-rendering.md) and
102
105
  [client benchmarks](docs/benchmarks.md) for workloads, comparisons, and limits.
103
106
 
107
+ ### Performance tips
108
+
109
+ Numbers are measurements from one ~30 MB/s link to one warehouse, not guarantees.
110
+
111
+ - **Select only the columns you need.** Selecting 3 columns instead of `SELECT *` was 1.5-6x faster and moved 1.9-15.8x fewer bytes across 5 tables.
112
+ - **Keep `compress_results` at its default (`True`)** unless the link is very fast. Here it was 1.8-4.7x faster with 2-5.5x fewer bytes. Turning it off on a very fast link was not tested.
113
+ - **Keep the Thrift default for small lookups.** It was about 0.15 s faster per tiny statement than `protocol="sea"` (a constant gap in these runs).
114
+
104
115
  ## Learn more
105
116
 
106
117
  - [Usage and API reference](docs/usage.md): authentication, errors, cancellation, observability, and Arrow interop.
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "arrowbricks"
3
- version = "6.1.0"
3
+ version = "6.3.0"
4
4
  description = "Rust-powered Databricks SQL client and agent CLI with Arrow/NDJSON streaming, bounded Markdown previews, actionable JSON diagnostics, and zero required Python dependencies."
5
5
  readme = "README.md"
6
6
  license = "MIT"
@@ -103,3 +103,6 @@ select = ["E", "F", "I", "UP", "S", "B"]
103
103
  "tests/*" = ["S101", "S106"]
104
104
  "rust/arrowbricks_core/tests_py/*" = ["S101", "S106"]
105
105
  "tests/arro3_free/*" = ["S101", "S106"]
106
+ # One-off benchmark harness: terse scripts, seeded (non-crypto) randomness, asserts as checks.
107
+ "benchmarks/agent-output-format/*" = ["B905", "E501", "E731", "E741", "S101", "S311"]
108
+ "benchmarks/benchmark_duckdb.py" = ["E501", "S608"]
@@ -177,7 +177,7 @@ dependencies = [
177
177
 
178
178
  [[package]]
179
179
  name = "arrowbricks_core"
180
- version = "6.1.0"
180
+ version = "6.3.0"
181
181
  dependencies = [
182
182
  "arrow-array",
183
183
  "arrow-buffer",
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "arrowbricks_core"
3
- version = "6.1.0"
3
+ version = "6.3.0"
4
4
  edition = "2024"
5
5
  readme = "README.md"
6
6
 
@@ -1,7 +1,7 @@
1
1
  use arrow_array::types::Int32Type;
2
2
  use arrow_array::{ArrayRef, Float64Array, Int64Array, ListArray, RecordBatch, StringArray};
3
3
  use arrow_schema::{Field, Schema};
4
- use arrowbricks_core::agent::render::MarkdownRenderer;
4
+ use arrowbricks_core::agent::render::{AgentRenderer, OutputFormat};
5
5
  use std::{hint::black_box, sync::Arc, time::Instant};
6
6
 
7
7
  fn batch(columns: Vec<ArrayRef>) -> RecordBatch {
@@ -14,16 +14,17 @@ fn batch(columns: Vec<ArrayRef>) -> RecordBatch {
14
14
  }
15
15
  fn direct(b: &RecordBatch, bytes: usize, interrupts: bool) -> String {
16
16
  let mut renderer = if interrupts {
17
- MarkdownRenderer::new_with_interrupt(
17
+ AgentRenderer::new_with_interrupt(
18
18
  b.schema(),
19
19
  1000,
20
20
  bytes,
21
21
  500,
22
+ OutputFormat::Markdown,
22
23
  Instant::now() + std::time::Duration::from_secs(60),
23
24
  Arc::new(std::sync::atomic::AtomicBool::new(false)),
24
25
  )
25
26
  } else {
26
- MarkdownRenderer::new(b.schema(), 1000, bytes, 500)
27
+ AgentRenderer::new(b.schema(), 1000, bytes, 500, OutputFormat::Markdown)
27
28
  }
28
29
  .unwrap();
29
30
  renderer.push_batch(b).unwrap();
@@ -10,7 +10,8 @@ use serde_json::Value;
10
10
 
11
11
  use crate::client::{ApiError, DbClient, Protocol};
12
12
  use crate::pipeline;
13
- use render::MarkdownRenderer;
13
+ use render::AgentRenderer;
14
+ pub use render::OutputFormat;
14
15
 
15
16
  #[derive(Clone, Copy)]
16
17
  pub struct AgentOptions {
@@ -18,6 +19,7 @@ pub struct AgentOptions {
18
19
  pub max_bytes: usize,
19
20
  pub max_cell_chars: usize,
20
21
  pub total_timeout_s: f64,
22
+ pub format: OutputFormat,
21
23
  }
22
24
 
23
25
  impl Default for AgentOptions {
@@ -27,6 +29,7 @@ impl Default for AgentOptions {
27
29
  max_bytes: 10_000,
28
30
  max_cell_chars: 500,
29
31
  total_timeout_s: 60.0,
32
+ format: OutputFormat::default(),
30
33
  }
31
34
  }
32
35
  }
@@ -158,7 +161,7 @@ pub async fn query(
158
161
  while !finished {
159
162
  let want = renderer
160
163
  .as_ref()
161
- .map_or(128, |r: &MarkdownRenderer| {
164
+ .map_or(128, |r: &AgentRenderer| {
162
165
  options.max_rows.saturating_sub(r.rows()).min(128)
163
166
  })
164
167
  .min(options.max_rows);
@@ -167,11 +170,12 @@ pub async fn query(
167
170
  let mut render_task = tokio::task::spawn_blocking(move || {
168
171
  let mut renderer = match renderer {
169
172
  Some(renderer) => renderer,
170
- None => MarkdownRenderer::new_with_interrupt(
173
+ None => AgentRenderer::new_with_interrupt(
171
174
  schema.unwrap_or_else(|| Arc::new(Schema::empty())),
172
175
  options.max_rows,
173
176
  options.max_bytes,
174
177
  options.max_cell_chars,
178
+ options.format,
175
179
  deadline.into_std(),
176
180
  interrupted,
177
181
  )?,