arrowbricks 5.2.2__tar.gz → 6.0.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. arrowbricks-6.0.1/PKG-INFO +125 -0
  2. arrowbricks-6.0.1/README.md +112 -0
  3. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/pyproject.toml +5 -2
  4. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/Cargo.lock +2 -1
  5. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/Cargo.toml +2 -1
  6. arrowbricks-6.0.1/rust/arrowbricks_core/examples/agent_render_bench.rs +156 -0
  7. arrowbricks-6.0.1/rust/arrowbricks_core/src/agent/diagnostics.rs +188 -0
  8. arrowbricks-6.0.1/rust/arrowbricks_core/src/agent/mod.rs +355 -0
  9. arrowbricks-6.0.1/rust/arrowbricks_core/src/agent/render.rs +646 -0
  10. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/client/sea.rs +13 -1
  11. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/lib.rs +38 -0
  12. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/pipeline/ndjson.rs +1 -0
  13. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/pipeline/sea.rs +22 -2
  14. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/pipeline/stats.rs +2 -1
  15. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/pipeline/thrift_exec.rs +41 -24
  16. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/pipeline.rs +2 -0
  17. arrowbricks-6.0.1/rust/arrowbricks_core/tests/wiremock_agent.rs +317 -0
  18. arrowbricks-6.0.1/src/arrowbricks/__main__.py +5 -0
  19. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/src/arrowbricks/_core.pyi +12 -1
  20. arrowbricks-6.0.1/src/arrowbricks/cli.py +119 -0
  21. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/src/arrowbricks/client.py +32 -0
  22. arrowbricks-5.2.2/PKG-INFO +0 -381
  23. arrowbricks-5.2.2/README.md +0 -368
  24. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/LICENSE +0 -0
  25. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/.gitignore +0 -0
  26. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/README.md +0 -0
  27. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/examples/duckdb_query.py +0 -0
  28. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/examples/fastapi_sse.py +0 -0
  29. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/proptest-regressions/json_convert.txt +0 -0
  30. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/rustfmt.toml +0 -0
  31. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/alloc.rs +0 -0
  32. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/arrow_ffi.rs +0 -0
  33. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/client/call_context.rs +0 -0
  34. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/client/download.rs +0 -0
  35. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/client/error.rs +0 -0
  36. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/client/model.rs +0 -0
  37. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/client/thrift_rpc.rs +0 -0
  38. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/client/volume.rs +0 -0
  39. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/client.rs +0 -0
  40. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/heartbeat.rs +0 -0
  41. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/json_convert.rs +0 -0
  42. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/pipeline/reorder.rs +0 -0
  43. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/pipeline/test_support.rs +0 -0
  44. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/thrift.rs +0 -0
  45. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/tests/common/mod.rs +0 -0
  46. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/tests/wiremock_pipeline.rs +0 -0
  47. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/tests/wiremock_thrift.rs +0 -0
  48. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/tests/wiremock_volume_files.rs +0 -0
  49. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/tests_py/conftest.py +0 -0
  50. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/tests_py/test_ipc_stream.py +0 -0
  51. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/tests_py/test_parameters.py +0 -0
  52. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/tests_py/test_stream_ndjson_lines.py +0 -0
  53. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/tests_py/test_streaming.py +0 -0
  54. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/tests_py/test_thrift.py +0 -0
  55. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/tests_py/test_token_provider.py +0 -0
  56. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/tests_py/test_volume_files.py +0 -0
  57. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/tests_py/thrift_mock.py +0 -0
  58. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/src/arrowbricks/__init__.py +0 -0
  59. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/src/arrowbricks/_streaming.py +0 -0
  60. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/src/arrowbricks/cursor.py +0 -0
  61. {arrowbricks-5.2.2 → arrowbricks-6.0.1}/src/arrowbricks/py.typed +0 -0
@@ -0,0 +1,125 @@
1
+ Metadata-Version: 2.4
2
+ Name: arrowbricks
3
+ Version: 6.0.1
4
+ Requires-Dist: arro3-core>=0.8 ; extra == 'arro3'
5
+ Provides-Extra: arro3
6
+ License-File: LICENSE
7
+ Summary: Rust-powered Databricks SQL client and agent CLI with Arrow/NDJSON streaming, bounded Markdown previews, actionable JSON diagnostics, and zero required Python dependencies.
8
+ License-Expression: MIT
9
+ Requires-Python: >=3.11
10
+ Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
11
+ Project-URL: Repository, https://github.com/bmsuisse/arrowbricks
12
+
13
+ <p align="center">
14
+ <img src="assets/logo.svg" width="72" height="72" alt="arrowbricks logo">
15
+ </p>
16
+
17
+ <h1 align="center">arrowbricks</h1>
18
+ <p align="center">Fast Databricks SQL for Python applications and AI agents.</p>
19
+
20
+ - **Zero required Python dependencies.** One precompiled wheel. No Rust toolchain or cloud SDK needed.
21
+ - **Fast Rust core.** Concurrent chunk downloads, lazy Arrow decoding, and streaming NDJSON, with result order preserved.
22
+ - **Agent mode, new in 6.0.** A CLI and Python API that return compact typed Markdown, explicit truncation, and actionable JSON errors. Formatting stays in Rust.
23
+
24
+ Use Thrift by default or the Statement Execution API with `protocol="sea"`.
25
+ Bring a token or your own token-refresh callable.
26
+
27
+ ## Install
28
+
29
+ ```bash
30
+ pip install arrowbricks
31
+ ```
32
+
33
+ Wheels cover Linux, macOS, and Windows; Python 3.11+.
34
+ Arrow, NDJSON, and agent mode work without extras. Python row tuples need
35
+ `pip install 'arrowbricks[arro3]'`.
36
+
37
+ ## Query from an agent or terminal
38
+
39
+ ```bash
40
+ export DATABRICKS_HOST='your-workspace.azuredatabricks.net'
41
+ export DATABRICKS_WAREHOUSE_ID='your-warehouse-id'
42
+ export DATABRICKS_TOKEN='your-access-token'
43
+
44
+ arrowbricks -c 'SELECT 1 AS answer'
45
+ ```
46
+
47
+ ```markdown
48
+ | answer (Int32) |
49
+ | --- |
50
+ | 1 |
51
+
52
+ rows_shown=1; complete=true; total_rows=1
53
+ ```
54
+
55
+ Agent mode is the CLI default. SQL files and piped stdin work too:
56
+
57
+ ```bash
58
+ arrowbricks --file query.sql --max-rows 20 --max-bytes 4000
59
+ printf 'SELECT 1 AS answer' | arrowbricks
60
+ ```
61
+
62
+ Previews have row, byte, cell, and deadline limits. The footer tells you when
63
+ results are incomplete. Query errors go to stderr as JSON, preserving the
64
+ warehouse's diagnostic and suggestions with a recovery hint; the CLI exits 1.
65
+ Preview limits bound output and fetching, not warehouse scan cost.
66
+
67
+ For Python agents:
68
+
69
+ ```python
70
+ from arrowbricks import DatabricksClient
71
+
72
+ async with DatabricksClient(host=..., warehouse_id=..., token=...) as client:
73
+ text = await client.query_agent("SELECT 1 AS answer", max_bytes=4000)
74
+ print(text) # Markdown on success; JSON on query failure
75
+ ```
76
+
77
+ Give your agent the reusable [Arrowbricks skill](skills/arrowbricks/SKILL.md).
78
+ See [CLI options and skill installation](docs/usage.md#cli-quickstart-60) and
79
+ [agent budgets, completeness, and cancellation](docs/usage.md#agent-results).
80
+
81
+ ## Query from an application
82
+
83
+ The async cursor offers familiar `execute` and `fetch*` methods.
84
+ Arrow results work directly with DuckDB, PyArrow, and Polars:
85
+
86
+ ```python
87
+ from arrowbricks import connect
88
+
89
+ async with connect(host=..., warehouse_id=..., token=...) as conn:
90
+ cursor = conn.cursor()
91
+ await cursor.execute("SELECT * FROM my_catalog.my_schema.my_table LIMIT 100")
92
+ table = await cursor.fetchall_arrow()
93
+ ```
94
+
95
+ Or consume ready-to-send NDJSON without building Python rows:
96
+
97
+ ```python
98
+ from arrowbricks import HEARTBEAT, DatabricksClient
99
+
100
+ async with DatabricksClient(host=..., warehouse_id=..., token=...) as client:
101
+ async for item in client.stream_query_json("SELECT * FROM range(100)"):
102
+ if item is not HEARTBEAT:
103
+ print(item)
104
+ ```
105
+
106
+ ## Performance
107
+
108
+ The query and formatting work runs in Rust. Agent previews render directly from
109
+ Arrow, avoiding intermediate Python rows or JSON. In a real warehouse test,
110
+ a 100-row preview used **44% fewer bytes and 24% fewer tokens** than NDJSON.
111
+ See [agent measurements and methodology](docs/research/agent-rendering.md) and
112
+ [client benchmarks](docs/benchmarks.md) for workloads, comparisons, and limits.
113
+
114
+ ## Learn more
115
+
116
+ - [Usage and API reference](docs/usage.md): authentication, errors, cancellation, observability, and Arrow interop.
117
+ - [Runnable examples](examples/): cursors, agents, OAuth, and FastAPI streaming.
118
+ - [Rust core](rust/arrowbricks_core/README.md): implementation and native APIs.
119
+ - [Real warehouse verification](docs/usage.md#real-warehouse-verification).
120
+ - [Changelog](CHANGELOG.md).
121
+
122
+ ## License
123
+
124
+ MIT
125
+
@@ -0,0 +1,112 @@
1
+ <p align="center">
2
+ <img src="assets/logo.svg" width="72" height="72" alt="arrowbricks logo">
3
+ </p>
4
+
5
+ <h1 align="center">arrowbricks</h1>
6
+ <p align="center">Fast Databricks SQL for Python applications and AI agents.</p>
7
+
8
+ - **Zero required Python dependencies.** One precompiled wheel. No Rust toolchain or cloud SDK needed.
9
+ - **Fast Rust core.** Concurrent chunk downloads, lazy Arrow decoding, and streaming NDJSON, with result order preserved.
10
+ - **Agent mode, new in 6.0.** A CLI and Python API that return compact typed Markdown, explicit truncation, and actionable JSON errors. Formatting stays in Rust.
11
+
12
+ Use Thrift by default or the Statement Execution API with `protocol="sea"`.
13
+ Bring a token or your own token-refresh callable.
14
+
15
+ ## Install
16
+
17
+ ```bash
18
+ pip install arrowbricks
19
+ ```
20
+
21
+ Wheels cover Linux, macOS, and Windows; Python 3.11+.
22
+ Arrow, NDJSON, and agent mode work without extras. Python row tuples need
23
+ `pip install 'arrowbricks[arro3]'`.
24
+
25
+ ## Query from an agent or terminal
26
+
27
+ ```bash
28
+ export DATABRICKS_HOST='your-workspace.azuredatabricks.net'
29
+ export DATABRICKS_WAREHOUSE_ID='your-warehouse-id'
30
+ export DATABRICKS_TOKEN='your-access-token'
31
+
32
+ arrowbricks -c 'SELECT 1 AS answer'
33
+ ```
34
+
35
+ ```markdown
36
+ | answer (Int32) |
37
+ | --- |
38
+ | 1 |
39
+
40
+ rows_shown=1; complete=true; total_rows=1
41
+ ```
42
+
43
+ Agent mode is the CLI default. SQL files and piped stdin work too:
44
+
45
+ ```bash
46
+ arrowbricks --file query.sql --max-rows 20 --max-bytes 4000
47
+ printf 'SELECT 1 AS answer' | arrowbricks
48
+ ```
49
+
50
+ Previews have row, byte, cell, and deadline limits. The footer tells you when
51
+ results are incomplete. Query errors go to stderr as JSON, preserving the
52
+ warehouse's diagnostic and suggestions with a recovery hint; the CLI exits 1.
53
+ Preview limits bound output and fetching, not warehouse scan cost.
54
+
55
+ For Python agents:
56
+
57
+ ```python
58
+ from arrowbricks import DatabricksClient
59
+
60
+ async with DatabricksClient(host=..., warehouse_id=..., token=...) as client:
61
+ text = await client.query_agent("SELECT 1 AS answer", max_bytes=4000)
62
+ print(text) # Markdown on success; JSON on query failure
63
+ ```
64
+
65
+ Give your agent the reusable [Arrowbricks skill](skills/arrowbricks/SKILL.md).
66
+ See [CLI options and skill installation](docs/usage.md#cli-quickstart-60) and
67
+ [agent budgets, completeness, and cancellation](docs/usage.md#agent-results).
68
+
69
+ ## Query from an application
70
+
71
+ The async cursor offers familiar `execute` and `fetch*` methods.
72
+ Arrow results work directly with DuckDB, PyArrow, and Polars:
73
+
74
+ ```python
75
+ from arrowbricks import connect
76
+
77
+ async with connect(host=..., warehouse_id=..., token=...) as conn:
78
+ cursor = conn.cursor()
79
+ await cursor.execute("SELECT * FROM my_catalog.my_schema.my_table LIMIT 100")
80
+ table = await cursor.fetchall_arrow()
81
+ ```
82
+
83
+ Or consume ready-to-send NDJSON without building Python rows:
84
+
85
+ ```python
86
+ from arrowbricks import HEARTBEAT, DatabricksClient
87
+
88
+ async with DatabricksClient(host=..., warehouse_id=..., token=...) as client:
89
+ async for item in client.stream_query_json("SELECT * FROM range(100)"):
90
+ if item is not HEARTBEAT:
91
+ print(item)
92
+ ```
93
+
94
+ ## Performance
95
+
96
+ The query and formatting work runs in Rust. Agent previews render directly from
97
+ Arrow, avoiding intermediate Python rows or JSON. In a real warehouse test,
98
+ a 100-row preview used **44% fewer bytes and 24% fewer tokens** than NDJSON.
99
+ See [agent measurements and methodology](docs/research/agent-rendering.md) and
100
+ [client benchmarks](docs/benchmarks.md) for workloads, comparisons, and limits.
101
+
102
+ ## Learn more
103
+
104
+ - [Usage and API reference](docs/usage.md): authentication, errors, cancellation, observability, and Arrow interop.
105
+ - [Runnable examples](examples/): cursors, agents, OAuth, and FastAPI streaming.
106
+ - [Rust core](rust/arrowbricks_core/README.md): implementation and native APIs.
107
+ - [Real warehouse verification](docs/usage.md#real-warehouse-verification).
108
+ - [Changelog](CHANGELOG.md).
109
+
110
+ ## License
111
+
112
+ MIT
@@ -1,7 +1,7 @@
1
1
  [project]
2
2
  name = "arrowbricks"
3
- version = "5.2.2"
4
- description = "Runs SQL against a Databricks SQL warehouse via the Statement Execution API and hands you the result as Arrow -- a DB-API-ish Cursor (fetchone/fetchmany/fetchall/fetchall_arrow) or NDJSON streaming. Rust/PyO3 core throughout -- zero required runtime dependencies."
3
+ version = "6.0.1"
4
+ description = "Rust-powered Databricks SQL client and agent CLI with Arrow/NDJSON streaming, bounded Markdown previews, actionable JSON diagnostics, and zero required Python dependencies."
5
5
  readme = "README.md"
6
6
  license = "MIT"
7
7
  license-files = ["LICENSE"]
@@ -14,6 +14,9 @@ requires-python = ">=3.11"
14
14
  # Cursor.description all work with zero dependencies installed.
15
15
  dependencies = []
16
16
 
17
+ [project.scripts]
18
+ arrowbricks = "arrowbricks.cli:main"
19
+
17
20
  [project.optional-dependencies]
18
21
  # Row materialization converts our capsule table through arro3-core.
19
22
  # Arrow fetches and streaming do not need this optional dependency.
@@ -177,10 +177,11 @@ dependencies = [
177
177
 
178
178
  [[package]]
179
179
  name = "arrowbricks_core"
180
- version = "5.2.2"
180
+ version = "6.0.1"
181
181
  dependencies = [
182
182
  "arrow-array",
183
183
  "arrow-buffer",
184
+ "arrow-cast",
184
185
  "arrow-ipc",
185
186
  "arrow-json",
186
187
  "arrow-schema",
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "arrowbricks_core"
3
- version = "5.2.2"
3
+ version = "6.0.1"
4
4
  edition = "2024"
5
5
  readme = "README.md"
6
6
 
@@ -19,6 +19,7 @@ arrow-buffer = "59.1.0"
19
19
  arrow-ipc = "59.1.0"
20
20
  arrow-schema = "59.1.0"
21
21
  arrow-json = "59.1.0"
22
+ arrow-cast = "59.1.0"
22
23
  bytes = "1.12.1"
23
24
  # Already in the dependency graph; vectorized newline search in NDJSON output.
24
25
  memchr = "2"
@@ -0,0 +1,156 @@
1
+ use arrow_array::types::Int32Type;
2
+ use arrow_array::{ArrayRef, Float64Array, Int64Array, ListArray, RecordBatch, StringArray};
3
+ use arrow_schema::{Field, Schema};
4
+ use arrowbricks_core::agent::render::MarkdownRenderer;
5
+ use std::{hint::black_box, sync::Arc, time::Instant};
6
+
7
+ fn batch(columns: Vec<ArrayRef>) -> RecordBatch {
8
+ let fields: Vec<_> = columns
9
+ .iter()
10
+ .enumerate()
11
+ .map(|(i, a)| Field::new(format!("c{i}"), a.data_type().clone(), true))
12
+ .collect();
13
+ RecordBatch::try_new(Arc::new(Schema::new(fields)), columns).unwrap()
14
+ }
15
+ fn direct(b: &RecordBatch, bytes: usize, interrupts: bool) -> String {
16
+ let mut renderer = if interrupts {
17
+ MarkdownRenderer::new_with_interrupt(
18
+ b.schema(),
19
+ 1000,
20
+ bytes,
21
+ 500,
22
+ Instant::now() + std::time::Duration::from_secs(60),
23
+ Arc::new(std::sync::atomic::AtomicBool::new(false)),
24
+ )
25
+ } else {
26
+ MarkdownRenderer::new(b.schema(), 1000, bytes, 500)
27
+ }
28
+ .unwrap();
29
+ renderer.push_batch(b).unwrap();
30
+ renderer.finish(true)
31
+ }
32
+ fn json(b: &RecordBatch) -> Vec<serde_json::Value> {
33
+ let mut writer = arrow_json::ArrayWriter::new(Vec::new());
34
+ writer.write_batches(&[b]).unwrap();
35
+ writer.finish().unwrap();
36
+ serde_json::from_slice(&writer.into_inner()).unwrap()
37
+ }
38
+ fn json_markdown(b: &RecordBatch, bytes: usize) -> String {
39
+ let values = json(b);
40
+ let mut output = String::new();
41
+ for row in &values {
42
+ let mut line = String::from("|");
43
+ for field in b.schema().fields() {
44
+ let value = &row[field.name()];
45
+ let owned;
46
+ let text = if let Some(value) = value.as_str() {
47
+ value
48
+ } else {
49
+ owned = value.to_string();
50
+ &owned
51
+ };
52
+ line.push(' ');
53
+ if value.is_string() {
54
+ line.push('"');
55
+ }
56
+ for ch in text.chars().take(500) {
57
+ match ch {
58
+ '&' => line.push_str("&amp;"),
59
+ '<' => line.push_str("&lt;"),
60
+ '>' => line.push_str("&gt;"),
61
+ '\n' => line.push_str("\\n"),
62
+ '\r' => line.push_str("\\r"),
63
+ '\t' => line.push_str("\\t"),
64
+ '\\' | '|' | '`' | '*' | '_' | '[' | ']' | '~' | '"' => {
65
+ line.push('\\');
66
+ line.push(ch);
67
+ }
68
+ c => line.push(c),
69
+ }
70
+ }
71
+ if value.is_string() {
72
+ line.push('"');
73
+ }
74
+ line.push_str(" |");
75
+ }
76
+ line.push('\n');
77
+ if output.len() + line.len() > bytes.saturating_sub(160) {
78
+ break;
79
+ }
80
+ output.push_str(&line);
81
+ }
82
+ output
83
+ }
84
+ fn time(mut f: impl FnMut(), loops: usize) -> f64 {
85
+ let start = Instant::now();
86
+ for _ in 0..loops {
87
+ f();
88
+ }
89
+ start.elapsed().as_secs_f64() * 1e6 / loops as f64
90
+ }
91
+ fn main() {
92
+ let agent_only = std::env::args().any(|arg| arg == "--agent-only");
93
+ let n = 1000;
94
+ let numeric = batch(vec![
95
+ Arc::new(Int64Array::from_iter_values(0..n)),
96
+ Arc::new(Float64Array::from_iter_values((0..n).map(|i| i as f64 * 0.1))),
97
+ ]);
98
+ let unicode = batch(vec![
99
+ Arc::new(StringArray::from(vec!["é🦀|<>&\\".repeat(100); n as usize])),
100
+ Arc::new(ListArray::from_iter_primitive::<Int32Type, _, _>(
101
+ (0..n).map(|_| Some(vec![Some(1), None, Some(3)])),
102
+ )),
103
+ ]);
104
+ let ascii = batch(vec![Arc::new(StringArray::from(vec![
105
+ "ordinary query results without markup ".repeat(20); n as usize
106
+ ]))]);
107
+ let controls = batch(vec![Arc::new(StringArray::from(vec![
108
+ "a\0\u{0001}b\t\n".repeat(30);
109
+ n as usize
110
+ ]))]);
111
+ let huge = batch(vec![Arc::new(StringArray::from(vec!["🦀".repeat(1_000_000)]))]);
112
+ let mut builder = arrow_array::builder::ListBuilder::new(arrow_array::builder::StringBuilder::new());
113
+ builder.values().append_value("🦀".repeat(2_500_000));
114
+ builder.append(true);
115
+ let nested_huge = batch(vec![Arc::new(builder.finish())]);
116
+ for (name, b, loops) in [
117
+ ("numeric", numeric, 1000),
118
+ ("unicode_nested", unicode, 100),
119
+ ("ascii_strings", ascii, 100),
120
+ ("controls", controls, 100),
121
+ ("4MB_string", huge, 20),
122
+ ("10MB_nested_string", nested_huge, 10),
123
+ ] {
124
+ let bounded = time(
125
+ || {
126
+ black_box(direct(&b, 10_000, agent_only));
127
+ },
128
+ if agent_only { 10000 } else { loops },
129
+ );
130
+ let full = time(
131
+ || {
132
+ black_box(direct(&b, 2_000_000, agent_only));
133
+ },
134
+ loops,
135
+ );
136
+ if agent_only {
137
+ println!("{name}: direct_10KB={bounded:.2}us direct_2MB={full:.2}us");
138
+ continue;
139
+ }
140
+ let json_render = time(
141
+ || {
142
+ black_box(json_markdown(&b, 10_000));
143
+ },
144
+ loops,
145
+ );
146
+ let via_json = time(
147
+ || {
148
+ black_box(json(&b));
149
+ },
150
+ loops,
151
+ );
152
+ println!(
153
+ "{name}: direct_10KB={bounded:.2}us direct_2MB={full:.2}us JSON_materialize={via_json:.2}us JSON_then_Markdown_10KB={json_render:.2}us"
154
+ );
155
+ }
156
+ }
@@ -0,0 +1,188 @@
1
+ use crate::client::{ApiError, ApiErrorKind};
2
+ use serde_json::{Value, json};
3
+
4
+ pub(super) fn error_json(error: &ApiError, max_bytes: usize, timeout: bool) -> String {
5
+ let mut prefix_end = error.message.len().min(max_bytes);
6
+ while !error.message.is_char_boundary(prefix_end) {
7
+ prefix_end -= 1;
8
+ }
9
+ let message = &error.message[..prefix_end];
10
+ let shortened = prefix_end < error.message.len();
11
+ let upper = message.to_ascii_uppercase();
12
+ let (category, next_step) = if timeout {
13
+ (
14
+ "timeout",
15
+ "Increase total_timeout_s or narrow the query before retrying.",
16
+ )
17
+ } else if error.kind == ApiErrorKind::Auth {
18
+ ("authentication", "Refresh credentials and verify warehouse access.")
19
+ } else if upper.contains("UNRESOLVED_COLUMN") || upper.contains("UNRESOLVED_FIELD") {
20
+ (
21
+ "missing_column",
22
+ "Check column names against the table schema and database suggestions.",
23
+ )
24
+ } else if upper.contains("UNRESOLVED_ROUTINE") {
25
+ (
26
+ "missing_function",
27
+ "Check the function name and namespace with SHOW FUNCTIONS or DESCRIBE FUNCTION.",
28
+ )
29
+ } else if upper.contains("SCHEMA_NOT_FOUND") || upper.contains("CATALOG_NOT_FOUND") {
30
+ (
31
+ "missing_namespace",
32
+ "Verify catalog and schema names and access before submitting again.",
33
+ )
34
+ } else if upper.contains("TABLE_OR_VIEW_NOT_FOUND") {
35
+ ("missing_table", "Verify the table name, catalog, schema and access.")
36
+ } else if upper.contains("PARSE_SYNTAX_ERROR") || upper.contains("SYNTAX_ERROR") {
37
+ (
38
+ "syntax_error",
39
+ "Correct the SQL using the database diagnostic before submitting again.",
40
+ )
41
+ } else if upper.contains("PERMISSION") || upper.contains("INSUFFICIENT_PRIVILEGES") {
42
+ (
43
+ "permission_denied",
44
+ "Request access to the referenced objects or use an authorized object.",
45
+ )
46
+ } else if error.transient {
47
+ ("transient", "Check warehouse and network availability before retrying.")
48
+ } else {
49
+ (
50
+ "execution_error",
51
+ "Inspect the database diagnostic and verify the statement and warehouse.",
52
+ )
53
+ };
54
+ let error_classes = message.split('[').skip(1).filter_map(|part| {
55
+ let candidate = part.split(']').next()?;
56
+ (!candidate.is_empty()
57
+ && candidate.len() <= 128
58
+ && candidate
59
+ .bytes()
60
+ .all(|c| c.is_ascii_uppercase() || c.is_ascii_digit() || b"_.".contains(&c)))
61
+ .then_some(candidate)
62
+ });
63
+ let error_class = error_classes
64
+ .clone()
65
+ .find(|class| {
66
+ ["UNRESOLVED_", "TABLE_OR_VIEW_", "PARSE_", "SYNTAX_", "INSUFFICIENT_"]
67
+ .iter()
68
+ .any(|prefix| class.starts_with(prefix))
69
+ })
70
+ .or_else(|| error_classes.into_iter().next())
71
+ .or_else(|| {
72
+ message.split([' ', ':']).find(|token| {
73
+ ["UNRESOLVED_", "TABLE_OR_VIEW_", "PARSE_", "SYNTAX_", "INSUFFICIENT_"]
74
+ .iter()
75
+ .any(|prefix| token.starts_with(prefix))
76
+ })
77
+ });
78
+ let mut value = json!({
79
+ "category":category, "message":message, "next_step":next_step,
80
+ "error_class":error_class,
81
+ });
82
+ if shortened {
83
+ value["truncated"] = Value::Bool(true);
84
+ }
85
+ let output = value.to_string();
86
+ if output.len() <= max_bytes {
87
+ return output;
88
+ }
89
+ value["truncated"] = Value::Bool(true);
90
+ // Preserve the original diagnostic prefix within the caller's byte budget.
91
+ value.as_object_mut().unwrap().remove("error_class");
92
+ let fixed_bytes = {
93
+ value["message"] = Value::String(String::new());
94
+ value.to_string().len()
95
+ };
96
+ if fixed_bytes > max_bytes {
97
+ value["next_step"] = Value::String("Inspect the database diagnostic before retrying.".into());
98
+ }
99
+ let mut low = 0;
100
+ let mut high = message.len();
101
+ while low < high {
102
+ let middle = low + (high - low).div_ceil(2);
103
+ let mut end = middle;
104
+ while !message.is_char_boundary(end) {
105
+ end -= 1;
106
+ }
107
+ value["message"] = Value::String(message[..end].into());
108
+ if value.to_string().len() <= max_bytes {
109
+ low = middle;
110
+ } else {
111
+ high = middle - 1;
112
+ }
113
+ }
114
+ let mut end = low;
115
+ while !message.is_char_boundary(end) {
116
+ end -= 1;
117
+ }
118
+ value["message"] = Value::String(message[..end].into());
119
+ value.to_string()
120
+ }
121
+
122
+ #[cfg(test)]
123
+ mod tests {
124
+ use super::*;
125
+
126
+ proptest::proptest! {
127
+ #[test]
128
+ fn arbitrary_unicode_diagnostics_keep_original_prefix_and_output_budget(
129
+ message in ".{0,4096}", budget in 256usize..10_001,
130
+ ) {
131
+ let error = ApiError::permanent(message);
132
+ let output = error_json(&error, budget, false);
133
+ proptest::prop_assert!(output.len() <= budget);
134
+ let value: Value = serde_json::from_str(&output).unwrap();
135
+ let prefix = value["message"].as_str().unwrap();
136
+ proptest::prop_assert!(error.message.starts_with(prefix));
137
+ if prefix.len() < error.message.len() {
138
+ proptest::prop_assert_eq!(&value["truncated"], &Value::Bool(true));
139
+ }
140
+ }
141
+ }
142
+
143
+ #[test]
144
+ fn huge_diagnostic_only_classifies_the_bounded_original_prefix() {
145
+ let message = format!("{}[PARSE_SYNTAX_ERROR]", "é🦀".repeat(1_400_000));
146
+ let error = ApiError::statement_failed(message);
147
+ let output = error_json(&error, 10_000, false);
148
+ let value: Value = serde_json::from_str(&output).unwrap();
149
+ assert!(output.len() <= 10_000);
150
+ assert_eq!(value["category"], "execution_error");
151
+ assert_eq!(value["truncated"], true);
152
+ let message = value["message"].as_str().unwrap();
153
+ assert!(error.message.starts_with(message));
154
+ assert!(!message.is_empty());
155
+ }
156
+
157
+ #[test]
158
+ #[ignore = "manual diagnostic throughput measurement"]
159
+ fn benchmark_eight_megabyte_diagnostic_with_ten_kilobyte_budget() {
160
+ let error = ApiError::statement_failed(format!("[PARSE_SYNTAX_ERROR] {}", "x".repeat(8 * 1024 * 1024)));
161
+ let start = std::time::Instant::now();
162
+ let mut output_bytes = 0;
163
+ for _ in 0..20 {
164
+ let output = error_json(std::hint::black_box(&error), 10_000, false);
165
+ output_bytes += std::hint::black_box(output.len());
166
+ assert!(output.len() <= 10_000);
167
+ }
168
+ eprintln!(
169
+ "8MiB error / 10kB budget: {:.1}us/call, {} output bytes/call",
170
+ start.elapsed().as_secs_f64() * 1_000_000.0 / 20.0,
171
+ output_bytes / 20
172
+ );
173
+ }
174
+
175
+ #[test]
176
+ fn diagnostic_is_valid_bounded_json_even_with_escaped_unicode() {
177
+ let error = ApiError::statement_failed(format!("[PARSE_SYNTAX_ERROR] {}", "\"é\\\n".repeat(1000)));
178
+ for budget in [256, 257, 500, 10_000] {
179
+ let output = error_json(&error, budget, false);
180
+ assert!(output.len() <= budget);
181
+ let value: Value = serde_json::from_str(&output).unwrap();
182
+ assert_eq!(value["category"], "syntax_error");
183
+ if budget < 1000 {
184
+ assert_eq!(value["truncated"], true);
185
+ }
186
+ }
187
+ }
188
+ }