arrowbricks 5.2.2__tar.gz → 6.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- arrowbricks-6.0.1/PKG-INFO +125 -0
- arrowbricks-6.0.1/README.md +112 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/pyproject.toml +5 -2
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/Cargo.lock +2 -1
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/Cargo.toml +2 -1
- arrowbricks-6.0.1/rust/arrowbricks_core/examples/agent_render_bench.rs +156 -0
- arrowbricks-6.0.1/rust/arrowbricks_core/src/agent/diagnostics.rs +188 -0
- arrowbricks-6.0.1/rust/arrowbricks_core/src/agent/mod.rs +355 -0
- arrowbricks-6.0.1/rust/arrowbricks_core/src/agent/render.rs +646 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/client/sea.rs +13 -1
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/lib.rs +38 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/pipeline/ndjson.rs +1 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/pipeline/sea.rs +22 -2
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/pipeline/stats.rs +2 -1
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/pipeline/thrift_exec.rs +41 -24
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/pipeline.rs +2 -0
- arrowbricks-6.0.1/rust/arrowbricks_core/tests/wiremock_agent.rs +317 -0
- arrowbricks-6.0.1/src/arrowbricks/__main__.py +5 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/src/arrowbricks/_core.pyi +12 -1
- arrowbricks-6.0.1/src/arrowbricks/cli.py +119 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/src/arrowbricks/client.py +32 -0
- arrowbricks-5.2.2/PKG-INFO +0 -381
- arrowbricks-5.2.2/README.md +0 -368
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/LICENSE +0 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/.gitignore +0 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/README.md +0 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/examples/duckdb_query.py +0 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/examples/fastapi_sse.py +0 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/proptest-regressions/json_convert.txt +0 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/rustfmt.toml +0 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/alloc.rs +0 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/arrow_ffi.rs +0 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/client/call_context.rs +0 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/client/download.rs +0 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/client/error.rs +0 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/client/model.rs +0 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/client/thrift_rpc.rs +0 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/client/volume.rs +0 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/client.rs +0 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/heartbeat.rs +0 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/json_convert.rs +0 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/pipeline/reorder.rs +0 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/pipeline/test_support.rs +0 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/src/thrift.rs +0 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/tests/common/mod.rs +0 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/tests/wiremock_pipeline.rs +0 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/tests/wiremock_thrift.rs +0 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/tests/wiremock_volume_files.rs +0 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/tests_py/conftest.py +0 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/tests_py/test_ipc_stream.py +0 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/tests_py/test_parameters.py +0 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/tests_py/test_stream_ndjson_lines.py +0 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/tests_py/test_streaming.py +0 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/tests_py/test_thrift.py +0 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/tests_py/test_token_provider.py +0 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/tests_py/test_volume_files.py +0 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/rust/arrowbricks_core/tests_py/thrift_mock.py +0 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/src/arrowbricks/__init__.py +0 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/src/arrowbricks/_streaming.py +0 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/src/arrowbricks/cursor.py +0 -0
- {arrowbricks-5.2.2 → arrowbricks-6.0.1}/src/arrowbricks/py.typed +0 -0
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: arrowbricks
|
|
3
|
+
Version: 6.0.1
|
|
4
|
+
Requires-Dist: arro3-core>=0.8 ; extra == 'arro3'
|
|
5
|
+
Provides-Extra: arro3
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
Summary: Rust-powered Databricks SQL client and agent CLI with Arrow/NDJSON streaming, bounded Markdown previews, actionable JSON diagnostics, and zero required Python dependencies.
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
Requires-Python: >=3.11
|
|
10
|
+
Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
|
|
11
|
+
Project-URL: Repository, https://github.com/bmsuisse/arrowbricks
|
|
12
|
+
|
|
13
|
+
<p align="center">
|
|
14
|
+
<img src="assets/logo.svg" width="72" height="72" alt="arrowbricks logo">
|
|
15
|
+
</p>
|
|
16
|
+
|
|
17
|
+
<h1 align="center">arrowbricks</h1>
|
|
18
|
+
<p align="center">Fast Databricks SQL for Python applications and AI agents.</p>
|
|
19
|
+
|
|
20
|
+
- **Zero required Python dependencies.** One precompiled wheel. No Rust toolchain or cloud SDK needed.
|
|
21
|
+
- **Fast Rust core.** Concurrent chunk downloads, lazy Arrow decoding, and streaming NDJSON, with result order preserved.
|
|
22
|
+
- **Agent mode, new in 6.0.** A CLI and Python API that return compact typed Markdown, explicit truncation, and actionable JSON errors. Formatting stays in Rust.
|
|
23
|
+
|
|
24
|
+
Use Thrift by default or the Statement Execution API with `protocol="sea"`.
|
|
25
|
+
Bring a token or your own token-refresh callable.
|
|
26
|
+
|
|
27
|
+
## Install
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
pip install arrowbricks
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Wheels cover Linux, macOS, and Windows; Python 3.11+.
|
|
34
|
+
Arrow, NDJSON, and agent mode work without extras. Python row tuples need
|
|
35
|
+
`pip install 'arrowbricks[arro3]'`.
|
|
36
|
+
|
|
37
|
+
## Query from an agent or terminal
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
export DATABRICKS_HOST='your-workspace.azuredatabricks.net'
|
|
41
|
+
export DATABRICKS_WAREHOUSE_ID='your-warehouse-id'
|
|
42
|
+
export DATABRICKS_TOKEN='your-access-token'
|
|
43
|
+
|
|
44
|
+
arrowbricks -c 'SELECT 1 AS answer'
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
```markdown
|
|
48
|
+
| answer (Int32) |
|
|
49
|
+
| --- |
|
|
50
|
+
| 1 |
|
|
51
|
+
|
|
52
|
+
rows_shown=1; complete=true; total_rows=1
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
Agent mode is the CLI default. SQL files and piped stdin work too:
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
arrowbricks --file query.sql --max-rows 20 --max-bytes 4000
|
|
59
|
+
printf 'SELECT 1 AS answer' | arrowbricks
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
Previews have row, byte, cell, and deadline limits. The footer tells you when
|
|
63
|
+
results are incomplete. Query errors go to stderr as JSON, preserving the
|
|
64
|
+
warehouse's diagnostic and suggestions with a recovery hint; the CLI exits 1.
|
|
65
|
+
Preview limits bound output and fetching, not warehouse scan cost.
|
|
66
|
+
|
|
67
|
+
For Python agents:
|
|
68
|
+
|
|
69
|
+
```python
|
|
70
|
+
from arrowbricks import DatabricksClient
|
|
71
|
+
|
|
72
|
+
async with DatabricksClient(host=..., warehouse_id=..., token=...) as client:
|
|
73
|
+
text = await client.query_agent("SELECT 1 AS answer", max_bytes=4000)
|
|
74
|
+
print(text) # Markdown on success; JSON on query failure
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
Give your agent the reusable [Arrowbricks skill](skills/arrowbricks/SKILL.md).
|
|
78
|
+
See [CLI options and skill installation](docs/usage.md#cli-quickstart-60) and
|
|
79
|
+
[agent budgets, completeness, and cancellation](docs/usage.md#agent-results).
|
|
80
|
+
|
|
81
|
+
## Query from an application
|
|
82
|
+
|
|
83
|
+
The async cursor offers familiar `execute` and `fetch*` methods.
|
|
84
|
+
Arrow results work directly with DuckDB, PyArrow, and Polars:
|
|
85
|
+
|
|
86
|
+
```python
|
|
87
|
+
from arrowbricks import connect
|
|
88
|
+
|
|
89
|
+
async with connect(host=..., warehouse_id=..., token=...) as conn:
|
|
90
|
+
cursor = conn.cursor()
|
|
91
|
+
await cursor.execute("SELECT * FROM my_catalog.my_schema.my_table LIMIT 100")
|
|
92
|
+
table = await cursor.fetchall_arrow()
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
Or consume ready-to-send NDJSON without building Python rows:
|
|
96
|
+
|
|
97
|
+
```python
|
|
98
|
+
from arrowbricks import HEARTBEAT, DatabricksClient
|
|
99
|
+
|
|
100
|
+
async with DatabricksClient(host=..., warehouse_id=..., token=...) as client:
|
|
101
|
+
async for item in client.stream_query_json("SELECT * FROM range(100)"):
|
|
102
|
+
if item is not HEARTBEAT:
|
|
103
|
+
print(item)
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
## Performance
|
|
107
|
+
|
|
108
|
+
The query and formatting work runs in Rust. Agent previews render directly from
|
|
109
|
+
Arrow, avoiding intermediate Python rows or JSON. In a real warehouse test,
|
|
110
|
+
a 100-row preview used **44% fewer bytes and 24% fewer tokens** than NDJSON.
|
|
111
|
+
See [agent measurements and methodology](docs/research/agent-rendering.md) and
|
|
112
|
+
[client benchmarks](docs/benchmarks.md) for workloads, comparisons, and limits.
|
|
113
|
+
|
|
114
|
+
## Learn more
|
|
115
|
+
|
|
116
|
+
- [Usage and API reference](docs/usage.md): authentication, errors, cancellation, observability, and Arrow interop.
|
|
117
|
+
- [Runnable examples](examples/): cursors, agents, OAuth, and FastAPI streaming.
|
|
118
|
+
- [Rust core](rust/arrowbricks_core/README.md): implementation and native APIs.
|
|
119
|
+
- [Real warehouse verification](docs/usage.md#real-warehouse-verification).
|
|
120
|
+
- [Changelog](CHANGELOG.md).
|
|
121
|
+
|
|
122
|
+
## License
|
|
123
|
+
|
|
124
|
+
MIT
|
|
125
|
+
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="assets/logo.svg" width="72" height="72" alt="arrowbricks logo">
|
|
3
|
+
</p>
|
|
4
|
+
|
|
5
|
+
<h1 align="center">arrowbricks</h1>
|
|
6
|
+
<p align="center">Fast Databricks SQL for Python applications and AI agents.</p>
|
|
7
|
+
|
|
8
|
+
- **Zero required Python dependencies.** One precompiled wheel. No Rust toolchain or cloud SDK needed.
|
|
9
|
+
- **Fast Rust core.** Concurrent chunk downloads, lazy Arrow decoding, and streaming NDJSON, with result order preserved.
|
|
10
|
+
- **Agent mode, new in 6.0.** A CLI and Python API that return compact typed Markdown, explicit truncation, and actionable JSON errors. Formatting stays in Rust.
|
|
11
|
+
|
|
12
|
+
Use Thrift by default or the Statement Execution API with `protocol="sea"`.
|
|
13
|
+
Bring a token or your own token-refresh callable.
|
|
14
|
+
|
|
15
|
+
## Install
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
pip install arrowbricks
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
Wheels cover Linux, macOS, and Windows; Python 3.11+.
|
|
22
|
+
Arrow, NDJSON, and agent mode work without extras. Python row tuples need
|
|
23
|
+
`pip install 'arrowbricks[arro3]'`.
|
|
24
|
+
|
|
25
|
+
## Query from an agent or terminal
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
export DATABRICKS_HOST='your-workspace.azuredatabricks.net'
|
|
29
|
+
export DATABRICKS_WAREHOUSE_ID='your-warehouse-id'
|
|
30
|
+
export DATABRICKS_TOKEN='your-access-token'
|
|
31
|
+
|
|
32
|
+
arrowbricks -c 'SELECT 1 AS answer'
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
```markdown
|
|
36
|
+
| answer (Int32) |
|
|
37
|
+
| --- |
|
|
38
|
+
| 1 |
|
|
39
|
+
|
|
40
|
+
rows_shown=1; complete=true; total_rows=1
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
Agent mode is the CLI default. SQL files and piped stdin work too:
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
arrowbricks --file query.sql --max-rows 20 --max-bytes 4000
|
|
47
|
+
printf 'SELECT 1 AS answer' | arrowbricks
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
Previews have row, byte, cell, and deadline limits. The footer tells you when
|
|
51
|
+
results are incomplete. Query errors go to stderr as JSON, preserving the
|
|
52
|
+
warehouse's diagnostic and suggestions with a recovery hint; the CLI exits 1.
|
|
53
|
+
Preview limits bound output and fetching, not warehouse scan cost.
|
|
54
|
+
|
|
55
|
+
For Python agents:
|
|
56
|
+
|
|
57
|
+
```python
|
|
58
|
+
from arrowbricks import DatabricksClient
|
|
59
|
+
|
|
60
|
+
async with DatabricksClient(host=..., warehouse_id=..., token=...) as client:
|
|
61
|
+
text = await client.query_agent("SELECT 1 AS answer", max_bytes=4000)
|
|
62
|
+
print(text) # Markdown on success; JSON on query failure
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
Give your agent the reusable [Arrowbricks skill](skills/arrowbricks/SKILL.md).
|
|
66
|
+
See [CLI options and skill installation](docs/usage.md#cli-quickstart-60) and
|
|
67
|
+
[agent budgets, completeness, and cancellation](docs/usage.md#agent-results).
|
|
68
|
+
|
|
69
|
+
## Query from an application
|
|
70
|
+
|
|
71
|
+
The async cursor offers familiar `execute` and `fetch*` methods.
|
|
72
|
+
Arrow results work directly with DuckDB, PyArrow, and Polars:
|
|
73
|
+
|
|
74
|
+
```python
|
|
75
|
+
from arrowbricks import connect
|
|
76
|
+
|
|
77
|
+
async with connect(host=..., warehouse_id=..., token=...) as conn:
|
|
78
|
+
cursor = conn.cursor()
|
|
79
|
+
await cursor.execute("SELECT * FROM my_catalog.my_schema.my_table LIMIT 100")
|
|
80
|
+
table = await cursor.fetchall_arrow()
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
Or consume ready-to-send NDJSON without building Python rows:
|
|
84
|
+
|
|
85
|
+
```python
|
|
86
|
+
from arrowbricks import HEARTBEAT, DatabricksClient
|
|
87
|
+
|
|
88
|
+
async with DatabricksClient(host=..., warehouse_id=..., token=...) as client:
|
|
89
|
+
async for item in client.stream_query_json("SELECT * FROM range(100)"):
|
|
90
|
+
if item is not HEARTBEAT:
|
|
91
|
+
print(item)
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
## Performance
|
|
95
|
+
|
|
96
|
+
The query and formatting work runs in Rust. Agent previews render directly from
|
|
97
|
+
Arrow, avoiding intermediate Python rows or JSON. In a real warehouse test,
|
|
98
|
+
a 100-row preview used **44% fewer bytes and 24% fewer tokens** than NDJSON.
|
|
99
|
+
See [agent measurements and methodology](docs/research/agent-rendering.md) and
|
|
100
|
+
[client benchmarks](docs/benchmarks.md) for workloads, comparisons, and limits.
|
|
101
|
+
|
|
102
|
+
## Learn more
|
|
103
|
+
|
|
104
|
+
- [Usage and API reference](docs/usage.md): authentication, errors, cancellation, observability, and Arrow interop.
|
|
105
|
+
- [Runnable examples](examples/): cursors, agents, OAuth, and FastAPI streaming.
|
|
106
|
+
- [Rust core](rust/arrowbricks_core/README.md): implementation and native APIs.
|
|
107
|
+
- [Real warehouse verification](docs/usage.md#real-warehouse-verification).
|
|
108
|
+
- [Changelog](CHANGELOG.md).
|
|
109
|
+
|
|
110
|
+
## License
|
|
111
|
+
|
|
112
|
+
MIT
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "arrowbricks"
|
|
3
|
-
version = "
|
|
4
|
-
description = "
|
|
3
|
+
version = "6.0.1"
|
|
4
|
+
description = "Rust-powered Databricks SQL client and agent CLI with Arrow/NDJSON streaming, bounded Markdown previews, actionable JSON diagnostics, and zero required Python dependencies."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "MIT"
|
|
7
7
|
license-files = ["LICENSE"]
|
|
@@ -14,6 +14,9 @@ requires-python = ">=3.11"
|
|
|
14
14
|
# Cursor.description all work with zero dependencies installed.
|
|
15
15
|
dependencies = []
|
|
16
16
|
|
|
17
|
+
[project.scripts]
|
|
18
|
+
arrowbricks = "arrowbricks.cli:main"
|
|
19
|
+
|
|
17
20
|
[project.optional-dependencies]
|
|
18
21
|
# Row materialization converts our capsule table through arro3-core.
|
|
19
22
|
# Arrow fetches and streaming do not need this optional dependency.
|
|
@@ -177,10 +177,11 @@ dependencies = [
|
|
|
177
177
|
|
|
178
178
|
[[package]]
|
|
179
179
|
name = "arrowbricks_core"
|
|
180
|
-
version = "
|
|
180
|
+
version = "6.0.1"
|
|
181
181
|
dependencies = [
|
|
182
182
|
"arrow-array",
|
|
183
183
|
"arrow-buffer",
|
|
184
|
+
"arrow-cast",
|
|
184
185
|
"arrow-ipc",
|
|
185
186
|
"arrow-json",
|
|
186
187
|
"arrow-schema",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[package]
|
|
2
2
|
name = "arrowbricks_core"
|
|
3
|
-
version = "
|
|
3
|
+
version = "6.0.1"
|
|
4
4
|
edition = "2024"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
|
|
@@ -19,6 +19,7 @@ arrow-buffer = "59.1.0"
|
|
|
19
19
|
arrow-ipc = "59.1.0"
|
|
20
20
|
arrow-schema = "59.1.0"
|
|
21
21
|
arrow-json = "59.1.0"
|
|
22
|
+
arrow-cast = "59.1.0"
|
|
22
23
|
bytes = "1.12.1"
|
|
23
24
|
# Already in the dependency graph; vectorized newline search in NDJSON output.
|
|
24
25
|
memchr = "2"
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
use arrow_array::types::Int32Type;
|
|
2
|
+
use arrow_array::{ArrayRef, Float64Array, Int64Array, ListArray, RecordBatch, StringArray};
|
|
3
|
+
use arrow_schema::{Field, Schema};
|
|
4
|
+
use arrowbricks_core::agent::render::MarkdownRenderer;
|
|
5
|
+
use std::{hint::black_box, sync::Arc, time::Instant};
|
|
6
|
+
|
|
7
|
+
fn batch(columns: Vec<ArrayRef>) -> RecordBatch {
|
|
8
|
+
let fields: Vec<_> = columns
|
|
9
|
+
.iter()
|
|
10
|
+
.enumerate()
|
|
11
|
+
.map(|(i, a)| Field::new(format!("c{i}"), a.data_type().clone(), true))
|
|
12
|
+
.collect();
|
|
13
|
+
RecordBatch::try_new(Arc::new(Schema::new(fields)), columns).unwrap()
|
|
14
|
+
}
|
|
15
|
+
fn direct(b: &RecordBatch, bytes: usize, interrupts: bool) -> String {
|
|
16
|
+
let mut renderer = if interrupts {
|
|
17
|
+
MarkdownRenderer::new_with_interrupt(
|
|
18
|
+
b.schema(),
|
|
19
|
+
1000,
|
|
20
|
+
bytes,
|
|
21
|
+
500,
|
|
22
|
+
Instant::now() + std::time::Duration::from_secs(60),
|
|
23
|
+
Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
|
24
|
+
)
|
|
25
|
+
} else {
|
|
26
|
+
MarkdownRenderer::new(b.schema(), 1000, bytes, 500)
|
|
27
|
+
}
|
|
28
|
+
.unwrap();
|
|
29
|
+
renderer.push_batch(b).unwrap();
|
|
30
|
+
renderer.finish(true)
|
|
31
|
+
}
|
|
32
|
+
fn json(b: &RecordBatch) -> Vec<serde_json::Value> {
|
|
33
|
+
let mut writer = arrow_json::ArrayWriter::new(Vec::new());
|
|
34
|
+
writer.write_batches(&[b]).unwrap();
|
|
35
|
+
writer.finish().unwrap();
|
|
36
|
+
serde_json::from_slice(&writer.into_inner()).unwrap()
|
|
37
|
+
}
|
|
38
|
+
fn json_markdown(b: &RecordBatch, bytes: usize) -> String {
|
|
39
|
+
let values = json(b);
|
|
40
|
+
let mut output = String::new();
|
|
41
|
+
for row in &values {
|
|
42
|
+
let mut line = String::from("|");
|
|
43
|
+
for field in b.schema().fields() {
|
|
44
|
+
let value = &row[field.name()];
|
|
45
|
+
let owned;
|
|
46
|
+
let text = if let Some(value) = value.as_str() {
|
|
47
|
+
value
|
|
48
|
+
} else {
|
|
49
|
+
owned = value.to_string();
|
|
50
|
+
&owned
|
|
51
|
+
};
|
|
52
|
+
line.push(' ');
|
|
53
|
+
if value.is_string() {
|
|
54
|
+
line.push('"');
|
|
55
|
+
}
|
|
56
|
+
for ch in text.chars().take(500) {
|
|
57
|
+
match ch {
|
|
58
|
+
'&' => line.push_str("&"),
|
|
59
|
+
'<' => line.push_str("<"),
|
|
60
|
+
'>' => line.push_str(">"),
|
|
61
|
+
'\n' => line.push_str("\\n"),
|
|
62
|
+
'\r' => line.push_str("\\r"),
|
|
63
|
+
'\t' => line.push_str("\\t"),
|
|
64
|
+
'\\' | '|' | '`' | '*' | '_' | '[' | ']' | '~' | '"' => {
|
|
65
|
+
line.push('\\');
|
|
66
|
+
line.push(ch);
|
|
67
|
+
}
|
|
68
|
+
c => line.push(c),
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
if value.is_string() {
|
|
72
|
+
line.push('"');
|
|
73
|
+
}
|
|
74
|
+
line.push_str(" |");
|
|
75
|
+
}
|
|
76
|
+
line.push('\n');
|
|
77
|
+
if output.len() + line.len() > bytes.saturating_sub(160) {
|
|
78
|
+
break;
|
|
79
|
+
}
|
|
80
|
+
output.push_str(&line);
|
|
81
|
+
}
|
|
82
|
+
output
|
|
83
|
+
}
|
|
84
|
+
fn time(mut f: impl FnMut(), loops: usize) -> f64 {
|
|
85
|
+
let start = Instant::now();
|
|
86
|
+
for _ in 0..loops {
|
|
87
|
+
f();
|
|
88
|
+
}
|
|
89
|
+
start.elapsed().as_secs_f64() * 1e6 / loops as f64
|
|
90
|
+
}
|
|
91
|
+
fn main() {
|
|
92
|
+
let agent_only = std::env::args().any(|arg| arg == "--agent-only");
|
|
93
|
+
let n = 1000;
|
|
94
|
+
let numeric = batch(vec![
|
|
95
|
+
Arc::new(Int64Array::from_iter_values(0..n)),
|
|
96
|
+
Arc::new(Float64Array::from_iter_values((0..n).map(|i| i as f64 * 0.1))),
|
|
97
|
+
]);
|
|
98
|
+
let unicode = batch(vec![
|
|
99
|
+
Arc::new(StringArray::from(vec!["é🦀|<>&\\".repeat(100); n as usize])),
|
|
100
|
+
Arc::new(ListArray::from_iter_primitive::<Int32Type, _, _>(
|
|
101
|
+
(0..n).map(|_| Some(vec![Some(1), None, Some(3)])),
|
|
102
|
+
)),
|
|
103
|
+
]);
|
|
104
|
+
let ascii = batch(vec![Arc::new(StringArray::from(vec![
|
|
105
|
+
"ordinary query results without markup ".repeat(20); n as usize
|
|
106
|
+
]))]);
|
|
107
|
+
let controls = batch(vec![Arc::new(StringArray::from(vec![
|
|
108
|
+
"a\0\u{0001}b\t\n".repeat(30);
|
|
109
|
+
n as usize
|
|
110
|
+
]))]);
|
|
111
|
+
let huge = batch(vec![Arc::new(StringArray::from(vec!["🦀".repeat(1_000_000)]))]);
|
|
112
|
+
let mut builder = arrow_array::builder::ListBuilder::new(arrow_array::builder::StringBuilder::new());
|
|
113
|
+
builder.values().append_value("🦀".repeat(2_500_000));
|
|
114
|
+
builder.append(true);
|
|
115
|
+
let nested_huge = batch(vec![Arc::new(builder.finish())]);
|
|
116
|
+
for (name, b, loops) in [
|
|
117
|
+
("numeric", numeric, 1000),
|
|
118
|
+
("unicode_nested", unicode, 100),
|
|
119
|
+
("ascii_strings", ascii, 100),
|
|
120
|
+
("controls", controls, 100),
|
|
121
|
+
("4MB_string", huge, 20),
|
|
122
|
+
("10MB_nested_string", nested_huge, 10),
|
|
123
|
+
] {
|
|
124
|
+
let bounded = time(
|
|
125
|
+
|| {
|
|
126
|
+
black_box(direct(&b, 10_000, agent_only));
|
|
127
|
+
},
|
|
128
|
+
if agent_only { 10000 } else { loops },
|
|
129
|
+
);
|
|
130
|
+
let full = time(
|
|
131
|
+
|| {
|
|
132
|
+
black_box(direct(&b, 2_000_000, agent_only));
|
|
133
|
+
},
|
|
134
|
+
loops,
|
|
135
|
+
);
|
|
136
|
+
if agent_only {
|
|
137
|
+
println!("{name}: direct_10KB={bounded:.2}us direct_2MB={full:.2}us");
|
|
138
|
+
continue;
|
|
139
|
+
}
|
|
140
|
+
let json_render = time(
|
|
141
|
+
|| {
|
|
142
|
+
black_box(json_markdown(&b, 10_000));
|
|
143
|
+
},
|
|
144
|
+
loops,
|
|
145
|
+
);
|
|
146
|
+
let via_json = time(
|
|
147
|
+
|| {
|
|
148
|
+
black_box(json(&b));
|
|
149
|
+
},
|
|
150
|
+
loops,
|
|
151
|
+
);
|
|
152
|
+
println!(
|
|
153
|
+
"{name}: direct_10KB={bounded:.2}us direct_2MB={full:.2}us JSON_materialize={via_json:.2}us JSON_then_Markdown_10KB={json_render:.2}us"
|
|
154
|
+
);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
use crate::client::{ApiError, ApiErrorKind};
|
|
2
|
+
use serde_json::{Value, json};
|
|
3
|
+
|
|
4
|
+
pub(super) fn error_json(error: &ApiError, max_bytes: usize, timeout: bool) -> String {
|
|
5
|
+
let mut prefix_end = error.message.len().min(max_bytes);
|
|
6
|
+
while !error.message.is_char_boundary(prefix_end) {
|
|
7
|
+
prefix_end -= 1;
|
|
8
|
+
}
|
|
9
|
+
let message = &error.message[..prefix_end];
|
|
10
|
+
let shortened = prefix_end < error.message.len();
|
|
11
|
+
let upper = message.to_ascii_uppercase();
|
|
12
|
+
let (category, next_step) = if timeout {
|
|
13
|
+
(
|
|
14
|
+
"timeout",
|
|
15
|
+
"Increase total_timeout_s or narrow the query before retrying.",
|
|
16
|
+
)
|
|
17
|
+
} else if error.kind == ApiErrorKind::Auth {
|
|
18
|
+
("authentication", "Refresh credentials and verify warehouse access.")
|
|
19
|
+
} else if upper.contains("UNRESOLVED_COLUMN") || upper.contains("UNRESOLVED_FIELD") {
|
|
20
|
+
(
|
|
21
|
+
"missing_column",
|
|
22
|
+
"Check column names against the table schema and database suggestions.",
|
|
23
|
+
)
|
|
24
|
+
} else if upper.contains("UNRESOLVED_ROUTINE") {
|
|
25
|
+
(
|
|
26
|
+
"missing_function",
|
|
27
|
+
"Check the function name and namespace with SHOW FUNCTIONS or DESCRIBE FUNCTION.",
|
|
28
|
+
)
|
|
29
|
+
} else if upper.contains("SCHEMA_NOT_FOUND") || upper.contains("CATALOG_NOT_FOUND") {
|
|
30
|
+
(
|
|
31
|
+
"missing_namespace",
|
|
32
|
+
"Verify catalog and schema names and access before submitting again.",
|
|
33
|
+
)
|
|
34
|
+
} else if upper.contains("TABLE_OR_VIEW_NOT_FOUND") {
|
|
35
|
+
("missing_table", "Verify the table name, catalog, schema and access.")
|
|
36
|
+
} else if upper.contains("PARSE_SYNTAX_ERROR") || upper.contains("SYNTAX_ERROR") {
|
|
37
|
+
(
|
|
38
|
+
"syntax_error",
|
|
39
|
+
"Correct the SQL using the database diagnostic before submitting again.",
|
|
40
|
+
)
|
|
41
|
+
} else if upper.contains("PERMISSION") || upper.contains("INSUFFICIENT_PRIVILEGES") {
|
|
42
|
+
(
|
|
43
|
+
"permission_denied",
|
|
44
|
+
"Request access to the referenced objects or use an authorized object.",
|
|
45
|
+
)
|
|
46
|
+
} else if error.transient {
|
|
47
|
+
("transient", "Check warehouse and network availability before retrying.")
|
|
48
|
+
} else {
|
|
49
|
+
(
|
|
50
|
+
"execution_error",
|
|
51
|
+
"Inspect the database diagnostic and verify the statement and warehouse.",
|
|
52
|
+
)
|
|
53
|
+
};
|
|
54
|
+
let error_classes = message.split('[').skip(1).filter_map(|part| {
|
|
55
|
+
let candidate = part.split(']').next()?;
|
|
56
|
+
(!candidate.is_empty()
|
|
57
|
+
&& candidate.len() <= 128
|
|
58
|
+
&& candidate
|
|
59
|
+
.bytes()
|
|
60
|
+
.all(|c| c.is_ascii_uppercase() || c.is_ascii_digit() || b"_.".contains(&c)))
|
|
61
|
+
.then_some(candidate)
|
|
62
|
+
});
|
|
63
|
+
let error_class = error_classes
|
|
64
|
+
.clone()
|
|
65
|
+
.find(|class| {
|
|
66
|
+
["UNRESOLVED_", "TABLE_OR_VIEW_", "PARSE_", "SYNTAX_", "INSUFFICIENT_"]
|
|
67
|
+
.iter()
|
|
68
|
+
.any(|prefix| class.starts_with(prefix))
|
|
69
|
+
})
|
|
70
|
+
.or_else(|| error_classes.into_iter().next())
|
|
71
|
+
.or_else(|| {
|
|
72
|
+
message.split([' ', ':']).find(|token| {
|
|
73
|
+
["UNRESOLVED_", "TABLE_OR_VIEW_", "PARSE_", "SYNTAX_", "INSUFFICIENT_"]
|
|
74
|
+
.iter()
|
|
75
|
+
.any(|prefix| token.starts_with(prefix))
|
|
76
|
+
})
|
|
77
|
+
});
|
|
78
|
+
let mut value = json!({
|
|
79
|
+
"category":category, "message":message, "next_step":next_step,
|
|
80
|
+
"error_class":error_class,
|
|
81
|
+
});
|
|
82
|
+
if shortened {
|
|
83
|
+
value["truncated"] = Value::Bool(true);
|
|
84
|
+
}
|
|
85
|
+
let output = value.to_string();
|
|
86
|
+
if output.len() <= max_bytes {
|
|
87
|
+
return output;
|
|
88
|
+
}
|
|
89
|
+
value["truncated"] = Value::Bool(true);
|
|
90
|
+
// Preserve the original diagnostic prefix within the caller's byte budget.
|
|
91
|
+
value.as_object_mut().unwrap().remove("error_class");
|
|
92
|
+
let fixed_bytes = {
|
|
93
|
+
value["message"] = Value::String(String::new());
|
|
94
|
+
value.to_string().len()
|
|
95
|
+
};
|
|
96
|
+
if fixed_bytes > max_bytes {
|
|
97
|
+
value["next_step"] = Value::String("Inspect the database diagnostic before retrying.".into());
|
|
98
|
+
}
|
|
99
|
+
let mut low = 0;
|
|
100
|
+
let mut high = message.len();
|
|
101
|
+
while low < high {
|
|
102
|
+
let middle = low + (high - low).div_ceil(2);
|
|
103
|
+
let mut end = middle;
|
|
104
|
+
while !message.is_char_boundary(end) {
|
|
105
|
+
end -= 1;
|
|
106
|
+
}
|
|
107
|
+
value["message"] = Value::String(message[..end].into());
|
|
108
|
+
if value.to_string().len() <= max_bytes {
|
|
109
|
+
low = middle;
|
|
110
|
+
} else {
|
|
111
|
+
high = middle - 1;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
let mut end = low;
|
|
115
|
+
while !message.is_char_boundary(end) {
|
|
116
|
+
end -= 1;
|
|
117
|
+
}
|
|
118
|
+
value["message"] = Value::String(message[..end].into());
|
|
119
|
+
value.to_string()
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
#[cfg(test)]
|
|
123
|
+
mod tests {
|
|
124
|
+
use super::*;
|
|
125
|
+
|
|
126
|
+
proptest::proptest! {
|
|
127
|
+
#[test]
|
|
128
|
+
fn arbitrary_unicode_diagnostics_keep_original_prefix_and_output_budget(
|
|
129
|
+
message in ".{0,4096}", budget in 256usize..10_001,
|
|
130
|
+
) {
|
|
131
|
+
let error = ApiError::permanent(message);
|
|
132
|
+
let output = error_json(&error, budget, false);
|
|
133
|
+
proptest::prop_assert!(output.len() <= budget);
|
|
134
|
+
let value: Value = serde_json::from_str(&output).unwrap();
|
|
135
|
+
let prefix = value["message"].as_str().unwrap();
|
|
136
|
+
proptest::prop_assert!(error.message.starts_with(prefix));
|
|
137
|
+
if prefix.len() < error.message.len() {
|
|
138
|
+
proptest::prop_assert_eq!(&value["truncated"], &Value::Bool(true));
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
#[test]
|
|
144
|
+
fn huge_diagnostic_only_classifies_the_bounded_original_prefix() {
|
|
145
|
+
let message = format!("{}[PARSE_SYNTAX_ERROR]", "é🦀".repeat(1_400_000));
|
|
146
|
+
let error = ApiError::statement_failed(message);
|
|
147
|
+
let output = error_json(&error, 10_000, false);
|
|
148
|
+
let value: Value = serde_json::from_str(&output).unwrap();
|
|
149
|
+
assert!(output.len() <= 10_000);
|
|
150
|
+
assert_eq!(value["category"], "execution_error");
|
|
151
|
+
assert_eq!(value["truncated"], true);
|
|
152
|
+
let message = value["message"].as_str().unwrap();
|
|
153
|
+
assert!(error.message.starts_with(message));
|
|
154
|
+
assert!(!message.is_empty());
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
#[test]
|
|
158
|
+
#[ignore = "manual diagnostic throughput measurement"]
|
|
159
|
+
fn benchmark_eight_megabyte_diagnostic_with_ten_kilobyte_budget() {
|
|
160
|
+
let error = ApiError::statement_failed(format!("[PARSE_SYNTAX_ERROR] {}", "x".repeat(8 * 1024 * 1024)));
|
|
161
|
+
let start = std::time::Instant::now();
|
|
162
|
+
let mut output_bytes = 0;
|
|
163
|
+
for _ in 0..20 {
|
|
164
|
+
let output = error_json(std::hint::black_box(&error), 10_000, false);
|
|
165
|
+
output_bytes += std::hint::black_box(output.len());
|
|
166
|
+
assert!(output.len() <= 10_000);
|
|
167
|
+
}
|
|
168
|
+
eprintln!(
|
|
169
|
+
"8MiB error / 10kB budget: {:.1}us/call, {} output bytes/call",
|
|
170
|
+
start.elapsed().as_secs_f64() * 1_000_000.0 / 20.0,
|
|
171
|
+
output_bytes / 20
|
|
172
|
+
);
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
#[test]
|
|
176
|
+
fn diagnostic_is_valid_bounded_json_even_with_escaped_unicode() {
|
|
177
|
+
let error = ApiError::statement_failed(format!("[PARSE_SYNTAX_ERROR] {}", "\"é\\\n".repeat(1000)));
|
|
178
|
+
for budget in [256, 257, 500, 10_000] {
|
|
179
|
+
let output = error_json(&error, budget, false);
|
|
180
|
+
assert!(output.len() <= budget);
|
|
181
|
+
let value: Value = serde_json::from_str(&output).unwrap();
|
|
182
|
+
assert_eq!(value["category"], "syntax_error");
|
|
183
|
+
if budget < 1000 {
|
|
184
|
+
assert_eq!(value["truncated"], true);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
}
|