arrowbricks 6.0.0__tar.gz → 6.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- arrowbricks-6.1.0/PKG-INFO +129 -0
- arrowbricks-6.1.0/README.md +114 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/pyproject.toml +5 -1
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/Cargo.lock +1 -1
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/Cargo.toml +1 -1
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/src/arrowbricks/__init__.py +3 -0
- arrowbricks-6.1.0/src/arrowbricks/_sql.py +259 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/src/arrowbricks/_streaming.py +5 -3
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/src/arrowbricks/cli.py +5 -3
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/src/arrowbricks/client.py +8 -5
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/src/arrowbricks/cursor.py +10 -5
- arrowbricks-6.0.0/PKG-INFO +0 -512
- arrowbricks-6.0.0/README.md +0 -499
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/LICENSE +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/.gitignore +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/README.md +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/examples/agent_render_bench.rs +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/examples/duckdb_query.py +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/examples/fastapi_sse.py +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/proptest-regressions/json_convert.txt +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/rustfmt.toml +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/src/agent/diagnostics.rs +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/src/agent/mod.rs +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/src/agent/render.rs +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/src/alloc.rs +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/src/arrow_ffi.rs +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/src/client/call_context.rs +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/src/client/download.rs +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/src/client/error.rs +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/src/client/model.rs +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/src/client/sea.rs +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/src/client/thrift_rpc.rs +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/src/client/volume.rs +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/src/client.rs +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/src/heartbeat.rs +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/src/json_convert.rs +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/src/lib.rs +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/src/pipeline/ndjson.rs +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/src/pipeline/reorder.rs +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/src/pipeline/sea.rs +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/src/pipeline/stats.rs +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/src/pipeline/test_support.rs +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/src/pipeline/thrift_exec.rs +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/src/pipeline.rs +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/src/thrift.rs +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/tests/common/mod.rs +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/tests/wiremock_agent.rs +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/tests/wiremock_pipeline.rs +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/tests/wiremock_thrift.rs +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/tests/wiremock_volume_files.rs +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/tests_py/conftest.py +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/tests_py/test_ipc_stream.py +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/tests_py/test_parameters.py +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/tests_py/test_stream_ndjson_lines.py +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/tests_py/test_streaming.py +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/tests_py/test_thrift.py +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/tests_py/test_token_provider.py +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/tests_py/test_volume_files.py +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/rust/arrowbricks_core/tests_py/thrift_mock.py +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/src/arrowbricks/__main__.py +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/src/arrowbricks/_core.pyi +0 -0
- {arrowbricks-6.0.0 → arrowbricks-6.1.0}/src/arrowbricks/py.typed +0 -0
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: arrowbricks
|
|
3
|
+
Version: 6.1.0
|
|
4
|
+
Requires-Dist: arro3-core>=0.8 ; extra == 'arro3'
|
|
5
|
+
Requires-Dist: sqlglot>=27 ; extra == 'sqlglot'
|
|
6
|
+
Provides-Extra: arro3
|
|
7
|
+
Provides-Extra: sqlglot
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Summary: Rust-powered Databricks SQL client and agent CLI with Arrow/NDJSON streaming, bounded Markdown previews, actionable JSON diagnostics, and zero required Python dependencies.
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
Requires-Python: >=3.11
|
|
12
|
+
Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
|
|
13
|
+
Project-URL: Repository, https://github.com/bmsuisse/arrowbricks
|
|
14
|
+
|
|
15
|
+
<p align="center">
|
|
16
|
+
<img src="assets/logo.svg" width="72" height="72" alt="arrowbricks logo">
|
|
17
|
+
</p>
|
|
18
|
+
|
|
19
|
+
<h1 align="center">arrowbricks</h1>
|
|
20
|
+
<p align="center">Fast Databricks SQL for Python applications and AI agents.</p>
|
|
21
|
+
|
|
22
|
+
- **Zero required Python dependencies.** One precompiled wheel. No Rust toolchain or cloud SDK needed.
|
|
23
|
+
- **Fast Rust core.** Concurrent chunk downloads, lazy Arrow decoding, and streaming NDJSON, with result order preserved.
|
|
24
|
+
- **Agent mode, new in 6.0.** A CLI and Python API that return compact typed Markdown, explicit truncation, and actionable JSON errors. Formatting stays in Rust.
|
|
25
|
+
|
|
26
|
+
Use Thrift by default or the Statement Execution API with `protocol="sea"`.
|
|
27
|
+
Bring a token or your own token-refresh callable.
|
|
28
|
+
|
|
29
|
+
## Install
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
pip install arrowbricks
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
Wheels cover Linux, macOS, and Windows; Python 3.11+.
|
|
36
|
+
Arrow, NDJSON, and agent mode work without extras. Python row tuples need
|
|
37
|
+
`pip install 'arrowbricks[arro3]'`; sqlglot expressions as SQL need `[sqlglot]`.
|
|
38
|
+
On Python 3.14+ pass values as a t-string (`t"... WHERE id = {x}"`) and they are
|
|
39
|
+
bound as parameters -- see [SQL safety](docs/usage.md#sql-safety-t-strings-and-literalstring).
|
|
40
|
+
|
|
41
|
+
## Query from an agent or terminal
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
export DATABRICKS_HOST='your-workspace.azuredatabricks.net'
|
|
45
|
+
export DATABRICKS_WAREHOUSE_ID='your-warehouse-id'
|
|
46
|
+
export DATABRICKS_TOKEN='your-access-token'
|
|
47
|
+
|
|
48
|
+
arrowbricks -c 'SELECT 1 AS answer'
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
```markdown
|
|
52
|
+
| answer (Int32) |
|
|
53
|
+
| --- |
|
|
54
|
+
| 1 |
|
|
55
|
+
|
|
56
|
+
rows_shown=1; complete=true; total_rows=1
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
Agent mode is the CLI default. SQL files and piped stdin work too:
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
arrowbricks --file query.sql --max-rows 20 --max-bytes 4000
|
|
63
|
+
printf 'SELECT 1 AS answer' | arrowbricks
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
Previews have row, byte, cell, and deadline limits. The footer tells you when
|
|
67
|
+
results are incomplete. Query errors go to stderr as JSON, preserving the
|
|
68
|
+
warehouse's diagnostic and suggestions with a recovery hint; the CLI exits 1.
|
|
69
|
+
Preview limits bound output and fetching, not warehouse scan cost.
|
|
70
|
+
|
|
71
|
+
For Python agents:
|
|
72
|
+
|
|
73
|
+
```python
|
|
74
|
+
from arrowbricks import DatabricksClient
|
|
75
|
+
|
|
76
|
+
async with DatabricksClient(host=..., warehouse_id=..., token=...) as client:
|
|
77
|
+
text = await client.query_agent("SELECT 1 AS answer", max_bytes=4000)
|
|
78
|
+
print(text) # Markdown on success; JSON on query failure
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
Give your agent the reusable [Arrowbricks skill](skills/arrowbricks/SKILL.md).
|
|
82
|
+
See [CLI options and skill installation](docs/usage.md#cli-quickstart-60) and
|
|
83
|
+
[agent budgets, completeness, and cancellation](docs/usage.md#agent-results).
|
|
84
|
+
|
|
85
|
+
## Query from an application
|
|
86
|
+
|
|
87
|
+
The async cursor offers familiar `execute` and `fetch*` methods.
|
|
88
|
+
Arrow results work directly with DuckDB, PyArrow, and Polars:
|
|
89
|
+
|
|
90
|
+
```python
|
|
91
|
+
from arrowbricks import connect
|
|
92
|
+
|
|
93
|
+
async with connect(host=..., warehouse_id=..., token=...) as conn:
|
|
94
|
+
cursor = conn.cursor()
|
|
95
|
+
await cursor.execute("SELECT * FROM my_catalog.my_schema.my_table LIMIT 100")
|
|
96
|
+
table = await cursor.fetchall_arrow()
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
Or consume ready-to-send NDJSON without building Python rows:
|
|
100
|
+
|
|
101
|
+
```python
|
|
102
|
+
from arrowbricks import HEARTBEAT, DatabricksClient
|
|
103
|
+
|
|
104
|
+
async with DatabricksClient(host=..., warehouse_id=..., token=...) as client:
|
|
105
|
+
async for item in client.stream_query_json("SELECT * FROM range(100)"):
|
|
106
|
+
if item is not HEARTBEAT:
|
|
107
|
+
print(item)
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
## Performance
|
|
111
|
+
|
|
112
|
+
The query and formatting work runs in Rust. Agent previews render directly from
|
|
113
|
+
Arrow, avoiding intermediate Python rows or JSON. In a real warehouse test,
|
|
114
|
+
a 100-row preview used **44% fewer bytes and 24% fewer tokens** than NDJSON.
|
|
115
|
+
See [agent measurements and methodology](docs/research/agent-rendering.md) and
|
|
116
|
+
[client benchmarks](docs/benchmarks.md) for workloads, comparisons, and limits.
|
|
117
|
+
|
|
118
|
+
## Learn more
|
|
119
|
+
|
|
120
|
+
- [Usage and API reference](docs/usage.md): authentication, errors, cancellation, observability, and Arrow interop.
|
|
121
|
+
- [Runnable examples](examples/): cursors, agents, OAuth, and FastAPI streaming.
|
|
122
|
+
- [Rust core](rust/arrowbricks_core/README.md): implementation and native APIs.
|
|
123
|
+
- [Real warehouse verification](docs/usage.md#real-warehouse-verification).
|
|
124
|
+
- [Changelog](CHANGELOG.md).
|
|
125
|
+
|
|
126
|
+
## License
|
|
127
|
+
|
|
128
|
+
MIT
|
|
129
|
+
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="assets/logo.svg" width="72" height="72" alt="arrowbricks logo">
|
|
3
|
+
</p>
|
|
4
|
+
|
|
5
|
+
<h1 align="center">arrowbricks</h1>
|
|
6
|
+
<p align="center">Fast Databricks SQL for Python applications and AI agents.</p>
|
|
7
|
+
|
|
8
|
+
- **Zero required Python dependencies.** One precompiled wheel. No Rust toolchain or cloud SDK needed.
|
|
9
|
+
- **Fast Rust core.** Concurrent chunk downloads, lazy Arrow decoding, and streaming NDJSON, with result order preserved.
|
|
10
|
+
- **Agent mode, new in 6.0.** A CLI and Python API that return compact typed Markdown, explicit truncation, and actionable JSON errors. Formatting stays in Rust.
|
|
11
|
+
|
|
12
|
+
Use Thrift by default or the Statement Execution API with `protocol="sea"`.
|
|
13
|
+
Bring a token or your own token-refresh callable.
|
|
14
|
+
|
|
15
|
+
## Install
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
pip install arrowbricks
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
Wheels cover Linux, macOS, and Windows; Python 3.11+.
|
|
22
|
+
Arrow, NDJSON, and agent mode work without extras. Python row tuples need
|
|
23
|
+
`pip install 'arrowbricks[arro3]'`; sqlglot expressions as SQL need `[sqlglot]`.
|
|
24
|
+
On Python 3.14+ pass values as a t-string (`t"... WHERE id = {x}"`) and they are
|
|
25
|
+
bound as parameters -- see [SQL safety](docs/usage.md#sql-safety-t-strings-and-literalstring).
|
|
26
|
+
|
|
27
|
+
## Query from an agent or terminal
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
export DATABRICKS_HOST='your-workspace.azuredatabricks.net'
|
|
31
|
+
export DATABRICKS_WAREHOUSE_ID='your-warehouse-id'
|
|
32
|
+
export DATABRICKS_TOKEN='your-access-token'
|
|
33
|
+
|
|
34
|
+
arrowbricks -c 'SELECT 1 AS answer'
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
```markdown
|
|
38
|
+
| answer (Int32) |
|
|
39
|
+
| --- |
|
|
40
|
+
| 1 |
|
|
41
|
+
|
|
42
|
+
rows_shown=1; complete=true; total_rows=1
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
Agent mode is the CLI default. SQL files and piped stdin work too:
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
arrowbricks --file query.sql --max-rows 20 --max-bytes 4000
|
|
49
|
+
printf 'SELECT 1 AS answer' | arrowbricks
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
Previews have row, byte, cell, and deadline limits. The footer tells you when
|
|
53
|
+
results are incomplete. Query errors go to stderr as JSON, preserving the
|
|
54
|
+
warehouse's diagnostic and suggestions with a recovery hint; the CLI exits 1.
|
|
55
|
+
Preview limits bound output and fetching, not warehouse scan cost.
|
|
56
|
+
|
|
57
|
+
For Python agents:
|
|
58
|
+
|
|
59
|
+
```python
|
|
60
|
+
from arrowbricks import DatabricksClient
|
|
61
|
+
|
|
62
|
+
async with DatabricksClient(host=..., warehouse_id=..., token=...) as client:
|
|
63
|
+
text = await client.query_agent("SELECT 1 AS answer", max_bytes=4000)
|
|
64
|
+
print(text) # Markdown on success; JSON on query failure
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Give your agent the reusable [Arrowbricks skill](skills/arrowbricks/SKILL.md).
|
|
68
|
+
See [CLI options and skill installation](docs/usage.md#cli-quickstart-60) and
|
|
69
|
+
[agent budgets, completeness, and cancellation](docs/usage.md#agent-results).
|
|
70
|
+
|
|
71
|
+
## Query from an application
|
|
72
|
+
|
|
73
|
+
The async cursor offers familiar `execute` and `fetch*` methods.
|
|
74
|
+
Arrow results work directly with DuckDB, PyArrow, and Polars:
|
|
75
|
+
|
|
76
|
+
```python
|
|
77
|
+
from arrowbricks import connect
|
|
78
|
+
|
|
79
|
+
async with connect(host=..., warehouse_id=..., token=...) as conn:
|
|
80
|
+
cursor = conn.cursor()
|
|
81
|
+
await cursor.execute("SELECT * FROM my_catalog.my_schema.my_table LIMIT 100")
|
|
82
|
+
table = await cursor.fetchall_arrow()
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
Or consume ready-to-send NDJSON without building Python rows:
|
|
86
|
+
|
|
87
|
+
```python
|
|
88
|
+
from arrowbricks import HEARTBEAT, DatabricksClient
|
|
89
|
+
|
|
90
|
+
async with DatabricksClient(host=..., warehouse_id=..., token=...) as client:
|
|
91
|
+
async for item in client.stream_query_json("SELECT * FROM range(100)"):
|
|
92
|
+
if item is not HEARTBEAT:
|
|
93
|
+
print(item)
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
## Performance
|
|
97
|
+
|
|
98
|
+
The query and formatting work runs in Rust. Agent previews render directly from
|
|
99
|
+
Arrow, avoiding intermediate Python rows or JSON. In a real warehouse test,
|
|
100
|
+
a 100-row preview used **44% fewer bytes and 24% fewer tokens** than NDJSON.
|
|
101
|
+
See [agent measurements and methodology](docs/research/agent-rendering.md) and
|
|
102
|
+
[client benchmarks](docs/benchmarks.md) for workloads, comparisons, and limits.
|
|
103
|
+
|
|
104
|
+
## Learn more
|
|
105
|
+
|
|
106
|
+
- [Usage and API reference](docs/usage.md): authentication, errors, cancellation, observability, and Arrow interop.
|
|
107
|
+
- [Runnable examples](examples/): cursors, agents, OAuth, and FastAPI streaming.
|
|
108
|
+
- [Rust core](rust/arrowbricks_core/README.md): implementation and native APIs.
|
|
109
|
+
- [Real warehouse verification](docs/usage.md#real-warehouse-verification).
|
|
110
|
+
- [Changelog](CHANGELOG.md).
|
|
111
|
+
|
|
112
|
+
## License
|
|
113
|
+
|
|
114
|
+
MIT
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "arrowbricks"
|
|
3
|
-
version = "6.
|
|
3
|
+
version = "6.1.0"
|
|
4
4
|
description = "Rust-powered Databricks SQL client and agent CLI with Arrow/NDJSON streaming, bounded Markdown previews, actionable JSON diagnostics, and zero required Python dependencies."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "MIT"
|
|
@@ -21,6 +21,9 @@ arrowbricks = "arrowbricks.cli:main"
|
|
|
21
21
|
# Row materialization converts our capsule table through arro3-core.
|
|
22
22
|
# Arrow fetches and streaming do not need this optional dependency.
|
|
23
23
|
arro3 = ["arro3-core>=0.8"]
|
|
24
|
+
# Only for passing sqlglot expressions as SQL (`Cursor.execute(expr)`); they
|
|
25
|
+
# are rendered in the Databricks dialect. Nothing imports sqlglot at runtime.
|
|
26
|
+
sqlglot = ["sqlglot>=27"]
|
|
24
27
|
|
|
25
28
|
[project.urls]
|
|
26
29
|
Repository = "https://github.com/bmsuisse/arrowbricks"
|
|
@@ -74,6 +77,7 @@ dev = [
|
|
|
74
77
|
"pytest>=8.0",
|
|
75
78
|
"pytest-asyncio>=0.24",
|
|
76
79
|
"ruff>=0.6",
|
|
80
|
+
"sqlglot>=27",
|
|
77
81
|
"ty>=0.0.65",
|
|
78
82
|
]
|
|
79
83
|
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
__lazy_modules__ = ["arrowbricks._streaming", "arrowbricks.client", "arrowbricks.cursor"]
|
|
4
4
|
|
|
5
5
|
from ._core import ArrowbricksError, AuthError, QueryStats, StatementError, TransientError
|
|
6
|
+
from ._sql import Sql, unsafe_sql
|
|
6
7
|
from ._streaming import (
|
|
7
8
|
HEARTBEAT,
|
|
8
9
|
QueryTimeout,
|
|
@@ -24,10 +25,12 @@ __all__ = [
|
|
|
24
25
|
"QueryStats",
|
|
25
26
|
"QueryTimeout",
|
|
26
27
|
"ReplayableArrowChunk",
|
|
28
|
+
"Sql",
|
|
27
29
|
"StatementError",
|
|
28
30
|
"TransientError",
|
|
29
31
|
"await_with_heartbeat",
|
|
30
32
|
"connect",
|
|
31
33
|
"stream_query_json",
|
|
34
|
+
"unsafe_sql",
|
|
32
35
|
"write_ipc_stream",
|
|
33
36
|
]
|
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
"""SQL input handling: `LiteralString`, PEP 750 t-strings and sqlglot expressions.
|
|
2
|
+
|
|
3
|
+
Every public entry point that takes SQL accepts the `Sql` type below. A plain
|
|
4
|
+
`str` still works at runtime (it is sent unchanged), but is typed as
|
|
5
|
+
`LiteralString` so a type checker flags SQL built from runtime values -- the
|
|
6
|
+
injection bug class. Dynamic *values* go in a t-string (Python 3.14+), whose
|
|
7
|
+
interpolations are bound as Databricks named parameters, the same way
|
|
8
|
+
psycopg 3.3's t-string support works, never spliced into the SQL text:
|
|
9
|
+
|
|
10
|
+
await cur.execute(t"SELECT * FROM {table:i} WHERE id = {user_id}")
|
|
11
|
+
|
|
12
|
+
`{x}`/`{x:s}` binds a parameter, `{x:i}` quotes an identifier, `{t:q}` splices a
|
|
13
|
+
nested t-string; conversions (`!r`) and other specs raise, as in psycopg.
|
|
14
|
+
A t-string takes no separate `parameters` -- its interpolations are the
|
|
15
|
+
parameters. Never build a `Template(...)` from runtime strings: unlike a t-string
|
|
16
|
+
literal, the type system can't tell it apart.
|
|
17
|
+
|
|
18
|
+
`prepare_sql` turns any accepted input into the `(sql, parameters)` pair the
|
|
19
|
+
Rust core takes -- `parameters` being Databricks' own
|
|
20
|
+
`[{"name": ..., "value": ..., "type": ...}]` format. Nothing below the Python
|
|
21
|
+
facade knows about t-strings.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import datetime
|
|
27
|
+
import re
|
|
28
|
+
import sys
|
|
29
|
+
import uuid
|
|
30
|
+
from decimal import Decimal
|
|
31
|
+
from typing import Any, LiteralString, Protocol, TypeAlias, cast
|
|
32
|
+
|
|
33
|
+
if sys.version_info >= (3, 14):
|
|
34
|
+
from string.templatelib import Interpolation, Template
|
|
35
|
+
else:
|
|
36
|
+
# Stand-ins so type checkers (and `Sql` below) see a real `Template` type
|
|
37
|
+
# on older Pythons, instead of an unresolved name that would silently turn
|
|
38
|
+
# the whole `Sql` union into `Any` and switch the LiteralString guard off.
|
|
39
|
+
# Nothing at runtime ever builds or matches these.
|
|
40
|
+
from collections.abc import Iterator
|
|
41
|
+
from dataclasses import dataclass
|
|
42
|
+
from typing import Generic, Literal, TypeVar
|
|
43
|
+
|
|
44
|
+
_T = TypeVar("_T")
|
|
45
|
+
|
|
46
|
+
class Template:
|
|
47
|
+
strings: tuple[str, ...]
|
|
48
|
+
interpolations: tuple[Interpolation[Any], ...]
|
|
49
|
+
|
|
50
|
+
def __new__(cls, *args: str | Interpolation[Any]) -> Template:
|
|
51
|
+
raise NotImplementedError("t-strings need Python 3.14+")
|
|
52
|
+
|
|
53
|
+
def __iter__(self) -> Iterator[str | Interpolation[Any]]:
|
|
54
|
+
raise NotImplementedError
|
|
55
|
+
|
|
56
|
+
@dataclass
|
|
57
|
+
class Interpolation(Generic[_T]):
|
|
58
|
+
value: _T
|
|
59
|
+
expression: str
|
|
60
|
+
conversion: Literal["a", "r", "s"] | None
|
|
61
|
+
format_spec: str
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class _SqlExpression(Protocol):
|
|
65
|
+
"""Structural stand-in for a sqlglot expression (`pip install arrowbricks[sqlglot]`),
|
|
66
|
+
so the `Sql` union doesn't depend on sqlglot being installed. A `str` has no
|
|
67
|
+
`.sql()`, so it still doesn't match."""
|
|
68
|
+
|
|
69
|
+
def sql(self, dialect: str | None = None, **opts: Any) -> str: ...
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
Sql: TypeAlias = LiteralString | Template | _SqlExpression
|
|
73
|
+
|
|
74
|
+
__all__ = ["Sql", "prepare_sql", "unsafe_sql"]
|
|
75
|
+
|
|
76
|
+
_PARAM_PREFIX = "ab_p"
|
|
77
|
+
|
|
78
|
+
_INT64_MIN = -(2**63)
|
|
79
|
+
_INT64_MAX = 2**63 - 1
|
|
80
|
+
_DECIMAL_MAX = 10**38
|
|
81
|
+
|
|
82
|
+
# What may not directly follow a `:ab_pN` marker: it would extend the name (or,
|
|
83
|
+
# for `:`, turn into Databricks' JSON-path operator).
|
|
84
|
+
_MARKER_CONTINUES = re.compile(r"[A-Za-z0-9_:]")
|
|
85
|
+
_ENDS_WITH_MARKER = re.compile(rf":{_PARAM_PREFIX}\d+$")
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def unsafe_sql(sql: str) -> LiteralString:
|
|
89
|
+
"""Marks a runtime `str` as trusted SQL for type checkers -- the explicit,
|
|
90
|
+
greppable escape hatch for SQL that is genuinely not a literal (a CLI
|
|
91
|
+
argument, a `.sql` file read from disk). Does nothing at runtime. Never use
|
|
92
|
+
it on text that contains request data; bind values with a t-string."""
|
|
93
|
+
return cast("LiteralString", sql)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def prepare_sql(sql: Any, parameters: list[dict[str, Any]] | None = None) -> tuple[str, list[dict[str, Any]] | None]:
|
|
97
|
+
"""Returns the SQL text and the named-parameter list to submit. `str` is
|
|
98
|
+
returned as-is with `parameters`; a t-string is rendered with one
|
|
99
|
+
`:ab_p<n>` marker per interpolation (and may not be combined with
|
|
100
|
+
`parameters`); a sqlglot expression is rendered in the Databricks dialect."""
|
|
101
|
+
if isinstance(sql, str):
|
|
102
|
+
return sql, parameters
|
|
103
|
+
if sys.version_info >= (3, 14) and isinstance(sql, Template):
|
|
104
|
+
if parameters:
|
|
105
|
+
raise TypeError("a t-string carries its own parameters; don't also pass `parameters`")
|
|
106
|
+
parts: list[str] = []
|
|
107
|
+
bound: list[dict[str, Any]] = []
|
|
108
|
+
_render(sql, parts, bound)
|
|
109
|
+
return "".join(parts), bound or parameters
|
|
110
|
+
if _is_sqlglot_expression(sql):
|
|
111
|
+
return sql.sql(dialect="databricks"), parameters
|
|
112
|
+
raise TypeError(
|
|
113
|
+
f"sql must be a str, a t-string or a sqlglot expression, not {type(sql).__name__}"
|
|
114
|
+
+ (" (t-strings need Python 3.14+)" if sys.version_info < (3, 14) else "")
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _is_sqlglot_expression(value: object) -> bool:
|
|
119
|
+
# Duck-typed so sqlglot stays an optional dependency, never imported here.
|
|
120
|
+
return type(value).__module__.partition(".")[0] == "sqlglot" and callable(getattr(value, "sql", None))
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _render(template: Any, parts: list[str], bound: list[dict[str, Any]]) -> None:
|
|
124
|
+
interpolations = template.interpolations
|
|
125
|
+
for index, literal in enumerate(template.strings):
|
|
126
|
+
parts.append(literal)
|
|
127
|
+
if index == len(interpolations):
|
|
128
|
+
break
|
|
129
|
+
_render_interpolation(interpolations[index], template.strings[index + 1], parts, bound)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _render_interpolation(interpolation: Any, following: str, parts: list[str], bound: list[dict[str, Any]]) -> None:
|
|
133
|
+
spec = interpolation.format_spec
|
|
134
|
+
conversion = f"!{interpolation.conversion}" if interpolation.conversion else ""
|
|
135
|
+
shown = f"{{{interpolation.expression or '<value>'}{conversion}{':' + spec if spec else ''}}}"
|
|
136
|
+
if interpolation.conversion is not None:
|
|
137
|
+
raise TypeError(f"conversions not supported in SQL t-string; got '{shown}'")
|
|
138
|
+
if spec not in ("", "s", "i", "q"):
|
|
139
|
+
raise ValueError(f"format spec not supported in SQL t-string; got '{shown}' (use none, 's', 'i' or 'q')")
|
|
140
|
+
text = "".join(parts)
|
|
141
|
+
if _in_quoted_context(text):
|
|
142
|
+
raise ValueError(
|
|
143
|
+
f"{shown} sits inside a quoted string, identifier or comment, where it would not be bound; "
|
|
144
|
+
"build the value instead, e.g. {prefix + '%'} outside the quotes"
|
|
145
|
+
)
|
|
146
|
+
value = interpolation.value
|
|
147
|
+
if spec == "q" or isinstance(value, Template):
|
|
148
|
+
if not (spec == "q" and isinstance(value, Template)):
|
|
149
|
+
raise ValueError(f"{shown}: a nested t-string needs the ':q' spec, and ':q' accepts only a t-string")
|
|
150
|
+
_render(value, parts, bound)
|
|
151
|
+
return
|
|
152
|
+
if spec == "i":
|
|
153
|
+
parts.append(_quote_identifier(shown, value))
|
|
154
|
+
return
|
|
155
|
+
if text.endswith(":") or _ENDS_WITH_MARKER.search(text) or _MARKER_CONTINUES.match(following):
|
|
156
|
+
raise ValueError(f"{shown} must be separated from adjacent text by a space or punctuation (parameter marker)")
|
|
157
|
+
name = f"{_PARAM_PREFIX}{len(bound)}"
|
|
158
|
+
bound.append({"name": name, **_to_parameter(shown, value)})
|
|
159
|
+
parts.append(f":{name}")
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def _in_quoted_context(text: str) -> bool:
|
|
163
|
+
"""True if the end of `text` is inside a '...'/"..."/`...` literal, a
|
|
164
|
+
`-- ...` line comment or a `/* ... */` block comment. Backslash escapes the
|
|
165
|
+
next character inside quotes, as in Databricks' default string parsing."""
|
|
166
|
+
quote: str | None = None
|
|
167
|
+
i, n = 0, len(text)
|
|
168
|
+
while i < n:
|
|
169
|
+
c = text[i]
|
|
170
|
+
if quote == "--":
|
|
171
|
+
if c == "\n":
|
|
172
|
+
quote = None
|
|
173
|
+
elif quote == "/*":
|
|
174
|
+
if text.startswith("*/", i):
|
|
175
|
+
quote = None
|
|
176
|
+
i += 1
|
|
177
|
+
elif quote is not None:
|
|
178
|
+
if c == "\\" and quote != "`":
|
|
179
|
+
i += 1
|
|
180
|
+
elif c == quote:
|
|
181
|
+
quote = None
|
|
182
|
+
elif c in "'\"`":
|
|
183
|
+
quote = c
|
|
184
|
+
elif text.startswith("--", i):
|
|
185
|
+
quote = "--"
|
|
186
|
+
i += 1
|
|
187
|
+
elif text.startswith("/*", i):
|
|
188
|
+
quote = "/*"
|
|
189
|
+
i += 1
|
|
190
|
+
i += 1
|
|
191
|
+
return quote is not None
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _quote_identifier(shown: str, value: object) -> str:
|
|
195
|
+
# A tuple/list of parts renders `a`.`b`.`c`; a lone str is ONE identifier
|
|
196
|
+
# (a dotted string is not split), as with psycopg.
|
|
197
|
+
names = value if isinstance(value, (tuple, list)) else [value]
|
|
198
|
+
if not names or not all(isinstance(n, str) and n for n in names):
|
|
199
|
+
raise TypeError(f"identifier {shown} must be a non-empty str or a tuple of them, not {type(value).__name__}")
|
|
200
|
+
return ".".join("`" + str.__str__(n).replace("`", "``") + "`" for n in cast("list[str]", names))
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _to_parameter(shown: str, value: object) -> dict[str, Any]:
|
|
204
|
+
"""Databricks named-parameter `value`/`type` for a Python value -- the same
|
|
205
|
+
scalar types psycopg adapts, minus the ones Databricks can't bind (BINARY is
|
|
206
|
+
not a parameter type there). Text is taken through the base type's own
|
|
207
|
+
methods so a subclass (numpy.float64, a `str` Enum) binds its value, not its
|
|
208
|
+
repr or member name."""
|
|
209
|
+
if value is None:
|
|
210
|
+
return {"value": None, "type": "STRING"}
|
|
211
|
+
if isinstance(value, bool):
|
|
212
|
+
return {"value": "true" if value else "false", "type": "BOOLEAN"}
|
|
213
|
+
if isinstance(value, int):
|
|
214
|
+
if _INT64_MIN <= value <= _INT64_MAX:
|
|
215
|
+
return {"value": int.__repr__(value), "type": "BIGINT"}
|
|
216
|
+
if abs(value) >= _DECIMAL_MAX:
|
|
217
|
+
raise OverflowError(f"integer {shown} does not fit Databricks' DECIMAL(38,0)")
|
|
218
|
+
return {"value": int.__repr__(value), "type": "DECIMAL(38,0)"}
|
|
219
|
+
if isinstance(value, float):
|
|
220
|
+
return {"value": _float_text(value), "type": "DOUBLE"}
|
|
221
|
+
if isinstance(value, Decimal):
|
|
222
|
+
return _decimal_parameter(shown, value)
|
|
223
|
+
if isinstance(value, str):
|
|
224
|
+
return {"value": str.__str__(value), "type": "STRING"}
|
|
225
|
+
if isinstance(value, datetime.datetime): # before date: datetime is a date subclass
|
|
226
|
+
kind = "TIMESTAMP_NTZ" if value.utcoffset() is None else "TIMESTAMP"
|
|
227
|
+
return {"value": value.isoformat(sep=" "), "type": kind}
|
|
228
|
+
if isinstance(value, datetime.date):
|
|
229
|
+
return {"value": value.isoformat(), "type": "DATE"}
|
|
230
|
+
if isinstance(value, uuid.UUID):
|
|
231
|
+
return {"value": str(value), "type": "STRING"}
|
|
232
|
+
raise TypeError(
|
|
233
|
+
f"can't bind {shown} of type {type(value).__name__} as a SQL parameter; "
|
|
234
|
+
"supported: None, bool, int, float, Decimal, str, date, datetime, UUID "
|
|
235
|
+
"(convert other values explicitly)"
|
|
236
|
+
)
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _float_text(value: float) -> str:
|
|
240
|
+
if value != value: # NaN
|
|
241
|
+
return "NaN"
|
|
242
|
+
if value in (float("inf"), float("-inf")):
|
|
243
|
+
return "Infinity" if value > 0 else "-Infinity"
|
|
244
|
+
return float.__repr__(value)
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def _decimal_parameter(shown: str, value: Decimal) -> dict[str, Any]:
|
|
248
|
+
if not value.is_finite():
|
|
249
|
+
raise ValueError(f"can't bind non-finite Decimal {shown}; Databricks DECIMAL has no NaN/Infinity")
|
|
250
|
+
_, digits, exponent = value.as_tuple()
|
|
251
|
+
exponent = cast("int", exponent) # an int, now that NaN/Infinity are ruled out
|
|
252
|
+
if exponent >= 0:
|
|
253
|
+
precision, scale = len(digits) + exponent, 0
|
|
254
|
+
else:
|
|
255
|
+
precision, scale = max(len(digits), -exponent), -exponent
|
|
256
|
+
precision = max(precision, 1)
|
|
257
|
+
if precision > 38:
|
|
258
|
+
raise OverflowError(f"Decimal {shown} needs precision {precision}, Databricks allows at most 38")
|
|
259
|
+
return {"value": Decimal.__format__(value, "f"), "type": f"DECIMAL({precision},{scale})"}
|
|
@@ -17,6 +17,7 @@ from collections.abc import AsyncIterator, Awaitable, Iterator
|
|
|
17
17
|
from typing import Any, BinaryIO, TypeVar, cast
|
|
18
18
|
|
|
19
19
|
from . import _core
|
|
20
|
+
from ._sql import Sql, prepare_sql
|
|
20
21
|
from .client import DatabricksClient
|
|
21
22
|
|
|
22
23
|
__all__ = [
|
|
@@ -192,7 +193,7 @@ def _window_value(name: str, value: int | None) -> int | None:
|
|
|
192
193
|
|
|
193
194
|
async def stream_query_json(
|
|
194
195
|
client: DatabricksClient,
|
|
195
|
-
sql:
|
|
196
|
+
sql: Sql,
|
|
196
197
|
*,
|
|
197
198
|
params: list[dict[str, Any]] | None = None,
|
|
198
199
|
row_limit: int | None = None,
|
|
@@ -236,12 +237,13 @@ async def stream_query_json(
|
|
|
236
237
|
then yields each row individually."""
|
|
237
238
|
if non_finite_floats not in ("null", "string"):
|
|
238
239
|
raise ValueError(f"non_finite_floats must be 'null' or 'string', got {non_finite_floats!r}")
|
|
239
|
-
|
|
240
|
+
text, params = prepare_sql(sql, params)
|
|
241
|
+
text = windowed_sql(text, row_limit=row_limit, offset=offset)
|
|
240
242
|
core_client = client._core_client # noqa: SLF001 -- same package, see client.py
|
|
241
243
|
|
|
242
244
|
with rust_timeout_as_query_timeout():
|
|
243
245
|
async for item in core_client.stream_ndjson_lines(
|
|
244
|
-
|
|
246
|
+
text,
|
|
245
247
|
catalog=catalog,
|
|
246
248
|
schema=schema,
|
|
247
249
|
parameters=params,
|
|
@@ -53,6 +53,7 @@ def _parser() -> argparse.ArgumentParser:
|
|
|
53
53
|
|
|
54
54
|
|
|
55
55
|
async def _run(args: argparse.Namespace, sql: str) -> int:
|
|
56
|
+
from ._sql import unsafe_sql
|
|
56
57
|
from .client import DatabricksClient
|
|
57
58
|
|
|
58
59
|
try:
|
|
@@ -63,7 +64,7 @@ async def _run(args: argparse.Namespace, sql: str) -> int:
|
|
|
63
64
|
protocol=args.protocol,
|
|
64
65
|
) as client:
|
|
65
66
|
result = await client.query_agent(
|
|
66
|
-
sql,
|
|
67
|
+
unsafe_sql(sql), # operator-supplied SQL, sent unchanged
|
|
67
68
|
max_rows=args.max_rows,
|
|
68
69
|
max_bytes=args.max_bytes,
|
|
69
70
|
max_cell_chars=args.max_cell_chars,
|
|
@@ -74,10 +75,11 @@ async def _run(args: argparse.Namespace, sql: str) -> int:
|
|
|
74
75
|
except (ValueError, OverflowError, TypeError) as exc:
|
|
75
76
|
_configuration_error(str(exc))
|
|
76
77
|
return 2
|
|
78
|
+
# Locale encoding and newline translation must not change the byte budget.
|
|
77
79
|
if result.lstrip().startswith("{"):
|
|
78
|
-
sys.stderr.write(result)
|
|
80
|
+
sys.stderr.buffer.write(result.encode("utf-8"))
|
|
79
81
|
return 1
|
|
80
|
-
sys.stdout.write(result)
|
|
82
|
+
sys.stdout.buffer.write(result.encode("utf-8"))
|
|
81
83
|
return 0
|
|
82
84
|
|
|
83
85
|
|
|
@@ -11,6 +11,7 @@ from collections.abc import AsyncIterator, Awaitable, Callable
|
|
|
11
11
|
from typing import Any
|
|
12
12
|
|
|
13
13
|
from . import _core
|
|
14
|
+
from ._sql import Sql, prepare_sql
|
|
14
15
|
|
|
15
16
|
TokenProvider = Callable[[], "str | Awaitable[str]"]
|
|
16
17
|
|
|
@@ -148,7 +149,7 @@ class DatabricksClient:
|
|
|
148
149
|
|
|
149
150
|
async def query_agent(
|
|
150
151
|
self,
|
|
151
|
-
sql:
|
|
152
|
+
sql: Sql,
|
|
152
153
|
*,
|
|
153
154
|
max_rows: int = 1000,
|
|
154
155
|
max_bytes: int = 10000,
|
|
@@ -164,11 +165,13 @@ class DatabricksClient:
|
|
|
164
165
|
deadline across submission and fetching. The completeness footer marks
|
|
165
166
|
clipping and gives an exact total only after exhausting the result.
|
|
166
167
|
Invalid budgets raise before submitting SQL; asyncio cancellation remains
|
|
167
|
-
cancellation.
|
|
168
|
-
|
|
168
|
+
cancellation. A `str` is sent unchanged (a t-string's values are bound
|
|
169
|
+
as named parameters), with best-effort server cancellation when
|
|
170
|
+
stopped. Output limits do not limit the warehouse's scan cost.
|
|
169
171
|
"""
|
|
172
|
+
text, params = prepare_sql(sql, params)
|
|
170
173
|
return await self._core_client.query_agent(
|
|
171
|
-
|
|
174
|
+
text,
|
|
172
175
|
max_rows=max_rows,
|
|
173
176
|
max_bytes=max_bytes,
|
|
174
177
|
max_cell_chars=max_cell_chars,
|
|
@@ -180,7 +183,7 @@ class DatabricksClient:
|
|
|
180
183
|
|
|
181
184
|
def stream_query_json(
|
|
182
185
|
self,
|
|
183
|
-
sql:
|
|
186
|
+
sql: Sql,
|
|
184
187
|
*,
|
|
185
188
|
params: list[dict[str, Any]] | None = None,
|
|
186
189
|
row_limit: int | None = None,
|