arrowbricks 0.1.0__tar.gz

arrowbricks-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,22 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.13", "3.14"]
+    steps:
+      - uses: actions/checkout@v7
+      - uses: astral-sh/setup-uv@v9.0.0
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: uv sync --all-extras
+      - run: uv run ruff check .
+      - run: uv run ty check src tests
+      - run: uv run pytest -q

arrowbricks-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,32 @@
+name: Release
+
+on:
+  push:
+    tags: ["v*.*.*"]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: astral-sh/setup-uv@v9.0.0
+        with:
+          python-version: "3.14"
+      - run: uv sync --all-extras
+      - run: uv run ruff check .
+      - run: uv run ty check src tests
+      - run: uv run pytest -q
+
+  publish:
+    needs: test
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write # PyPI trusted publishing (OIDC) -- no stored API token
+    steps:
+      - uses: actions/checkout@v7
+      - uses: astral-sh/setup-uv@v9.0.0
+        with:
+          python-version: "3.14"
+      - run: uv build
+      - uses: pypa/gh-action-pypi-publish@release/v1

arrowbricks-0.1.0/.gitignore

@@ -0,0 +1,218 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#   Usually these files are written by a python script from a template
+#   before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+# Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+# uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+# poetry.lock
+# poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+# pdm.lock
+# pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+# pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# Redis
+*.rdb
+*.aof
+*.pid
+
+# RabbitMQ
+mnesia/
+rabbitmq/
+rabbitmq-data/
+
+# ActiveMQ
+activemq-data/
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#   JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#   be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#   and can be added to the global gitignore or merged into this file.  For a more nuclear
+#   option (not recommended) you can uncomment the following to ignore the entire idea folder.
+# .idea/
+
+# Abstra
+#   Abstra is an AI-powered process automation framework.
+#   Ignore directories containing user credentials, local state, and settings.
+#   Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#   Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#   that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#   and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#   you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+# Temporary file for partial code execution
+tempCodeRunnerFile.py
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/
+
+# Streamlit
+.streamlit/secrets.toml

arrowbricks-0.1.0/AGENTS.md

@@ -0,0 +1,44 @@
+# arrowbricks
+
+Runs SQL against a Databricks SQL warehouse via the Statement Execution API and hands the result back as Arrow (a `Cursor` shaped like `databricks-sql-python`'s) or streaming NDJSON. See `README.md` for the user-facing API; this file is about working *on* the package.
+
+## Layout
+
+- `src/arrowbricks/client.py` -- pure REST client (auth, statement submission/polling, backpressure-bounded concurrent chunk download). No Arrow dependency at all, intentionally: someone who only wants `execute_json_statement` or `upload_volume_file`/`delete_volume_file` shouldn't need arro3 pulled in conceptually either, even though it's a hard dependency of the package as a whole. Retries are a small hand-rolled `_retry_call` loop, not a dependency (see "Design invariants" below).
+- `src/arrowbricks/_streaming.py` -- Arrow (de)serialization via arro3, always: `ReplayableArrowChunk`, `write_ipc_stream` (always uncompressed, see below), heartbeat helpers, chunk fetching, and `stream_query_json` (arro3's `write_ndjson`). One Arrow engine, no pluggable backend -- unlike duckbricks, there's nothing here to make pluggable; arro3 *is* the whole point.
+- `src/arrowbricks/cursor.py` -- `Connection`/`Cursor`, the DB-API-ish surface (`execute`/`execute_streamed`, `fetchone`/`fetchmany`/`fetchall`, `fetchall_arrow`/`fetchmany_arrow`). `_ResultSet` buffers at the Arrow-Table level (not materialized Python rows) so the Arrow-native fetch methods stay zero-copy; row-based fetches materialize lazily off that buffer.
+- `tests/` -- respx mocks the Databricks REST endpoints (warehouse status, statement submit, chunk-link resolution, external-link byte download); no real warehouse or credentials needed to run the suite.
+
+## Commands
+
+```bash
+uv sync --all-extras
+uv run pytest -q
+uv run ruff check .
+uv run ty check src tests
+```
+
+One-time setup per clone: `prek install` (needs `uv tool install prek` first if not already on PATH).
+
+## Design invariants -- don't casually undo these
+
+- **No cloud-SDK dependency.** Auth is `token: str` or `token_provider: Callable[[], str | Awaitable[str]]`. Do not add `azure-identity`/`boto3`/etc. as a real dependency -- that belongs in the caller's app.
+- **No hardcoded catalog/schema.** `catalog`/`schema` default to `None` everywhere. This package has zero knowledge of any specific Databricks workspace's naming.
+- **Chunk order is not fetch order.** `DatabricksClient` fetches chunks concurrently (bounded, with backpressure) and they can complete out of order. `_ResultSet`/`stream_query_json` both hold a `pending: dict[int, chunk]` reorder buffer keyed by `chunk_index`, releasing in order as the next expected index shows up. If you touch either, keep a test proving order survives out-of-order arrival (see `test_fetchall_preserves_order_despite_out_of_order_chunks`, `test_stream_query_json_preserves_order_despite_out_of_order_chunks`).
+- **Chunks are fetched lazily, not all upfront.** `_ResultSet` only pulls the next chunk from `_chunk_aiter` when the caller's `fetchone`/`fetchmany`/`fetchall` actually needs more rows than are already buffered. Don't "simplify" this into draining the whole chunk iterator inside `execute()` -- that defeats the point of a paginated cursor.
+- **No silent row caps.** There's no `ABSOLUTE_ROW_LIMIT`-style ceiling baked in. If a caller wants one, that's `row_limit`, which they pass explicitly.
+- **No retry dependency.** `client.py`'s `_retry_call` is a ~10-line hand-rolled exponential-backoff loop, replacing tenacity on purpose -- it's the only retry pattern in the whole client, so a dependency for it wasn't worth it.
+- **One Arrow engine, no pluggable backend.** Unlike duckbricks (which supports nanoarrow *or* arro3 *or* bring-your-own), arrowbricks is arro3-only by design -- that's the entire "single responsibility" pitch. Don't add a backend-abstraction layer back in; if a caller needs a different Arrow engine, that's duckbricks' `set_arrow_backend()`, not this package.
+- **`write_ipc_stream` (and everything built on it) always writes uncompressed Arrow-IPC bodies.** `aio.write_ipc_stream(..., compression=None)` explicitly, everywhere. arro3's own default (`compression="LZ4"`) is transparently decompressed by DuckDB's Arrow reader but not necessarily by other Arrow IPC readers -- `duckdb-wasm`'s browser-side decoder silently fails to parse LZ4-compressed bodies (this was a real bug in duckbricks 0.3.0, fixed in 0.3.1 -- see its CHANGELOG/git history). Never remove the explicit `compression=None`.
+
+## Testing
+
+Mock the Databricks endpoints with `respx` (see `tests/conftest.py`'s `mock_warehouse` fixture) rather than hitting a real warehouse. The fixture builds real Arrow-IPC chunk bytes via arro3 directly, so tests exercise the actual Arrow IPC round trip, not a stand-in. Pass `reverse_arrival=True` to force genuine out-of-order chunk completion when a test needs to prove ordering survives it.
+
+## Releasing
+
+1. Bump `version` in `pyproject.toml`.
+2. `git tag vX.Y.Z && git push origin vX.Y.Z`.
+3. `.github/workflows/release.yml` runs the test job, then builds and publishes to PyPI via trusted publishing (OIDC) -- no stored token.
+
+One-time, outside this repo: register this GitHub repo + `release.yml` workflow as a **trusted publisher** on the `arrowbricks` PyPI project (PyPI project settings -> Publishing). Without that, the `publish` job's OIDC exchange fails even though tests pass.

arrowbricks-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 BMS Suisse AG
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

arrowbricks-0.1.0/PKG-INFO

@@ -0,0 +1,114 @@
+Metadata-Version: 2.4
+Name: arrowbricks
+Version: 0.1.0
+Summary: Runs SQL against a Databricks SQL warehouse via the Statement Execution API and hands you the result as Arrow -- a DB-API-ish Cursor (fetchone/fetchmany/fetchall/fetchall_arrow) or NDJSON streaming. Single Arrow engine (arro3), no DuckDB.
+Project-URL: Repository, https://github.com/bmsuisse/arrowbricks
+License-Expression: MIT
+License-File: LICENSE
+Requires-Python: >=3.11
+Requires-Dist: arro3-core>=0.8
+Requires-Dist: arro3-io>=0.8
+Requires-Dist: httpx>=0.27
+Description-Content-Type: text/markdown
+
+# arrowbricks
+
+Runs SQL against a Databricks SQL warehouse via the Statement Execution API and hands you the result as Arrow -- a `Cursor` shaped like [`databricks-sql-python`](https://github.com/databricks/databricks-sql-python)'s (`execute`, `fetchone`/`fetchmany`/`fetchall`, `fetchall_arrow`/`fetchmany_arrow`), or `stream_query_json` for streaming NDJSON. One Arrow engine ([arro3](https://github.com/kylebarron/arro3)), no DuckDB, no pandas/pyarrow.
+
+- Single responsibility: Databricks to Arrow via arro3. No embedded query engine -- that's [duckbricks](https://github.com/bmsuisse/duckbricks), built on top of this.
+- Bring-your-own-auth -- a static token or your own token-refresh callable. No cloud-SDK dependency baked in.
+- Result-order preserved even though chunks can complete out of order over the network.
+- Chunks are fetched lazily as `fetchone`/`fetchmany`/`fetchall` actually need them, not all upfront.
+- Heartbeats between slow chunks (`execute_streamed`/`stream_query_json`), so a caller streaming this over e.g. SSE never goes silent during a cold warehouse start.
+
+## Install
+
+```bash
+pip install arrowbricks
+```
+
+Dependencies: `httpx` + `arro3-core` + `arro3-io`. That's the whole tree.
+
+## Quickstart
+
+```python
+import asyncio
+from arrowbricks import connect
+
+
+async def main():
+    conn = connect(
+        host="adb-1234567890.1.azuredatabricks.net",
+        warehouse_id="abcd1234efgh5678",
+        token="dapi...",  # or token_provider=... -- see Auth below
+    )
+    cursor = conn.cursor()
+
+    await cursor.execute("SELECT * FROM my_catalog.my_schema.my_table LIMIT 100")
+    async for row in cursor:
+        print(row)
+
+    await cursor.execute("SELECT * FROM my_catalog.my_schema.my_table LIMIT 100")
+    table = await cursor.fetchall_arrow()  # an arro3 Table
+
+
+asyncio.run(main())
+```
+
+For streaming NDJSON (e.g. a FastAPI SSE endpoint, first row out as soon as its chunk arrives):
+
+```python
+from arrowbricks import HEARTBEAT, DatabricksClient, stream_query_json
+
+client = DatabricksClient(host=..., warehouse_id=..., token=...)
+
+async for item in stream_query_json(client, "SELECT * FROM my_catalog.my_schema.big_table"):
+    if item is HEARTBEAT:
+        continue  # forward as an SSE keep-alive comment, e.g.
+    print(item)  # one ready-to-send JSON string per row
+```
+
+## Why not `databricks-sql-connector`?
+
+The [official driver](https://github.com/databricks/databricks-sql-python) is the right choice if you need full DB-API 2.0 compatibility over Databricks' Thrift/ODBC-style protocol. If you just want a query result as Arrow/JSON in your own async app, it drags in a lot for that: `pandas`, `thrift`, `openpyxl`, `pybreaker`, `pyjwt`, `oauthlib`, `lz4`, `requests`, `urllib3` as hard dependencies. arrowbricks talks to the plain REST Statement Execution API instead, and its whole dependency tree is `httpx` + `arro3-core` + `arro3-io`. The `Cursor` API is deliberately shaped like the official driver's so switching between them is mostly a constructor change, but arrowbricks is async throughout (`execute`, `fetchone`, etc. are all coroutines) -- there's no sync escape hatch.
+
+## Why not `duckbricks`?
+
+[duckbricks](https://github.com/bmsuisse/duckbricks) does the same Databricks-to-Arrow work, then goes further: it uses a real embedded DuckDB engine to materialize results into your own DuckDB connection/table (`feed_select_to_duckdb_table`), or push a DuckDB query's result *up* to Databricks (`feed_duckdb_table_to_databricks`). If you need that -- a real local SQL engine sitting on top, not just "run this query, get Arrow/JSON back" -- use duckbricks; it depends on arrowbricks for the Databricks/Arrow half. If you don't need DuckDB at all, arrowbricks alone is the smaller, single-responsibility half.
+
+## Auth
+
+`connect`/`DatabricksClient` take either:
+
+- `token: str` -- a static personal access token or pre-issued OAuth token, or
+- `token_provider` -- a callable (sync or async) returning a token string, called on every request.
+
+arrowbricks has no opinion on *how* you get a token and no cloud-SDK dependency of its own. If your provider is expensive to call, cache/refresh inside it -- arrowbricks does no caching on your behalf.
+
+```python
+conn = connect(host=..., warehouse_id=..., token_provider=my_token_provider)
+```
+
+## API
+
+- `connect(host, warehouse_id, *, token=None, token_provider=None, ...) -> Connection`
+- `Connection.cursor() -> Cursor`
+- `Cursor.execute(sql, parameters=None, *, row_limit=None, offset=None, catalog=None, schema=None, total_timeout_s=None) -> Cursor` -- submits and waits for the statement, like a real DB-API cursor. `parameters`, if given, is Databricks' own named-parameter format -- `[{"name": ..., "value": ..., "type": ...}]` bound against `:name` markers in `sql`.
+- `Cursor.execute_streamed(...)` -- same args, but an async generator yielding `HEARTBEAT` while waiting on a slow cold start, then the ready `Cursor` -- for bridging e.g. an SSE connection.
+- `Cursor.fetchone() -> tuple | None`, `Cursor.fetchmany(size) -> list[tuple]`, `Cursor.fetchall() -> list[tuple]`
+- `Cursor.fetchmany_arrow(size) -> arro3.core.Table`, `Cursor.fetchall_arrow() -> arro3.core.Table`
+- `Cursor` is an async iterator, yielding one row (tuple) at a time.
+- `Cursor.description` -- DB-API-style `[(name, type_name, None, None, None, None, None), ...]` after `execute()`.
+- `stream_query_json(client, sql, **kwargs)` -- yields `HEARTBEAT`, then each row as a JSON string, as soon as its chunk arrives. Timestamps come out as full ISO-8601, every column key is always present (`"col":null` for a null value, never an omitted key).
+- `DatabricksClient(host, warehouse_id, *, token=None, token_provider=None, ...)` -- the lower-level client `Connection` wraps. `client.execute_json_statement(sql, ...)` for plain JSON rows with no Arrow parse at all; `client.upload_volume_file(volume_path, data)`/`client.delete_volume_file(volume_path)` for the Files API.
+- `write_ipc_stream(table_or_chunk, buf)` -- thin wrapper around `arro3.io.write_ipc_stream` that always writes uncompressed bodies (see below).
+
+`Cursor.execute`/`execute_streamed`/`stream_query_json` all accept `catalog`, `schema`, `row_limit`, `offset`, and `total_timeout_s`.
+
+## A note on Arrow IPC compression
+
+`write_ipc_stream` (and everything in this package that serializes Arrow-IPC bytes) always writes **uncompressed** bodies. arro3's own default (`compression="LZ4"`) is transparently decompressed by DuckDB's Arrow reader, but not necessarily by every other Arrow IPC reader -- notably, `duckdb-wasm`'s browser-side decoder silently fails to parse LZ4-compressed bodies. If you're producing bytes that might be consumed by something other than a Python DuckDB connection, this default matters.
+
+## License
+
+MIT

arrowbricks-0.1.0/README.md

@@ -0,0 +1,101 @@
+# arrowbricks
+
+Runs SQL against a Databricks SQL warehouse via the Statement Execution API and hands you the result as Arrow -- a `Cursor` shaped like [`databricks-sql-python`](https://github.com/databricks/databricks-sql-python)'s (`execute`, `fetchone`/`fetchmany`/`fetchall`, `fetchall_arrow`/`fetchmany_arrow`), or `stream_query_json` for streaming NDJSON. One Arrow engine ([arro3](https://github.com/kylebarron/arro3)), no DuckDB, no pandas/pyarrow.
+
+- Single responsibility: Databricks to Arrow via arro3. No embedded query engine -- that's [duckbricks](https://github.com/bmsuisse/duckbricks), built on top of this.
+- Bring-your-own-auth -- a static token or your own token-refresh callable. No cloud-SDK dependency baked in.
+- Result-order preserved even though chunks can complete out of order over the network.
+- Chunks are fetched lazily as `fetchone`/`fetchmany`/`fetchall` actually need them, not all upfront.
+- Heartbeats between slow chunks (`execute_streamed`/`stream_query_json`), so a caller streaming this over e.g. SSE never goes silent during a cold warehouse start.
+
+## Install
+
+```bash
+pip install arrowbricks
+```
+
+Dependencies: `httpx` + `arro3-core` + `arro3-io`. That's the whole tree.
+
+## Quickstart
+
+```python
+import asyncio
+from arrowbricks import connect
+
+
+async def main():
+    conn = connect(
+        host="adb-1234567890.1.azuredatabricks.net",
+        warehouse_id="abcd1234efgh5678",
+        token="dapi...",  # or token_provider=... -- see Auth below
+    )
+    cursor = conn.cursor()
+
+    await cursor.execute("SELECT * FROM my_catalog.my_schema.my_table LIMIT 100")
+    async for row in cursor:
+        print(row)
+
+    await cursor.execute("SELECT * FROM my_catalog.my_schema.my_table LIMIT 100")
+    table = await cursor.fetchall_arrow()  # an arro3 Table
+
+
+asyncio.run(main())
+```
+
+For streaming NDJSON (e.g. a FastAPI SSE endpoint, first row out as soon as its chunk arrives):
+
+```python
+from arrowbricks import HEARTBEAT, DatabricksClient, stream_query_json
+
+client = DatabricksClient(host=..., warehouse_id=..., token=...)
+
+async for item in stream_query_json(client, "SELECT * FROM my_catalog.my_schema.big_table"):
+    if item is HEARTBEAT:
+        continue  # forward as an SSE keep-alive comment, e.g.
+    print(item)  # one ready-to-send JSON string per row
+```
+
+## Why not `databricks-sql-connector`?
+
+The [official driver](https://github.com/databricks/databricks-sql-python) is the right choice if you need full DB-API 2.0 compatibility over Databricks' Thrift/ODBC-style protocol. If you just want a query result as Arrow/JSON in your own async app, it drags in a lot for that: `pandas`, `thrift`, `openpyxl`, `pybreaker`, `pyjwt`, `oauthlib`, `lz4`, `requests`, `urllib3` as hard dependencies. arrowbricks talks to the plain REST Statement Execution API instead, and its whole dependency tree is `httpx` + `arro3-core` + `arro3-io`. The `Cursor` API is deliberately shaped like the official driver's so switching between them is mostly a constructor change, but arrowbricks is async throughout (`execute`, `fetchone`, etc. are all coroutines) -- there's no sync escape hatch.
+
+## Why not `duckbricks`?
+
+[duckbricks](https://github.com/bmsuisse/duckbricks) does the same Databricks-to-Arrow work, then goes further: it uses a real embedded DuckDB engine to materialize results into your own DuckDB connection/table (`feed_select_to_duckdb_table`), or push a DuckDB query's result *up* to Databricks (`feed_duckdb_table_to_databricks`). If you need that -- a real local SQL engine sitting on top, not just "run this query, get Arrow/JSON back" -- use duckbricks; it depends on arrowbricks for the Databricks/Arrow half. If you don't need DuckDB at all, arrowbricks alone is the smaller, single-responsibility half.
+
+## Auth
+
+`connect`/`DatabricksClient` take either:
+
+- `token: str` -- a static personal access token or pre-issued OAuth token, or
+- `token_provider` -- a callable (sync or async) returning a token string, called on every request.
+
+arrowbricks has no opinion on *how* you get a token and no cloud-SDK dependency of its own. If your provider is expensive to call, cache/refresh inside it -- arrowbricks does no caching on your behalf.
+
+```python
+conn = connect(host=..., warehouse_id=..., token_provider=my_token_provider)
+```
+
+## API
+
+- `connect(host, warehouse_id, *, token=None, token_provider=None, ...) -> Connection`
+- `Connection.cursor() -> Cursor`
+- `Cursor.execute(sql, parameters=None, *, row_limit=None, offset=None, catalog=None, schema=None, total_timeout_s=None) -> Cursor` -- submits and waits for the statement, like a real DB-API cursor. `parameters`, if given, is Databricks' own named-parameter format -- `[{"name": ..., "value": ..., "type": ...}]` bound against `:name` markers in `sql`.
+- `Cursor.execute_streamed(...)` -- same args, but an async generator yielding `HEARTBEAT` while waiting on a slow cold start, then the ready `Cursor` -- for bridging e.g. an SSE connection.
+- `Cursor.fetchone() -> tuple | None`, `Cursor.fetchmany(size) -> list[tuple]`, `Cursor.fetchall() -> list[tuple]`
+- `Cursor.fetchmany_arrow(size) -> arro3.core.Table`, `Cursor.fetchall_arrow() -> arro3.core.Table`
+- `Cursor` is an async iterator, yielding one row (tuple) at a time.
+- `Cursor.description` -- DB-API-style `[(name, type_name, None, None, None, None, None), ...]` after `execute()`.
+- `stream_query_json(client, sql, **kwargs)` -- yields `HEARTBEAT`, then each row as a JSON string, as soon as its chunk arrives. Timestamps come out as full ISO-8601, every column key is always present (`"col":null` for a null value, never an omitted key).
+- `DatabricksClient(host, warehouse_id, *, token=None, token_provider=None, ...)` -- the lower-level client `Connection` wraps. `client.execute_json_statement(sql, ...)` for plain JSON rows with no Arrow parse at all; `client.upload_volume_file(volume_path, data)`/`client.delete_volume_file(volume_path)` for the Files API.
+- `write_ipc_stream(table_or_chunk, buf)` -- thin wrapper around `arro3.io.write_ipc_stream` that always writes uncompressed bodies (see below).
+
+`Cursor.execute`/`execute_streamed`/`stream_query_json` all accept `catalog`, `schema`, `row_limit`, `offset`, and `total_timeout_s`.
+
+## A note on Arrow IPC compression
+
+`write_ipc_stream` (and everything in this package that serializes Arrow-IPC bytes) always writes **uncompressed** bodies. arro3's own default (`compression="LZ4"`) is transparently decompressed by DuckDB's Arrow reader, but not necessarily by every other Arrow IPC reader -- notably, `duckdb-wasm`'s browser-side decoder silently fails to parse LZ4-compressed bodies. If you're producing bytes that might be consumed by something other than a Python DuckDB connection, this default matters.
+
+## License
+
+MIT

arrowbricks-0.1.0/prek.toml

@@ -0,0 +1,30 @@
+# prek.toml — pre-commit hook configuration
+# Install git hook: prek install
+# Format all files: prek run --all-files
+
+[[repos]]
+repo = "https://github.com/astral-sh/ruff-pre-commit"
+rev = "v0.15.1"
+hooks = [
+    { id = "ruff-check", args = ["--fix"] },
+    { id = "ruff-format" },
+]
+
+[[repos]]
+repo = "local"
+hooks = [
+    { id = "ty", name = "ty check", language = "system", entry = "uv run --frozen ty check src tests", pass_filenames = false, files = '\.py$' },
+]
+
+[[repos]]
+repo = "https://github.com/pre-commit/pre-commit-hooks"
+rev = "v5.0.0"
+hooks = [
+    { id = "trailing-whitespace" },
+    { id = "end-of-file-fixer" },
+    { id = "check-yaml" },
+    { id = "check-toml" },
+    { id = "check-added-large-files", args = ["--maxkb=1000"] },
+    { id = "check-merge-conflict" },
+    { id = "detect-private-key" },
+]

arrowbricks-0.1.0/pyproject.toml

@@ -0,0 +1,44 @@
+[project]
+name = "arrowbricks"
+version = "0.1.0"
+description = "Runs SQL against a Databricks SQL warehouse via the Statement Execution API and hands you the result as Arrow -- a DB-API-ish Cursor (fetchone/fetchmany/fetchall/fetchall_arrow) or NDJSON streaming. Single Arrow engine (arro3), no DuckDB."
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.11"
+dependencies = [
+    "httpx>=0.27",
+    "arro3-core>=0.8",
+    "arro3-io>=0.8",
+]
+
+[project.urls]
+Repository = "https://github.com/bmsuisse/arrowbricks"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/arrowbricks"]
+
+[dependency-groups]
+dev = [
+    "pytest>=8.0",
+    "pytest-asyncio>=0.24",
+    "respx>=0.21",
+    "ruff>=0.6",
+    "ty>=0.0.65",
+]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+
+[tool.ruff]
+line-length = 120
+target-version = "py311"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "S", "B"]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/*" = ["S101", "S106"]

arrowbricks-0.1.0/src/arrowbricks/__init__.py

@@ -0,0 +1,23 @@
+from ._streaming import (
+    HEARTBEAT,
+    QueryTimeout,
+    ReplayableArrowChunk,
+    await_with_heartbeat,
+    stream_query_json,
+    write_ipc_stream,
+)
+from .client import DatabricksClient
+from .cursor import Connection, Cursor, connect
+
+__all__ = [
+    "HEARTBEAT",
+    "Connection",
+    "Cursor",
+    "DatabricksClient",
+    "QueryTimeout",
+    "ReplayableArrowChunk",
+    "await_with_heartbeat",
+    "connect",
+    "stream_query_json",
+    "write_ipc_stream",
+]