ob-databricks 2.36.0__tar.gz → 3.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ob_databricks-2.36.0 → ob_databricks-3.0.0}/PKG-INFO +3 -2
- {ob_databricks-2.36.0 → ob_databricks-3.0.0}/pyproject.toml +9 -2
- ob_databricks-3.0.0/src/ob_databricks/arrow_stream.py +48 -0
- {ob_databricks-2.36.0 → ob_databricks-3.0.0}/src/ob_databricks/cursor.py +21 -0
- ob_databricks-3.0.0/tests/test_stream.py +61 -0
- {ob_databricks-2.36.0 → ob_databricks-3.0.0}/.gitignore +0 -0
- {ob_databricks-2.36.0 → ob_databricks-3.0.0}/AGENTS.md +0 -0
- {ob_databricks-2.36.0 → ob_databricks-3.0.0}/CLAUDE.md +0 -0
- {ob_databricks-2.36.0 → ob_databricks-3.0.0}/LICENSE +0 -0
- {ob_databricks-2.36.0 → ob_databricks-3.0.0}/README.md +0 -0
- {ob_databricks-2.36.0 → ob_databricks-3.0.0}/src/ob_databricks/__init__.py +0 -0
- {ob_databricks-2.36.0 → ob_databricks-3.0.0}/src/ob_databricks/compiler.py +0 -0
- {ob_databricks-2.36.0 → ob_databricks-3.0.0}/src/ob_databricks/connection.py +0 -0
- {ob_databricks-2.36.0 → ob_databricks-3.0.0}/src/ob_databricks/exceptions.py +0 -0
- {ob_databricks-2.36.0 → ob_databricks-3.0.0}/src/ob_databricks/type_codes.py +0 -0
- {ob_databricks-2.36.0 → ob_databricks-3.0.0}/tests/test_driver.py +0 -0
- {ob_databricks-2.36.0 → ob_databricks-3.0.0}/tests/test_yaml_detection.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: ob-databricks
|
|
3
|
-
Version:
|
|
3
|
+
Version: 3.0.0
|
|
4
4
|
Summary: OrionBelt Semantic Layer driver for Databricks (DB-API 2.0)
|
|
5
5
|
Author-email: Ralf Becher <info@orionbelt.ai>
|
|
6
6
|
License-Expression: BUSL-1.1
|
|
@@ -8,7 +8,8 @@ License-File: LICENSE
|
|
|
8
8
|
Keywords: databricks,db-api,obml,orionbelt,semantic-layer
|
|
9
9
|
Requires-Python: >=3.12
|
|
10
10
|
Requires-Dist: databricks-sql-connector>=3.0
|
|
11
|
-
Requires-Dist: ob-driver-core>=
|
|
11
|
+
Requires-Dist: ob-driver-core>=3.0.0
|
|
12
|
+
Requires-Dist: pyarrow>=16.0
|
|
12
13
|
Provides-Extra: dev
|
|
13
14
|
Requires-Dist: mypy>=1.10; extra == 'dev'
|
|
14
15
|
Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "ob-databricks"
|
|
7
|
-
version = "
|
|
7
|
+
version = "3.0.0"
|
|
8
8
|
description = "OrionBelt Semantic Layer driver for Databricks (DB-API 2.0)"
|
|
9
9
|
license = "BUSL-1.1"
|
|
10
10
|
license-files = ["LICENSE"]
|
|
@@ -13,8 +13,11 @@ authors = [{ name = "Ralf Becher", email = "info@orionbelt.ai" }]
|
|
|
13
13
|
keywords = ["databricks", "semantic-layer", "obml", "db-api", "orionbelt"]
|
|
14
14
|
requires-python = ">=3.12"
|
|
15
15
|
dependencies = [
|
|
16
|
-
"ob-driver-core>=
|
|
16
|
+
"ob-driver-core>=3.0.0",
|
|
17
17
|
"databricks-sql-connector>=3.0",
|
|
18
|
+
# Arrow results (fetchall_arrow, fetchmany_arrow): optional in the
|
|
19
|
+
# connector since 4.0.
|
|
20
|
+
"pyarrow>=16.0",
|
|
18
21
|
]
|
|
19
22
|
|
|
20
23
|
[project.optional-dependencies]
|
|
@@ -56,6 +59,10 @@ python_version = "3.12"
|
|
|
56
59
|
strict = true
|
|
57
60
|
files = ["src/ob_databricks"]
|
|
58
61
|
|
|
62
|
+
[[tool.mypy.overrides]]
|
|
63
|
+
module = ["pyarrow.*"]
|
|
64
|
+
ignore_missing_imports = true
|
|
65
|
+
|
|
59
66
|
[tool.pytest.ini_options]
|
|
60
67
|
asyncio_mode = "auto"
|
|
61
68
|
testpaths = ["tests"]
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""A result's pages as one Arrow stream, for a consumer that reads it batch by batch.
|
|
2
|
+
|
|
3
|
+
A ``pyarrow.RecordBatchReader`` has one schema, and the pages of one result
|
|
4
|
+
need not agree on it: a page's types can depend on its values. Every page is
|
|
5
|
+
cast to the first page's schema, so a reader never changes type midway; a page
|
|
6
|
+
whose values that schema cannot hold fails the read and says so, rather than
|
|
7
|
+
streaming a value changed to fit.
|
|
8
|
+
|
|
9
|
+
Each driver package keeps its own copy, so a driver needs no newer
|
|
10
|
+
``ob-driver-core`` than the one it already requires.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import itertools
|
|
16
|
+
from collections.abc import Iterator
|
|
17
|
+
|
|
18
|
+
import pyarrow as pa
|
|
19
|
+
|
|
20
|
+
from ob_databricks.exceptions import DataError
|
|
21
|
+
|
|
22
|
+
#: Rows per page asked of a driver that pages by row count.
|
|
23
|
+
BATCH_ROWS = 65_536
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def one_schema_reader(pages: Iterator[pa.Table], empty: pa.Schema) -> pa.RecordBatchReader:
|
|
27
|
+
"""*pages* as one reader under the first page's schema; *empty*'s without a page.
|
|
28
|
+
|
|
29
|
+
The first page is read here, so an error the query raises on its first
|
|
30
|
+
page surfaces at once; later pages are read as the reader is.
|
|
31
|
+
"""
|
|
32
|
+
first = next(pages, None)
|
|
33
|
+
if first is None:
|
|
34
|
+
return pa.RecordBatchReader.from_batches(empty, [])
|
|
35
|
+
schema = first.schema
|
|
36
|
+
|
|
37
|
+
def batches() -> Iterator[pa.RecordBatch]:
|
|
38
|
+
for page in itertools.chain([first], pages):
|
|
39
|
+
if page.schema != schema:
|
|
40
|
+
try:
|
|
41
|
+
page = page.cast(schema)
|
|
42
|
+
except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as exc:
|
|
43
|
+
raise DataError(
|
|
44
|
+
f"A page of the result does not fit the first page's types: {exc}"
|
|
45
|
+
) from exc
|
|
46
|
+
yield from page.to_batches()
|
|
47
|
+
|
|
48
|
+
return pa.RecordBatchReader.from_batches(schema, batches())
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
+
from collections.abc import Iterator
|
|
5
6
|
from typing import Any
|
|
6
7
|
|
|
7
8
|
from ob_databricks.compiler import compile_obml, is_obml, parse_obml
|
|
@@ -127,6 +128,26 @@ class Cursor:
|
|
|
127
128
|
self._check_open()
|
|
128
129
|
return self._native.fetchall_arrow()
|
|
129
130
|
|
|
131
|
+
def fetch_record_batch(self) -> Any: # noqa: ANN401
|
|
132
|
+
"""Stream the remaining rows as a ``pyarrow.RecordBatchReader``.
|
|
133
|
+
|
|
134
|
+
Page by page through the connector's ``fetchmany_arrow``, so a large
|
|
135
|
+
result is never held whole; every page is held to the first page's
|
|
136
|
+
types (see :mod:`ob_databricks.arrow_stream`).
|
|
137
|
+
"""
|
|
138
|
+
self._check_open()
|
|
139
|
+
from ob_databricks.arrow_stream import BATCH_ROWS, one_schema_reader
|
|
140
|
+
|
|
141
|
+
first = self._native.fetchmany_arrow(BATCH_ROWS)
|
|
142
|
+
|
|
143
|
+
def pages() -> Iterator[Any]:
|
|
144
|
+
page = first
|
|
145
|
+
while page.num_rows:
|
|
146
|
+
yield page
|
|
147
|
+
page = self._native.fetchmany_arrow(BATCH_ROWS)
|
|
148
|
+
|
|
149
|
+
return one_schema_reader(pages(), first.schema)
|
|
150
|
+
|
|
130
151
|
def close(self) -> None:
|
|
131
152
|
"""Close the cursor."""
|
|
132
153
|
if not self._closed:
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""``fetch_record_batch()``: the result page by page, under one schema."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from unittest.mock import MagicMock
|
|
6
|
+
|
|
7
|
+
import pyarrow as pa
|
|
8
|
+
|
|
9
|
+
from ob_databricks.arrow_stream import BATCH_ROWS
|
|
10
|
+
from ob_databricks.connection import Connection
|
|
11
|
+
from ob_databricks.cursor import Cursor
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _cursor(pages: list[pa.Table]) -> tuple[Cursor, MagicMock]:
|
|
15
|
+
native = MagicMock()
|
|
16
|
+
native.cursor.return_value.fetchmany_arrow.side_effect = pages
|
|
17
|
+
cursor = Connection(native).cursor()
|
|
18
|
+
cursor.execute("SELECT n FROM big")
|
|
19
|
+
return cursor, native.cursor.return_value
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def test_pages_until_an_empty_one() -> None:
|
|
23
|
+
schema = pa.schema([("n", pa.int64())])
|
|
24
|
+
cursor, native = _cursor(
|
|
25
|
+
[
|
|
26
|
+
pa.table({"n": [1, 2]}, schema=schema),
|
|
27
|
+
pa.table({"n": [3]}, schema=schema),
|
|
28
|
+
pa.table({"n": []}, schema=schema),
|
|
29
|
+
]
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
table = cursor.fetch_record_batch().read_all()
|
|
33
|
+
|
|
34
|
+
assert table.column("n").to_pylist() == [1, 2, 3]
|
|
35
|
+
assert native.fetchmany_arrow.call_count == 3
|
|
36
|
+
native.fetchmany_arrow.assert_called_with(BATCH_ROWS)
|
|
37
|
+
native.fetchall_arrow.assert_not_called()
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def test_a_later_page_is_held_to_the_first_pages_types() -> None:
|
|
41
|
+
cursor, _ = _cursor(
|
|
42
|
+
[
|
|
43
|
+
pa.table({"n": pa.array([1], type=pa.int32())}),
|
|
44
|
+
pa.table({"n": pa.array([2], type=pa.int64())}),
|
|
45
|
+
pa.table({"n": pa.array([], type=pa.int32())}),
|
|
46
|
+
]
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
reader = cursor.fetch_record_batch()
|
|
50
|
+
|
|
51
|
+
assert reader.schema.field("n").type == pa.int32()
|
|
52
|
+
assert reader.read_all().column("n").to_pylist() == [1, 2]
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def test_an_empty_result_streams_its_schema() -> None:
|
|
56
|
+
cursor, _ = _cursor([pa.table({"n": pa.array([], type=pa.int64())})])
|
|
57
|
+
|
|
58
|
+
reader = cursor.fetch_record_batch()
|
|
59
|
+
|
|
60
|
+
assert reader.schema == pa.schema([("n", pa.int64())])
|
|
61
|
+
assert reader.read_all().num_rows == 0
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|