ob-databricks 2.36.0__tar.gz → 3.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -10,6 +10,11 @@ build/
10
10
  .venv/
11
11
  venv/
12
12
  .env
13
+ # The Compose setup's own settings, credentials, models and data (deploy/compose/README.md).
14
+ deploy/compose/obsl.env
15
+ deploy/compose/secrets/
16
+ deploy/compose/models/
17
+ deploy/compose/data/
13
18
  *.db
14
19
  *.duckdb
15
20
  # Generated FOCUS export (20 MB); rebuilt by scripts/build_finops_duckdb.py.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: ob-databricks
3
- Version: 2.36.0
3
+ Version: 3.1.0
4
4
  Summary: OrionBelt Semantic Layer driver for Databricks (DB-API 2.0)
5
5
  Author-email: Ralf Becher <info@orionbelt.ai>
6
6
  License-Expression: BUSL-1.1
@@ -8,7 +8,8 @@ License-File: LICENSE
8
8
  Keywords: databricks,db-api,obml,orionbelt,semantic-layer
9
9
  Requires-Python: >=3.12
10
10
  Requires-Dist: databricks-sql-connector>=3.0
11
- Requires-Dist: ob-driver-core>=2.36.0
11
+ Requires-Dist: ob-driver-core>=3.0.0
12
+ Requires-Dist: pyarrow>=16.0
12
13
  Provides-Extra: dev
13
14
  Requires-Dist: mypy>=1.10; extra == 'dev'
14
15
  Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "ob-databricks"
7
- version = "2.36.0"
7
+ version = "3.1.0"
8
8
  description = "OrionBelt Semantic Layer driver for Databricks (DB-API 2.0)"
9
9
  license = "BUSL-1.1"
10
10
  license-files = ["LICENSE"]
@@ -13,8 +13,11 @@ authors = [{ name = "Ralf Becher", email = "info@orionbelt.ai" }]
13
13
  keywords = ["databricks", "semantic-layer", "obml", "db-api", "orionbelt"]
14
14
  requires-python = ">=3.12"
15
15
  dependencies = [
16
- "ob-driver-core>=2.36.0",
16
+ "ob-driver-core>=3.0.0",
17
17
  "databricks-sql-connector>=3.0",
18
+ # Arrow results (fetchall_arrow, fetchmany_arrow): optional in the
19
+ # connector since 4.0.
20
+ "pyarrow>=16.0",
18
21
  ]
19
22
 
20
23
  [project.optional-dependencies]
@@ -56,6 +59,10 @@ python_version = "3.12"
56
59
  strict = true
57
60
  files = ["src/ob_databricks"]
58
61
 
62
+ [[tool.mypy.overrides]]
63
+ module = ["pyarrow.*"]
64
+ ignore_missing_imports = true
65
+
59
66
  [tool.pytest.ini_options]
60
67
  asyncio_mode = "auto"
61
68
  testpaths = ["tests"]
@@ -0,0 +1,48 @@
1
+ """A result's pages as one Arrow stream, for a consumer that reads it batch by batch.
2
+
3
+ A ``pyarrow.RecordBatchReader`` has one schema, and the pages of one result
4
+ need not agree on it: a page's types can depend on its values. Every page is
5
+ cast to the first page's schema, so a reader never changes type midway; a page
6
+ whose values that schema cannot hold fails the read and says so, rather than
7
+ streaming a value changed to fit.
8
+
9
+ Each driver package keeps its own copy, so a driver needs no newer
10
+ ``ob-driver-core`` than the one it already requires.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import itertools
16
+ from collections.abc import Iterator
17
+
18
+ import pyarrow as pa
19
+
20
+ from ob_databricks.exceptions import DataError
21
+
22
+ #: Rows per page asked of a driver that pages by row count.
23
+ BATCH_ROWS = 65_536
24
+
25
+
26
+ def one_schema_reader(pages: Iterator[pa.Table], empty: pa.Schema) -> pa.RecordBatchReader:
27
+ """*pages* as one reader under the first page's schema; *empty*'s without a page.
28
+
29
+ The first page is read here, so an error the query raises on its first
30
+ page surfaces at once; later pages are read as the reader is.
31
+ """
32
+ first = next(pages, None)
33
+ if first is None:
34
+ return pa.RecordBatchReader.from_batches(empty, [])
35
+ schema = first.schema
36
+
37
+ def batches() -> Iterator[pa.RecordBatch]:
38
+ for page in itertools.chain([first], pages):
39
+ if page.schema != schema:
40
+ try:
41
+ page = page.cast(schema)
42
+ except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as exc:
43
+ raise DataError(
44
+ f"A page of the result does not fit the first page's types: {exc}"
45
+ ) from exc
46
+ yield from page.to_batches()
47
+
48
+ return pa.RecordBatchReader.from_batches(schema, batches())
@@ -2,6 +2,7 @@
2
2
 
3
3
  from __future__ import annotations
4
4
 
5
+ from collections.abc import Iterator
5
6
  from typing import Any
6
7
 
7
8
  from ob_databricks.compiler import compile_obml, is_obml, parse_obml
@@ -127,6 +128,26 @@ class Cursor:
127
128
  self._check_open()
128
129
  return self._native.fetchall_arrow()
129
130
 
131
+ def fetch_record_batch(self) -> Any: # noqa: ANN401
132
+ """Stream the remaining rows as a ``pyarrow.RecordBatchReader``.
133
+
134
+ Page by page through the connector's ``fetchmany_arrow``, so a large
135
+ result is never held whole; every page is held to the first page's
136
+ types (see :mod:`ob_databricks.arrow_stream`).
137
+ """
138
+ self._check_open()
139
+ from ob_databricks.arrow_stream import BATCH_ROWS, one_schema_reader
140
+
141
+ first = self._native.fetchmany_arrow(BATCH_ROWS)
142
+
143
+ def pages() -> Iterator[Any]:
144
+ page = first
145
+ while page.num_rows:
146
+ yield page
147
+ page = self._native.fetchmany_arrow(BATCH_ROWS)
148
+
149
+ return one_schema_reader(pages(), first.schema)
150
+
130
151
  def close(self) -> None:
131
152
  """Close the cursor."""
132
153
  if not self._closed:
@@ -0,0 +1,61 @@
1
+ """``fetch_record_batch()``: the result page by page, under one schema."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from unittest.mock import MagicMock
6
+
7
+ import pyarrow as pa
8
+
9
+ from ob_databricks.arrow_stream import BATCH_ROWS
10
+ from ob_databricks.connection import Connection
11
+ from ob_databricks.cursor import Cursor
12
+
13
+
14
+ def _cursor(pages: list[pa.Table]) -> tuple[Cursor, MagicMock]:
15
+ native = MagicMock()
16
+ native.cursor.return_value.fetchmany_arrow.side_effect = pages
17
+ cursor = Connection(native).cursor()
18
+ cursor.execute("SELECT n FROM big")
19
+ return cursor, native.cursor.return_value
20
+
21
+
22
+ def test_pages_until_an_empty_one() -> None:
23
+ schema = pa.schema([("n", pa.int64())])
24
+ cursor, native = _cursor(
25
+ [
26
+ pa.table({"n": [1, 2]}, schema=schema),
27
+ pa.table({"n": [3]}, schema=schema),
28
+ pa.table({"n": []}, schema=schema),
29
+ ]
30
+ )
31
+
32
+ table = cursor.fetch_record_batch().read_all()
33
+
34
+ assert table.column("n").to_pylist() == [1, 2, 3]
35
+ assert native.fetchmany_arrow.call_count == 3
36
+ native.fetchmany_arrow.assert_called_with(BATCH_ROWS)
37
+ native.fetchall_arrow.assert_not_called()
38
+
39
+
40
+ def test_a_later_page_is_held_to_the_first_pages_types() -> None:
41
+ cursor, _ = _cursor(
42
+ [
43
+ pa.table({"n": pa.array([1], type=pa.int32())}),
44
+ pa.table({"n": pa.array([2], type=pa.int64())}),
45
+ pa.table({"n": pa.array([], type=pa.int32())}),
46
+ ]
47
+ )
48
+
49
+ reader = cursor.fetch_record_batch()
50
+
51
+ assert reader.schema.field("n").type == pa.int32()
52
+ assert reader.read_all().column("n").to_pylist() == [1, 2]
53
+
54
+
55
+ def test_an_empty_result_streams_its_schema() -> None:
56
+ cursor, _ = _cursor([pa.table({"n": pa.array([], type=pa.int64())})])
57
+
58
+ reader = cursor.fetch_record_batch()
59
+
60
+ assert reader.schema == pa.schema([("n", pa.int64())])
61
+ assert reader.read_all().num_rows == 0
File without changes
File without changes
File without changes
File without changes