ob-databricks 2.1.0__tar.gz → 3.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,67 @@
1
+ __pycache__/
2
+ *.py[cod]
3
+ *$py.class
4
+ *.so
5
+ dist/
6
+ build/
7
+ *.egg-info/
8
+ *.egg
9
+ .eggs/
10
+ .venv/
11
+ venv/
12
+ .env
13
+ *.db
14
+ *.duckdb
15
+ # Generated FOCUS export (20 MB); rebuilt by scripts/build_finops_duckdb.py.
16
+ # The committed excerpt is examples/finops_charges_sample.json.
17
+ examples/finops_data/
18
+ *.sqlite3
19
+ .mypy_cache/
20
+ .ruff_cache/
21
+ .pytest_cache/
22
+ htmlcov/
23
+ .coverage
24
+ .coverage.*
25
+ *.log
26
+ .DS_Store
27
+ .claude/
28
+ # Playwright MCP scratch output: accessibility snapshots and console logs
29
+ # written per browser session. The console logs were already caught by
30
+ # *.log above, which left only the .yml page dumps showing as untracked.
31
+ .playwright-mcp/
32
+ # Playwright writes screenshots to the working directory when the tool is
33
+ # given a bare filename, which is the repo root. Twice now one has been
34
+ # swept into a commit by ``git add -A``.
35
+ *.playwright.png
36
+ gradio*-production.png
37
+ ui-gradio*.png
38
+ /cache/
39
+ /usage/
40
+ security/
41
+ infra/
42
+ site/
43
+ scripts-infra
44
+ .mcpregistry_github_token
45
+ .mcpregistry_registry_token
46
+ _*/
47
+ blog/
48
+ review*
49
+ design/
50
+ examples/test*
51
+ # TPC-DS sweep harness. The queries, the runner and the compiled SQL are
52
+ # published so the coverage is auditable; the generated database, the working
53
+ # notes and the drafts that do not yet match the reference are not.
54
+ examples/tpcds_queries/*
55
+ !examples/tpcds_queries/README.md
56
+ !examples/tpcds_queries/sweep.py
57
+ !examples/tpcds_queries/Q*.yml
58
+ !examples/tpcds_queries/sql/
59
+ examples/tpcds_queries/*.duckdb
60
+ examples/tpcds_queries/REPORT.md
61
+ examples/tpcds_queries/tpcds_run.py
62
+ examples/tpcds_queries/drafts/
63
+ semantic-layer-spec.md
64
+
65
+ # Generated per-vendor seed scripts (backup of what the drift seed executed)
66
+ tests/integration/drift/vendor_exec/seed_sql/
67
+ examples/tpcds_queries/results/
@@ -0,0 +1 @@
1
+ @AGENTS.md
@@ -3,8 +3,8 @@ Business Source License 1.1
3
3
  Parameters
4
4
 
5
5
  Licensor: RALFORION d.o.o.
6
- Licensed Work: OrionBelt Semantic Layer 1.0
7
- The Licensed Work is (c) 2025 RALFORION d.o.o.
6
+ Licensed Work: OrionBelt Semantic Layer (all versions)
7
+ The Licensed Work is (c) 2025-2026 RALFORION d.o.o.
8
8
  Additional Use Grant: You may use the Licensed Work for any purpose,
9
9
  including production use, PROVIDED THAT you may
10
10
  not embed OrionBelt in a commercial product or
@@ -1,14 +1,15 @@
1
- Metadata-Version: 2.4
1
+ Metadata-Version: 2.5
2
2
  Name: ob-databricks
3
- Version: 2.1.0
3
+ Version: 3.0.0
4
4
  Summary: OrionBelt Semantic Layer driver for Databricks (DB-API 2.0)
5
5
  Author-email: Ralf Becher <info@orionbelt.ai>
6
- License: BSL-1.1
6
+ License-Expression: BUSL-1.1
7
7
  License-File: LICENSE
8
8
  Keywords: databricks,db-api,obml,orionbelt,semantic-layer
9
9
  Requires-Python: >=3.12
10
10
  Requires-Dist: databricks-sql-connector>=3.0
11
- Requires-Dist: ob-driver-core>=0.1
11
+ Requires-Dist: ob-driver-core>=3.0.0
12
+ Requires-Dist: pyarrow>=16.0
12
13
  Provides-Extra: dev
13
14
  Requires-Dist: mypy>=1.10; extra == 'dev'
14
15
  Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
@@ -23,7 +24,7 @@ Description-Content-Type: text/markdown
23
24
 
24
25
  OrionBelt Semantic Layer driver for Databricks (DB-API 2.0)
25
26
 
26
- Part of the [OrionBelt Semantic Layer](https://github.com/ralfbecher/orionbelt-semantic-layer) project.
27
+ Part of the [OrionBelt Semantic Layer](https://github.com/ralforion/orionbelt-semantic-layer) project.
27
28
 
28
29
  ## Installation
29
30
 
@@ -33,4 +34,4 @@ pip install ob-databricks
33
34
 
34
35
  ## License
35
36
 
36
- BSL-1.1 — see [LICENSE](LICENSE) for details.
37
+ BUSL-1.1 — see [LICENSE](LICENSE) for details.
@@ -0,0 +1,15 @@
1
+ # ob-databricks
2
+
3
+ OrionBelt Semantic Layer driver for Databricks (DB-API 2.0)
4
+
5
+ Part of the [OrionBelt Semantic Layer](https://github.com/ralforion/orionbelt-semantic-layer) project.
6
+
7
+ ## Installation
8
+
9
+ ```bash
10
+ pip install ob-databricks
11
+ ```
12
+
13
+ ## License
14
+
15
+ BUSL-1.1 — see [LICENSE](LICENSE) for details.
@@ -4,16 +4,20 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "ob-databricks"
7
- version = "2.1.0"
7
+ version = "3.0.0"
8
8
  description = "OrionBelt Semantic Layer driver for Databricks (DB-API 2.0)"
9
- license = { text = "BSL-1.1" }
9
+ license = "BUSL-1.1"
10
+ license-files = ["LICENSE"]
10
11
  readme = "README.md"
11
12
  authors = [{ name = "Ralf Becher", email = "info@orionbelt.ai" }]
12
13
  keywords = ["databricks", "semantic-layer", "obml", "db-api", "orionbelt"]
13
14
  requires-python = ">=3.12"
14
15
  dependencies = [
15
- "ob-driver-core>=0.1",
16
+ "ob-driver-core>=3.0.0",
16
17
  "databricks-sql-connector>=3.0",
18
+ # Arrow results (fetchall_arrow, fetchmany_arrow): optional in the
19
+ # connector since 4.0.
20
+ "pyarrow>=16.0",
17
21
  ]
18
22
 
19
23
  [project.optional-dependencies]
@@ -38,12 +42,27 @@ target-version = "py312"
38
42
 
39
43
  [tool.ruff.lint]
40
44
  select = ["E", "F", "I", "UP", "B", "SIM", "ANN"]
45
+ ignore = [
46
+ # PEP 249 is an untyped boundary by specification: ``execute()`` takes
47
+ # whatever the caller passes, and every cursor wraps a native object no
48
+ # vendor annotates. ``Any`` there is the honest signature, not a missing one.
49
+ "ANN401",
50
+ ]
51
+
52
+ [tool.ruff.lint.per-file-ignores]
53
+ # A test reads better with its patches nested, and its return type says nothing
54
+ # a reader needs.
55
+ "tests/**" = ["ANN001", "ANN201", "SIM117"]
41
56
 
42
57
  [tool.mypy]
43
58
  python_version = "3.12"
44
59
  strict = true
45
60
  files = ["src/ob_databricks"]
46
61
 
62
+ [[tool.mypy.overrides]]
63
+ module = ["pyarrow.*"]
64
+ ignore_missing_imports = true
65
+
47
66
  [tool.pytest.ini_options]
48
67
  asyncio_mode = "auto"
49
68
  testpaths = ["tests"]
@@ -0,0 +1,48 @@
1
+ """A result's pages as one Arrow stream, for a consumer that reads it batch by batch.
2
+
3
+ A ``pyarrow.RecordBatchReader`` has one schema, and the pages of one result
4
+ need not agree on it: a page's types can depend on its values. Every page is
5
+ cast to the first page's schema, so a reader never changes type midway; a page
6
+ whose values that schema cannot hold fails the read and says so, rather than
7
+ streaming a value changed to fit.
8
+
9
+ Each driver package keeps its own copy, so a driver needs no newer
10
+ ``ob-driver-core`` than the one it already requires.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import itertools
16
+ from collections.abc import Iterator
17
+
18
+ import pyarrow as pa
19
+
20
+ from ob_databricks.exceptions import DataError
21
+
22
+ #: Rows per page asked of a driver that pages by row count.
23
+ BATCH_ROWS = 65_536
24
+
25
+
26
+ def one_schema_reader(pages: Iterator[pa.Table], empty: pa.Schema) -> pa.RecordBatchReader:
27
+ """*pages* as one reader under the first page's schema; *empty*'s without a page.
28
+
29
+ The first page is read here, so an error the query raises on its first
30
+ page surfaces at once; later pages are read as the reader is.
31
+ """
32
+ first = next(pages, None)
33
+ if first is None:
34
+ return pa.RecordBatchReader.from_batches(empty, [])
35
+ schema = first.schema
36
+
37
+ def batches() -> Iterator[pa.RecordBatch]:
38
+ for page in itertools.chain([first], pages):
39
+ if page.schema != schema:
40
+ try:
41
+ page = page.cast(schema)
42
+ except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as exc:
43
+ raise DataError(
44
+ f"A page of the result does not fit the first page's types: {exc}"
45
+ ) from exc
46
+ yield from page.to_batches()
47
+
48
+ return pa.RecordBatchReader.from_batches(schema, batches())
@@ -2,6 +2,7 @@
2
2
 
3
3
  from __future__ import annotations
4
4
 
5
+ from collections.abc import Iterator
5
6
  from typing import Any
6
7
 
7
8
  from ob_databricks.compiler import compile_obml, is_obml, parse_obml
@@ -53,7 +54,14 @@ class Cursor:
53
54
 
54
55
  @property
55
56
  def rowcount(self) -> int:
56
- return self._native.rowcount
57
+ """Rows the last statement produced, or -1 where the driver says nothing.
58
+
59
+ ``self._native`` is untyped, so the value arrives as ``Any``; PEP 249
60
+ promises an ``int`` and -1 is what it reserves for "unknown", which is
61
+ what a connector answering ``None`` means.
62
+ """
63
+ count = self._native.rowcount
64
+ return int(count) if isinstance(count, int) else -1
57
65
 
58
66
  def _check_open(self) -> None:
59
67
  if self._closed:
@@ -120,6 +128,26 @@ class Cursor:
120
128
  self._check_open()
121
129
  return self._native.fetchall_arrow()
122
130
 
131
+ def fetch_record_batch(self) -> Any: # noqa: ANN401
132
+ """Stream the remaining rows as a ``pyarrow.RecordBatchReader``.
133
+
134
+ Page by page through the connector's ``fetchmany_arrow``, so a large
135
+ result is never held whole; every page is held to the first page's
136
+ types (see :mod:`ob_databricks.arrow_stream`).
137
+ """
138
+ self._check_open()
139
+ from ob_databricks.arrow_stream import BATCH_ROWS, one_schema_reader
140
+
141
+ first = self._native.fetchmany_arrow(BATCH_ROWS)
142
+
143
+ def pages() -> Iterator[Any]:
144
+ page = first
145
+ while page.num_rows:
146
+ yield page
147
+ page = self._native.fetchmany_arrow(BATCH_ROWS)
148
+
149
+ return one_schema_reader(pages(), first.schema)
150
+
123
151
  def close(self) -> None:
124
152
  """Close the cursor."""
125
153
  if not self._closed:
@@ -8,7 +8,7 @@ from __future__ import annotations
8
8
 
9
9
  from ob_driver_core.type_codes import BINARY, DATETIME, NUMBER, ROWID, STRING
10
10
 
11
- __all__ = ["STRING", "BINARY", "NUMBER", "DATETIME", "ROWID", "DBR_TYPE_MAP"]
11
+ __all__ = ["BINARY", "DATETIME", "DBR_TYPE_MAP", "NUMBER", "ROWID", "STRING"]
12
12
 
13
13
  # Databricks SQL connector type names from cursor.description
14
14
  # https://docs.databricks.com/en/sql/language-manual/sql-ref-datatypes.html
@@ -0,0 +1,61 @@
1
+ """``fetch_record_batch()``: the result page by page, under one schema."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from unittest.mock import MagicMock
6
+
7
+ import pyarrow as pa
8
+
9
+ from ob_databricks.arrow_stream import BATCH_ROWS
10
+ from ob_databricks.connection import Connection
11
+ from ob_databricks.cursor import Cursor
12
+
13
+
14
+ def _cursor(pages: list[pa.Table]) -> tuple[Cursor, MagicMock]:
15
+ native = MagicMock()
16
+ native.cursor.return_value.fetchmany_arrow.side_effect = pages
17
+ cursor = Connection(native).cursor()
18
+ cursor.execute("SELECT n FROM big")
19
+ return cursor, native.cursor.return_value
20
+
21
+
22
+ def test_pages_until_an_empty_one() -> None:
23
+ schema = pa.schema([("n", pa.int64())])
24
+ cursor, native = _cursor(
25
+ [
26
+ pa.table({"n": [1, 2]}, schema=schema),
27
+ pa.table({"n": [3]}, schema=schema),
28
+ pa.table({"n": []}, schema=schema),
29
+ ]
30
+ )
31
+
32
+ table = cursor.fetch_record_batch().read_all()
33
+
34
+ assert table.column("n").to_pylist() == [1, 2, 3]
35
+ assert native.fetchmany_arrow.call_count == 3
36
+ native.fetchmany_arrow.assert_called_with(BATCH_ROWS)
37
+ native.fetchall_arrow.assert_not_called()
38
+
39
+
40
+ def test_a_later_page_is_held_to_the_first_pages_types() -> None:
41
+ cursor, _ = _cursor(
42
+ [
43
+ pa.table({"n": pa.array([1], type=pa.int32())}),
44
+ pa.table({"n": pa.array([2], type=pa.int64())}),
45
+ pa.table({"n": pa.array([], type=pa.int32())}),
46
+ ]
47
+ )
48
+
49
+ reader = cursor.fetch_record_batch()
50
+
51
+ assert reader.schema.field("n").type == pa.int32()
52
+ assert reader.read_all().column("n").to_pylist() == [1, 2]
53
+
54
+
55
+ def test_an_empty_result_streams_its_schema() -> None:
56
+ cursor, _ = _cursor([pa.table({"n": pa.array([], type=pa.int64())})])
57
+
58
+ reader = cursor.fetch_record_batch()
59
+
60
+ assert reader.schema == pa.schema([("n", pa.int64())])
61
+ assert reader.read_all().num_rows == 0
@@ -1,37 +0,0 @@
1
- __pycache__/
2
- *.py[cod]
3
- *$py.class
4
- *.so
5
- dist/
6
- build/
7
- *.egg-info/
8
- *.egg
9
- .eggs/
10
- .venv/
11
- venv/
12
- .env
13
- *.db
14
- *.duckdb
15
- *.sqlite3
16
- .mypy_cache/
17
- .ruff_cache/
18
- .pytest_cache/
19
- htmlcov/
20
- .coverage
21
- .coverage.*
22
- *.log
23
- .DS_Store
24
- .claude/
25
- security/
26
- infra/
27
- site/
28
- scripts/
29
- .mcpregistry_github_token
30
- .mcpregistry_registry_token
31
- _*/
32
- blog/
33
- review*
34
- design/
35
- examples/test*
36
- examples/tpcds_queries/
37
- semantic-layer-spec.md
@@ -1,15 +0,0 @@
1
- # ob-databricks
2
-
3
- OrionBelt Semantic Layer driver for Databricks (DB-API 2.0)
4
-
5
- Part of the [OrionBelt Semantic Layer](https://github.com/ralfbecher/orionbelt-semantic-layer) project.
6
-
7
- ## Installation
8
-
9
- ```bash
10
- pip install ob-databricks
11
- ```
12
-
13
- ## License
14
-
15
- BSL-1.1 — see [LICENSE](LICENSE) for details.
@@ -95,19 +95,19 @@ def connect(
95
95
 
96
96
 
97
97
  __all__ = [
98
- "apilevel",
99
- "threadsafety",
100
- "paramstyle",
101
- "connect",
102
98
  "Connection",
103
- "Warning",
104
- "Error",
105
- "InterfaceError",
106
- "DatabaseError",
107
99
  "DataError",
108
- "OperationalError",
100
+ "DatabaseError",
101
+ "Error",
109
102
  "IntegrityError",
103
+ "InterfaceError",
110
104
  "InternalError",
111
- "ProgrammingError",
112
105
  "NotSupportedError",
106
+ "OperationalError",
107
+ "ProgrammingError",
108
+ "Warning",
109
+ "apilevel",
110
+ "connect",
111
+ "paramstyle",
112
+ "threadsafety",
113
113
  ]
@@ -19,14 +19,14 @@ from ob_driver_core.exceptions import (
19
19
  )
20
20
 
21
21
  __all__ = [
22
- "Warning",
23
- "Error",
24
- "InterfaceError",
25
- "DatabaseError",
26
22
  "DataError",
27
- "OperationalError",
23
+ "DatabaseError",
24
+ "Error",
28
25
  "IntegrityError",
26
+ "InterfaceError",
29
27
  "InternalError",
30
- "ProgrammingError",
31
28
  "NotSupportedError",
29
+ "OperationalError",
30
+ "ProgrammingError",
31
+ "Warning",
32
32
  ]