dbt-benodb 0.17.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dbt_benodb-0.17.0/PKG-INFO +109 -0
- dbt_benodb-0.17.0/README.md +82 -0
- dbt_benodb-0.17.0/dbt/adapters/benodb/__init__.py +12 -0
- dbt_benodb-0.17.0/dbt/adapters/benodb/__version__.py +57 -0
- dbt_benodb-0.17.0/dbt/adapters/benodb/connections.py +206 -0
- dbt_benodb-0.17.0/dbt/adapters/benodb/impl.py +96 -0
- dbt_benodb-0.17.0/dbt/include/benodb/__init__.py +3 -0
- dbt_benodb-0.17.0/dbt/include/benodb/dbt_project.yml +10 -0
- dbt_benodb-0.17.0/dbt/include/benodb/macros/adapters.sql +90 -0
- dbt_benodb-0.17.0/dbt/include/benodb/macros/ddl.sql +89 -0
- dbt_benodb-0.17.0/dbt/include/benodb/macros/graph.sql +503 -0
- dbt_benodb-0.17.0/dbt/include/benodb/macros/json.sql +115 -0
- dbt_benodb-0.17.0/dbt/include/benodb/macros/materializations/incremental.sql +108 -0
- dbt_benodb-0.17.0/dbt/include/benodb/macros/materializations/table.sql +19 -0
- dbt_benodb-0.17.0/dbt/include/benodb/macros/search.sql +55 -0
- dbt_benodb-0.17.0/dbt/include/benodb/macros/vector.sql +349 -0
- dbt_benodb-0.17.0/dbt_benodb.egg-info/PKG-INFO +109 -0
- dbt_benodb-0.17.0/dbt_benodb.egg-info/SOURCES.txt +22 -0
- dbt_benodb-0.17.0/dbt_benodb.egg-info/dependency_links.txt +1 -0
- dbt_benodb-0.17.0/dbt_benodb.egg-info/requires.txt +5 -0
- dbt_benodb-0.17.0/dbt_benodb.egg-info/top_level.txt +1 -0
- dbt_benodb-0.17.0/pyproject.toml +53 -0
- dbt_benodb-0.17.0/setup.cfg +4 -0
- dbt_benodb-0.17.0/setup.py +10 -0
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: dbt-benodb
|
|
3
|
+
Version: 0.17.0
|
|
4
|
+
Summary: The BenoDB adapter plugin for dbt
|
|
5
|
+
Author-email: Richard Albright <rla3rd@gmail.com>
|
|
6
|
+
License: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://github.com/benolabsai/benodb
|
|
8
|
+
Project-URL: Repository, https://github.com/benolabsai/benodb
|
|
9
|
+
Project-URL: Documentation, https://github.com/benolabsai/benodb/blob/main/dbt-benodb/README.md
|
|
10
|
+
Keywords: dbt,benodb,iceberg,vector-search,analytics
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
20
|
+
Requires-Python: >=3.10
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
Requires-Dist: dbt-adapters<2.0,>=1.0
|
|
23
|
+
Requires-Dist: dbt-core<2.0,>=1.8
|
|
24
|
+
Requires-Dist: benodb>=0.17.0
|
|
25
|
+
Requires-Dist: pyarrow>=15.0.0
|
|
26
|
+
Requires-Dist: pandas
|
|
27
|
+
|
|
28
|
+
# dbt Adapter for BenoDB (`dbt-benodb`)
|
|
29
|
+
|
|
30
|
+
`dbt-benodb` is the official dbt adapter enabling analytics engineering, transformation models, and data lineage directly against BenoDB.
|
|
31
|
+
|
|
32
|
+
## Features
|
|
33
|
+
|
|
34
|
+
- **Table & View Materializations**: Build standard and optimized Iceberg-format tables.
|
|
35
|
+
- **Incremental Materializations**: Efficient append and merge updates leveraging BenoDB's primary key upserts and delete vector acceleration.
|
|
36
|
+
- **Vector & Graph Extensions**: Built-in macros for vector distance scoring, hybrid search, and graph relationship traversals (`benodb.vector_search(...)`, `benodb.graph_neighbors(...)`).
|
|
37
|
+
|
|
38
|
+
## Installation
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
pip install dbt-benodb
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
## Configuration (`profiles.yml`)
|
|
45
|
+
|
|
46
|
+
```yaml
|
|
47
|
+
benodb_profile:
|
|
48
|
+
target: dev
|
|
49
|
+
outputs:
|
|
50
|
+
dev:
|
|
51
|
+
type: benodb
|
|
52
|
+
database: default
|
|
53
|
+
schema: public
|
|
54
|
+
threads: 4
|
|
55
|
+
uri: "file:///path/to/benodb/warehouse"
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
## Remote / containerised runs: use `dbt-trino`
|
|
59
|
+
|
|
60
|
+
This adapter is an **embedded** adapter: it opens the warehouse through the
|
|
61
|
+
engine's JNI/`benodb` Python module, so the warehouse path has to be
|
|
62
|
+
mounted in the process running dbt.
|
|
63
|
+
|
|
64
|
+
For Airflow, Cloud Run, CI, or any topology where dbt should not mount the
|
|
65
|
+
warehouse, point `dbt-trino` at a Trino coordinator that has the
|
|
66
|
+
`trino-benodb` plugin installed. This is the supported remote path and
|
|
67
|
+
needs no extra transport in this adapter:
|
|
68
|
+
|
|
69
|
+
```yaml
|
|
70
|
+
# profiles.yml — remote access through the BenoDB Trino connector
|
|
71
|
+
benodb_remote:
|
|
72
|
+
target: prod
|
|
73
|
+
outputs:
|
|
74
|
+
prod:
|
|
75
|
+
type: trino
|
|
76
|
+
method: none # or ldap / jwt, per your coordinator
|
|
77
|
+
host: trino.internal
|
|
78
|
+
port: 8080
|
|
79
|
+
catalog: benodb # the catalog backed by trino-benodb
|
|
80
|
+
schema: default
|
|
81
|
+
threads: 8
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
Notes:
|
|
85
|
+
|
|
86
|
+
- The catalog name must match the entry in the coordinator's
|
|
87
|
+
`etc/catalog/benodb.properties` (`connector.name=benodb`).
|
|
88
|
+
- Table functions work remotely too: `system.vector_search`,
|
|
89
|
+
`system.hybrid_search`, `system.list_indexes`, `system.regional_drift_search`.
|
|
90
|
+
`hybrid_search` now honours its `filter` argument, so a filtered hybrid query
|
|
91
|
+
is pushed to the engine instead of being rejected.
|
|
92
|
+
- `vector_search` accepts a sixth `metric` argument (`cosine` by default, plus
|
|
93
|
+
`l2`, `dot`, `l1`, `hamming`, `jaccard`).
|
|
94
|
+
- Keep the embedded adapter for local development and for features that need the
|
|
95
|
+
Python API directly (index building, graph traversal from Python).
|
|
96
|
+
|
|
97
|
+
## Cluster Indexing & Cache Warming Macros
|
|
98
|
+
|
|
99
|
+
Use post-hooks in your model files to automatically rebuild table-level Voronoi cluster indexes and prime memory caches following data ingestion:
|
|
100
|
+
|
|
101
|
+
```sql+jinja
|
|
102
|
+
{{ config(
|
|
103
|
+
materialized='table',
|
|
104
|
+
post_hook=[
|
|
105
|
+
rebuild_cluster_index(this, 'embedding', k=1024, m=16, ef_construction=200, target_shard_mb=512),
|
|
106
|
+
warmup(this, n_probes=500, max_memory_gb=8.0)
|
|
107
|
+
]
|
|
108
|
+
) }}
|
|
109
|
+
```
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
# dbt Adapter for BenoDB (`dbt-benodb`)
|
|
2
|
+
|
|
3
|
+
`dbt-benodb` is the official dbt adapter enabling analytics engineering, transformation models, and data lineage directly against BenoDB.
|
|
4
|
+
|
|
5
|
+
## Features
|
|
6
|
+
|
|
7
|
+
- **Table & View Materializations**: Build standard and optimized Iceberg-format tables.
|
|
8
|
+
- **Incremental Materializations**: Efficient append and merge updates leveraging BenoDB's primary key upserts and delete vector acceleration.
|
|
9
|
+
- **Vector & Graph Extensions**: Built-in macros for vector distance scoring, hybrid search, and graph relationship traversals (`benodb.vector_search(...)`, `benodb.graph_neighbors(...)`).
|
|
10
|
+
|
|
11
|
+
## Installation
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
pip install dbt-benodb
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
## Configuration (`profiles.yml`)
|
|
18
|
+
|
|
19
|
+
```yaml
|
|
20
|
+
benodb_profile:
|
|
21
|
+
target: dev
|
|
22
|
+
outputs:
|
|
23
|
+
dev:
|
|
24
|
+
type: benodb
|
|
25
|
+
database: default
|
|
26
|
+
schema: public
|
|
27
|
+
threads: 4
|
|
28
|
+
uri: "file:///path/to/benodb/warehouse"
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
## Remote / containerised runs: use `dbt-trino`
|
|
32
|
+
|
|
33
|
+
This adapter is an **embedded** adapter: it opens the warehouse through the
|
|
34
|
+
engine's JNI/`benodb` Python module, so the warehouse path has to be
|
|
35
|
+
mounted in the process running dbt.
|
|
36
|
+
|
|
37
|
+
For Airflow, Cloud Run, CI, or any topology where dbt should not mount the
|
|
38
|
+
warehouse, point `dbt-trino` at a Trino coordinator that has the
|
|
39
|
+
`trino-benodb` plugin installed. This is the supported remote path and
|
|
40
|
+
needs no extra transport in this adapter:
|
|
41
|
+
|
|
42
|
+
```yaml
|
|
43
|
+
# profiles.yml — remote access through the BenoDB Trino connector
|
|
44
|
+
benodb_remote:
|
|
45
|
+
target: prod
|
|
46
|
+
outputs:
|
|
47
|
+
prod:
|
|
48
|
+
type: trino
|
|
49
|
+
method: none # or ldap / jwt, per your coordinator
|
|
50
|
+
host: trino.internal
|
|
51
|
+
port: 8080
|
|
52
|
+
catalog: benodb # the catalog backed by trino-benodb
|
|
53
|
+
schema: default
|
|
54
|
+
threads: 8
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
Notes:
|
|
58
|
+
|
|
59
|
+
- The catalog name must match the entry in the coordinator's
|
|
60
|
+
`etc/catalog/benodb.properties` (`connector.name=benodb`).
|
|
61
|
+
- Table functions work remotely too: `system.vector_search`,
|
|
62
|
+
`system.hybrid_search`, `system.list_indexes`, `system.regional_drift_search`.
|
|
63
|
+
`hybrid_search` now honours its `filter` argument, so a filtered hybrid query
|
|
64
|
+
is pushed to the engine instead of being rejected.
|
|
65
|
+
- `vector_search` accepts a sixth `metric` argument (`cosine` by default, plus
|
|
66
|
+
`l2`, `dot`, `l1`, `hamming`, `jaccard`).
|
|
67
|
+
- Keep the embedded adapter for local development and for features that need the
|
|
68
|
+
Python API directly (index building, graph traversal from Python).
|
|
69
|
+
|
|
70
|
+
## Cluster Indexing & Cache Warming Macros
|
|
71
|
+
|
|
72
|
+
Use post-hooks in your model files to automatically rebuild table-level Voronoi cluster indexes and prime memory caches following data ingestion:
|
|
73
|
+
|
|
74
|
+
```sql+jinja
|
|
75
|
+
{{ config(
|
|
76
|
+
materialized='table',
|
|
77
|
+
post_hook=[
|
|
78
|
+
rebuild_cluster_index(this, 'embedding', k=1024, m=16, ef_construction=200, target_shard_mb=512),
|
|
79
|
+
warmup(this, n_probes=500, max_memory_gb=8.0)
|
|
80
|
+
]
|
|
81
|
+
) }}
|
|
82
|
+
```
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
from dbt.adapters.base import AdapterPlugin
|
|
2
|
+
from dbt.include import benodb
|
|
3
|
+
|
|
4
|
+
from .connections import BenoDBConnectionManager
|
|
5
|
+
from .connections import BenoDBCredentials
|
|
6
|
+
from .impl import BenoDBAdapter
|
|
7
|
+
|
|
8
|
+
Plugin = AdapterPlugin(
|
|
9
|
+
adapter=BenoDBAdapter,
|
|
10
|
+
credentials=BenoDBCredentials,
|
|
11
|
+
include_path=benodb.PACKAGE_PATH,
|
|
12
|
+
)
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""Version of the BenoDB dbt adapter.
|
|
2
|
+
|
|
3
|
+
Since dbt 1.8 decoupled adapters from dbt-core, an adapter uses independent
|
|
4
|
+
SemVer and does NOT mirror dbt-core's version. This package therefore simply
|
|
5
|
+
carries the BenoDB engine version, whose single source of truth is the
|
|
6
|
+
repository's ``Cargo.toml`` ``[workspace.package].version``.
|
|
7
|
+
|
|
8
|
+
dbt-core compatibility is declared separately, not through this number:
|
|
9
|
+
|
|
10
|
+
* the ``dbt-adapters`` dependency in ``pyproject.toml``, and
|
|
11
|
+
* ``require-dbt-version`` in ``dbt/include/benodb/dbt_project.yml``.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import os
|
|
17
|
+
import re
|
|
18
|
+
from importlib.metadata import PackageNotFoundError, version as _dist_version
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _from_cargo() -> str | None:
|
|
23
|
+
"""Read ``[workspace.package].version`` from the nearest Cargo.toml.
|
|
24
|
+
|
|
25
|
+
Works while building from the repo (the source tree is present); returns
|
|
26
|
+
``None`` from an installed wheel, where no Cargo.toml exists.
|
|
27
|
+
"""
|
|
28
|
+
for parent in Path(__file__).resolve().parents:
|
|
29
|
+
cargo = parent / "Cargo.toml"
|
|
30
|
+
if cargo.is_file():
|
|
31
|
+
match = re.search(
|
|
32
|
+
r'(?ms)^\[workspace\.package\].*?^version\s*=\s*"([0-9][^"]*)"',
|
|
33
|
+
cargo.read_text(),
|
|
34
|
+
)
|
|
35
|
+
if match:
|
|
36
|
+
return match.group(1)
|
|
37
|
+
return None
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _detect() -> str:
|
|
41
|
+
# Explicit release override wins: the release workflow sets this so a PyPI
|
|
42
|
+
# repack can be <base>.postN while crates.io stays at <base>.
|
|
43
|
+
forced = os.environ.get("BDB_RELEASE_VERSION")
|
|
44
|
+
if forced:
|
|
45
|
+
return forced
|
|
46
|
+
# Building from source: the repo Cargo.toml is authoritative.
|
|
47
|
+
from_cargo = _from_cargo()
|
|
48
|
+
if from_cargo:
|
|
49
|
+
return from_cargo
|
|
50
|
+
# Installed wheel: the distribution metadata is authoritative.
|
|
51
|
+
try:
|
|
52
|
+
return _dist_version("dbt-benodb")
|
|
53
|
+
except PackageNotFoundError:
|
|
54
|
+
return "0.0.0"
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
version = _detect()
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
from contextlib import contextmanager
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
from typing import Optional, Tuple, Any
|
|
4
|
+
|
|
5
|
+
from dbt.adapters.contracts.connection import Credentials, Connection, AdapterResponse
|
|
6
|
+
from dbt.adapters.sql import SQLConnectionManager
|
|
7
|
+
from dbt.exceptions import DbtRuntimeError
|
|
8
|
+
from dbt.adapters.events.logging import AdapterLogger
|
|
9
|
+
|
|
10
|
+
import benodb
|
|
11
|
+
|
|
12
|
+
logger = AdapterLogger("BenoDB")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass
|
|
16
|
+
class BenoDBCredentials(Credentials):
|
|
17
|
+
path: str = ":memory:"
|
|
18
|
+
database: Optional[str] = "benodb"
|
|
19
|
+
# Unified lakehouse default schema (matches the engine's DEFAULT_SCHEMA,
|
|
20
|
+
# Spark's default namespace, and Trino's default schema).
|
|
21
|
+
schema: str = "default"
|
|
22
|
+
# Optional GPU device for the embedded engine:
|
|
23
|
+
# auto | cpu | cuda[:N] | mps | intel | rocm.
|
|
24
|
+
gpu_device: Optional[str] = None
|
|
25
|
+
|
|
26
|
+
@property
|
|
27
|
+
def type(self):
|
|
28
|
+
return "benodb"
|
|
29
|
+
|
|
30
|
+
@property
|
|
31
|
+
def unique_field(self):
|
|
32
|
+
return self.path
|
|
33
|
+
|
|
34
|
+
def _connection_keys(self):
|
|
35
|
+
return ("path", "database", "schema", "gpu_device")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class BenoDBCursor:
|
|
39
|
+
def __init__(self, session):
|
|
40
|
+
self.session = session
|
|
41
|
+
self.description = None
|
|
42
|
+
self._results = []
|
|
43
|
+
|
|
44
|
+
@staticmethod
|
|
45
|
+
def _prepare_sql(sql: str) -> str:
|
|
46
|
+
"""Normalize a dbt-issued SQL script for the engine's single-statement
|
|
47
|
+
`session.sql` guard (which rejects ';' and '--' anywhere).
|
|
48
|
+
|
|
49
|
+
dbt materializations legitimately emit line comments and a trailing
|
|
50
|
+
semicolon. We strip `--` line comments (quote-aware, so `--` inside a
|
|
51
|
+
string/identifier survives), drop a single trailing semicolon, and
|
|
52
|
+
still reject any *embedded* semicolon so the multi-statement injection
|
|
53
|
+
protection is preserved end to end.
|
|
54
|
+
"""
|
|
55
|
+
out = []
|
|
56
|
+
i, n = 0, len(sql)
|
|
57
|
+
in_str = None
|
|
58
|
+
while i < n:
|
|
59
|
+
c = sql[i]
|
|
60
|
+
if in_str is not None:
|
|
61
|
+
out.append(c)
|
|
62
|
+
if c == in_str:
|
|
63
|
+
if i + 1 < n and sql[i + 1] == in_str: # doubled-quote escape
|
|
64
|
+
out.append(sql[i + 1])
|
|
65
|
+
i += 2
|
|
66
|
+
continue
|
|
67
|
+
in_str = None
|
|
68
|
+
i += 1
|
|
69
|
+
continue
|
|
70
|
+
if c in ("'", '"'):
|
|
71
|
+
in_str = c
|
|
72
|
+
out.append(c)
|
|
73
|
+
i += 1
|
|
74
|
+
continue
|
|
75
|
+
if c == "-" and i + 1 < n and sql[i + 1] == "-":
|
|
76
|
+
nl = sql.find("\n", i)
|
|
77
|
+
i = n if nl == -1 else nl # skip comment, keep the newline
|
|
78
|
+
continue
|
|
79
|
+
out.append(c)
|
|
80
|
+
i += 1
|
|
81
|
+
prepared = "".join(out).strip()
|
|
82
|
+
if prepared.endswith(";"):
|
|
83
|
+
prepared = prepared[:-1].rstrip()
|
|
84
|
+
if ";" in prepared:
|
|
85
|
+
raise DbtRuntimeError(
|
|
86
|
+
"BenoDB executes one statement at a time; "
|
|
87
|
+
"embedded ';' is not supported."
|
|
88
|
+
)
|
|
89
|
+
return prepared
|
|
90
|
+
|
|
91
|
+
def execute(self, sql: str, bindings=None):
|
|
92
|
+
if bindings:
|
|
93
|
+
# Note: dbt-core handles most string formatting. For complex bindings, DBAPI compliance would be needed.
|
|
94
|
+
pass
|
|
95
|
+
|
|
96
|
+
sql = self._prepare_sql(sql)
|
|
97
|
+
try:
|
|
98
|
+
arrow_table = self.session.sql(sql)
|
|
99
|
+
if arrow_table is not None and hasattr(arrow_table, "schema"):
|
|
100
|
+
self.description = [(f.name, f.type) for f in arrow_table.schema]
|
|
101
|
+
df = arrow_table.to_pandas()
|
|
102
|
+
self._results = [tuple(x) for x in df.to_numpy()]
|
|
103
|
+
else:
|
|
104
|
+
self.description = None
|
|
105
|
+
self._results = []
|
|
106
|
+
except Exception as e:
|
|
107
|
+
raise Exception(f"Error executing {sql}: {e}")
|
|
108
|
+
|
|
109
|
+
def fetchall(self):
|
|
110
|
+
return self._results
|
|
111
|
+
|
|
112
|
+
def fetchone(self):
|
|
113
|
+
if self._results:
|
|
114
|
+
return self._results.pop(0)
|
|
115
|
+
return None
|
|
116
|
+
|
|
117
|
+
def close(self):
|
|
118
|
+
pass
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
class BenoDBConnectionWrapper:
|
|
122
|
+
def __init__(self, session):
|
|
123
|
+
self.session = session
|
|
124
|
+
|
|
125
|
+
def cursor(self):
|
|
126
|
+
return BenoDBCursor(self.session)
|
|
127
|
+
|
|
128
|
+
def commit(self):
|
|
129
|
+
pass
|
|
130
|
+
|
|
131
|
+
def rollback(self):
|
|
132
|
+
pass
|
|
133
|
+
|
|
134
|
+
def close(self):
|
|
135
|
+
pass
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
class BenoDBConnectionManager(SQLConnectionManager):
|
|
139
|
+
TYPE = "benodb"
|
|
140
|
+
_session = None
|
|
141
|
+
|
|
142
|
+
@classmethod
|
|
143
|
+
def get_session(cls, credentials):
|
|
144
|
+
if cls._session is None:
|
|
145
|
+
# Instantiate a single embedded engine session for all connections.
|
|
146
|
+
# The profile `path` is the warehouse base location so CREATE
|
|
147
|
+
# TABLE / CREATE TABLE AS SELECT can derive table URIs; the
|
|
148
|
+
# default ":memory:" maps onto the engine's in-memory store.
|
|
149
|
+
path = getattr(credentials, "path", None) or ":memory:"
|
|
150
|
+
warehouse = "memory:/dbt" if path in (":memory:", "") else path
|
|
151
|
+
cls._session = benodb.Session(warehouse=warehouse)
|
|
152
|
+
# Pin the GPU device for the embedded engine (profile `gpu_device`).
|
|
153
|
+
gpu_device = getattr(credentials, "gpu_device", None)
|
|
154
|
+
if gpu_device:
|
|
155
|
+
try:
|
|
156
|
+
resolved = benodb.set_gpu_device(gpu_device)
|
|
157
|
+
logger.debug(f"GPU device set to {resolved}")
|
|
158
|
+
except Exception as e:
|
|
159
|
+
logger.warning(f"Could not set GPU device '{gpu_device}': {e}")
|
|
160
|
+
return cls._session
|
|
161
|
+
|
|
162
|
+
@classmethod
|
|
163
|
+
def open(cls, connection: Connection) -> Connection:
|
|
164
|
+
if connection.state == "open":
|
|
165
|
+
logger.debug("Connection is already open, skipping open.")
|
|
166
|
+
return connection
|
|
167
|
+
|
|
168
|
+
credentials = connection.credentials
|
|
169
|
+
|
|
170
|
+
try:
|
|
171
|
+
session = cls.get_session(credentials)
|
|
172
|
+
connection.handle = BenoDBConnectionWrapper(session)
|
|
173
|
+
connection.state = "open"
|
|
174
|
+
except Exception as e:
|
|
175
|
+
logger.error(f"Error initializing BenoDB embedded engine: {e}")
|
|
176
|
+
connection.handle = None
|
|
177
|
+
connection.state = "fail"
|
|
178
|
+
raise DbtRuntimeError(f"Failed to connect to BenoDB: {e}")
|
|
179
|
+
|
|
180
|
+
return connection
|
|
181
|
+
|
|
182
|
+
@classmethod
|
|
183
|
+
def get_response(cls, cursor: Any) -> AdapterResponse:
|
|
184
|
+
return AdapterResponse(_message="OK")
|
|
185
|
+
|
|
186
|
+
def cancel(self, connection: Connection):
|
|
187
|
+
pass
|
|
188
|
+
|
|
189
|
+
def begin(self):
|
|
190
|
+
pass
|
|
191
|
+
|
|
192
|
+
def commit(self):
|
|
193
|
+
pass
|
|
194
|
+
|
|
195
|
+
def clear_transaction(self):
|
|
196
|
+
pass
|
|
197
|
+
|
|
198
|
+
@contextmanager
|
|
199
|
+
def exception_handler(self, sql: str):
|
|
200
|
+
try:
|
|
201
|
+
yield
|
|
202
|
+
except Exception as e:
|
|
203
|
+
logger.error(f"Error running SQL: {sql}")
|
|
204
|
+
logger.error(f"Exception: {e}")
|
|
205
|
+
raise DbtRuntimeError(str(e))
|
|
206
|
+
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
# Portions of this code were inspired by fal-ai/dbt-datafusion
|
|
2
|
+
# Copyright (c) 2022 fal-ai
|
|
3
|
+
# Licensed under the MIT License (or corresponding open source license)
|
|
4
|
+
|
|
5
|
+
from typing import Optional, List, Any
|
|
6
|
+
|
|
7
|
+
from dbt.adapters.base.relation import BaseRelation, RelationType
|
|
8
|
+
from dbt.adapters.sql import SQLAdapter
|
|
9
|
+
from dbt.adapters.events.logging import AdapterLogger
|
|
10
|
+
import pandas as pd
|
|
11
|
+
|
|
12
|
+
from .connections import BenoDBConnectionManager
|
|
13
|
+
|
|
14
|
+
logger = AdapterLogger("BenoDB")
|
|
15
|
+
|
|
16
|
+
class BenoDBAdapter(SQLAdapter):
|
|
17
|
+
ConnectionManager = BenoDBConnectionManager
|
|
18
|
+
|
|
19
|
+
@classmethod
|
|
20
|
+
def date_function(cls):
|
|
21
|
+
return "datenow()"
|
|
22
|
+
|
|
23
|
+
@classmethod
|
|
24
|
+
def is_cancelable(cls) -> bool:
|
|
25
|
+
return False
|
|
26
|
+
|
|
27
|
+
def convert_boolean_type(self, column=None):
|
|
28
|
+
return "boolean"
|
|
29
|
+
|
|
30
|
+
def convert_date_type(self, column=None):
|
|
31
|
+
return "date"
|
|
32
|
+
|
|
33
|
+
def convert_datetime_type(self, column=None):
|
|
34
|
+
# The engine's SQL surface (and the Trino connector over it) exposes
|
|
35
|
+
# TIMESTAMP for both `timestamp` and `timestamptz` columns.
|
|
36
|
+
return "timestamp"
|
|
37
|
+
|
|
38
|
+
def convert_number_type(self, column=None):
|
|
39
|
+
return "double"
|
|
40
|
+
|
|
41
|
+
def convert_text_type(self, column=None):
|
|
42
|
+
return "string"
|
|
43
|
+
|
|
44
|
+
def convert_time_type(self, column=None):
|
|
45
|
+
return "time"
|
|
46
|
+
|
|
47
|
+
def debug_query(self) -> None:
|
|
48
|
+
self.execute("SELECT 1 as id")
|
|
49
|
+
|
|
50
|
+
# Datafusion / Flight SQL currently prefers standard unquoted or double-quoted identifiers
|
|
51
|
+
def quote(self, identifier: str) -> str:
|
|
52
|
+
return f'"{identifier}"'
|
|
53
|
+
|
|
54
|
+
def list_schemas(self, database: str) -> List[str]:
|
|
55
|
+
# Using standard information schema to fetch schemas
|
|
56
|
+
# DataFusion natively supports information_schema.schemata
|
|
57
|
+
try:
|
|
58
|
+
_, results = self.execute(
|
|
59
|
+
f"SELECT schema_name FROM information_schema.schemata",
|
|
60
|
+
fetch=True
|
|
61
|
+
)
|
|
62
|
+
return [row[0] for row in results]
|
|
63
|
+
except Exception as e:
|
|
64
|
+
logger.error(f"Failed to list schemas: {e}")
|
|
65
|
+
return ["public"]
|
|
66
|
+
|
|
67
|
+
def list_relations_without_caching(
|
|
68
|
+
self, schema_relation: BaseRelation
|
|
69
|
+
) -> List[BaseRelation]:
|
|
70
|
+
schema = schema_relation.schema
|
|
71
|
+
if not schema:
|
|
72
|
+
schema = "public"
|
|
73
|
+
|
|
74
|
+
sql = f"""
|
|
75
|
+
SELECT table_name, table_type
|
|
76
|
+
FROM information_schema.tables
|
|
77
|
+
WHERE table_schema = '{schema}'
|
|
78
|
+
"""
|
|
79
|
+
try:
|
|
80
|
+
_, results = self.execute(sql, fetch=True)
|
|
81
|
+
relations = []
|
|
82
|
+
for row in results:
|
|
83
|
+
table_name = row[0]
|
|
84
|
+
table_type = row[1]
|
|
85
|
+
|
|
86
|
+
rel_type = RelationType.Table if "TABLE" in table_type else RelationType.View
|
|
87
|
+
relations.append(self.Relation.create(
|
|
88
|
+
database=schema_relation.database,
|
|
89
|
+
schema=schema,
|
|
90
|
+
identifier=table_name,
|
|
91
|
+
type=rel_type
|
|
92
|
+
))
|
|
93
|
+
return relations
|
|
94
|
+
except Exception as e:
|
|
95
|
+
logger.error(f"Failed to list relations: {e}")
|
|
96
|
+
return []
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
name: dbt_benostreamdb
|
|
2
|
+
# Macro package version; tracks the BenoDB engine version.
|
|
3
|
+
version: 0.17.0
|
|
4
|
+
config-version: 2
|
|
5
|
+
|
|
6
|
+
# Supported dbt-core range. Adapters declare compatibility with ranges here
|
|
7
|
+
# instead of mirroring dbt-core's version number in their own version.
|
|
8
|
+
require-dbt-version: [">=1.8.0", "<2.0.0"]
|
|
9
|
+
|
|
10
|
+
macro-paths: ["macros"]
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
{% macro benodb__create_table_as(temporary, relation, sql) -%}
|
|
2
|
+
{%- set sql_header = config.get('sql_header', none) -%}
|
|
3
|
+
{%- set partition_by = config.get('partition_by') -%}
|
|
4
|
+
{%- if partition_by is none -%}
|
|
5
|
+
{%- set partition_by = config.get('partitioned_by') -%}
|
|
6
|
+
{%- endif -%}
|
|
7
|
+
|
|
8
|
+
{{ sql_header if sql_header is not none }}
|
|
9
|
+
|
|
10
|
+
create table {{ relation }}
|
|
11
|
+
{% if partition_by %}
|
|
12
|
+
{% if partition_by is string %}
|
|
13
|
+
PARTITIONED BY ({{ partition_by }})
|
|
14
|
+
{% else %}
|
|
15
|
+
PARTITIONED BY ({{ partition_by | join(', ') }})
|
|
16
|
+
{% endif %}
|
|
17
|
+
{% endif %}
|
|
18
|
+
as (
|
|
19
|
+
{{ sql }}
|
|
20
|
+
);
|
|
21
|
+
{%- endmacro %}
|
|
22
|
+
|
|
23
|
+
{% macro benodb__create_view_as(relation, sql) -%}
|
|
24
|
+
{%- set sql_header = config.get('sql_header', none) -%}
|
|
25
|
+
|
|
26
|
+
{{ sql_header if sql_header is not none }}
|
|
27
|
+
|
|
28
|
+
create view {{ relation }} as (
|
|
29
|
+
{{ sql }}
|
|
30
|
+
);
|
|
31
|
+
{%- endmacro %}
|
|
32
|
+
|
|
33
|
+
{% macro benodb__drop_relation(relation) -%}
|
|
34
|
+
{% call statement('drop_relation', auto_begin=False) -%}
|
|
35
|
+
drop {{ relation.type or 'table' }} if exists {{ relation }}
|
|
36
|
+
{%- endcall %}
|
|
37
|
+
{% endmacro %}
|
|
38
|
+
|
|
39
|
+
{% macro benodb__rename_relation(from_relation, to_relation) -%}
|
|
40
|
+
{% call statement('rename_relation') -%}
|
|
41
|
+
alter {{ from_relation.type }} {{ from_relation }} rename to {{ to_relation }}
|
|
42
|
+
{%- endcall %}
|
|
43
|
+
{% endmacro %}
|
|
44
|
+
|
|
45
|
+
{% macro benodb__create_schema(relation) -%}
|
|
46
|
+
{%- call statement('create_schema') -%}
|
|
47
|
+
create schema if not exists {{ relation.without_identifier().include(database=False) }}
|
|
48
|
+
{%- endcall -%}
|
|
49
|
+
{% endmacro %}
|
|
50
|
+
|
|
51
|
+
{% macro benodb__drop_schema(relation) -%}
|
|
52
|
+
{%- call statement('drop_schema') -%}
|
|
53
|
+
drop schema if exists {{ relation.without_identifier().include(database=False) }}
|
|
54
|
+
{%- endcall -%}
|
|
55
|
+
{% endmacro %}
|
|
56
|
+
|
|
57
|
+
{% macro benodb__check_schema_exists(information_schema, schema) -%}
|
|
58
|
+
{% call statement('check_schema_exists', fetch_result=True, auto_begin=False) %}
|
|
59
|
+
select count(*)
|
|
60
|
+
from information_schema.schemata
|
|
61
|
+
where schema_name = '{{ schema }}'
|
|
62
|
+
{% endcall %}
|
|
63
|
+
{{ return(load_result('check_schema_exists').table) }}
|
|
64
|
+
{% endmacro %}
|
|
65
|
+
|
|
66
|
+
{% macro benodb__get_columns_in_relation(relation) -%}
|
|
67
|
+
{% call statement('get_columns_in_relation', fetch_result=True) %}
|
|
68
|
+
select
|
|
69
|
+
column_name,
|
|
70
|
+
data_type,
|
|
71
|
+
character_maximum_length,
|
|
72
|
+
numeric_precision,
|
|
73
|
+
numeric_scale
|
|
74
|
+
from information_schema.columns
|
|
75
|
+
where table_name = '{{ relation.identifier }}'
|
|
76
|
+
{% if relation.schema %}
|
|
77
|
+
and table_schema = '{{ relation.schema }}'
|
|
78
|
+
{% endif %}
|
|
79
|
+
order by ordinal_position
|
|
80
|
+
{% endcall %}
|
|
81
|
+
{% set table = load_result('get_columns_in_relation').table %}
|
|
82
|
+
{{ return(sql_convert_columns_in_relation(table)) }}
|
|
83
|
+
{% endmacro %}
|
|
84
|
+
|
|
85
|
+
{% macro benodb__get_create_index_sql(relation, index_dict) -%}
|
|
86
|
+
{%- set index_columns = index_dict.get('columns', []) -%}
|
|
87
|
+
{%- set index_name = index_dict.get('name', relation.identifier ~ '_' ~ index_columns | join('_') ~ '_idx') -%}
|
|
88
|
+
{{ log("BenoDB manages indices natively; skipping CREATE INDEX for " ~ index_name, info=True) }}
|
|
89
|
+
{% do return(None) %}
|
|
90
|
+
{%- endmacro %}
|