dbt-lakecompute 0.1.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dbt/__init__.py +3 -0
- dbt/adapters/lakecompute/__init__.py +17 -0
- dbt/adapters/lakecompute/__version__.py +1 -0
- dbt/adapters/lakecompute/auth.py +80 -0
- dbt/adapters/lakecompute/column.py +56 -0
- dbt/adapters/lakecompute/connections.py +415 -0
- dbt/adapters/lakecompute/credentials.py +80 -0
- dbt/adapters/lakecompute/driver.py +61 -0
- dbt/adapters/lakecompute/impl.py +184 -0
- dbt/adapters/lakecompute/install.py +309 -0
- dbt/adapters/lakecompute/relation.py +30 -0
- dbt/adapters/lakecompute/seed.py +67 -0
- dbt/adapters/lakecompute/type_mapping.py +132 -0
- dbt/include/lakecompute/__init__.py +3 -0
- dbt/include/lakecompute/dbt_project.yml +5 -0
- dbt/include/lakecompute/macros/adapters.sql +37 -0
- dbt/include/lakecompute/macros/apply_grants.sql +8 -0
- dbt/include/lakecompute/macros/hooks.sql +15 -0
- dbt/include/lakecompute/macros/materializations/incremental.sql +191 -0
- dbt/include/lakecompute/macros/materializations/seed.sql +47 -0
- dbt/include/lakecompute/macros/materializations/table.sql +28 -0
- dbt/include/lakecompute/macros/materializations/unsupported.sql +58 -0
- dbt/include/lakecompute/macros/persist_docs.sql +11 -0
- dbt/include/lakecompute/macros/relations/table/create.sql +26 -0
- dbt/include/lakecompute/macros/schemas.sql +94 -0
- dbt/include/lakecompute/macros/timestamps.sql +3 -0
- dbt/include/lakecompute/macros/utils/_README.md +9 -0
- dbt/include/lakecompute/macros/utils/date.sql +5 -0
- dbt/include/lakecompute/macros/utils/dateadd.sql +16 -0
- dbt/include/lakecompute/macros/utils/datediff.sql +17 -0
- dbt/include/lakecompute/macros/utils/generate_series.sql +7 -0
- dbt/include/lakecompute/macros/utils/listagg.sql +24 -0
- dbt/include/lakecompute/macros/utils/split_part.sql +6 -0
- dbt/include/lakecompute/profile_template.yml +17 -0
- dbt_lakecompute-0.1.0.dev1.dist-info/METADATA +158 -0
- dbt_lakecompute-0.1.0.dev1.dist-info/RECORD +38 -0
- dbt_lakecompute-0.1.0.dev1.dist-info/WHEEL +4 -0
- dbt_lakecompute-0.1.0.dev1.dist-info/licenses/LICENSE +201 -0
dbt/__init__.py
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
from dbt.adapters.base import AdapterPlugin
|
|
2
|
+
|
|
3
|
+
from dbt.adapters.lakecompute.connections import ( # noqa: F401
|
|
4
|
+
LakeComputeAdapterResponse,
|
|
5
|
+
LakeComputeConnectionManager,
|
|
6
|
+
)
|
|
7
|
+
from dbt.adapters.lakecompute.credentials import LakeComputeCredentials # noqa: F401
|
|
8
|
+
from dbt.adapters.lakecompute.relation import LakeComputeRelation # noqa: F401
|
|
9
|
+
from dbt.adapters.lakecompute.impl import LakeComputeAdapter
|
|
10
|
+
from dbt.include import lakecompute
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
Plugin: AdapterPlugin = AdapterPlugin(
|
|
14
|
+
adapter=LakeComputeAdapter, # type: ignore[arg-type]
|
|
15
|
+
credentials=LakeComputeCredentials,
|
|
16
|
+
include_path=lakecompute.PACKAGE_PATH,
|
|
17
|
+
)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
version = "0.1.0.dev1"
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"""Mapping from the profile's ``method`` to the driver's ``adbc.dbt.auth.*`` options."""
|
|
2
|
+
|
|
3
|
+
from typing import Dict, TYPE_CHECKING
|
|
4
|
+
|
|
5
|
+
from dbt.adapters.exceptions import FailedToConnectError
|
|
6
|
+
|
|
7
|
+
from dbt.adapters.lakecompute.credentials import LakeComputeAuthMethod
|
|
8
|
+
|
|
9
|
+
if TYPE_CHECKING:
|
|
10
|
+
from dbt.adapters.lakecompute.credentials import LakeComputeCredentials
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
OPTION_AUTH_METHOD = "adbc.dbt.auth.method"
|
|
14
|
+
OPTION_AUTH_TOKEN = "adbc.dbt.auth.token"
|
|
15
|
+
OPTION_AUTH_API_KEY = "adbc.dbt.auth.api_key"
|
|
16
|
+
OPTION_AUTH_OKTA_AUTH_URL = "adbc.dbt.auth.okta.auth_url"
|
|
17
|
+
OPTION_AUTH_OKTA_TOKEN_URL = "adbc.dbt.auth.okta.token_url"
|
|
18
|
+
OPTION_AUTH_OKTA_CLIENT_ID = "adbc.dbt.auth.okta.client_id"
|
|
19
|
+
OPTION_AUTH_FIVETRAN_CREDENTIAL = "adbc.dbt.auth.fivetran.credential"
|
|
20
|
+
OPTION_AUTH_FIVETRAN_AUTH_URL = "adbc.dbt.auth.fivetran.auth_url"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
REQUIRED_FIELD_BY_METHOD: Dict[str, str] = {
|
|
24
|
+
LakeComputeAuthMethod.TOKEN: "token",
|
|
25
|
+
LakeComputeAuthMethod.API_KEY: "api_key",
|
|
26
|
+
LakeComputeAuthMethod.FIVETRAN: "fivetran_credential",
|
|
27
|
+
# okta_browser needs no pre-shared secret: the browser flow supplies it
|
|
28
|
+
LakeComputeAuthMethod.OKTA_BROWSER: "",
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def validate_auth_credentials(credentials: "LakeComputeCredentials") -> None:
|
|
33
|
+
"""Raise if ``method`` is unknown or its required field is missing."""
|
|
34
|
+
method = credentials.method
|
|
35
|
+
|
|
36
|
+
if method not in REQUIRED_FIELD_BY_METHOD:
|
|
37
|
+
supported = ", ".join(sorted(REQUIRED_FIELD_BY_METHOD))
|
|
38
|
+
raise FailedToConnectError(
|
|
39
|
+
f"Unknown lakecompute auth method {method!r}. Supported methods: {supported}."
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
required = REQUIRED_FIELD_BY_METHOD[method]
|
|
43
|
+
if required and not getattr(credentials, required, None):
|
|
44
|
+
raise FailedToConnectError(
|
|
45
|
+
f"lakecompute auth method {method!r} requires `{required}` to be set in the profile."
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def auth_options(credentials: "LakeComputeCredentials") -> Dict[str, str]:
|
|
50
|
+
"""Build the ``adbc.dbt.auth.*`` database options for this profile.
|
|
51
|
+
|
|
52
|
+
Optional fields are emitted only when present: the driver distinguishes "unset" from
|
|
53
|
+
"set to empty".
|
|
54
|
+
"""
|
|
55
|
+
validate_auth_credentials(credentials)
|
|
56
|
+
|
|
57
|
+
method = credentials.method
|
|
58
|
+
options: Dict[str, str] = {OPTION_AUTH_METHOD: str(method)}
|
|
59
|
+
|
|
60
|
+
if method == LakeComputeAuthMethod.TOKEN:
|
|
61
|
+
options[OPTION_AUTH_TOKEN] = str(credentials.token)
|
|
62
|
+
|
|
63
|
+
elif method == LakeComputeAuthMethod.API_KEY:
|
|
64
|
+
options[OPTION_AUTH_API_KEY] = str(credentials.api_key)
|
|
65
|
+
|
|
66
|
+
elif method == LakeComputeAuthMethod.OKTA_BROWSER:
|
|
67
|
+
_set_if_present(options, OPTION_AUTH_OKTA_AUTH_URL, credentials.okta_auth_url)
|
|
68
|
+
_set_if_present(options, OPTION_AUTH_OKTA_TOKEN_URL, credentials.okta_token_url)
|
|
69
|
+
_set_if_present(options, OPTION_AUTH_OKTA_CLIENT_ID, credentials.okta_client_id)
|
|
70
|
+
|
|
71
|
+
elif method == LakeComputeAuthMethod.FIVETRAN:
|
|
72
|
+
options[OPTION_AUTH_FIVETRAN_CREDENTIAL] = str(credentials.fivetran_credential)
|
|
73
|
+
_set_if_present(options, OPTION_AUTH_FIVETRAN_AUTH_URL, credentials.fivetran_auth_url)
|
|
74
|
+
|
|
75
|
+
return options
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _set_if_present(options: Dict[str, str], key: str, value: object) -> None:
|
|
79
|
+
if value is not None and value != "":
|
|
80
|
+
options[key] = str(value)
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""Column types are kept as ``describe`` reports them; the base class would rebuild
|
|
2
|
+
``VARCHAR`` as ``character varying(256)``."""
|
|
3
|
+
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from typing import ClassVar, Dict
|
|
6
|
+
|
|
7
|
+
from dbt.adapters.base.column import Column
|
|
8
|
+
|
|
9
|
+
_STRING_TYPES = frozenset({"varchar", "text", "string", "char", "bpchar"})
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass
|
|
13
|
+
class LakeComputeColumn(Column):
|
|
14
|
+
# no STRING -> TEXT translation: contracts compare the literal type names
|
|
15
|
+
TYPE_LABELS: ClassVar[Dict[str, str]] = {}
|
|
16
|
+
|
|
17
|
+
@property
|
|
18
|
+
def data_type(self) -> str:
|
|
19
|
+
return self.dtype
|
|
20
|
+
|
|
21
|
+
def _scalar_type_name(self) -> str:
|
|
22
|
+
"""``decimal`` for ``DECIMAL(38,2)``; empty for a list or array like ``DECIMAL(38,2)[]``."""
|
|
23
|
+
if self.dtype.rstrip().endswith("]"):
|
|
24
|
+
return ""
|
|
25
|
+
return self.dtype.lower().split("(")[0].strip()
|
|
26
|
+
|
|
27
|
+
def is_string(self) -> bool:
|
|
28
|
+
return self._scalar_type_name() in _STRING_TYPES
|
|
29
|
+
|
|
30
|
+
def is_numeric(self) -> bool:
|
|
31
|
+
return self._scalar_type_name() in ("decimal", "numeric")
|
|
32
|
+
|
|
33
|
+
def is_float(self) -> bool:
|
|
34
|
+
return self.dtype.lower() in ("real", "float", "float4", "float8", "double")
|
|
35
|
+
|
|
36
|
+
def is_integer(self) -> bool:
|
|
37
|
+
return self.dtype.lower() in (
|
|
38
|
+
"tinyint",
|
|
39
|
+
"smallint",
|
|
40
|
+
"integer",
|
|
41
|
+
"int",
|
|
42
|
+
"bigint",
|
|
43
|
+
"hugeint",
|
|
44
|
+
"utinyint",
|
|
45
|
+
"usmallint",
|
|
46
|
+
"uinteger",
|
|
47
|
+
"ubigint",
|
|
48
|
+
"uhugeint",
|
|
49
|
+
"int1",
|
|
50
|
+
"int2",
|
|
51
|
+
"int4",
|
|
52
|
+
"int8",
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
def __repr__(self) -> str:
|
|
56
|
+
return "<LakeComputeColumn {} ({})>".format(self.name, self.data_type)
|
|
@@ -0,0 +1,415 @@
|
|
|
1
|
+
"""Connection management for lake compute.
|
|
2
|
+
|
|
3
|
+
* One shared ``AdbcDatabase`` per target, one ``AdbcConnection`` per thread. Auth runs once
|
|
4
|
+
per database, and the driver allows concurrent ``NewConnection``.
|
|
5
|
+
* Connections are built by hand with ``autocommit=True``. ``Connection.adbc_clone()`` would
|
|
6
|
+
hardcode ``autocommit=False``, and the service has no transactions, so ``begin``/``commit``
|
|
7
|
+
are no-ops and ``transaction_open`` stays ``False``.
|
|
8
|
+
* ``get_response`` is pure: query metadata is read once and stashed on the cursor for the
|
|
9
|
+
caller and the structured query-status event.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import atexit
|
|
13
|
+
import threading
|
|
14
|
+
import time
|
|
15
|
+
from contextlib import contextmanager
|
|
16
|
+
from dataclasses import dataclass
|
|
17
|
+
from typing import Any, cast, Dict, Iterable, Iterator, Optional, Tuple, TYPE_CHECKING
|
|
18
|
+
|
|
19
|
+
from dbt.adapters.contracts.connection import (
|
|
20
|
+
AdapterResponse,
|
|
21
|
+
Connection,
|
|
22
|
+
ConnectionState,
|
|
23
|
+
)
|
|
24
|
+
from dbt.adapters.events.logging import AdapterLogger
|
|
25
|
+
from dbt.adapters.events.types import ConnectionUsed, SQLQuery, SQLQueryStatus
|
|
26
|
+
from dbt.adapters.sql import SQLConnectionManager
|
|
27
|
+
from dbt_common.events.contextvars import get_node_info
|
|
28
|
+
from dbt_common.events.functions import fire_event
|
|
29
|
+
from dbt_common.exceptions import DbtDatabaseError, DbtRuntimeError
|
|
30
|
+
from dbt_common.invocation import get_invocation_id
|
|
31
|
+
from dbt_common.utils import cast_to_str
|
|
32
|
+
|
|
33
|
+
from dbt.adapters.lakecompute.credentials import LakeComputeCredentials
|
|
34
|
+
from dbt.adapters.lakecompute.driver import ResolvedDriver, resolve_driver
|
|
35
|
+
from dbt.adapters.lakecompute.type_mapping import data_type_code_to_name, normalize_value
|
|
36
|
+
|
|
37
|
+
if TYPE_CHECKING:
|
|
38
|
+
import agate
|
|
39
|
+
from adbc_driver_manager import dbapi
|
|
40
|
+
|
|
41
|
+
logger = AdapterLogger("LakeCompute")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
OPTION_LAST_QUERY_ID = "adbc.dbt.last_query_id"
|
|
45
|
+
OPTION_LAST_WARNINGS = "adbc.dbt.last_warnings"
|
|
46
|
+
OPTION_PROJECT = "adbc.dbt.project"
|
|
47
|
+
OPTION_MODEL = "adbc.dbt.model"
|
|
48
|
+
OPTION_RUN_ID = "adbc.dbt.run_id"
|
|
49
|
+
|
|
50
|
+
_ATTR_QUERY_ID = "_lakecompute_query_id"
|
|
51
|
+
_ATTR_WARNINGS = "_lakecompute_warnings"
|
|
52
|
+
_ATTR_ACTIVE_STATEMENT = "_lakecompute_active_statement"
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@dataclass
|
|
56
|
+
class LakeComputeAdapterResponse(AdapterResponse):
|
|
57
|
+
warning: Optional[str] = None
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
# keyed by (path, entrypoint, sorted db_kwargs) so two targets never share a database
|
|
61
|
+
_REGISTRY: Dict[Tuple[str, str, Tuple[Tuple[str, str], ...]], "dbapi._SharedDatabase"] = {}
|
|
62
|
+
_REGISTRY_LOCK = threading.Lock()
|
|
63
|
+
_CANCEL_LOCK = threading.Lock()
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _close_all_databases() -> None:
|
|
67
|
+
"""Release every shared database at interpreter exit.
|
|
68
|
+
|
|
69
|
+
Not wired into ``cleanup_all()``: dbt can call that mid-process, and dbt-tests-adapter
|
|
70
|
+
expects the databases to stay live.
|
|
71
|
+
"""
|
|
72
|
+
with _REGISTRY_LOCK:
|
|
73
|
+
shared_databases = list(_REGISTRY.values())
|
|
74
|
+
_REGISTRY.clear()
|
|
75
|
+
for shared in shared_databases:
|
|
76
|
+
try:
|
|
77
|
+
shared.close()
|
|
78
|
+
except Exception as exc: # pragma: no cover - best effort at shutdown
|
|
79
|
+
logger.debug(f"Error closing shared lakecompute database: {exc}")
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
atexit.register(_close_all_databases)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _get_or_create_shared_database(resolved: ResolvedDriver) -> "dbapi._SharedDatabase":
|
|
86
|
+
"""Return the refcounted ``_SharedDatabase`` for ``resolved``, creating it once."""
|
|
87
|
+
from adbc_driver_manager import _lib, dbapi
|
|
88
|
+
|
|
89
|
+
key = (resolved.path, resolved.entrypoint, tuple(sorted(resolved.db_kwargs.items())))
|
|
90
|
+
with _REGISTRY_LOCK:
|
|
91
|
+
if key not in _REGISTRY:
|
|
92
|
+
database = _lib.AdbcDatabase(
|
|
93
|
+
driver=resolved.path, entrypoint=resolved.entrypoint, **resolved.db_kwargs
|
|
94
|
+
)
|
|
95
|
+
_REGISTRY[key] = dbapi._SharedDatabase(database)
|
|
96
|
+
return _REGISTRY[key]
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _new_thread_connection(shared: "dbapi._SharedDatabase") -> "dbapi.Connection":
|
|
100
|
+
"""Build one autocommit ``dbapi.Connection`` over ``shared``.
|
|
101
|
+
|
|
102
|
+
``Connection.__init__`` clones the ``_SharedDatabase`` itself, so do not pre-clone.
|
|
103
|
+
"""
|
|
104
|
+
from adbc_driver_manager import _lib, dbapi
|
|
105
|
+
|
|
106
|
+
return dbapi.Connection(shared, _lib.AdbcConnection(shared._db), {}, autocommit=True)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
@contextmanager
|
|
110
|
+
def _cancellable(connection: Connection, cursor: "dbapi.Cursor") -> Iterator[None]:
|
|
111
|
+
"""Publish the in-flight statement on ``connection`` for the duration of the call.
|
|
112
|
+
|
|
113
|
+
``cancel()`` needs the ``AdbcStatement``, because the driver only implements
|
|
114
|
+
statement-level cancel.
|
|
115
|
+
"""
|
|
116
|
+
# A dbt connection belongs to one thread, so at most one statement is in flight on it.
|
|
117
|
+
with _CANCEL_LOCK:
|
|
118
|
+
setattr(connection, _ATTR_ACTIVE_STATEMENT, cursor.adbc_statement)
|
|
119
|
+
try:
|
|
120
|
+
yield
|
|
121
|
+
finally:
|
|
122
|
+
with _CANCEL_LOCK:
|
|
123
|
+
setattr(connection, _ATTR_ACTIVE_STATEMENT, None)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
class LakeComputeConnectionManager(SQLConnectionManager):
|
|
127
|
+
TYPE = "lakecompute"
|
|
128
|
+
|
|
129
|
+
def set_query_header(self, query_header_context: Dict[str, Any]) -> None:
|
|
130
|
+
# off unless the profile opts in; `_add_query_comment` is a no-op when this is None
|
|
131
|
+
credentials = cast(LakeComputeCredentials, self.profile.credentials)
|
|
132
|
+
if credentials.query_comment_enabled:
|
|
133
|
+
super().set_query_header(query_header_context)
|
|
134
|
+
else:
|
|
135
|
+
self.query_header = None
|
|
136
|
+
|
|
137
|
+
# The connection is autocommit and the service has no transactions. Like Athena and
|
|
138
|
+
# BigQuery, begin/commit do nothing, so `transaction_open` stays False and the base
|
|
139
|
+
# rollback/close paths never call `handle.rollback()`.
|
|
140
|
+
|
|
141
|
+
def begin(self) -> None:
|
|
142
|
+
pass
|
|
143
|
+
|
|
144
|
+
def commit(self) -> None:
|
|
145
|
+
pass
|
|
146
|
+
|
|
147
|
+
@classmethod
|
|
148
|
+
def open(cls, connection: Connection) -> Connection:
|
|
149
|
+
if connection.state == ConnectionState.OPEN:
|
|
150
|
+
logger.debug("Connection is already open, skipping open.")
|
|
151
|
+
return connection
|
|
152
|
+
|
|
153
|
+
credentials: LakeComputeCredentials = connection.credentials
|
|
154
|
+
resolved = resolve_driver(credentials)
|
|
155
|
+
|
|
156
|
+
from adbc_driver_manager import OperationalError
|
|
157
|
+
|
|
158
|
+
return cls.retry_connection(
|
|
159
|
+
connection,
|
|
160
|
+
connect=lambda: _new_thread_connection(_get_or_create_shared_database(resolved)),
|
|
161
|
+
logger=logger,
|
|
162
|
+
retry_limit=credentials.retries,
|
|
163
|
+
retryable_exceptions=(OperationalError,),
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
def cancel(self, connection: Connection) -> None:
|
|
167
|
+
"""Cancel the statement currently in flight on ``connection``.
|
|
168
|
+
|
|
169
|
+
Statement-level, because the driver returns ``NOT_IMPLEMENTED`` for
|
|
170
|
+
``AdbcConnectionCancel``. ``AdbcStatementCancel`` is safe to call from another thread.
|
|
171
|
+
A cancelled query surfaces on the blocked thread as ADBC ``INTERNAL`` with "... ended
|
|
172
|
+
with status Canceled", not as ``CANCELLED``.
|
|
173
|
+
|
|
174
|
+
The cancel runs under ``_CANCEL_LOCK``: ``AdbcStatement.cancel`` takes no lock of its
|
|
175
|
+
own, so without it the owning thread could leave ``_cancellable`` and release the
|
|
176
|
+
statement while the cancel is still inside the driver.
|
|
177
|
+
"""
|
|
178
|
+
with _CANCEL_LOCK:
|
|
179
|
+
statement = getattr(connection, _ATTR_ACTIVE_STATEMENT, None)
|
|
180
|
+
if statement is None:
|
|
181
|
+
logger.debug(f"No in-flight lakecompute query to cancel on {connection.name}")
|
|
182
|
+
return
|
|
183
|
+
try:
|
|
184
|
+
statement.cancel()
|
|
185
|
+
except Exception as exc:
|
|
186
|
+
# don't mask the failure that prompted the cancel
|
|
187
|
+
logger.debug(f"Could not cancel lakecompute statement: {exc}")
|
|
188
|
+
|
|
189
|
+
@contextmanager
|
|
190
|
+
def exception_handler(self, sql: str) -> Iterator[None]:
|
|
191
|
+
try:
|
|
192
|
+
yield
|
|
193
|
+
except Exception as exc:
|
|
194
|
+
# no transaction, so no rollback_if_open() here
|
|
195
|
+
if isinstance(exc, DbtRuntimeError):
|
|
196
|
+
raise
|
|
197
|
+
|
|
198
|
+
status = getattr(getattr(exc, "status_code", None), "name", "UNKNOWN")
|
|
199
|
+
message = str(exc).strip()
|
|
200
|
+
|
|
201
|
+
if status in ("UNAUTHENTICATED", "UNAUTHORIZED"):
|
|
202
|
+
credentials = cast(LakeComputeCredentials, self.profile.credentials)
|
|
203
|
+
raise DbtDatabaseError(
|
|
204
|
+
f"lakecompute rejected the credentials for auth method "
|
|
205
|
+
f"'{credentials.method}': {message}"
|
|
206
|
+
) from exc
|
|
207
|
+
|
|
208
|
+
if status in ("TIMEOUT", "CANCELLED"):
|
|
209
|
+
raise DbtDatabaseError(
|
|
210
|
+
f"lakecompute query {status.lower()}: {message}. "
|
|
211
|
+
f"Consider raising `timeout_seconds` in the profile."
|
|
212
|
+
) from exc
|
|
213
|
+
|
|
214
|
+
raise DbtDatabaseError(message) from exc
|
|
215
|
+
|
|
216
|
+
def execute(
|
|
217
|
+
self,
|
|
218
|
+
sql: str,
|
|
219
|
+
auto_begin: bool = False,
|
|
220
|
+
fetch: bool = False,
|
|
221
|
+
limit: Optional[int] = None,
|
|
222
|
+
) -> Tuple[AdapterResponse, "agate.Table"]:
|
|
223
|
+
"""Run ``sql`` and, when asked, materialise its rows.
|
|
224
|
+
|
|
225
|
+
Keep the statement cancellable and translate driver errors until fetching finishes.
|
|
226
|
+
Each cursor owns an ``AdbcStatement`` and is closed before returning or raising.
|
|
227
|
+
"""
|
|
228
|
+
from dbt_common.clients.agate_helper import empty_table
|
|
229
|
+
|
|
230
|
+
sql = self._add_query_comment(sql)
|
|
231
|
+
with self._query(sql, fetch=fetch, close_cursor=True) as (_, cursor):
|
|
232
|
+
table = self.get_result_from_cursor(cursor, limit) if fetch else empty_table()
|
|
233
|
+
return self.get_response(cursor), table
|
|
234
|
+
|
|
235
|
+
def add_select_query(self, sql: str) -> Tuple[Connection, "dbapi.Cursor"]:
|
|
236
|
+
# Same as execute: `get_column_schema_from_query` reads `cursor.description`, which is
|
|
237
|
+
# None on the no-fetch path. The cursor is handed back open, as the base contract says.
|
|
238
|
+
return self.add_query(self._add_query_comment(sql), fetch=True)
|
|
239
|
+
|
|
240
|
+
def add_query(
|
|
241
|
+
self,
|
|
242
|
+
sql: str,
|
|
243
|
+
auto_begin: bool = True,
|
|
244
|
+
bindings: Optional[Any] = None,
|
|
245
|
+
abridge_sql_log: bool = False,
|
|
246
|
+
retryable_exceptions: Tuple[type, ...] = tuple(),
|
|
247
|
+
retry_limit: int = 1,
|
|
248
|
+
*,
|
|
249
|
+
fetch: bool = False,
|
|
250
|
+
) -> Tuple[Connection, "dbapi.Cursor"]:
|
|
251
|
+
"""Send ``sql`` and return the cursor.
|
|
252
|
+
|
|
253
|
+
``fetch`` is keyword-only so ``adapter.add_query(...)`` macro calls keep working. A
|
|
254
|
+
``;``-joined batch is sent verbatim as one request; the service runs every statement.
|
|
255
|
+
"""
|
|
256
|
+
if bindings is not None:
|
|
257
|
+
raise DbtRuntimeError(
|
|
258
|
+
"lakecompute does not support parameter binding; render literals into the SQL "
|
|
259
|
+
"instead."
|
|
260
|
+
)
|
|
261
|
+
|
|
262
|
+
# auto_begin and the retry arguments are accepted for signature compatibility only:
|
|
263
|
+
# there are no transactions, and queries are not retried.
|
|
264
|
+
with self._query(sql, abridge_sql_log, fetch=fetch) as (connection, cursor):
|
|
265
|
+
return connection, cursor
|
|
266
|
+
|
|
267
|
+
@contextmanager
|
|
268
|
+
def _query(
|
|
269
|
+
self,
|
|
270
|
+
sql: str,
|
|
271
|
+
abridge_sql_log: bool = False,
|
|
272
|
+
*,
|
|
273
|
+
fetch: bool = False,
|
|
274
|
+
close_cursor: bool = False,
|
|
275
|
+
) -> Iterator[Tuple[Connection, "dbapi.Cursor"]]:
|
|
276
|
+
"""Guard execution and optional result consumption as one operation.
|
|
277
|
+
|
|
278
|
+
``add_query`` transfers ownership of a successful cursor to its caller. ``execute``
|
|
279
|
+
consumes it inside this context and requests cleanup on success too. Failures always
|
|
280
|
+
close the cursor, after deregistration has waited for any concurrent cancel.
|
|
281
|
+
"""
|
|
282
|
+
connection = self.get_thread_connection()
|
|
283
|
+
fire_event(
|
|
284
|
+
ConnectionUsed(
|
|
285
|
+
conn_type=self.TYPE,
|
|
286
|
+
conn_name=cast_to_str(connection.name),
|
|
287
|
+
node_info=get_node_info(),
|
|
288
|
+
)
|
|
289
|
+
)
|
|
290
|
+
|
|
291
|
+
with self.exception_handler(sql):
|
|
292
|
+
log_sql = f"{sql[:512]}..." if abridge_sql_log else sql
|
|
293
|
+
fire_event(
|
|
294
|
+
SQLQuery(
|
|
295
|
+
conn_name=cast_to_str(connection.name),
|
|
296
|
+
sql=log_sql,
|
|
297
|
+
node_info=get_node_info(),
|
|
298
|
+
)
|
|
299
|
+
)
|
|
300
|
+
started = time.perf_counter()
|
|
301
|
+
|
|
302
|
+
cursor: "dbapi.Cursor" = connection.handle.cursor()
|
|
303
|
+
try:
|
|
304
|
+
with _cancellable(connection, cursor):
|
|
305
|
+
self._apply_statement_options(cursor)
|
|
306
|
+
if fetch:
|
|
307
|
+
cursor.execute(sql)
|
|
308
|
+
else:
|
|
309
|
+
# execute_update keeps rowcount without producing a result set.
|
|
310
|
+
cursor.executemany(sql, None)
|
|
311
|
+
|
|
312
|
+
self._collect_metadata(cursor)
|
|
313
|
+
yield connection, cursor
|
|
314
|
+
|
|
315
|
+
response = self.get_response(cursor)
|
|
316
|
+
fire_event(
|
|
317
|
+
SQLQueryStatus(
|
|
318
|
+
status=str(response),
|
|
319
|
+
elapsed=time.perf_counter() - started,
|
|
320
|
+
node_info=get_node_info(),
|
|
321
|
+
query_id=response.query_id,
|
|
322
|
+
)
|
|
323
|
+
)
|
|
324
|
+
except BaseException:
|
|
325
|
+
# Preserve the query error even if releasing the statement also fails.
|
|
326
|
+
try:
|
|
327
|
+
cursor.close()
|
|
328
|
+
except Exception as exc:
|
|
329
|
+
logger.debug(f"Error closing failed lakecompute cursor: {exc}")
|
|
330
|
+
raise
|
|
331
|
+
else:
|
|
332
|
+
if close_cursor:
|
|
333
|
+
cursor.close()
|
|
334
|
+
|
|
335
|
+
def _apply_statement_options(self, cursor: "dbapi.Cursor") -> None:
|
|
336
|
+
"""Set ``adbc.dbt.*`` attribution options on the statement.
|
|
337
|
+
|
|
338
|
+
Set after cursor construction so a rejected option cannot leave a half-built cursor,
|
|
339
|
+
and a rejection only logs: attribution must never fail a run.
|
|
340
|
+
"""
|
|
341
|
+
options = self._statement_options()
|
|
342
|
+
if not options:
|
|
343
|
+
return
|
|
344
|
+
try:
|
|
345
|
+
cursor.adbc_statement.set_options(**options)
|
|
346
|
+
except Exception as exc:
|
|
347
|
+
logger.debug(f"lakecompute driver rejected statement options {sorted(options)}: {exc}")
|
|
348
|
+
|
|
349
|
+
def _statement_options(self) -> Dict[str, str]:
|
|
350
|
+
options: Dict[str, str] = {}
|
|
351
|
+
|
|
352
|
+
project_name = getattr(self.profile, "project_name", None)
|
|
353
|
+
if project_name:
|
|
354
|
+
options[OPTION_PROJECT] = str(project_name)
|
|
355
|
+
|
|
356
|
+
# empty for on-run-start/on-run-end hooks and for `dbt debug`
|
|
357
|
+
node_info = get_node_info() or {}
|
|
358
|
+
node_name = node_info.get("node_name")
|
|
359
|
+
if node_name:
|
|
360
|
+
options[OPTION_MODEL] = str(node_name)
|
|
361
|
+
|
|
362
|
+
invocation_id = get_invocation_id()
|
|
363
|
+
if invocation_id:
|
|
364
|
+
options[OPTION_RUN_ID] = str(invocation_id)
|
|
365
|
+
|
|
366
|
+
return options
|
|
367
|
+
|
|
368
|
+
@classmethod
|
|
369
|
+
def _collect_metadata(cls, cursor: "dbapi.Cursor") -> None:
|
|
370
|
+
# read now: a closed statement raises. Each read is best effort.
|
|
371
|
+
setattr(cursor, _ATTR_QUERY_ID, _try_get_option(cursor, OPTION_LAST_QUERY_ID))
|
|
372
|
+
setattr(cursor, _ATTR_WARNINGS, _try_get_option(cursor, OPTION_LAST_WARNINGS))
|
|
373
|
+
|
|
374
|
+
@classmethod
|
|
375
|
+
def get_response(cls, cursor: "dbapi.Cursor") -> LakeComputeAdapterResponse:
|
|
376
|
+
"""Build the response; must stay pure (see the module docstring)."""
|
|
377
|
+
# dbapi reports -1 when the row count is unknown
|
|
378
|
+
rows_affected = cursor.rowcount if cursor.rowcount >= 0 else None
|
|
379
|
+
|
|
380
|
+
warning = getattr(cursor, _ATTR_WARNINGS, None)
|
|
381
|
+
|
|
382
|
+
message = "SUCCESS" if rows_affected is None else f"SUCCESS {rows_affected}"
|
|
383
|
+
if warning:
|
|
384
|
+
message = f"{message} [{warning}]"
|
|
385
|
+
|
|
386
|
+
return LakeComputeAdapterResponse(
|
|
387
|
+
_message=message,
|
|
388
|
+
code="SUCCESS",
|
|
389
|
+
rows_affected=rows_affected,
|
|
390
|
+
query_id=getattr(cursor, _ATTR_QUERY_ID, None),
|
|
391
|
+
warning=warning,
|
|
392
|
+
)
|
|
393
|
+
|
|
394
|
+
@classmethod
|
|
395
|
+
def data_type_code_to_name(cls, type_code: Any) -> str:
|
|
396
|
+
# Lives on the connection manager because `get_column_schema_from_query` calls it as
|
|
397
|
+
# `self.connections.data_type_code_to_name`. ADBC supplies a pyarrow DataType.
|
|
398
|
+
return data_type_code_to_name(type_code)
|
|
399
|
+
|
|
400
|
+
@classmethod
|
|
401
|
+
def process_results(
|
|
402
|
+
cls, column_names: Iterable[str], rows: Iterable[Any]
|
|
403
|
+
) -> Iterator[Dict[str, Any]]:
|
|
404
|
+
for row in super().process_results(column_names, rows):
|
|
405
|
+
yield {key: normalize_value(value) for key, value in row.items()}
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
def _try_get_option(cursor: "dbapi.Cursor", option: str) -> Optional[str]:
|
|
409
|
+
try:
|
|
410
|
+
value = cursor.adbc_statement.get_option(option)
|
|
411
|
+
except Exception:
|
|
412
|
+
return None
|
|
413
|
+
if value in (None, ""):
|
|
414
|
+
return None
|
|
415
|
+
return str(value)
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
from typing import Optional, Tuple
|
|
4
|
+
|
|
5
|
+
from dbt.adapters.contracts.connection import Credentials
|
|
6
|
+
from dbt_common.dataclass_schema import StrEnum
|
|
7
|
+
|
|
8
|
+
DEFAULT_BASE_URL = "https://api.lake-compute.fivetran.com/"
|
|
9
|
+
BASE_URL_ENV_VAR = "DBT_COMPUTE_BASE_URL"
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class LakeComputeAuthMethod(StrEnum):
|
|
13
|
+
TOKEN = "token"
|
|
14
|
+
API_KEY = "api_key"
|
|
15
|
+
OKTA_BROWSER = "okta_browser"
|
|
16
|
+
FIVETRAN = "fivetran"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass
|
|
20
|
+
class LakeComputeCredentials(Credentials):
|
|
21
|
+
"""Profile fields for a ``type: lakecompute`` target.
|
|
22
|
+
|
|
23
|
+
Which auth fields are required depends on ``method``, so they are validated at connect
|
|
24
|
+
time (see ``auth.validate_auth_credentials``), where ``dbt debug`` can report them.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
base_url: Optional[str] = None
|
|
28
|
+
method: str = LakeComputeAuthMethod.TOKEN
|
|
29
|
+
|
|
30
|
+
token: Optional[str] = None
|
|
31
|
+
api_key: Optional[str] = None
|
|
32
|
+
okta_auth_url: Optional[str] = None
|
|
33
|
+
okta_token_url: Optional[str] = None
|
|
34
|
+
okta_client_id: Optional[str] = None
|
|
35
|
+
fivetran_credential: Optional[str] = None
|
|
36
|
+
fivetran_auth_url: Optional[str] = None
|
|
37
|
+
|
|
38
|
+
warehouse: Optional[str] = None
|
|
39
|
+
dialect: Optional[str] = None
|
|
40
|
+
timeout_seconds: Optional[int] = None
|
|
41
|
+
|
|
42
|
+
retries: int = 1
|
|
43
|
+
|
|
44
|
+
driver_path: Optional[str] = None
|
|
45
|
+
|
|
46
|
+
query_comment_enabled: bool = False
|
|
47
|
+
|
|
48
|
+
_ALIASES = {"dbname": "database"}
|
|
49
|
+
|
|
50
|
+
@property
|
|
51
|
+
def type(self) -> str:
|
|
52
|
+
return "lakecompute"
|
|
53
|
+
|
|
54
|
+
@property
|
|
55
|
+
def unique_field(self) -> str:
|
|
56
|
+
return self.resolved_base_url
|
|
57
|
+
|
|
58
|
+
@property
|
|
59
|
+
def resolved_base_url(self) -> str:
|
|
60
|
+
"""profile -> $DBT_COMPUTE_BASE_URL -> public endpoint."""
|
|
61
|
+
return self.base_url or os.getenv(BASE_URL_ENV_VAR) or DEFAULT_BASE_URL
|
|
62
|
+
|
|
63
|
+
def _connection_keys(self) -> Tuple[str, ...]:
|
|
64
|
+
# secrets (token, api_key, fivetran_credential) are deliberately absent
|
|
65
|
+
return (
|
|
66
|
+
"base_url",
|
|
67
|
+
"database",
|
|
68
|
+
"schema",
|
|
69
|
+
"method",
|
|
70
|
+
"warehouse",
|
|
71
|
+
"dialect",
|
|
72
|
+
"timeout_seconds",
|
|
73
|
+
"okta_auth_url",
|
|
74
|
+
"okta_token_url",
|
|
75
|
+
"okta_client_id",
|
|
76
|
+
"fivetran_auth_url",
|
|
77
|
+
"retries",
|
|
78
|
+
"driver_path",
|
|
79
|
+
"query_comment_enabled",
|
|
80
|
+
)
|