databricks-lakeflow-integrations 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- databricks/__init__.py +7 -0
- databricks/lakeflow/__init__.py +6 -0
- databricks/lakeflow/integrations/__init__.py +15 -0
- databricks/lakeflow/integrations/_api.py +73 -0
- databricks/lakeflow/integrations/_integration.py +50 -0
- databricks/lakeflow/integrations/_runtime.py +196 -0
- databricks/lakeflow/integrations/generate.py +282 -0
- databricks_lakeflow_integrations-0.0.1.dist-info/METADATA +18 -0
- databricks_lakeflow_integrations-0.0.1.dist-info/RECORD +11 -0
- databricks_lakeflow_integrations-0.0.1.dist-info/WHEEL +4 -0
- databricks_lakeflow_integrations-0.0.1.dist-info/licenses/LICENSE.md +30 -0
databricks/__init__.py
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
# See: https://packaging.python.org/guides/packaging-namespace-packages/#pkgutil-style-namespace-packages
|
|
2
|
+
#
|
|
3
|
+
# This file must only contain the following line, or other packages in the databricks.* namespace
|
|
4
|
+
# may not be importable. The contents of this file must be byte-for-byte equivalent across all packages.
|
|
5
|
+
# If they are not, parallel package installation may lead to clobbered and invalid files.
|
|
6
|
+
# Also see https://github.com/databricks/databricks-sdk-py/issues/343.
|
|
7
|
+
__path__ = __import__("pkgutil").extend_path(__path__, __name__)
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
# See: https://packaging.python.org/guides/packaging-namespace-packages/#pkgutil-style-namespace-packages
|
|
2
|
+
#
|
|
3
|
+
# This file must only contain the following line, or other packages in the databricks.lakeflow.* namespace
|
|
4
|
+
# may not be importable. The contents of this file must be byte-for-byte equivalent across all packages.
|
|
5
|
+
# If they are not, parallel package installation may lead to clobbered and invalid files.
|
|
6
|
+
__path__ = __import__("pkgutil").extend_path(__path__, __name__)
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""Databricks Lakeflow integrations."""
|
|
2
|
+
|
|
3
|
+
from databricks.lakeflow.integrations._api import Context, Sensor, SensorResult
|
|
4
|
+
from databricks.lakeflow.integrations._integration import integration
|
|
5
|
+
from databricks.lakeflow.integrations._runtime import _entrypoint
|
|
6
|
+
|
|
7
|
+
__version__ = "0.0.1"
|
|
8
|
+
|
|
9
|
+
__all__ = [
|
|
10
|
+
"Context",
|
|
11
|
+
"Sensor",
|
|
12
|
+
"SensorResult",
|
|
13
|
+
"integration",
|
|
14
|
+
"_entrypoint",
|
|
15
|
+
]
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""Core API types for Databricks Lakeflow integrations."""
|
|
2
|
+
|
|
3
|
+
import datetime
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from typing import Literal, Protocol, runtime_checkable
|
|
6
|
+
|
|
7
|
+
__all__ = [
|
|
8
|
+
"Context",
|
|
9
|
+
"Sensor",
|
|
10
|
+
"SensorResult",
|
|
11
|
+
]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass(frozen=True, kw_only=True)
|
|
15
|
+
class Context:
|
|
16
|
+
"""Carries the context of the job task run.
|
|
17
|
+
|
|
18
|
+
It is passed to :meth:`Sensor.poll` so sensor implementations can correlate
|
|
19
|
+
their polling with the run they belong to.
|
|
20
|
+
|
|
21
|
+
Attributes:
|
|
22
|
+
main: Fully qualified name of the task entry point.
|
|
23
|
+
task_key: Key of the task within the job.
|
|
24
|
+
job_run_id: Identifier of the job run.
|
|
25
|
+
task_run_id: Identifier of the task run.
|
|
26
|
+
job_id: Identifier of the job.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
main: str
|
|
30
|
+
task_key: str
|
|
31
|
+
job_run_id: int
|
|
32
|
+
task_run_id: int
|
|
33
|
+
job_id: int
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass(frozen=True)
|
|
37
|
+
class SensorResult:
|
|
38
|
+
"""
|
|
39
|
+
SensorResult is returned by Sensor.poll method to indicate whether
|
|
40
|
+
the sensor has completed or should be deferred. When the sensor is
|
|
41
|
+
deferred, compute resources are released.
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
status: Literal["completed", "deferred"]
|
|
45
|
+
defer_for: datetime.timedelta | None = None
|
|
46
|
+
|
|
47
|
+
@classmethod
|
|
48
|
+
def completed(cls) -> "SensorResult":
|
|
49
|
+
return cls(status="completed")
|
|
50
|
+
|
|
51
|
+
@classmethod
|
|
52
|
+
def deferred(cls, duration: datetime.timedelta) -> "SensorResult":
|
|
53
|
+
return cls(status="deferred", defer_for=duration)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@runtime_checkable
|
|
57
|
+
class Sensor(Protocol):
|
|
58
|
+
"""
|
|
59
|
+
Sensor is responsible for polling a condition and returning a SensorResult
|
|
60
|
+
indicating whether the condition has been met or if the sensor should be deferred.
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
def poll(self, ctx: "Context") -> SensorResult:
|
|
64
|
+
"""Check the status of the condition being polled.
|
|
65
|
+
|
|
66
|
+
Args:
|
|
67
|
+
ctx: Context of the enclosing job task run.
|
|
68
|
+
|
|
69
|
+
Returns:
|
|
70
|
+
:meth:`SensorResult.completed` if the condition has been met, or
|
|
71
|
+
:meth:`SensorResult.deferred` if the sensor should be deferred.
|
|
72
|
+
"""
|
|
73
|
+
...
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import inspect
|
|
2
|
+
from collections.abc import Callable
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from typing import Any, Optional, TypeVar
|
|
5
|
+
|
|
6
|
+
from databricks.lakeflow.integrations._api import Sensor
|
|
7
|
+
|
|
8
|
+
F = TypeVar("F", bound=Callable[..., Any])
|
|
9
|
+
|
|
10
|
+
_INTEGRATION_METADATA_ATTR = "_lakeflow_integration_metadata"
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass(frozen=True, kw_only=True)
|
|
14
|
+
class IntegrationMetadata:
|
|
15
|
+
display_name: str
|
|
16
|
+
description: Optional[str] = None
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def integration(*, display_name: str, description: Optional[str] = None) -> Callable[[F], F]:
|
|
20
|
+
"""Mark a Sensor subclass or a function to appear as an integration.
|
|
21
|
+
|
|
22
|
+
Example::
|
|
23
|
+
|
|
24
|
+
@integration(display_name="Refresh view")
|
|
25
|
+
def refresh_mv(view: str): ...
|
|
26
|
+
|
|
27
|
+
@integration(display_name="Orders")
|
|
28
|
+
class OrdersSensor(Sensor): ...
|
|
29
|
+
"""
|
|
30
|
+
metadata = IntegrationMetadata(display_name=display_name, description=description)
|
|
31
|
+
|
|
32
|
+
def decorate(target: F) -> F:
|
|
33
|
+
is_sensor = isinstance(target, type) and issubclass(target, Sensor)
|
|
34
|
+
if not (is_sensor or inspect.isfunction(target)):
|
|
35
|
+
raise TypeError("@integration can decorate only Sensor subclasses or functions")
|
|
36
|
+
setattr(target, _INTEGRATION_METADATA_ATTR, metadata)
|
|
37
|
+
return target
|
|
38
|
+
|
|
39
|
+
return decorate
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def get_integration_metadata(target: object) -> Optional[IntegrationMetadata]:
|
|
43
|
+
"""Return the entry point's metadata, or None if it is not tagged."""
|
|
44
|
+
# __dict__ for a class, not getattr, so a subclass of a tagged Sensor does
|
|
45
|
+
# not inherit the tag.
|
|
46
|
+
if isinstance(target, type):
|
|
47
|
+
metadata = target.__dict__.get(_INTEGRATION_METADATA_ATTR)
|
|
48
|
+
else:
|
|
49
|
+
metadata = getattr(target, _INTEGRATION_METADATA_ATTR, None)
|
|
50
|
+
return metadata if isinstance(metadata, IntegrationMetadata) else None
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
"""Runtime entry point for executing Lakeflow Python operator tasks."""
|
|
2
|
+
|
|
3
|
+
import inspect
|
|
4
|
+
import importlib
|
|
5
|
+
import json
|
|
6
|
+
from enum import Enum
|
|
7
|
+
from typing import Optional, get_type_hints
|
|
8
|
+
|
|
9
|
+
from databricks.lakeflow.integrations._api import Context, Sensor, SensorResult
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"Context",
|
|
13
|
+
"Sensor",
|
|
14
|
+
"SensorResult",
|
|
15
|
+
"_entrypoint",
|
|
16
|
+
"_run_function",
|
|
17
|
+
]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _entrypoint(ctx: dict):
|
|
21
|
+
# `databricks.sdk.runtime` is only available inside the user's cluster
|
|
22
|
+
# Python environment. Import lazily so this module can be imported (and
|
|
23
|
+
# unit-tested) without the SDK present.
|
|
24
|
+
from databricks.sdk.runtime import dbutils
|
|
25
|
+
|
|
26
|
+
bindings = dbutils.widgets.getAll()
|
|
27
|
+
return _run_function(bindings=bindings, ctx=ctx)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class _Outcome(Enum):
|
|
31
|
+
COMPLETED = "COMPLETED"
|
|
32
|
+
DEFERRED = "DEFERRED"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class _PythonOperatorClientMessage:
|
|
36
|
+
MIME_TYPE = "application/vnd.databricks.pythonoperatortask+json"
|
|
37
|
+
|
|
38
|
+
def __init__(self, outcome: _Outcome, deferred_duration_millis: Optional[int]):
|
|
39
|
+
self.payload: dict = {"outcome": outcome.name}
|
|
40
|
+
if deferred_duration_millis is not None:
|
|
41
|
+
self.payload["deferred_duration_millis"] = deferred_duration_millis
|
|
42
|
+
|
|
43
|
+
def _repr_mimebundle_(self, **kwargs):
|
|
44
|
+
return {self.MIME_TYPE: json.dumps(self.payload)}, {}
|
|
45
|
+
|
|
46
|
+
def __repr__(self):
|
|
47
|
+
return f"PythonOperatorClientMessage({self.payload})"
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _run_function(bindings: dict[str, str], ctx: dict):
|
|
51
|
+
context = _parse_context(ctx)
|
|
52
|
+
main = context.main
|
|
53
|
+
|
|
54
|
+
parameters = {k: v for k, v in bindings.items() if not k.startswith("__databricks")}
|
|
55
|
+
|
|
56
|
+
# main is a string like "a.b.c.function". `rsplit` on a bare name (no dot)
|
|
57
|
+
# would raise an unfriendly unpacking error, so reject that up front.
|
|
58
|
+
if "." not in main:
|
|
59
|
+
raise ValueError("main is not a valid qualified name")
|
|
60
|
+
|
|
61
|
+
module_name, func_name = main.rsplit(".", 1)
|
|
62
|
+
|
|
63
|
+
if not module_name or not func_name:
|
|
64
|
+
raise ValueError("main is not a valid qualified name")
|
|
65
|
+
|
|
66
|
+
module = importlib.import_module(module_name)
|
|
67
|
+
obj = getattr(module, func_name, None)
|
|
68
|
+
|
|
69
|
+
if obj is None:
|
|
70
|
+
raise ValueError(f"Function {func_name} not found in module {module_name}")
|
|
71
|
+
|
|
72
|
+
if inspect.isfunction(obj):
|
|
73
|
+
_run_callable(obj, parameters)
|
|
74
|
+
|
|
75
|
+
return _PythonOperatorClientMessage(
|
|
76
|
+
outcome=_Outcome.COMPLETED,
|
|
77
|
+
deferred_duration_millis=None,
|
|
78
|
+
)
|
|
79
|
+
elif _is_sensor_class(obj):
|
|
80
|
+
instance = _run_callable(obj, parameters)
|
|
81
|
+
|
|
82
|
+
if not isinstance(instance, Sensor):
|
|
83
|
+
raise ValueError(f"{func_name} is not a Sensor")
|
|
84
|
+
|
|
85
|
+
result = instance.poll(context)
|
|
86
|
+
|
|
87
|
+
return _client_message_from_result(result)
|
|
88
|
+
else:
|
|
89
|
+
raise ValueError(f"{func_name} is not a function or Sensor")
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _parse_context(ctx: dict) -> Context:
|
|
93
|
+
return Context(
|
|
94
|
+
main=ctx["main"],
|
|
95
|
+
task_key=ctx["task_key"],
|
|
96
|
+
job_run_id=ctx["job_run_id"],
|
|
97
|
+
task_run_id=ctx["task_run_id"],
|
|
98
|
+
job_id=ctx["job_id"],
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _client_message_from_result(result: SensorResult) -> _PythonOperatorClientMessage:
|
|
103
|
+
if result.status == "completed":
|
|
104
|
+
return _PythonOperatorClientMessage(
|
|
105
|
+
outcome=_Outcome.COMPLETED,
|
|
106
|
+
deferred_duration_millis=None,
|
|
107
|
+
)
|
|
108
|
+
else:
|
|
109
|
+
deferred_duration_millis = None
|
|
110
|
+
if result.defer_for:
|
|
111
|
+
deferred_duration_millis = int(result.defer_for.total_seconds() * 1000)
|
|
112
|
+
|
|
113
|
+
return _PythonOperatorClientMessage(
|
|
114
|
+
outcome=_Outcome.DEFERRED,
|
|
115
|
+
deferred_duration_millis=deferred_duration_millis,
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _is_sensor_class(obj: object) -> bool:
|
|
120
|
+
return inspect.isclass(obj) and issubclass(obj, Sensor)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _run_callable(obj, parameters: dict[str, str]):
|
|
124
|
+
signature = inspect.signature(obj)
|
|
125
|
+
converted_kwargs: dict[str, object] = {}
|
|
126
|
+
|
|
127
|
+
# `inspect.signature(...).parameters[i].annotation` returns the raw
|
|
128
|
+
# `__annotations__` entry, which is a literal string under PEP 563
|
|
129
|
+
# (`from __future__ import annotations`) or PEP 484 explicit forward
|
|
130
|
+
# references. `typing.get_type_hints` evaluates those strings against
|
|
131
|
+
# the callable's module globals to produce the actual type objects.
|
|
132
|
+
# For classes, annotations live on `__init__`.
|
|
133
|
+
hints_target = obj.__init__ if inspect.isclass(obj) else obj
|
|
134
|
+
type_hints = get_type_hints(hints_target)
|
|
135
|
+
|
|
136
|
+
has_kwargs = any(param.kind == inspect.Parameter.VAR_KEYWORD for param in signature.parameters.values())
|
|
137
|
+
if has_kwargs:
|
|
138
|
+
# capture all parameters as kwargs if signature has kwargs
|
|
139
|
+
converted_kwargs = {**parameters}
|
|
140
|
+
|
|
141
|
+
# inspect func signature and validate parameters
|
|
142
|
+
for param in signature.parameters.values():
|
|
143
|
+
if param.kind in [
|
|
144
|
+
inspect.Parameter.VAR_POSITIONAL, # non-deterministic, don't pass anything
|
|
145
|
+
inspect.Parameter.VAR_KEYWORD, # already captured as kwargs
|
|
146
|
+
inspect.Parameter.POSITIONAL_ONLY, # this will result in error when we call the function
|
|
147
|
+
]:
|
|
148
|
+
continue
|
|
149
|
+
|
|
150
|
+
if param.kind in [
|
|
151
|
+
inspect.Parameter.POSITIONAL_OR_KEYWORD,
|
|
152
|
+
inspect.Parameter.KEYWORD_ONLY,
|
|
153
|
+
]:
|
|
154
|
+
if param.name in parameters:
|
|
155
|
+
annotation = type_hints.get(param.name, param.annotation)
|
|
156
|
+
converted_kwargs[param.name] = _convert_parameter(param.name, annotation, parameters[param.name])
|
|
157
|
+
|
|
158
|
+
return obj(**converted_kwargs)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _convert_parameter(name: str, annotation, value: str):
|
|
162
|
+
# No annotation: pass the raw widget string through unchanged. Otherwise a
|
|
163
|
+
# plain `def fn(x, **kwargs)` would error before the user code ever runs.
|
|
164
|
+
if annotation is inspect.Parameter.empty or annotation is str:
|
|
165
|
+
return value
|
|
166
|
+
if annotation is int:
|
|
167
|
+
try:
|
|
168
|
+
return int(value)
|
|
169
|
+
except ValueError as e:
|
|
170
|
+
raise ValueError(f"Parameter {name}: invalid int value '{value}'") from e
|
|
171
|
+
if annotation is float:
|
|
172
|
+
try:
|
|
173
|
+
return float(value)
|
|
174
|
+
except ValueError as e:
|
|
175
|
+
raise ValueError(f"Parameter {name}: invalid float value '{value}'") from e
|
|
176
|
+
if annotation is bool:
|
|
177
|
+
if value not in ["true", "false"]:
|
|
178
|
+
raise ValueError(f"Parameter {name}: invalid boolean value '{value}'")
|
|
179
|
+
return value == "true"
|
|
180
|
+
if annotation is list:
|
|
181
|
+
result = json.loads(value)
|
|
182
|
+
if not isinstance(result, list):
|
|
183
|
+
raise ValueError(f"Parameter {name}: invalid list value '{value}'")
|
|
184
|
+
return result
|
|
185
|
+
if annotation is dict:
|
|
186
|
+
result = json.loads(value)
|
|
187
|
+
if not isinstance(result, dict):
|
|
188
|
+
raise ValueError(f"Parameter {name}: invalid dict value '{value}'")
|
|
189
|
+
return result
|
|
190
|
+
|
|
191
|
+
# TODO: support more types, including generics
|
|
192
|
+
|
|
193
|
+
raise ValueError(
|
|
194
|
+
f"Parameter {name}: unsupported type '{annotation}',"
|
|
195
|
+
f"only str, int, float, bool, list, and dict are supported."
|
|
196
|
+
)
|
|
@@ -0,0 +1,282 @@
|
|
|
1
|
+
"""Generate Lakeflow integration definitions from Python Operator entry points."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import importlib
|
|
5
|
+
import inspect
|
|
6
|
+
import json
|
|
7
|
+
import pkgutil
|
|
8
|
+
from collections.abc import Callable
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from types import ModuleType
|
|
11
|
+
from typing import Optional, Union, get_args, get_origin, get_type_hints
|
|
12
|
+
|
|
13
|
+
from databricks.lakeflow.integrations._integration import get_integration_metadata
|
|
14
|
+
|
|
15
|
+
_SCHEMA_VERSION = "lakeflow-integration-v0.1.0"
|
|
16
|
+
|
|
17
|
+
_EntryPoint = Union[type, Callable]
|
|
18
|
+
|
|
19
|
+
# Parameter kinds that cannot be surfaced as named config fields: *args and
|
|
20
|
+
# **kwargs have no stable name to bind a widget to, and positional-only params
|
|
21
|
+
# cannot be passed by keyword (which is how the operator injects values).
|
|
22
|
+
_UNSUPPORTED_PARAMETER_KINDS = frozenset(
|
|
23
|
+
{
|
|
24
|
+
inspect.Parameter.VAR_POSITIONAL,
|
|
25
|
+
inspect.Parameter.VAR_KEYWORD,
|
|
26
|
+
inspect.Parameter.POSITIONAL_ONLY,
|
|
27
|
+
}
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _generate(package_module_name: str, environment: Optional[dict] = None) -> list[dict]:
|
|
32
|
+
"""Return an integration definition for every entry point in the package.
|
|
33
|
+
|
|
34
|
+
Definitions are ordered by their ``main`` entry point for deterministic
|
|
35
|
+
output. ``environment`` is copied into every definition (omitted when None).
|
|
36
|
+
"""
|
|
37
|
+
return [_build_definition(obj, environment) for obj in _discover_entry_points(package_module_name)]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _discover_entry_points(package_module_name: str) -> list[_EntryPoint]:
|
|
41
|
+
"""Import the package recursively and collect its ``@integration`` entry points.
|
|
42
|
+
|
|
43
|
+
Results are deduplicated by entry point name and sorted by it.
|
|
44
|
+
"""
|
|
45
|
+
package = importlib.import_module(package_module_name)
|
|
46
|
+
|
|
47
|
+
entry_points: dict[str, _EntryPoint] = {}
|
|
48
|
+
for module in _load_modules(package):
|
|
49
|
+
for _name, member in inspect.getmembers(module):
|
|
50
|
+
if _is_entry_point(member, module.__name__):
|
|
51
|
+
entry_points[f"{member.__module__}.{member.__qualname__}"] = member
|
|
52
|
+
|
|
53
|
+
return [entry_points[main] for main in sorted(entry_points)]
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _load_modules(package_module: ModuleType) -> list[ModuleType]:
|
|
57
|
+
"""Import ``package_module`` and all of its submodules recursively.
|
|
58
|
+
|
|
59
|
+
Simplified from the reference loader: import errors propagate rather than
|
|
60
|
+
being collected, since this is a build-time tool where failing loudly is
|
|
61
|
+
preferable to silently emitting a partial definition set.
|
|
62
|
+
"""
|
|
63
|
+
spec = package_module.__spec__
|
|
64
|
+
if spec is None:
|
|
65
|
+
raise ValueError(f"Module '{package_module.__name__}' has no __spec__")
|
|
66
|
+
|
|
67
|
+
module_names = []
|
|
68
|
+
# A package with an __init__.py has an origin; a namespace package does not.
|
|
69
|
+
if spec.origin:
|
|
70
|
+
module_names.append(spec.name)
|
|
71
|
+
|
|
72
|
+
if spec.submodule_search_locations:
|
|
73
|
+
prefix = spec.name + "."
|
|
74
|
+
for _finder, name, _is_pkg in pkgutil.walk_packages(spec.submodule_search_locations, prefix):
|
|
75
|
+
module_names.append(name)
|
|
76
|
+
|
|
77
|
+
# Sort for a deterministic import order.
|
|
78
|
+
return [importlib.import_module(name) for name in sorted(set(module_names))]
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _is_entry_point(obj: object, module_name: str) -> bool:
|
|
82
|
+
# Restrict to entry points physically defined in this module so one
|
|
83
|
+
# imported into another module is not reported twice.
|
|
84
|
+
if getattr(obj, "__module__", None) != module_name:
|
|
85
|
+
return False
|
|
86
|
+
return get_integration_metadata(obj) is not None
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _build_definition(obj: _EntryPoint, environment: Optional[dict] = None) -> dict:
|
|
90
|
+
"""Build a single ``lakeflow-integration-v0.1.0`` definition for ``obj``.
|
|
91
|
+
|
|
92
|
+
``environment`` is copied in verbatim when provided, otherwise omitted.
|
|
93
|
+
"""
|
|
94
|
+
metadata = get_integration_metadata(obj)
|
|
95
|
+
assert metadata is not None # discovery only yields tagged objects
|
|
96
|
+
|
|
97
|
+
definition = {
|
|
98
|
+
"schema": _SCHEMA_VERSION,
|
|
99
|
+
"name": metadata.display_name,
|
|
100
|
+
"description": metadata.description or _docstring(obj) or metadata.display_name,
|
|
101
|
+
"main": f"{obj.__module__}.{obj.__qualname__}",
|
|
102
|
+
"config": _build_config(obj),
|
|
103
|
+
}
|
|
104
|
+
if environment is not None:
|
|
105
|
+
definition["environment"] = environment
|
|
106
|
+
return definition
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _build_environment(
|
|
110
|
+
dependencies: list[str],
|
|
111
|
+
environment_version: Optional[str],
|
|
112
|
+
environment_key: Optional[str],
|
|
113
|
+
) -> Optional[dict]:
|
|
114
|
+
"""Build the shared ``environment`` block from CLI inputs.
|
|
115
|
+
|
|
116
|
+
Key order matches the schema: ``environment_key``, ``environment_version``,
|
|
117
|
+
``dependencies``. Returns None when no environment inputs were supplied, so
|
|
118
|
+
the block is omitted rather than emitted empty.
|
|
119
|
+
"""
|
|
120
|
+
if not dependencies and environment_version is None and environment_key is None:
|
|
121
|
+
return None
|
|
122
|
+
|
|
123
|
+
environment: dict = {}
|
|
124
|
+
if environment_key is not None:
|
|
125
|
+
environment["environment_key"] = environment_key
|
|
126
|
+
if environment_version is not None:
|
|
127
|
+
environment["environment_version"] = environment_version
|
|
128
|
+
environment["dependencies"] = dependencies
|
|
129
|
+
return environment
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _build_config(obj: _EntryPoint) -> dict:
|
|
133
|
+
"""Build the ``config`` JSON Schema from the entry point's signature.
|
|
134
|
+
|
|
135
|
+
Parameters without a default are listed under ``required``; scalar defaults
|
|
136
|
+
are carried through.
|
|
137
|
+
"""
|
|
138
|
+
properties: dict[str, dict] = {}
|
|
139
|
+
required: list[str] = []
|
|
140
|
+
|
|
141
|
+
annotations = _annotations(obj)
|
|
142
|
+
for param in _config_parameters(obj):
|
|
143
|
+
annotation = annotations.get(param.name, param.annotation)
|
|
144
|
+
prop = _config_property(annotation)
|
|
145
|
+
|
|
146
|
+
if param.default is inspect.Parameter.empty:
|
|
147
|
+
required.append(param.name)
|
|
148
|
+
elif _is_scalar(param.default):
|
|
149
|
+
prop["default"] = param.default
|
|
150
|
+
|
|
151
|
+
properties[param.name] = prop
|
|
152
|
+
|
|
153
|
+
config: dict = {"type": "object", "properties": properties}
|
|
154
|
+
if required:
|
|
155
|
+
config["required"] = required
|
|
156
|
+
return config
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _config_parameters(obj: _EntryPoint) -> list[inspect.Parameter]:
|
|
160
|
+
try:
|
|
161
|
+
signature = inspect.signature(obj)
|
|
162
|
+
except (ValueError, TypeError):
|
|
163
|
+
# Builtins and some C-level types have no introspectable signature.
|
|
164
|
+
return []
|
|
165
|
+
|
|
166
|
+
return [param for param in signature.parameters.values() if param.kind not in _UNSUPPORTED_PARAMETER_KINDS]
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def _annotations(obj: _EntryPoint) -> dict:
|
|
170
|
+
# get_type_hints resolves deferred (PEP 563) / forward-reference annotations
|
|
171
|
+
# against the module globals. For a class they live on __init__.
|
|
172
|
+
target = obj.__init__ if inspect.isclass(obj) else obj
|
|
173
|
+
try:
|
|
174
|
+
return get_type_hints(target)
|
|
175
|
+
except (NameError, TypeError):
|
|
176
|
+
# Unresolvable forward reference: fall back to raw annotations rather
|
|
177
|
+
# than aborting the run.
|
|
178
|
+
return {}
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def _config_property(annotation: object) -> dict:
|
|
182
|
+
# int gets the number widget; every other type is emitted with only its JSON
|
|
183
|
+
# Schema type, letting the UI pick its default control.
|
|
184
|
+
annotation = _unwrap_optional(annotation)
|
|
185
|
+
|
|
186
|
+
if annotation is int:
|
|
187
|
+
return {"type": "integer", "x-ui": {"widget": "number"}}
|
|
188
|
+
if annotation is float:
|
|
189
|
+
return {"type": "number"}
|
|
190
|
+
if annotation is bool:
|
|
191
|
+
return {"type": "boolean"}
|
|
192
|
+
return {"type": "string"}
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _unwrap_optional(annotation: object) -> object:
|
|
196
|
+
# Optional[T] is Union[T, None]; unwrap the single non-None arm.
|
|
197
|
+
if get_origin(annotation) is Union:
|
|
198
|
+
args = [arg for arg in get_args(annotation) if arg is not type(None)]
|
|
199
|
+
if len(args) == 1:
|
|
200
|
+
return args[0]
|
|
201
|
+
return annotation
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _docstring(obj: _EntryPoint) -> Optional[str]:
|
|
205
|
+
# __doc__ directly, not inspect.getdoc, which would inherit the Sensor
|
|
206
|
+
# protocol's docstring for an undocumented sensor.
|
|
207
|
+
doc = obj.__doc__
|
|
208
|
+
if doc:
|
|
209
|
+
for line in inspect.cleandoc(doc).splitlines():
|
|
210
|
+
stripped = line.strip()
|
|
211
|
+
if stripped:
|
|
212
|
+
return stripped
|
|
213
|
+
return None
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _is_scalar(value: object) -> bool:
|
|
217
|
+
# bool is a subclass of int; all four are JSON scalars the schema allows.
|
|
218
|
+
return isinstance(value, (str, int, float, bool))
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _write_definitions(
|
|
222
|
+
package_module_name: str,
|
|
223
|
+
output_dir: Path,
|
|
224
|
+
environment: Optional[dict] = None,
|
|
225
|
+
) -> list[Path]:
|
|
226
|
+
"""Write one ``<main>.yml`` file per entry point into ``output_dir``.
|
|
227
|
+
|
|
228
|
+
``main`` is the entry point's unique dotted path, so it is a collision-free
|
|
229
|
+
file name. The body is formatted JSON, a valid YAML subset. Returns the
|
|
230
|
+
written paths, sorted.
|
|
231
|
+
"""
|
|
232
|
+
output_dir.mkdir(parents=True, exist_ok=True)
|
|
233
|
+
|
|
234
|
+
written = []
|
|
235
|
+
for definition in _generate(package_module_name, environment):
|
|
236
|
+
path = output_dir / f"{definition['main']}.yml"
|
|
237
|
+
path.write_text(json.dumps(definition, indent=2) + "\n")
|
|
238
|
+
written.append(path)
|
|
239
|
+
return written
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def main(argv: list[str] | None = None) -> None:
|
|
243
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
244
|
+
parser.add_argument(
|
|
245
|
+
"--package-module",
|
|
246
|
+
required=True,
|
|
247
|
+
help="Dotted name of the package to load recursively, e.g. 'my_pkg'.",
|
|
248
|
+
)
|
|
249
|
+
parser.add_argument(
|
|
250
|
+
"--output-dir",
|
|
251
|
+
required=True,
|
|
252
|
+
help="Directory to write the '<main>.yml' files into.",
|
|
253
|
+
)
|
|
254
|
+
parser.add_argument(
|
|
255
|
+
"--dependency",
|
|
256
|
+
action="append",
|
|
257
|
+
default=[],
|
|
258
|
+
dest="dependencies",
|
|
259
|
+
metavar="SPEC",
|
|
260
|
+
help="pip-style dependency or workspace .whl path for the environment "
|
|
261
|
+
"block. Repeatable; must include the wheel that provides the entry points.",
|
|
262
|
+
)
|
|
263
|
+
parser.add_argument(
|
|
264
|
+
"--environment-version",
|
|
265
|
+
help="environment_version for the environment block, e.g. '5'.",
|
|
266
|
+
)
|
|
267
|
+
parser.add_argument(
|
|
268
|
+
"--environment-key",
|
|
269
|
+
help="environment_key for the environment block.",
|
|
270
|
+
)
|
|
271
|
+
args = parser.parse_args(argv)
|
|
272
|
+
|
|
273
|
+
environment = _build_environment(
|
|
274
|
+
dependencies=args.dependencies,
|
|
275
|
+
environment_version=args.environment_version,
|
|
276
|
+
environment_key=args.environment_key,
|
|
277
|
+
)
|
|
278
|
+
_write_definitions(args.package_module, Path(args.output_dir), environment)
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
if __name__ == "__main__":
|
|
282
|
+
main()
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: databricks-lakeflow-integrations
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: Extend Lakeflow with custom integrations using Python.
|
|
5
|
+
Author: Gleb Kanterov
|
|
6
|
+
Author-email: Gleb Kanterov <gleb.kanterov@databricks.com>
|
|
7
|
+
License-Expression: LicenseRef-Databricks
|
|
8
|
+
License-File: LICENSE.md
|
|
9
|
+
Requires-Python: >=3.12
|
|
10
|
+
Description-Content-Type: text/markdown
|
|
11
|
+
|
|
12
|
+
# databricks-lakeflow-integrations
|
|
13
|
+
|
|
14
|
+
Extend Lakeflow with custom integrations using Python.
|
|
15
|
+
|
|
16
|
+
## Usage
|
|
17
|
+
|
|
18
|
+
Follow instructions received by email from the Databricks representative.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
databricks/__init__.py,sha256=CF2MJcZFwbpn9TwQER8qnCDhkPooBGQNVkX4v7g6p3g,537
|
|
2
|
+
databricks/lakeflow/__init__.py,sha256=iZWKnlKLkb1JhqJHYHyEK2BUo2AbMRoc-jEA_aisfVI,475
|
|
3
|
+
databricks/lakeflow/integrations/__init__.py,sha256=fFqHzRYix2wGKLuDIeO5nDMhYcagXsdW5SqkPXlJLVE,382
|
|
4
|
+
databricks/lakeflow/integrations/_api.py,sha256=FkKwrn0VdUlvOBN19TqFpE890JLeU2kEC6_fLuLGuS8,2016
|
|
5
|
+
databricks/lakeflow/integrations/_integration.py,sha256=7T4yXlQhCZ42CxZIjnXVd7APM_XGwdk6lrq4lF0jzIo,1756
|
|
6
|
+
databricks/lakeflow/integrations/_runtime.py,sha256=iab-GQz_pZHwf2zQ6L0B3YEupFLfNDihLTdDYHMJ-m0,6900
|
|
7
|
+
databricks/lakeflow/integrations/generate.py,sha256=79uh6ibOs6yeB6N9uibiPJLHZDvUE-1EJSV_dpImWsk,9978
|
|
8
|
+
databricks_lakeflow_integrations-0.0.1.dist-info/licenses/LICENSE.md,sha256=TTZEJNJMEwwTNZdZXD4V3iPuUjU10jBJwoSzRCc4J34,3442
|
|
9
|
+
databricks_lakeflow_integrations-0.0.1.dist-info/WHEEL,sha256=CoDSoyhtC_eO_tlxRYzsTraPv1fPJRXFx91k6ISeAvA,81
|
|
10
|
+
databricks_lakeflow_integrations-0.0.1.dist-info/METADATA,sha256=E8a49UNg4disllmUS_ONyDWM6CMEFoFFxKH8ILLvVO0,529
|
|
11
|
+
databricks_lakeflow_integrations-0.0.1.dist-info/RECORD,,
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# Databricks License
|
|
2
|
+
|
|
3
|
+
## Definitions
|
|
4
|
+
|
|
5
|
+
**Agreement:** The agreement between Databricks, Inc., and you governing the use of the Databricks Services, as that term is defined in the Master Cloud Services Agreement (MCSA) located at [www.databricks.com/legal/mcsa](https://www.databricks.com/legal/mcsa).
|
|
6
|
+
|
|
7
|
+
**Licensed Materials:** The source code, object code, data, and/or other works to which this license applies.
|
|
8
|
+
|
|
9
|
+
## Scope of Use
|
|
10
|
+
|
|
11
|
+
You may not use the Licensed Materials except in connection with your use of the Databricks Services pursuant to the Agreement. Your use of the Licensed Materials must comply at all times with any restrictions applicable to the Databricks Services, generally, and must be used in accordance with any applicable documentation. You may view, use, copy, modify, publish, and/or distribute the Licensed Materials solely for the purposes of using the Licensed Materials within or connecting to the Databricks Services. If you do not agree to these terms, you may not view, use, copy, modify, publish, and/or distribute the Licensed Materials.
|
|
12
|
+
|
|
13
|
+
## Redistribution
|
|
14
|
+
|
|
15
|
+
You may redistribute and sublicense the Licensed Materials so long as all use is in compliance with these terms. In addition:
|
|
16
|
+
|
|
17
|
+
- You must give any other recipients a copy of this License;
|
|
18
|
+
- You must cause any modified files to carry prominent notices stating that you changed the files;
|
|
19
|
+
- You must retain, in any derivative works that you distribute, all copyright, patent, trademark, and attribution notices, excluding those notices that do not pertain to any part of the derivative works; and
|
|
20
|
+
- If a "NOTICE" text file is provided as part of its distribution, then any derivative works that you distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the derivative works.
|
|
21
|
+
|
|
22
|
+
You may add your own copyright statement to your modifications and may provide additional license terms and conditions for use, reproduction, or distribution of your modifications, or for any such derivative works as a whole, provided your use, reproduction, and distribution of the Licensed Materials otherwise complies with the conditions stated in this License.
|
|
23
|
+
|
|
24
|
+
## Termination
|
|
25
|
+
|
|
26
|
+
This license terminates automatically upon your breach of these terms or upon the termination of your Agreement. Additionally, Databricks may terminate this license at any time on notice. Upon termination, you must permanently delete the Licensed Materials and all copies thereof.
|
|
27
|
+
|
|
28
|
+
## Disclaimer; Limitation of Liability
|
|
29
|
+
|
|
30
|
+
THE LICENSED MATERIALS ARE PROVIDED "AS-IS" AND WITH ALL FAULTS. DATABRICKS, ON BEHALF OF ITSELF AND ITS LICENSORS, SPECIFICALLY DISCLAIMS ALL WARRANTIES RELATING TO THE LICENSED MATERIALS, EXPRESS AND IMPLIED, INCLUDING, WITHOUT LIMITATION, IMPLIED WARRANTIES, CONDITIONS AND OTHER TERMS OF MERCHANTABILITY, SATISFACTORY QUALITY OR FITNESS FOR A PARTICULAR PURPOSE, AND NON-INFRINGEMENT. DATABRICKS AND ITS LICENSORS TOTAL AGGREGATE LIABILITY RELATING TO OR ARISING OUT OF YOUR USE OF OR DATABRICKS' PROVISIONING OF THE LICENSED MATERIALS SHALL BE LIMITED TO ONE THOUSAND ($1,000) DOLLARS. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE LICENSED MATERIALS OR THE USE OR OTHER DEALINGS IN THE LICENSED MATERIALS.
|