databricks-lakeflow-integrations 0.0.2__tar.gz → 0.0.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {databricks_lakeflow_integrations-0.0.2 → databricks_lakeflow_integrations-0.0.4}/PKG-INFO +1 -1
- {databricks_lakeflow_integrations-0.0.2 → databricks_lakeflow_integrations-0.0.4}/pyproject.toml +1 -1
- {databricks_lakeflow_integrations-0.0.2 → databricks_lakeflow_integrations-0.0.4}/pyproject.toml.orig +1 -1
- {databricks_lakeflow_integrations-0.0.2 → databricks_lakeflow_integrations-0.0.4}/src/databricks/lakeflow/integrations/__init__.py +6 -1
- {databricks_lakeflow_integrations-0.0.2 → databricks_lakeflow_integrations-0.0.4}/src/databricks/lakeflow/integrations/generator/generate.py +20 -8
- {databricks_lakeflow_integrations-0.0.2 → databricks_lakeflow_integrations-0.0.4}/src/databricks/lakeflow/integrations/runtime/__init__.py +8 -3
- {databricks_lakeflow_integrations-0.0.2 → databricks_lakeflow_integrations-0.0.4}/src/databricks/lakeflow/integrations/shared/_parameters.py +4 -0
- {databricks_lakeflow_integrations-0.0.2 → databricks_lakeflow_integrations-0.0.4}/LICENSE.md +0 -0
- {databricks_lakeflow_integrations-0.0.2 → databricks_lakeflow_integrations-0.0.4}/README.md +0 -0
- {databricks_lakeflow_integrations-0.0.2 → databricks_lakeflow_integrations-0.0.4}/src/databricks/__init__.py +0 -0
- {databricks_lakeflow_integrations-0.0.2 → databricks_lakeflow_integrations-0.0.4}/src/databricks/lakeflow/__init__.py +0 -0
- {databricks_lakeflow_integrations-0.0.2 → databricks_lakeflow_integrations-0.0.4}/src/databricks/lakeflow/integrations/generator/__init__.py +0 -0
- {databricks_lakeflow_integrations-0.0.2 → databricks_lakeflow_integrations-0.0.4}/src/databricks/lakeflow/integrations/shared/__init__.py +0 -0
- {databricks_lakeflow_integrations-0.0.2 → databricks_lakeflow_integrations-0.0.4}/src/databricks/lakeflow/integrations/shared/_api.py +0 -0
- {databricks_lakeflow_integrations-0.0.2 → databricks_lakeflow_integrations-0.0.4}/src/databricks/lakeflow/integrations/shared/_integration.py +0 -0
- {databricks_lakeflow_integrations-0.0.2 → databricks_lakeflow_integrations-0.0.4}/src/databricks/lakeflow/integrations/shared/_paramtypes.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: databricks-lakeflow-integrations
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.4
|
|
4
4
|
Summary: Extend Lakeflow with custom integrations using Python.
|
|
5
5
|
Author: Gleb Kanterov, Cristian Jensen
|
|
6
6
|
Author-email: Gleb Kanterov <gleb.kanterov@databricks.com>, Cristian Jensen <cristian.jensen@databricks.com>
|
|
@@ -4,7 +4,7 @@ build-backend = "uv_build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "databricks-lakeflow-integrations"
|
|
7
|
-
version = "0.0.
|
|
7
|
+
version = "0.0.4"
|
|
8
8
|
description = "Extend Lakeflow with custom integrations using Python."
|
|
9
9
|
authors = [
|
|
10
10
|
{ name = "Gleb Kanterov", email = "gleb.kanterov@databricks.com" },
|
|
@@ -3,7 +3,11 @@ from importlib.metadata import PackageNotFoundError, version
|
|
|
3
3
|
|
|
4
4
|
from databricks.lakeflow.integrations.shared._api import Context, Operator, PollResult, Sensor, SensorResult
|
|
5
5
|
from databricks.lakeflow.integrations.shared._integration import IntegrationMetadata, integration
|
|
6
|
-
from databricks.lakeflow.integrations.shared._parameters import
|
|
6
|
+
from databricks.lakeflow.integrations.shared._parameters import (
|
|
7
|
+
HttpConnectionFilter,
|
|
8
|
+
HttpUcConnectionParameter,
|
|
9
|
+
TextAreaParameter,
|
|
10
|
+
)
|
|
7
11
|
|
|
8
12
|
try:
|
|
9
13
|
__version__ = version("databricks-lakeflow-integrations")
|
|
@@ -22,6 +26,7 @@ __all__ = [
|
|
|
22
26
|
"IntegrationMetadata",
|
|
23
27
|
"HttpConnectionFilter",
|
|
24
28
|
"HttpUcConnectionParameter",
|
|
29
|
+
"TextAreaParameter",
|
|
25
30
|
]
|
|
26
31
|
|
|
27
32
|
|
|
@@ -13,7 +13,11 @@ from types import ModuleType
|
|
|
13
13
|
from typing import Annotated, Optional, Union, get_args, get_origin, get_type_hints
|
|
14
14
|
|
|
15
15
|
from databricks.lakeflow.integrations.shared._integration import get_integration_metadata
|
|
16
|
-
from databricks.lakeflow.integrations.shared._parameters import
|
|
16
|
+
from databricks.lakeflow.integrations.shared._parameters import (
|
|
17
|
+
HttpConnectionFilter,
|
|
18
|
+
HttpUcConnectionParameter,
|
|
19
|
+
TextAreaParameter,
|
|
20
|
+
)
|
|
17
21
|
from databricks.lakeflow.integrations.shared._paramtypes import _SUPPORTED_PARAM_TYPES, _unwrap_optional
|
|
18
22
|
|
|
19
23
|
_SCHEMA_VERSION = "lakeflow-integration-v0.1.0"
|
|
@@ -262,9 +266,11 @@ def _config_property(name: str, annotation: object) -> dict:
|
|
|
262
266
|
return _build_http_connection_property(connection_filters)
|
|
263
267
|
if len(connection_filters) > 0:
|
|
264
268
|
raise ValueError(f"Parameter {name}: HttpConnectionFilter requires HttpUcConnectionParameter")
|
|
269
|
+
if isinstance(annotation, type) and issubclass(annotation, TextAreaParameter):
|
|
270
|
+
return {"type": "string", "x-ui": {"widget": "textarea"}}
|
|
265
271
|
|
|
266
272
|
if isinstance(annotation, type) and issubclass(annotation, Enum):
|
|
267
|
-
return {"type": "string"}
|
|
273
|
+
return {"type": "string", "x-ui": {"widget": "select", "optionsSource": _enum_options_source(annotation)}}
|
|
268
274
|
|
|
269
275
|
schema_type = _SUPPORTED_PARAM_TYPES.get(annotation) if isinstance(annotation, type) else None
|
|
270
276
|
if schema_type is None:
|
|
@@ -272,6 +278,8 @@ def _config_property(name: str, annotation: object) -> dict:
|
|
|
272
278
|
f"Parameter {name}: unsupported type '{annotation}', "
|
|
273
279
|
f"only str, int, float, bool, list, dict, and enums are supported."
|
|
274
280
|
)
|
|
281
|
+
if annotation is bool:
|
|
282
|
+
return {"type": schema_type, "x-ui": {"widget": "toggle"}}
|
|
275
283
|
return {"type": schema_type}
|
|
276
284
|
|
|
277
285
|
|
|
@@ -285,6 +293,10 @@ def _build_http_connection_property(connection_filters: list[HttpConnectionFilte
|
|
|
285
293
|
return {"type": "string", "x-ui": ui}
|
|
286
294
|
|
|
287
295
|
|
|
296
|
+
def _enum_options_source(enum_type: type[Enum]) -> dict:
|
|
297
|
+
return {"type": "static", "values": [member.name for member in enum_type]}
|
|
298
|
+
|
|
299
|
+
|
|
288
300
|
def _resolve_parameter_annotation(annotation: object) -> tuple[object, list[HttpConnectionFilter]]:
|
|
289
301
|
"""Unwrap Annotated/Optional layers to the underlying type, collecting any HttpConnectionFilter metadata."""
|
|
290
302
|
if get_origin(annotation) is Annotated:
|
|
@@ -322,13 +334,13 @@ def _write_definitions(
|
|
|
322
334
|
output_dir: Path,
|
|
323
335
|
environment: dict,
|
|
324
336
|
) -> list[Path]:
|
|
325
|
-
"""Write one ``<main>.
|
|
337
|
+
"""Write one ``<main>.yml`` file per entry point into ``output_dir``.
|
|
326
338
|
|
|
327
339
|
``main`` is the entry point's unique dotted path, so it is a collision-free
|
|
328
340
|
file name. The body is formatted JSON, a valid YAML subset. Returns the
|
|
329
341
|
written paths, sorted.
|
|
330
342
|
|
|
331
|
-
``output_dir`` is treated as owned by this tool: any ``*.
|
|
343
|
+
``output_dir`` is treated as owned by this tool: any ``*.yml`` left from a
|
|
332
344
|
prior run whose entry point no longer exists is removed, so a deleted or
|
|
333
345
|
renamed integration cannot linger and be picked up as live.
|
|
334
346
|
"""
|
|
@@ -336,7 +348,7 @@ def _write_definitions(
|
|
|
336
348
|
|
|
337
349
|
written = []
|
|
338
350
|
for definition in _generate(package_module_name, environment):
|
|
339
|
-
path = output_dir / f"{definition['main']}.
|
|
351
|
+
path = output_dir / f"{definition['main']}.yml"
|
|
340
352
|
path.write_text(json.dumps(definition, indent=2) + "\n")
|
|
341
353
|
written.append(path)
|
|
342
354
|
|
|
@@ -348,10 +360,10 @@ def _write_definitions(
|
|
|
348
360
|
|
|
349
361
|
|
|
350
362
|
def _prune_stale_definitions(output_dir: Path, keep: list[Path]) -> None:
|
|
351
|
-
# Remove any *.
|
|
363
|
+
# Remove any *.yml this run did not write, so a deleted/renamed entry point leaves
|
|
352
364
|
# no stale definition. Top-level only (glob, not rglob), matching what we emit.
|
|
353
365
|
current = set(keep)
|
|
354
|
-
for existing in output_dir.glob("*.
|
|
366
|
+
for existing in output_dir.glob("*.yml"):
|
|
355
367
|
if existing not in current:
|
|
356
368
|
existing.unlink()
|
|
357
369
|
|
|
@@ -366,7 +378,7 @@ def main(argv: list[str] | None = None) -> None:
|
|
|
366
378
|
parser.add_argument(
|
|
367
379
|
"--output-dir",
|
|
368
380
|
required=True,
|
|
369
|
-
help="Directory to write one '<main>.
|
|
381
|
+
help="Directory to write one '<main>.yml' file per sensor into.",
|
|
370
382
|
)
|
|
371
383
|
parser.add_argument(
|
|
372
384
|
"--dependency",
|
|
@@ -9,7 +9,7 @@ from types import TracebackType
|
|
|
9
9
|
from typing import Optional, get_type_hints
|
|
10
10
|
|
|
11
11
|
from databricks.lakeflow.integrations.shared._api import Context, Operator, PollResult, Sensor, SensorResult
|
|
12
|
-
from databricks.lakeflow.integrations.shared._parameters import HttpUcConnectionParameter
|
|
12
|
+
from databricks.lakeflow.integrations.shared._parameters import HttpUcConnectionParameter, TextAreaParameter
|
|
13
13
|
from databricks.lakeflow.integrations.shared._paramtypes import ParameterAnnotation, _unwrap_optional
|
|
14
14
|
|
|
15
15
|
__all__ = [
|
|
@@ -283,7 +283,10 @@ def _run_callable(obj, parameters: dict[str, str]):
|
|
|
283
283
|
inspect.Parameter.POSITIONAL_OR_KEYWORD,
|
|
284
284
|
inspect.Parameter.KEYWORD_ONLY,
|
|
285
285
|
]:
|
|
286
|
-
|
|
286
|
+
# The console persists an unfilled optional field as an empty string.
|
|
287
|
+
if parameters.get(param.name) == "" and param.default is not inspect.Parameter.empty:
|
|
288
|
+
converted_kwargs.pop(param.name, None)
|
|
289
|
+
elif param.name in parameters:
|
|
287
290
|
annotation = type_hints.get(param.name, param.annotation)
|
|
288
291
|
converted_kwargs[param.name] = _convert_parameter(param.name, annotation, parameters[param.name])
|
|
289
292
|
|
|
@@ -369,7 +372,9 @@ def _convert_parameter(name: str, type_annotation: ParameterAnnotation, value: s
|
|
|
369
372
|
if isinstance(type_annotation, type) and issubclass(type_annotation, Enum):
|
|
370
373
|
return _convert_enum(name, type_annotation, value)
|
|
371
374
|
|
|
372
|
-
if isinstance(type_annotation, type) and issubclass(
|
|
375
|
+
if isinstance(type_annotation, type) and issubclass(
|
|
376
|
+
type_annotation, (HttpUcConnectionParameter, TextAreaParameter)
|
|
377
|
+
):
|
|
373
378
|
return type_annotation(value)
|
|
374
379
|
|
|
375
380
|
convert = _CONVERTERS.get(type_annotation) if isinstance(type_annotation, type) else None
|
|
@@ -5,6 +5,10 @@ class HttpUcConnectionParameter(str):
|
|
|
5
5
|
"""The name of a UC HTTP connection."""
|
|
6
6
|
|
|
7
7
|
|
|
8
|
+
class TextAreaParameter(str):
|
|
9
|
+
"""A multiline string, edited in a textarea."""
|
|
10
|
+
|
|
11
|
+
|
|
8
12
|
@dataclass(frozen=True, kw_only=True)
|
|
9
13
|
class HttpConnectionFilter:
|
|
10
14
|
"""Optional exact-match filters for the UC HTTP connection picker."""
|
{databricks_lakeflow_integrations-0.0.2 → databricks_lakeflow_integrations-0.0.4}/LICENSE.md
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|