databricks-bundles 0.267.0__tar.gz → 0.269.0__tar.gz

{databricks_bundles-0.267.0 → databricks_bundles-0.269.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: databricks-bundles
-Version: 0.267.0
+Version: 0.269.0
 Summary: Python support for Databricks Asset Bundles
 Author-email: Gleb Kanterov <gleb.kanterov@databricks.com>
 Requires-Python: >=3.10
@@ -22,7 +22,7 @@ Reference documentation is available at https://databricks.github.io/cli/experim
 
 To use `databricks-bundles`, you must first:
 
-1. Install the [Databricks CLI](https://github.com/databricks/cli), version 0.267.0 or above
+1. Install the [Databricks CLI](https://github.com/databricks/cli), version 0.269.0 or above
 2. Authenticate to your Databricks workspace if you have not done so already:
 
    ```bash

{databricks_bundles-0.267.0 → databricks_bundles-0.269.0}/README.md

@@ -13,7 +13,7 @@ Reference documentation is available at https://databricks.github.io/cli/experim
 
 To use `databricks-bundles`, you must first:
 
-1. Install the [Databricks CLI](https://github.com/databricks/cli), version 0.267.0 or above
+1. Install the [Databricks CLI](https://github.com/databricks/cli), version 0.269.0 or above
 2. Authenticate to your Databricks workspace if you have not done so already:
 
    ```bash

{databricks_bundles-0.267.0 → databricks_bundles-0.269.0}/databricks/bundles/build.py

@@ -166,16 +166,12 @@ def _apply_mutators_for_type(
     return resources, Diagnostics()
 
 
-def python_mutator(
-    args: _Args,
-) -> tuple[dict, dict[tuple[str, ...], Location], Diagnostics]:
-    input = json.load(open(args.input, encoding="utf-8"))
+def _read_conf(input: dict) -> tuple[_Conf, Diagnostics]:
     experimental = input.get("experimental", {})
 
     if experimental.get("pydabs", {}) != {}:
         return (
-            {},
-            {},
+            _Conf(),
             Diagnostics.create_error(
                 "'experimental/pydabs' is not supported by 'databricks-bundles', use 'experimental/python' instead",
                 detail="",
@@ -184,8 +180,53 @@ def python_mutator(
             ),
         )
 
-    conf_dict = experimental.get("python", {})
+    experimental_conf_dict = experimental.get("python", {})
+    experimental_conf = _transform(_Conf, experimental_conf_dict)
+
+    conf_dict = input.get("python", {})
     conf = _transform(_Conf, conf_dict)
+
+    has_conf = conf != _Conf()
+    has_experimental_conf = experimental_conf != _Conf()
+
+    if has_conf and not has_experimental_conf:
+        return conf, Diagnostics()
+    elif not has_conf and has_experimental_conf:
+        # do not generate warning in Python code, if CLI supports non-experimental 'python',
+        # it should generate a warning
+        return experimental_conf, Diagnostics()
+    elif has_conf and has_experimental_conf:
+        # for backward-compatibility, CLI can copy contents of 'python' into 'experimental/python'
+        # if configs are equal, it isn't a problem
+        if conf != experimental_conf:
+            return (
+                _Conf(),
+                Diagnostics.create_error(
+                    "Both 'python' and 'experimental/python' sections are present, use 'python' section only",
+                    detail="",
+                    location=None,
+                    path=(
+                        "experimental",
+                        "python",
+                    ),
+                ),
+            )
+        else:
+            return conf, Diagnostics()
+    else:
+        return _Conf(), Diagnostics()
+
+
+def python_mutator(
+    args: _Args,
+) -> tuple[dict, dict[tuple[str, ...], Location], Diagnostics]:
+    input = json.load(open(args.input, encoding="utf-8"))
+    diagnostics = Diagnostics()
+
+    conf, diagnostics = diagnostics.extend_tuple(_read_conf(input))
+    if diagnostics.has_error():
+        return input, {}, diagnostics
+
     bundle = _parse_bundle_info(input)
 
     if args.phase == "load_resources":

{databricks_bundles-0.267.0 → databricks_bundles-0.269.0}/databricks/bundles/core/__init__.py

@@ -18,6 +18,7 @@ __all__ = [
     "load_resources_from_modules",
     "load_resources_from_package_module",
     "pipeline_mutator",
+    "schema_mutator",
     "variables",
     "volume_mutator",
 ]
@@ -40,6 +41,7 @@ from databricks.bundles.core._resource_mutator import (
     ResourceMutator,
     job_mutator,
     pipeline_mutator,
+    schema_mutator,
     volume_mutator,
 )
 from databricks.bundles.core._resources import Resources

{databricks_bundles-0.267.0 → databricks_bundles-0.269.0}/databricks/bundles/core/_diagnostics.py

@@ -134,6 +134,17 @@ class Diagnostics:
 
         return False
 
+    def has_warning(self) -> bool:
+        """
+        Returns True if there is at least one warning in diagnostics.
+        """
+
+        for item in self.items:
+            if item.severity == Severity.WARNING:
+                return True
+
+        return False
+
     @classmethod
     def create_error(
         cls,

{databricks_bundles-0.267.0 → databricks_bundles-0.269.0}/databricks/bundles/core/_resource_mutator.py

@@ -8,6 +8,7 @@ from databricks.bundles.core._resource import Resource
 if TYPE_CHECKING:
     from databricks.bundles.jobs._models.job import Job
     from databricks.bundles.pipelines._models.pipeline import Pipeline
+    from databricks.bundles.schemas._models.schema import Schema
     from databricks.bundles.volumes._models.volume import Volume
 
 _T = TypeVar("_T", bound=Resource)
@@ -130,6 +131,38 @@ def pipeline_mutator(function: Callable) -> ResourceMutator["Pipeline"]:
     return ResourceMutator(resource_type=Pipeline, function=function)
 
 
+@overload
+def schema_mutator(
+    function: Callable[[Bundle, "Schema"], "Schema"],
+) -> ResourceMutator["Schema"]: ...
+
+
+@overload
+def schema_mutator(
+    function: Callable[["Schema"], "Schema"],
+) -> ResourceMutator["Schema"]: ...
+
+
+def schema_mutator(function: Callable) -> ResourceMutator["Schema"]:
+    """
+    Decorator for defining a schema mutator. Function should return a new instance of the schema with the desired changes,
+    instead of mutating the input schema.
+
+    Example:
+
+    .. code-block:: python
+
+        @schema_mutator
+        def my_schema_mutator(bundle: Bundle, schema: Schema) -> Schema:
+            return replace(schema, name="my_schema")
+
+    :param function: Function that mutates a schema.
+    """
+    from databricks.bundles.schemas._models.schema import Schema
+
+    return ResourceMutator(resource_type=Schema, function=function)
+
+
 @overload
 def volume_mutator(
     function: Callable[[Bundle, "Volume"], "Volume"],

{databricks_bundles-0.267.0 → databricks_bundles-0.269.0}/databricks/bundles/core/_resource_type.py

@@ -2,7 +2,6 @@ from dataclasses import dataclass
 from typing import Type
 
 from databricks.bundles.core._resource import Resource
-from databricks.bundles.volumes._models.volume import Volume
 
 
 @dataclass(kw_only=True, frozen=True)
@@ -34,6 +33,8 @@ class _ResourceType:
 
         from databricks.bundles.jobs._models.job import Job
         from databricks.bundles.pipelines._models.pipeline import Pipeline
+        from databricks.bundles.schemas._models.schema import Schema
+        from databricks.bundles.volumes._models.volume import Volume
 
         return (
             _ResourceType(
@@ -51,4 +52,9 @@ class _ResourceType:
                 plural_name="volumes",
                 singular_name="volume",
             ),
+            _ResourceType(
+                resource_type=Schema,
+                plural_name="schemas",
+                singular_name="schema",
+            ),
         )

{databricks_bundles-0.267.0 → databricks_bundles-0.269.0}/databricks/bundles/core/_resources.py

@@ -8,6 +8,7 @@ from databricks.bundles.core._transform import _transform
 if TYPE_CHECKING:
     from databricks.bundles.jobs._models.job import Job, JobParam
     from databricks.bundles.pipelines._models.pipeline import Pipeline, PipelineParam
+    from databricks.bundles.schemas._models.schema import Schema, SchemaParam
     from databricks.bundles.volumes._models.volume import Volume, VolumeParam
 
 __all__ = ["Resources"]
@@ -58,6 +59,7 @@ class Resources:
     def __init__(self):
         self._jobs = dict[str, "Job"]()
         self._pipelines = dict[str, "Pipeline"]()
+        self._schemas = dict[str, "Schema"]()
         self._volumes = dict[str, "Volume"]()
         self._locations = dict[tuple[str, ...], Location]()
         self._diagnostics = Diagnostics()
@@ -70,6 +72,10 @@ class Resources:
     def pipelines(self) -> dict[str, "Pipeline"]:
         return self._pipelines
 
+    @property
+    def schemas(self) -> dict[str, "Schema"]:
+        return self._schemas
+
     @property
     def volumes(self) -> dict[str, "Volume"]:
         return self._volumes
@@ -99,6 +105,7 @@ class Resources:
 
         from databricks.bundles.jobs import Job
         from databricks.bundles.pipelines import Pipeline
+        from databricks.bundles.schemas import Schema
         from databricks.bundles.volumes import Volume
 
         location = location or Location.from_stack_frame(depth=1)
@@ -108,6 +115,8 @@ class Resources:
                 self.add_job(resource_name, resource, location=location)
             case Pipeline():
                 self.add_pipeline(resource_name, resource, location=location)
+            case Schema():
+                self.add_schema(resource_name, resource, location=location)
             case Volume():
                 self.add_volume(resource_name, resource, location=location)
             case _:
@@ -177,6 +186,38 @@ class Resources:
 
             self._pipelines[resource_name] = pipeline
 
+    def add_schema(
+        self,
+        resource_name: str,
+        schema: "SchemaParam",
+        *,
+        location: Optional[Location] = None,
+    ) -> None:
+        """
+        Adds a schema to the collection of resources. Resource name must be unique across all schemas.
+
+        :param resource_name: unique identifier for the schema
+        :param schema: the schema to add, can be Schema or dict
+        :param location: optional location of the schema in the source code
+        """
+        from databricks.bundles.schemas import Schema
+
+        schema = _transform(Schema, schema)
+        path = ("resources", "schemas", resource_name)
+        location = location or Location.from_stack_frame(depth=1)
+
+        if self._schemas.get(resource_name):
+            self.add_diagnostic_error(
+                msg=f"Duplicate resource name '{resource_name}' for a schema. Resource names must be unique.",
+                location=location,
+                path=path,
+            )
+        else:
+            if location:
+                self.add_location(path, location)
+
+            self._schemas[resource_name] = schema
+
     def add_volume(
         self,
         resource_name: str,
@@ -285,6 +326,9 @@ class Resources:
         for name, pipeline in other.pipelines.items():
             self.add_pipeline(name, pipeline)
 
+        for name, schema in other.schemas.items():
+            self.add_schema(name, schema)
+
         for name, volume in other.volumes.items():
             self.add_volume(name, volume)
 

{databricks_bundles-0.267.0 → databricks_bundles-0.269.0}/databricks/bundles/jobs/__init__.py

@@ -134,6 +134,9 @@ __all__ = [
     "Library",
     "LibraryDict",
     "LibraryParam",
+    "Lifecycle",
+    "LifecycleDict",
+    "LifecycleParam",
     "LocalFileInfo",
     "LocalFileInfoDict",
     "LocalFileInfoParam",
@@ -482,6 +485,11 @@ from databricks.bundles.jobs._models.jobs_health_rules import (
     JobsHealthRulesParam,
 )
 from databricks.bundles.jobs._models.library import Library, LibraryDict, LibraryParam
+from databricks.bundles.jobs._models.lifecycle import (
+    Lifecycle,
+    LifecycleDict,
+    LifecycleParam,
+)
 from databricks.bundles.jobs._models.local_file_info import (
     LocalFileInfo,
     LocalFileInfoDict,

{databricks_bundles-0.267.0 → databricks_bundles-0.269.0}/databricks/bundles/jobs/_models/job.py

@@ -44,6 +44,7 @@ from databricks.bundles.jobs._models.jobs_health_rules import (
     JobsHealthRules,
     JobsHealthRulesParam,
 )
+from databricks.bundles.jobs._models.lifecycle import Lifecycle, LifecycleParam
 from databricks.bundles.jobs._models.performance_target import (
     PerformanceTarget,
     PerformanceTargetParam,
@@ -116,6 +117,11 @@ class Job(Resource):
     A list of job cluster specifications that can be shared and reused by tasks of this job. Libraries cannot be declared in a shared job cluster. You must declare dependent libraries in task settings.
     """
 
+    lifecycle: VariableOrOptional[Lifecycle] = None
+    """
+    Lifecycle is a struct that contains the lifecycle settings for a resource. It controls the behavior of the resource when it is deployed or destroyed.
+    """
+
     max_concurrent_runs: VariableOrOptional[int] = None
     """
     An optional maximum allowed number of concurrent runs of the job.
@@ -256,6 +262,11 @@ class JobDict(TypedDict, total=False):
     A list of job cluster specifications that can be shared and reused by tasks of this job. Libraries cannot be declared in a shared job cluster. You must declare dependent libraries in task settings.
     """
 
+    lifecycle: VariableOrOptional[LifecycleParam]
+    """
+    Lifecycle is a struct that contains the lifecycle settings for a resource. It controls the behavior of the resource when it is deployed or destroyed.
+    """
+
     max_concurrent_runs: VariableOrOptional[int]
     """
     An optional maximum allowed number of concurrent runs of the job.

databricks_bundles-0.269.0/databricks/bundles/jobs/_models/lifecycle.py

@@ -0,0 +1,38 @@
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, TypedDict
+
+from databricks.bundles.core._transform import _transform
+from databricks.bundles.core._transform_to_json import _transform_to_json_value
+from databricks.bundles.core._variable import VariableOrOptional
+
+if TYPE_CHECKING:
+    from typing_extensions import Self
+
+
+@dataclass(kw_only=True)
+class Lifecycle:
+    """"""
+
+    prevent_destroy: VariableOrOptional[bool] = None
+    """
+    Lifecycle setting to prevent the resource from being destroyed.
+    """
+
+    @classmethod
+    def from_dict(cls, value: "LifecycleDict") -> "Self":
+        return _transform(cls, value)
+
+    def as_dict(self) -> "LifecycleDict":
+        return _transform_to_json_value(self)  # type:ignore
+
+
+class LifecycleDict(TypedDict, total=False):
+    """"""
+
+    prevent_destroy: VariableOrOptional[bool]
+    """
+    Lifecycle setting to prevent the resource from being destroyed.
+    """
+
+
+LifecycleParam = LifecycleDict | Lifecycle

{databricks_bundles-0.267.0 → databricks_bundles-0.269.0}/databricks/bundles/pipelines/__init__.py

@@ -56,6 +56,9 @@ __all__ = [
     "InitScriptInfo",
     "InitScriptInfoDict",
     "InitScriptInfoParam",
+    "Lifecycle",
+    "LifecycleDict",
+    "LifecycleParam",
     "LocalFileInfo",
     "LocalFileInfoDict",
     "LocalFileInfoParam",
@@ -236,6 +239,11 @@ from databricks.bundles.pipelines._models.init_script_info import (
     InitScriptInfoDict,
     InitScriptInfoParam,
 )
+from databricks.bundles.pipelines._models.lifecycle import (
+    Lifecycle,
+    LifecycleDict,
+    LifecycleParam,
+)
 from databricks.bundles.pipelines._models.local_file_info import (
     LocalFileInfo,
     LocalFileInfoDict,

databricks_bundles-0.269.0/databricks/bundles/pipelines/_models/lifecycle.py

@@ -0,0 +1,38 @@
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, TypedDict
+
+from databricks.bundles.core._transform import _transform
+from databricks.bundles.core._transform_to_json import _transform_to_json_value
+from databricks.bundles.core._variable import VariableOrOptional
+
+if TYPE_CHECKING:
+    from typing_extensions import Self
+
+
+@dataclass(kw_only=True)
+class Lifecycle:
+    """"""
+
+    prevent_destroy: VariableOrOptional[bool] = None
+    """
+    Lifecycle setting to prevent the resource from being destroyed.
+    """
+
+    @classmethod
+    def from_dict(cls, value: "LifecycleDict") -> "Self":
+        return _transform(cls, value)
+
+    def as_dict(self) -> "LifecycleDict":
+        return _transform_to_json_value(self)  # type:ignore
+
+
+class LifecycleDict(TypedDict, total=False):
+    """"""
+
+    prevent_destroy: VariableOrOptional[bool]
+    """
+    Lifecycle setting to prevent the resource from being destroyed.
+    """
+
+
+LifecycleParam = LifecycleDict | Lifecycle

{databricks_bundles-0.267.0 → databricks_bundles-0.269.0}/databricks/bundles/pipelines/_models/pipeline.py

@@ -25,6 +25,10 @@ from databricks.bundles.pipelines._models.ingestion_pipeline_definition import (
     IngestionPipelineDefinition,
     IngestionPipelineDefinitionParam,
 )
+from databricks.bundles.pipelines._models.lifecycle import (
+    Lifecycle,
+    LifecycleParam,
+)
 from databricks.bundles.pipelines._models.notifications import (
     Notifications,
     NotificationsParam,
@@ -143,6 +147,11 @@ class Pipeline(Resource):
     Libraries or code needed by this deployment.
     """
 
+    lifecycle: VariableOrOptional[Lifecycle] = None
+    """
+    Lifecycle is a struct that contains the lifecycle settings for a resource. It controls the behavior of the resource when it is deployed or destroyed.
+    """
+
     name: VariableOrOptional[str] = None
     """
     Friendly identifier for this pipeline.
@@ -301,6 +310,11 @@ class PipelineDict(TypedDict, total=False):
     Libraries or code needed by this deployment.
     """
 
+    lifecycle: VariableOrOptional[LifecycleParam]
+    """
+    Lifecycle is a struct that contains the lifecycle settings for a resource. It controls the behavior of the resource when it is deployed or destroyed.
+    """
+
     name: VariableOrOptional[str]
     """
     Friendly identifier for this pipeline.

databricks_bundles-0.269.0/databricks/bundles/schemas/__init__.py

@@ -0,0 +1,30 @@
+__all__ = [
+    "Lifecycle",
+    "LifecycleDict",
+    "LifecycleParam",
+    "Schema",
+    "SchemaDict",
+    "SchemaGrant",
+    "SchemaGrantDict",
+    "SchemaGrantParam",
+    "SchemaGrantPrivilege",
+    "SchemaGrantPrivilegeParam",
+    "SchemaParam",
+]
+
+
+from databricks.bundles.schemas._models.lifecycle import (
+    Lifecycle,
+    LifecycleDict,
+    LifecycleParam,
+)
+from databricks.bundles.schemas._models.schema import Schema, SchemaDict, SchemaParam
+from databricks.bundles.schemas._models.schema_grant import (
+    SchemaGrant,
+    SchemaGrantDict,
+    SchemaGrantParam,
+)
+from databricks.bundles.schemas._models.schema_grant_privilege import (
+    SchemaGrantPrivilege,
+    SchemaGrantPrivilegeParam,
+)

databricks_bundles-0.269.0/databricks/bundles/schemas/_models/lifecycle.py

@@ -0,0 +1,38 @@
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, TypedDict
+
+from databricks.bundles.core._transform import _transform
+from databricks.bundles.core._transform_to_json import _transform_to_json_value
+from databricks.bundles.core._variable import VariableOrOptional
+
+if TYPE_CHECKING:
+    from typing_extensions import Self
+
+
+@dataclass(kw_only=True)
+class Lifecycle:
+    """"""
+
+    prevent_destroy: VariableOrOptional[bool] = None
+    """
+    Lifecycle setting to prevent the resource from being destroyed.
+    """
+
+    @classmethod
+    def from_dict(cls, value: "LifecycleDict") -> "Self":
+        return _transform(cls, value)
+
+    def as_dict(self) -> "LifecycleDict":
+        return _transform_to_json_value(self)  # type:ignore
+
+
+class LifecycleDict(TypedDict, total=False):
+    """"""
+
+    prevent_destroy: VariableOrOptional[bool]
+    """
+    Lifecycle setting to prevent the resource from being destroyed.
+    """
+
+
+LifecycleParam = LifecycleDict | Lifecycle

databricks_bundles-0.269.0/databricks/bundles/schemas/_models/schema.py

@@ -0,0 +1,97 @@
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, TypedDict
+
+from databricks.bundles.core._resource import Resource
+from databricks.bundles.core._transform import _transform
+from databricks.bundles.core._transform_to_json import _transform_to_json_value
+from databricks.bundles.core._variable import (
+    VariableOr,
+    VariableOrDict,
+    VariableOrList,
+    VariableOrOptional,
+)
+from databricks.bundles.schemas._models.lifecycle import Lifecycle, LifecycleParam
+from databricks.bundles.schemas._models.schema_grant import (
+    SchemaGrant,
+    SchemaGrantParam,
+)
+
+if TYPE_CHECKING:
+    from typing_extensions import Self
+
+
+@dataclass(kw_only=True)
+class Schema(Resource):
+    """"""
+
+    catalog_name: VariableOr[str]
+    """
+    Name of parent catalog.
+    """
+
+    name: VariableOr[str]
+    """
+    Name of schema, relative to parent catalog.
+    """
+
+    comment: VariableOrOptional[str] = None
+    """
+    User-provided free-form text description.
+    """
+
+    grants: VariableOrList[SchemaGrant] = field(default_factory=list)
+
+    lifecycle: VariableOrOptional[Lifecycle] = None
+    """
+    Lifecycle is a struct that contains the lifecycle settings for a resource. It controls the behavior of the resource when it is deployed or destroyed.
+    """
+
+    properties: VariableOrDict[str] = field(default_factory=dict)
+
+    storage_root: VariableOrOptional[str] = None
+    """
+    Storage root URL for managed tables within schema.
+    """
+
+    @classmethod
+    def from_dict(cls, value: "SchemaDict") -> "Self":
+        return _transform(cls, value)
+
+    def as_dict(self) -> "SchemaDict":
+        return _transform_to_json_value(self)  # type:ignore
+
+
+class SchemaDict(TypedDict, total=False):
+    """"""
+
+    catalog_name: VariableOr[str]
+    """
+    Name of parent catalog.
+    """
+
+    name: VariableOr[str]
+    """
+    Name of schema, relative to parent catalog.
+    """
+
+    comment: VariableOrOptional[str]
+    """
+    User-provided free-form text description.
+    """
+
+    grants: VariableOrList[SchemaGrantParam]
+
+    lifecycle: VariableOrOptional[LifecycleParam]
+    """
+    Lifecycle is a struct that contains the lifecycle settings for a resource. It controls the behavior of the resource when it is deployed or destroyed.
+    """
+
+    properties: VariableOrDict[str]
+
+    storage_root: VariableOrOptional[str]
+    """
+    Storage root URL for managed tables within schema.
+    """
+
+
+SchemaParam = SchemaDict | Schema