databricks-bundles 0.266.0__py3-none-any.whl → 0.268.0__py3-none-any.whl

databricks/bundles/jobs/_models/task_email_notifications.py

@@ -3,7 +3,7 @@ from typing import TYPE_CHECKING, TypedDict
 
 from databricks.bundles.core._transform import _transform
 from databricks.bundles.core._transform_to_json import _transform_to_json_value
-from databricks.bundles.core._variable import VariableOrList
+from databricks.bundles.core._variable import VariableOrList, VariableOrOptional
 
 if TYPE_CHECKING:
     from typing_extensions import Self
@@ -13,6 +13,12 @@ if TYPE_CHECKING:
 class TaskEmailNotifications:
     """"""
 
+    no_alert_for_skipped_runs: VariableOrOptional[bool] = None
+    """
+    [DEPRECATED] If true, do not send email to recipients specified in `on_failure` if the run is skipped.
+    This field is `deprecated`. Please use the `notification_settings.no_alert_for_skipped_runs` field.
+    """
+
     on_duration_warning_threshold_exceeded: VariableOrList[str] = field(
         default_factory=list
     )
@@ -53,6 +59,12 @@ class TaskEmailNotifications:
 class TaskEmailNotificationsDict(TypedDict, total=False):
     """"""
 
+    no_alert_for_skipped_runs: VariableOrOptional[bool]
+    """
+    [DEPRECATED] If true, do not send email to recipients specified in `on_failure` if the run is skipped.
+    This field is `deprecated`. Please use the `notification_settings.no_alert_for_skipped_runs` field.
+    """
+
     on_duration_warning_threshold_exceeded: VariableOrList[str]
     """
     A list of email addresses to be notified when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. If no rule for the `RUN_DURATION_SECONDS` metric is specified in the `health` field for the job, notifications are not sent.

databricks/bundles/jobs/_models/task_retry_mode.py

@@ -0,0 +1,17 @@
+from enum import Enum
+from typing import Literal
+
+
+class TaskRetryMode(Enum):
+    """
+    task retry mode of the continuous job
+    * NEVER: The failed task will not be retried.
+    * ON_FAILURE: Retry a failed task if at least one other task in the job is still running its first attempt.
+    When this condition is no longer met or the retry limit is reached, the job run is cancelled and a new run is started.
+    """
+
+    NEVER = "NEVER"
+    ON_FAILURE = "ON_FAILURE"
+
+
+TaskRetryModeParam = Literal["NEVER", "ON_FAILURE"] | TaskRetryMode

databricks/bundles/pipelines/__init__.py

@@ -56,6 +56,9 @@ __all__ = [
     "InitScriptInfo",
     "InitScriptInfoDict",
     "InitScriptInfoParam",
+    "Lifecycle",
+    "LifecycleDict",
+    "LifecycleParam",
     "LocalFileInfo",
     "LocalFileInfoDict",
     "LocalFileInfoParam",
@@ -96,6 +99,12 @@ __all__ = [
     "PipelinesEnvironment",
     "PipelinesEnvironmentDict",
     "PipelinesEnvironmentParam",
+    "PostgresCatalogConfig",
+    "PostgresCatalogConfigDict",
+    "PostgresCatalogConfigParam",
+    "PostgresSlotConfig",
+    "PostgresSlotConfigDict",
+    "PostgresSlotConfigParam",
     "ReportSpec",
     "ReportSpecDict",
     "ReportSpecParam",
@@ -111,6 +120,12 @@ __all__ = [
     "SchemaSpec",
     "SchemaSpecDict",
     "SchemaSpecParam",
+    "SourceCatalogConfig",
+    "SourceCatalogConfigDict",
+    "SourceCatalogConfigParam",
+    "SourceConfig",
+    "SourceConfigDict",
+    "SourceConfigParam",
     "TableSpec",
     "TableSpecDict",
     "TableSpecParam",
@@ -224,6 +239,11 @@ from databricks.bundles.pipelines._models.init_script_info import (
     InitScriptInfoDict,
     InitScriptInfoParam,
 )
+from databricks.bundles.pipelines._models.lifecycle import (
+    Lifecycle,
+    LifecycleDict,
+    LifecycleParam,
+)
 from databricks.bundles.pipelines._models.local_file_info import (
     LocalFileInfo,
     LocalFileInfoDict,
@@ -292,6 +312,16 @@ from databricks.bundles.pipelines._models.pipelines_environment import (
     PipelinesEnvironmentDict,
     PipelinesEnvironmentParam,
 )
+from databricks.bundles.pipelines._models.postgres_catalog_config import (
+    PostgresCatalogConfig,
+    PostgresCatalogConfigDict,
+    PostgresCatalogConfigParam,
+)
+from databricks.bundles.pipelines._models.postgres_slot_config import (
+    PostgresSlotConfig,
+    PostgresSlotConfigDict,
+    PostgresSlotConfigParam,
+)
 from databricks.bundles.pipelines._models.report_spec import (
     ReportSpec,
     ReportSpecDict,
@@ -313,6 +343,16 @@ from databricks.bundles.pipelines._models.schema_spec import (
     SchemaSpecDict,
     SchemaSpecParam,
 )
+from databricks.bundles.pipelines._models.source_catalog_config import (
+    SourceCatalogConfig,
+    SourceCatalogConfigDict,
+    SourceCatalogConfigParam,
+)
+from databricks.bundles.pipelines._models.source_config import (
+    SourceConfig,
+    SourceConfigDict,
+    SourceConfigParam,
+)
 from databricks.bundles.pipelines._models.table_spec import (
     TableSpec,
     TableSpecDict,

databricks/bundles/pipelines/_models/gcp_attributes.py

@@ -53,6 +53,13 @@ class GcpAttributes:
     for the supported number of local SSDs for each instance type.
     """
 
+    use_preemptible_executors: VariableOrOptional[bool] = None
+    """
+    [DEPRECATED] This field determines whether the spark executors will be scheduled to run on preemptible
+    VMs (when set to true) versus standard compute engine VMs (when set to false; default).
+    Note: Soon to be deprecated, use the 'availability' field instead.
+    """
+
     zone_id: VariableOrOptional[str] = None
     """
     Identifier for the availability zone in which the cluster resides.
@@ -108,6 +115,13 @@ class GcpAttributesDict(TypedDict, total=False):
     for the supported number of local SSDs for each instance type.
     """
 
+    use_preemptible_executors: VariableOrOptional[bool]
+    """
+    [DEPRECATED] This field determines whether the spark executors will be scheduled to run on preemptible
+    VMs (when set to true) versus standard compute engine VMs (when set to false; default).
+    Note: Soon to be deprecated, use the 'availability' field instead.
+    """
+
     zone_id: VariableOrOptional[str]
     """
     Identifier for the availability zone in which the cluster resides.

databricks/bundles/pipelines/_models/ingestion_gateway_pipeline_definition.py

@@ -30,6 +30,11 @@ class IngestionGatewayPipelineDefinition:
     Required, Immutable. The name of the schema for the gateway pipelines's storage location.
     """
 
+    connection_id: VariableOrOptional[str] = None
+    """
+    [DEPRECATED] [Deprecated, use connection_name instead] Immutable. The Unity Catalog connection that this gateway pipeline uses to communicate with the source.
+    """
+
     gateway_storage_name: VariableOrOptional[str] = None
     """
     Optional. The Unity Catalog-compatible name for the gateway storage location.
@@ -63,6 +68,11 @@ class IngestionGatewayPipelineDefinitionDict(TypedDict, total=False):
     Required, Immutable. The name of the schema for the gateway pipelines's storage location.
     """
 
+    connection_id: VariableOrOptional[str]
+    """
+    [DEPRECATED] [Deprecated, use connection_name instead] Immutable. The Unity Catalog connection that this gateway pipeline uses to communicate with the source.
+    """
+
     gateway_storage_name: VariableOrOptional[str]
     """
     Optional. The Unity Catalog-compatible name for the gateway storage location.

databricks/bundles/pipelines/_models/ingestion_pipeline_definition.py

@@ -12,6 +12,10 @@ from databricks.bundles.pipelines._models.ingestion_source_type import (
     IngestionSourceType,
     IngestionSourceTypeParam,
 )
+from databricks.bundles.pipelines._models.source_config import (
+    SourceConfig,
+    SourceConfigParam,
+)
 from databricks.bundles.pipelines._models.table_specific_config import (
     TableSpecificConfig,
     TableSpecificConfigParam,
@@ -40,6 +44,13 @@ class IngestionPipelineDefinition:
     Required. Settings specifying tables to replicate and the destination for the replicated tables.
     """
 
+    source_configurations: VariableOrList[SourceConfig] = field(default_factory=list)
+    """
+    :meta private: [EXPERIMENTAL]
+    
+    Top-level source configurations
+    """
+
     source_type: VariableOrOptional[IngestionSourceType] = None
     """
     The type of the foreign source.
@@ -78,6 +89,13 @@ class IngestionPipelineDefinitionDict(TypedDict, total=False):
     Required. Settings specifying tables to replicate and the destination for the replicated tables.
     """
 
+    source_configurations: VariableOrList[SourceConfigParam]
+    """
+    :meta private: [EXPERIMENTAL]
+    
+    Top-level source configurations
+    """
+
     source_type: VariableOrOptional[IngestionSourceTypeParam]
     """
     The type of the foreign source.

databricks/bundles/pipelines/_models/init_script_info.py

@@ -8,6 +8,10 @@ from databricks.bundles.pipelines._models.adlsgen2_info import (
     Adlsgen2Info,
     Adlsgen2InfoParam,
 )
+from databricks.bundles.pipelines._models.dbfs_storage_info import (
+    DbfsStorageInfo,
+    DbfsStorageInfoParam,
+)
 from databricks.bundles.pipelines._models.gcs_storage_info import (
     GcsStorageInfo,
     GcsStorageInfoParam,
@@ -45,6 +49,12 @@ class InitScriptInfo:
     Contains the Azure Data Lake Storage destination path
     """
 
+    dbfs: VariableOrOptional[DbfsStorageInfo] = None
+    """
+    [DEPRECATED] destination needs to be provided. e.g.
+    `{ "dbfs": { "destination" : "dbfs:/home/cluster_log" } }`
+    """
+
     file: VariableOrOptional[LocalFileInfo] = None
     """
     destination needs to be provided, e.g.
@@ -93,6 +103,12 @@ class InitScriptInfoDict(TypedDict, total=False):
     Contains the Azure Data Lake Storage destination path
     """
 
+    dbfs: VariableOrOptional[DbfsStorageInfoParam]
+    """
+    [DEPRECATED] destination needs to be provided. e.g.
+    `{ "dbfs": { "destination" : "dbfs:/home/cluster_log" } }`
+    """
+
     file: VariableOrOptional[LocalFileInfoParam]
     """
     destination needs to be provided, e.g.

databricks/bundles/pipelines/_models/lifecycle.py

@@ -0,0 +1,38 @@
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, TypedDict
+
+from databricks.bundles.core._transform import _transform
+from databricks.bundles.core._transform_to_json import _transform_to_json_value
+from databricks.bundles.core._variable import VariableOrOptional
+
+if TYPE_CHECKING:
+    from typing_extensions import Self
+
+
+@dataclass(kw_only=True)
+class Lifecycle:
+    """"""
+
+    prevent_destroy: VariableOrOptional[bool] = None
+    """
+    Lifecycle setting to prevent the resource from being destroyed.
+    """
+
+    @classmethod
+    def from_dict(cls, value: "LifecycleDict") -> "Self":
+        return _transform(cls, value)
+
+    def as_dict(self) -> "LifecycleDict":
+        return _transform_to_json_value(self)  # type:ignore
+
+
+class LifecycleDict(TypedDict, total=False):
+    """"""
+
+    prevent_destroy: VariableOrOptional[bool]
+    """
+    Lifecycle setting to prevent the resource from being destroyed.
+    """
+
+
+LifecycleParam = LifecycleDict | Lifecycle

databricks/bundles/pipelines/_models/pipeline.py

@@ -25,6 +25,10 @@ from databricks.bundles.pipelines._models.ingestion_pipeline_definition import (
     IngestionPipelineDefinition,
     IngestionPipelineDefinitionParam,
 )
+from databricks.bundles.pipelines._models.lifecycle import (
+    Lifecycle,
+    LifecycleParam,
+)
 from databricks.bundles.pipelines._models.notifications import (
     Notifications,
     NotificationsParam,
@@ -143,6 +147,11 @@ class Pipeline(Resource):
     Libraries or code needed by this deployment.
     """
 
+    lifecycle: VariableOrOptional[Lifecycle] = None
+    """
+    Lifecycle is a struct that contains the lifecycle settings for a resource. It controls the behavior of the resource when it is deployed or destroyed.
+    """
+
     name: VariableOrOptional[str] = None
     """
     Friendly identifier for this pipeline.
@@ -203,7 +212,7 @@ class Pipeline(Resource):
 
     target: VariableOrOptional[str] = None
     """
-    Target schema (database) to add tables in this pipeline to. Exactly one of `schema` or `target` must be specified. To publish to Unity Catalog, also specify `catalog`. This legacy field is deprecated for pipeline creation in favor of the `schema` field.
+    [DEPRECATED] Target schema (database) to add tables in this pipeline to. Exactly one of `schema` or `target` must be specified. To publish to Unity Catalog, also specify `catalog`. This legacy field is deprecated for pipeline creation in favor of the `schema` field.
     """
 
     @classmethod
@@ -301,6 +310,11 @@ class PipelineDict(TypedDict, total=False):
     Libraries or code needed by this deployment.
     """
 
+    lifecycle: VariableOrOptional[LifecycleParam]
+    """
+    Lifecycle is a struct that contains the lifecycle settings for a resource. It controls the behavior of the resource when it is deployed or destroyed.
+    """
+
     name: VariableOrOptional[str]
     """
     Friendly identifier for this pipeline.
@@ -361,7 +375,7 @@ class PipelineDict(TypedDict, total=False):
 
     target: VariableOrOptional[str]
     """
-    Target schema (database) to add tables in this pipeline to. Exactly one of `schema` or `target` must be specified. To publish to Unity Catalog, also specify `catalog`. This legacy field is deprecated for pipeline creation in favor of the `schema` field.
+    [DEPRECATED] Target schema (database) to add tables in this pipeline to. Exactly one of `schema` or `target` must be specified. To publish to Unity Catalog, also specify `catalog`. This legacy field is deprecated for pipeline creation in favor of the `schema` field.
     """
 
 

databricks/bundles/pipelines/_models/postgres_catalog_config.py

@@ -0,0 +1,50 @@
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, TypedDict
+
+from databricks.bundles.core._transform import _transform
+from databricks.bundles.core._transform_to_json import _transform_to_json_value
+from databricks.bundles.core._variable import VariableOrOptional
+from databricks.bundles.pipelines._models.postgres_slot_config import (
+    PostgresSlotConfig,
+    PostgresSlotConfigParam,
+)
+
+if TYPE_CHECKING:
+    from typing_extensions import Self
+
+
+@dataclass(kw_only=True)
+class PostgresCatalogConfig:
+    """
+    :meta private: [EXPERIMENTAL]
+
+    PG-specific catalog-level configuration parameters
+    """
+
+    slot_config: VariableOrOptional[PostgresSlotConfig] = None
+    """
+    :meta private: [EXPERIMENTAL]
+    
+    Optional. The Postgres slot configuration to use for logical replication
+    """
+
+    @classmethod
+    def from_dict(cls, value: "PostgresCatalogConfigDict") -> "Self":
+        return _transform(cls, value)
+
+    def as_dict(self) -> "PostgresCatalogConfigDict":
+        return _transform_to_json_value(self)  # type:ignore
+
+
+class PostgresCatalogConfigDict(TypedDict, total=False):
+    """"""
+
+    slot_config: VariableOrOptional[PostgresSlotConfigParam]
+    """
+    :meta private: [EXPERIMENTAL]
+    
+    Optional. The Postgres slot configuration to use for logical replication
+    """
+
+
+PostgresCatalogConfigParam = PostgresCatalogConfigDict | PostgresCatalogConfig

databricks/bundles/pipelines/_models/postgres_slot_config.py

@@ -0,0 +1,60 @@
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, TypedDict
+
+from databricks.bundles.core._transform import _transform
+from databricks.bundles.core._transform_to_json import _transform_to_json_value
+from databricks.bundles.core._variable import VariableOrOptional
+
+if TYPE_CHECKING:
+    from typing_extensions import Self
+
+
+@dataclass(kw_only=True)
+class PostgresSlotConfig:
+    """
+    :meta private: [EXPERIMENTAL]
+
+    PostgresSlotConfig contains the configuration for a Postgres logical replication slot
+    """
+
+    publication_name: VariableOrOptional[str] = None
+    """
+    :meta private: [EXPERIMENTAL]
+    
+    The name of the publication to use for the Postgres source
+    """
+
+    slot_name: VariableOrOptional[str] = None
+    """
+    :meta private: [EXPERIMENTAL]
+    
+    The name of the logical replication slot to use for the Postgres source
+    """
+
+    @classmethod
+    def from_dict(cls, value: "PostgresSlotConfigDict") -> "Self":
+        return _transform(cls, value)
+
+    def as_dict(self) -> "PostgresSlotConfigDict":
+        return _transform_to_json_value(self)  # type:ignore
+
+
+class PostgresSlotConfigDict(TypedDict, total=False):
+    """"""
+
+    publication_name: VariableOrOptional[str]
+    """
+    :meta private: [EXPERIMENTAL]
+    
+    The name of the publication to use for the Postgres source
+    """
+
+    slot_name: VariableOrOptional[str]
+    """
+    :meta private: [EXPERIMENTAL]
+    
+    The name of the logical replication slot to use for the Postgres source
+    """
+
+
+PostgresSlotConfigParam = PostgresSlotConfigDict | PostgresSlotConfig

databricks/bundles/pipelines/_models/source_catalog_config.py

@@ -0,0 +1,64 @@
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, TypedDict
+
+from databricks.bundles.core._transform import _transform
+from databricks.bundles.core._transform_to_json import _transform_to_json_value
+from databricks.bundles.core._variable import VariableOrOptional
+from databricks.bundles.pipelines._models.postgres_catalog_config import (
+    PostgresCatalogConfig,
+    PostgresCatalogConfigParam,
+)
+
+if TYPE_CHECKING:
+    from typing_extensions import Self
+
+
+@dataclass(kw_only=True)
+class SourceCatalogConfig:
+    """
+    :meta private: [EXPERIMENTAL]
+
+    SourceCatalogConfig contains catalog-level custom configuration parameters for each source
+    """
+
+    postgres: VariableOrOptional[PostgresCatalogConfig] = None
+    """
+    :meta private: [EXPERIMENTAL]
+    
+    Postgres-specific catalog-level configuration parameters
+    """
+
+    source_catalog: VariableOrOptional[str] = None
+    """
+    :meta private: [EXPERIMENTAL]
+    
+    Source catalog name
+    """
+
+    @classmethod
+    def from_dict(cls, value: "SourceCatalogConfigDict") -> "Self":
+        return _transform(cls, value)
+
+    def as_dict(self) -> "SourceCatalogConfigDict":
+        return _transform_to_json_value(self)  # type:ignore
+
+
+class SourceCatalogConfigDict(TypedDict, total=False):
+    """"""
+
+    postgres: VariableOrOptional[PostgresCatalogConfigParam]
+    """
+    :meta private: [EXPERIMENTAL]
+    
+    Postgres-specific catalog-level configuration parameters
+    """
+
+    source_catalog: VariableOrOptional[str]
+    """
+    :meta private: [EXPERIMENTAL]
+    
+    Source catalog name
+    """
+
+
+SourceCatalogConfigParam = SourceCatalogConfigDict | SourceCatalogConfig

databricks/bundles/pipelines/_models/source_config.py

@@ -0,0 +1,48 @@
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, TypedDict
+
+from databricks.bundles.core._transform import _transform
+from databricks.bundles.core._transform_to_json import _transform_to_json_value
+from databricks.bundles.core._variable import VariableOrOptional
+from databricks.bundles.pipelines._models.source_catalog_config import (
+    SourceCatalogConfig,
+    SourceCatalogConfigParam,
+)
+
+if TYPE_CHECKING:
+    from typing_extensions import Self
+
+
+@dataclass(kw_only=True)
+class SourceConfig:
+    """
+    :meta private: [EXPERIMENTAL]
+    """
+
+    catalog: VariableOrOptional[SourceCatalogConfig] = None
+    """
+    :meta private: [EXPERIMENTAL]
+    
+    Catalog-level source configuration parameters
+    """
+
+    @classmethod
+    def from_dict(cls, value: "SourceConfigDict") -> "Self":
+        return _transform(cls, value)
+
+    def as_dict(self) -> "SourceConfigDict":
+        return _transform_to_json_value(self)  # type:ignore
+
+
+class SourceConfigDict(TypedDict, total=False):
+    """"""
+
+    catalog: VariableOrOptional[SourceCatalogConfigParam]
+    """
+    :meta private: [EXPERIMENTAL]
+    
+    Catalog-level source configuration parameters
+    """
+
+
+SourceConfigParam = SourceConfigDict | SourceConfig

databricks/bundles/schemas/__init__.py

@@ -0,0 +1,30 @@
+__all__ = [
+    "Lifecycle",
+    "LifecycleDict",
+    "LifecycleParam",
+    "Schema",
+    "SchemaDict",
+    "SchemaGrant",
+    "SchemaGrantDict",
+    "SchemaGrantParam",
+    "SchemaGrantPrivilege",
+    "SchemaGrantPrivilegeParam",
+    "SchemaParam",
+]
+
+
+from databricks.bundles.schemas._models.lifecycle import (
+    Lifecycle,
+    LifecycleDict,
+    LifecycleParam,
+)
+from databricks.bundles.schemas._models.schema import Schema, SchemaDict, SchemaParam
+from databricks.bundles.schemas._models.schema_grant import (
+    SchemaGrant,
+    SchemaGrantDict,
+    SchemaGrantParam,
+)
+from databricks.bundles.schemas._models.schema_grant_privilege import (
+    SchemaGrantPrivilege,
+    SchemaGrantPrivilegeParam,
+)

databricks/bundles/schemas/_models/lifecycle.py

@@ -0,0 +1,38 @@
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, TypedDict
+
+from databricks.bundles.core._transform import _transform
+from databricks.bundles.core._transform_to_json import _transform_to_json_value
+from databricks.bundles.core._variable import VariableOrOptional
+
+if TYPE_CHECKING:
+    from typing_extensions import Self
+
+
+@dataclass(kw_only=True)
+class Lifecycle:
+    """"""
+
+    prevent_destroy: VariableOrOptional[bool] = None
+    """
+    Lifecycle setting to prevent the resource from being destroyed.
+    """
+
+    @classmethod
+    def from_dict(cls, value: "LifecycleDict") -> "Self":
+        return _transform(cls, value)
+
+    def as_dict(self) -> "LifecycleDict":
+        return _transform_to_json_value(self)  # type:ignore
+
+
+class LifecycleDict(TypedDict, total=False):
+    """"""
+
+    prevent_destroy: VariableOrOptional[bool]
+    """
+    Lifecycle setting to prevent the resource from being destroyed.
+    """
+
+
+LifecycleParam = LifecycleDict | Lifecycle