databricks-bundles 0.284.0__tar.gz → 0.285.0__tar.gz

{databricks_bundles-0.284.0 → databricks_bundles-0.285.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: databricks-bundles
-Version: 0.284.0
+Version: 0.285.0
 Summary: Python support for Databricks Asset Bundles
 Author-email: Gleb Kanterov <gleb.kanterov@databricks.com>
 Requires-Python: >=3.10
@@ -22,7 +22,7 @@ Reference documentation is available at https://databricks.github.io/cli/python/
 
 To use `databricks-bundles`, you must first:
 
-1. Install the [Databricks CLI](https://github.com/databricks/cli), version 0.284.0 or above
+1. Install the [Databricks CLI](https://github.com/databricks/cli), version 0.285.0 or above
 2. Authenticate to your Databricks workspace if you have not done so already:
 
    ```bash

{databricks_bundles-0.284.0 → databricks_bundles-0.285.0}/README.md

@@ -13,7 +13,7 @@ Reference documentation is available at https://databricks.github.io/cli/python/
 
 To use `databricks-bundles`, you must first:
 
-1. Install the [Databricks CLI](https://github.com/databricks/cli), version 0.284.0 or above
+1. Install the [Databricks CLI](https://github.com/databricks/cli), version 0.285.0 or above
 2. Authenticate to your Databricks workspace if you have not done so already:
 
    ```bash

{databricks_bundles-0.284.0 → databricks_bundles-0.285.0}/databricks/bundles/jobs/__init__.py

@@ -153,6 +153,9 @@ __all__ = [
     "ModelTriggerConfigurationConditionParam",
     "ModelTriggerConfigurationDict",
     "ModelTriggerConfigurationParam",
+    "NodeTypeFlexibility",
+    "NodeTypeFlexibilityDict",
+    "NodeTypeFlexibilityParam",
     "NotebookTask",
     "NotebookTaskDict",
     "NotebookTaskParam",
@@ -522,6 +525,11 @@ from databricks.bundles.jobs._models.model_trigger_configuration_condition impor
     ModelTriggerConfigurationCondition,
     ModelTriggerConfigurationConditionParam,
 )
+from databricks.bundles.jobs._models.node_type_flexibility import (
+    NodeTypeFlexibility,
+    NodeTypeFlexibilityDict,
+    NodeTypeFlexibilityParam,
+)
 from databricks.bundles.jobs._models.notebook_task import (
     NotebookTask,
     NotebookTaskDict,

{databricks_bundles-0.284.0 → databricks_bundles-0.285.0}/databricks/bundles/jobs/_models/cluster_spec.py

@@ -38,6 +38,10 @@ from databricks.bundles.jobs._models.init_script_info import (
     InitScriptInfoParam,
 )
 from databricks.bundles.jobs._models.kind import Kind, KindParam
+from databricks.bundles.jobs._models.node_type_flexibility import (
+    NodeTypeFlexibility,
+    NodeTypeFlexibilityParam,
+)
 from databricks.bundles.jobs._models.runtime_engine import (
     RuntimeEngine,
     RuntimeEngineParam,
@@ -125,6 +129,11 @@ class ClusterSpec:
     assigned.
     """
 
+    driver_node_type_flexibility: VariableOrOptional[NodeTypeFlexibility] = None
+    """
+    Flexible node type configuration for the driver node.
+    """
+
     driver_node_type_id: VariableOrOptional[str] = None
     """
     The node type of the Spark driver.
@@ -258,6 +267,11 @@ class ClusterSpec:
     `effective_spark_version` is determined by `spark_version` (DBR release), this field `use_ml_runtime`, and whether `node_type_id` is gpu node or not.
     """
 
+    worker_node_type_flexibility: VariableOrOptional[NodeTypeFlexibility] = None
+    """
+    Flexible node type configuration for worker nodes.
+    """
+
     workload_type: VariableOrOptional[WorkloadType] = None
 
     @classmethod
@@ -339,6 +353,11 @@ class ClusterSpecDict(TypedDict, total=False):
     assigned.
     """
 
+    driver_node_type_flexibility: VariableOrOptional[NodeTypeFlexibilityParam]
+    """
+    Flexible node type configuration for the driver node.
+    """
+
     driver_node_type_id: VariableOrOptional[str]
     """
     The node type of the Spark driver.
@@ -472,6 +491,11 @@ class ClusterSpecDict(TypedDict, total=False):
     `effective_spark_version` is determined by `spark_version` (DBR release), this field `use_ml_runtime`, and whether `node_type_id` is gpu node or not.
     """
 
+    worker_node_type_flexibility: VariableOrOptional[NodeTypeFlexibilityParam]
+    """
+    Flexible node type configuration for worker nodes.
+    """
+
     workload_type: VariableOrOptional[WorkloadTypeParam]
 
 

{databricks_bundles-0.284.0 → databricks_bundles-0.285.0}/databricks/bundles/jobs/_models/dashboard_task.py

@@ -4,10 +4,7 @@ from typing import TYPE_CHECKING, TypedDict
 from databricks.bundles.core._transform import _transform
 from databricks.bundles.core._transform_to_json import _transform_to_json_value
 from databricks.bundles.core._variable import VariableOrOptional
-from databricks.bundles.jobs._models.subscription import (
-    Subscription,
-    SubscriptionParam,
-)
+from databricks.bundles.jobs._models.subscription import Subscription, SubscriptionParam
 
 if TYPE_CHECKING:
     from typing_extensions import Self

{databricks_bundles-0.284.0 → databricks_bundles-0.285.0}/databricks/bundles/jobs/_models/job.py

@@ -9,10 +9,7 @@ from databricks.bundles.core._variable import (
     VariableOrList,
     VariableOrOptional,
 )
-from databricks.bundles.jobs._models.continuous import (
-    Continuous,
-    ContinuousParam,
-)
+from databricks.bundles.jobs._models.continuous import Continuous, ContinuousParam
 from databricks.bundles.jobs._models.cron_schedule import (
     CronSchedule,
     CronScheduleParam,

databricks_bundles-0.285.0/databricks/bundles/jobs/_models/node_type_flexibility.py

@@ -0,0 +1,40 @@
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, TypedDict
+
+from databricks.bundles.core._transform import _transform
+from databricks.bundles.core._transform_to_json import _transform_to_json_value
+from databricks.bundles.core._variable import VariableOrList
+
+if TYPE_CHECKING:
+    from typing_extensions import Self
+
+
+@dataclass(kw_only=True)
+class NodeTypeFlexibility:
+    """
+    Configuration for flexible node types, allowing fallback to alternate node types during cluster launch and upscale.
+    """
+
+    alternate_node_type_ids: VariableOrList[str] = field(default_factory=list)
+    """
+    A list of node type IDs to use as fallbacks when the primary node type is unavailable.
+    """
+
+    @classmethod
+    def from_dict(cls, value: "NodeTypeFlexibilityDict") -> "Self":
+        return _transform(cls, value)
+
+    def as_dict(self) -> "NodeTypeFlexibilityDict":
+        return _transform_to_json_value(self)  # type:ignore
+
+
+class NodeTypeFlexibilityDict(TypedDict, total=False):
+    """"""
+
+    alternate_node_type_ids: VariableOrList[str]
+    """
+    A list of node type IDs to use as fallbacks when the primary node type is unavailable.
+    """
+
+
+NodeTypeFlexibilityParam = NodeTypeFlexibilityDict | NodeTypeFlexibility

{databricks_bundles-0.284.0 → databricks_bundles-0.285.0}/databricks/bundles/pipelines/__init__.py

@@ -2,6 +2,9 @@ __all__ = [
     "Adlsgen2Info",
     "Adlsgen2InfoDict",
     "Adlsgen2InfoParam",
+    "AutoFullRefreshPolicy",
+    "AutoFullRefreshPolicyDict",
+    "AutoFullRefreshPolicyParam",
     "AwsAttributes",
     "AwsAttributesDict",
     "AwsAttributesParam",
@@ -81,6 +84,9 @@ __all__ = [
     "Notifications",
     "NotificationsDict",
     "NotificationsParam",
+    "OperationTimeWindow",
+    "OperationTimeWindowDict",
+    "OperationTimeWindowParam",
     "PathPattern",
     "PathPatternDict",
     "PathPatternParam",
@@ -155,6 +161,11 @@ from databricks.bundles.pipelines._models.adlsgen2_info import (
     Adlsgen2InfoDict,
     Adlsgen2InfoParam,
 )
+from databricks.bundles.pipelines._models.auto_full_refresh_policy import (
+    AutoFullRefreshPolicy,
+    AutoFullRefreshPolicyDict,
+    AutoFullRefreshPolicyParam,
+)
 from databricks.bundles.pipelines._models.aws_attributes import (
     AwsAttributes,
     AwsAttributesDict,
@@ -287,6 +298,11 @@ from databricks.bundles.pipelines._models.notifications import (
     NotificationsDict,
     NotificationsParam,
 )
+from databricks.bundles.pipelines._models.operation_time_window import (
+    OperationTimeWindow,
+    OperationTimeWindowDict,
+    OperationTimeWindowParam,
+)
 from databricks.bundles.pipelines._models.path_pattern import (
     PathPattern,
     PathPatternDict,

databricks_bundles-0.285.0/databricks/bundles/pipelines/_models/auto_full_refresh_policy.py

@@ -0,0 +1,54 @@
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, TypedDict
+
+from databricks.bundles.core._transform import _transform
+from databricks.bundles.core._transform_to_json import _transform_to_json_value
+from databricks.bundles.core._variable import VariableOr, VariableOrOptional
+
+if TYPE_CHECKING:
+    from typing_extensions import Self
+
+
+@dataclass(kw_only=True)
+class AutoFullRefreshPolicy:
+    """
+    Policy for auto full refresh.
+    """
+
+    enabled: VariableOr[bool]
+    """
+    (Required, Mutable) Whether to enable auto full refresh or not.
+    """
+
+    min_interval_hours: VariableOrOptional[int] = None
+    """
+    (Optional, Mutable) Specify the minimum interval in hours between the timestamp
+    at which a table was last full refreshed and the current timestamp for triggering auto full
+    If unspecified and autoFullRefresh is enabled then by default min_interval_hours is 24 hours.
+    """
+
+    @classmethod
+    def from_dict(cls, value: "AutoFullRefreshPolicyDict") -> "Self":
+        return _transform(cls, value)
+
+    def as_dict(self) -> "AutoFullRefreshPolicyDict":
+        return _transform_to_json_value(self)  # type:ignore
+
+
+class AutoFullRefreshPolicyDict(TypedDict, total=False):
+    """"""
+
+    enabled: VariableOr[bool]
+    """
+    (Required, Mutable) Whether to enable auto full refresh or not.
+    """
+
+    min_interval_hours: VariableOrOptional[int]
+    """
+    (Optional, Mutable) Specify the minimum interval in hours between the timestamp
+    at which a table was last full refreshed and the current timestamp for triggering auto full
+    If unspecified and autoFullRefresh is enabled then by default min_interval_hours is 24 hours.
+    """
+
+
+AutoFullRefreshPolicyParam = AutoFullRefreshPolicyDict | AutoFullRefreshPolicy

{databricks_bundles-0.284.0 → databricks_bundles-0.285.0}/databricks/bundles/pipelines/_models/day_of_week.py

@@ -4,8 +4,6 @@ from typing import Literal
 
 class DayOfWeek(Enum):
     """
-    :meta private: [EXPERIMENTAL]
-
     Days of week in which the window is allowed to happen.
     If not specified all days of the week will be used.
     """

{databricks_bundles-0.284.0 → databricks_bundles-0.285.0}/databricks/bundles/pipelines/_models/ingestion_pipeline_definition.py

@@ -8,6 +8,10 @@ from databricks.bundles.pipelines._models.ingestion_config import (
     IngestionConfig,
     IngestionConfigParam,
 )
+from databricks.bundles.pipelines._models.operation_time_window import (
+    OperationTimeWindow,
+    OperationTimeWindowParam,
+)
 from databricks.bundles.pipelines._models.source_config import (
     SourceConfig,
     SourceConfigParam,
@@ -30,6 +34,11 @@ class IngestionPipelineDefinition:
     Immutable. The Unity Catalog connection that this ingestion pipeline uses to communicate with the source. This is used with connectors for applications like Salesforce, Workday, and so on.
     """
 
+    full_refresh_window: VariableOrOptional[OperationTimeWindow] = None
+    """
+    (Optional) A window that specifies a set of time ranges for snapshot queries in CDC.
+    """
+
     ingest_from_uc_foreign_catalog: VariableOrOptional[bool] = None
     """
     :meta private: [EXPERIMENTAL]
@@ -81,6 +90,11 @@ class IngestionPipelineDefinitionDict(TypedDict, total=False):
     Immutable. The Unity Catalog connection that this ingestion pipeline uses to communicate with the source. This is used with connectors for applications like Salesforce, Workday, and so on.
     """
 
+    full_refresh_window: VariableOrOptional[OperationTimeWindowParam]
+    """
+    (Optional) A window that specifies a set of time ranges for snapshot queries in CDC.
+    """
+
     ingest_from_uc_foreign_catalog: VariableOrOptional[bool]
     """
     :meta private: [EXPERIMENTAL]

databricks_bundles-0.285.0/databricks/bundles/pipelines/_models/operation_time_window.py

@@ -0,0 +1,69 @@
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, TypedDict
+
+from databricks.bundles.core._transform import _transform
+from databricks.bundles.core._transform_to_json import _transform_to_json_value
+from databricks.bundles.core._variable import (
+    VariableOr,
+    VariableOrList,
+    VariableOrOptional,
+)
+from databricks.bundles.pipelines._models.day_of_week import DayOfWeek, DayOfWeekParam
+
+if TYPE_CHECKING:
+    from typing_extensions import Self
+
+
+@dataclass(kw_only=True)
+class OperationTimeWindow:
+    """
+    Proto representing a window
+    """
+
+    start_hour: VariableOr[int]
+    """
+    An integer between 0 and 23 denoting the start hour for the window in the 24-hour day.
+    """
+
+    days_of_week: VariableOrList[DayOfWeek] = field(default_factory=list)
+    """
+    Days of week in which the window is allowed to happen
+    If not specified all days of the week will be used.
+    """
+
+    time_zone_id: VariableOrOptional[str] = None
+    """
+    Time zone id of window. See https://docs.databricks.com/sql/language-manual/sql-ref-syntax-aux-conf-mgmt-set-timezone.html for details.
+    If not specified, UTC will be used.
+    """
+
+    @classmethod
+    def from_dict(cls, value: "OperationTimeWindowDict") -> "Self":
+        return _transform(cls, value)
+
+    def as_dict(self) -> "OperationTimeWindowDict":
+        return _transform_to_json_value(self)  # type:ignore
+
+
+class OperationTimeWindowDict(TypedDict, total=False):
+    """"""
+
+    start_hour: VariableOr[int]
+    """
+    An integer between 0 and 23 denoting the start hour for the window in the 24-hour day.
+    """
+
+    days_of_week: VariableOrList[DayOfWeekParam]
+    """
+    Days of week in which the window is allowed to happen
+    If not specified all days of the week will be used.
+    """
+
+    time_zone_id: VariableOrOptional[str]
+    """
+    Time zone id of window. See https://docs.databricks.com/sql/language-manual/sql-ref-syntax-aux-conf-mgmt-set-timezone.html for details.
+    If not specified, UTC will be used.
+    """
+
+
+OperationTimeWindowParam = OperationTimeWindowDict | OperationTimeWindow

{databricks_bundles-0.284.0 → databricks_bundles-0.285.0}/databricks/bundles/pipelines/_models/table_specific_config.py

@@ -4,6 +4,10 @@ from typing import TYPE_CHECKING, TypedDict
 from databricks.bundles.core._transform import _transform
 from databricks.bundles.core._transform_to_json import _transform_to_json_value
 from databricks.bundles.core._variable import VariableOrList, VariableOrOptional
+from databricks.bundles.pipelines._models.auto_full_refresh_policy import (
+    AutoFullRefreshPolicy,
+    AutoFullRefreshPolicyParam,
+)
 from databricks.bundles.pipelines._models.ingestion_pipeline_definition_table_specific_config_query_based_connector_config import (
     IngestionPipelineDefinitionTableSpecificConfigQueryBasedConnectorConfig,
     IngestionPipelineDefinitionTableSpecificConfigQueryBasedConnectorConfigParam,
@@ -25,6 +29,21 @@ if TYPE_CHECKING:
 class TableSpecificConfig:
     """"""
 
+    auto_full_refresh_policy: VariableOrOptional[AutoFullRefreshPolicy] = None
+    """
+    (Optional, Mutable) Policy for auto full refresh, if enabled pipeline will automatically try
+    to fix issues by doing a full refresh on the table in the retry run. auto_full_refresh_policy
+    in table configuration will override the above level auto_full_refresh_policy.
+    For example,
+    {
+    "auto_full_refresh_policy": {
+    "enabled": true,
+    "min_interval_hours": 23,
+    }
+    }
+    If unspecified, auto full refresh is disabled.
+    """
+
     exclude_columns: VariableOrList[str] = field(default_factory=list)
     """
     A list of column names to be excluded for the ingestion.
@@ -102,6 +121,21 @@ class TableSpecificConfig:
 class TableSpecificConfigDict(TypedDict, total=False):
     """"""
 
+    auto_full_refresh_policy: VariableOrOptional[AutoFullRefreshPolicyParam]
+    """
+    (Optional, Mutable) Policy for auto full refresh, if enabled pipeline will automatically try
+    to fix issues by doing a full refresh on the table in the retry run. auto_full_refresh_policy
+    in table configuration will override the above level auto_full_refresh_policy.
+    For example,
+    {
+    "auto_full_refresh_policy": {
+    "enabled": true,
+    "min_interval_hours": 23,
+    }
+    }
+    If unspecified, auto full refresh is disabled.
+    """
+
     exclude_columns: VariableOrList[str]
     """
     A list of column names to be excluded for the ingestion.

databricks_bundles-0.285.0/databricks/bundles/version.py

@@ -0,0 +1 @@
+__version__ = "0.285.0"

{databricks_bundles-0.284.0 → databricks_bundles-0.285.0}/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "databricks-bundles"
 description = "Python support for Databricks Asset Bundles"
-version = "0.284.0"
+version = "0.285.0"
 
 authors = [
     { name = "Gleb Kanterov", email = "gleb.kanterov@databricks.com" },

databricks_bundles-0.284.0/databricks/bundles/version.py

@@ -1 +0,0 @@
-__version__ = "0.284.0"