databricks-bundles 0.270.0__tar.gz → 0.272.0__tar.gz

{databricks_bundles-0.270.0 → databricks_bundles-0.272.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: databricks-bundles
-Version: 0.270.0
+Version: 0.272.0
 Summary: Python support for Databricks Asset Bundles
 Author-email: Gleb Kanterov <gleb.kanterov@databricks.com>
 Requires-Python: >=3.10
@@ -22,7 +22,7 @@ Reference documentation is available at https://databricks.github.io/cli/experim
 
 To use `databricks-bundles`, you must first:
 
-1. Install the [Databricks CLI](https://github.com/databricks/cli), version 0.270.0 or above
+1. Install the [Databricks CLI](https://github.com/databricks/cli), version 0.272.0 or above
 2. Authenticate to your Databricks workspace if you have not done so already:
 
    ```bash

{databricks_bundles-0.270.0 → databricks_bundles-0.272.0}/README.md

@@ -13,7 +13,7 @@ Reference documentation is available at https://databricks.github.io/cli/experim
 
 To use `databricks-bundles`, you must first:
 
-1. Install the [Databricks CLI](https://github.com/databricks/cli), version 0.270.0 or above
+1. Install the [Databricks CLI](https://github.com/databricks/cli), version 0.272.0 or above
 2. Authenticate to your Databricks workspace if you have not done so already:
 
    ```bash

{databricks_bundles-0.270.0 → databricks_bundles-0.272.0}/databricks/bundles/build.py

@@ -528,11 +528,23 @@ def _relativize_location(location: Location) -> Location:
 
 
 def _relativize_path(path: str) -> str:
+    """
+    Attempt to relativize an absolute path to the current working directory.
+
+    If the path is not absolute or cannot be relativized, return it as is.
+    Used to relativize paths in locations to show shorter paths in diagnostics.
+    """
+
     if not os.path.isabs(path):
         return path
 
     cwd = os.getcwd()
-    common = os.path.commonpath([os.getcwd(), path])
+
+    try:
+        common = os.path.commonpath([cwd, path])
+    except ValueError:
+        # On Windows, paths on different drives don't have a common path
+        return path
 
     if common == cwd:
         return os.path.relpath(path, cwd)

{databricks_bundles-0.270.0 → databricks_bundles-0.272.0}/databricks/bundles/jobs/__init__.py

@@ -131,6 +131,8 @@ __all__ = [
     "JobsHealthRules",
     "JobsHealthRulesDict",
     "JobsHealthRulesParam",
+    "Kind",
+    "KindParam",
     "Library",
     "LibraryDict",
     "LibraryParam",
@@ -484,6 +486,7 @@ from databricks.bundles.jobs._models.jobs_health_rules import (
     JobsHealthRulesDict,
     JobsHealthRulesParam,
 )
+from databricks.bundles.jobs._models.kind import Kind, KindParam
 from databricks.bundles.jobs._models.library import Library, LibraryDict, LibraryParam
 from databricks.bundles.jobs._models.lifecycle import (
     Lifecycle,

{databricks_bundles-0.270.0 → databricks_bundles-0.272.0}/databricks/bundles/jobs/_models/cluster_spec.py

@@ -37,6 +37,7 @@ from databricks.bundles.jobs._models.init_script_info import (
     InitScriptInfo,
     InitScriptInfoParam,
 )
+from databricks.bundles.jobs._models.kind import Kind, KindParam
 from databricks.bundles.jobs._models.runtime_engine import (
     RuntimeEngine,
     RuntimeEngineParam,
@@ -171,6 +172,8 @@ class ClusterSpec:
     When set to true, Databricks will automatically set single node related `custom_tags`, `spark_conf`, and `num_workers`
     """
 
+    kind: VariableOrOptional[Kind] = None
+
     node_type_id: VariableOrOptional[str] = None
     """
     This field encodes, through a single value, the resources available to each of
@@ -384,6 +387,8 @@ class ClusterSpecDict(TypedDict, total=False):
     When set to true, Databricks will automatically set single node related `custom_tags`, `spark_conf`, and `num_workers`
     """
 
+    kind: VariableOrOptional[KindParam]
+
     node_type_id: VariableOrOptional[str]
     """
     This field encodes, through a single value, the resources available to each of

{databricks_bundles-0.270.0 → databricks_bundles-0.272.0}/databricks/bundles/jobs/_models/environment.py

@@ -33,7 +33,7 @@ class Environment:
     The version is a string, consisting of an integer.
     """
 
-    jar_dependencies: VariableOrList[str] = field(default_factory=list)
+    java_dependencies: VariableOrList[str] = field(default_factory=list)
     """
     :meta private: [EXPERIMENTAL]
     
@@ -68,7 +68,7 @@ class EnvironmentDict(TypedDict, total=False):
     The version is a string, consisting of an integer.
     """
 
-    jar_dependencies: VariableOrList[str]
+    java_dependencies: VariableOrList[str]
     """
     :meta private: [EXPERIMENTAL]
     

databricks_bundles-0.272.0/databricks/bundles/jobs/_models/kind.py

@@ -0,0 +1,9 @@
+from enum import Enum
+from typing import Literal
+
+
+class Kind(Enum):
+    CLASSIC_PREVIEW = "CLASSIC_PREVIEW"
+
+
+KindParam = Literal["CLASSIC_PREVIEW"] | Kind

{databricks_bundles-0.270.0 → databricks_bundles-0.272.0}/databricks/bundles/jobs/_models/spark_submit_task.py

@@ -11,7 +11,9 @@ if TYPE_CHECKING:
 
 @dataclass(kw_only=True)
 class SparkSubmitTask:
-    """"""
+    """
+    [DEPRECATED]
+    """
 
     parameters: VariableOrList[str] = field(default_factory=list)
     """

{databricks_bundles-0.270.0 → databricks_bundles-0.272.0}/databricks/bundles/jobs/_models/task.py

@@ -104,7 +104,7 @@ class Task:
 
     clean_rooms_notebook_task: VariableOrOptional[CleanRoomsNotebookTask] = None
     """
-    The task runs a [clean rooms](https://docs.databricks.com/en/clean-rooms/index.html) notebook
+    The task runs a [clean rooms](https://docs.databricks.com/clean-rooms/index.html) notebook
     when the `clean_rooms_notebook_task` field is present.
     """
 
@@ -145,6 +145,13 @@ class Task:
     An option to disable auto optimization in serverless
     """
 
+    disabled: VariableOrOptional[bool] = None
+    """
+    :meta private: [EXPERIMENTAL]
+    
+    An optional flag to disable the task. If set to true, the task will not run even if it is part of a job.
+    """
+
     email_notifications: VariableOrOptional[TaskEmailNotifications] = None
     """
     An optional set of email addresses that is notified when runs of this task begin or complete as well as when this task is deleted. The default behavior is to not send any emails.
@@ -261,15 +268,7 @@ class Task:
 
     spark_submit_task: VariableOrOptional[SparkSubmitTask] = None
     """
-    (Legacy) The task runs the spark-submit script when the `spark_submit_task` field is present. This task can run only on new clusters and is not compatible with serverless compute.
-    
-    In the `new_cluster` specification, `libraries` and `spark_conf` are not supported. Instead, use `--jars` and `--py-files` to add Java and Python libraries and `--conf` to set the Spark configurations.
-    
-    `master`, `deploy-mode`, and `executor-cores` are automatically configured by Databricks; you _cannot_ specify them in parameters.
-    
-    By default, the Spark submit job uses all available memory (excluding reserved memory for Databricks services). You can set `--driver-memory`, and `--executor-memory` to a smaller value to leave some room for off-heap usage.
-    
-    The `--jars`, `--py-files`, `--files` arguments support DBFS and S3 paths.
+    [DEPRECATED] (Legacy) The task runs the spark-submit script when the spark_submit_task field is present. Databricks recommends using the spark_jar_task instead; see [Spark Submit task for jobs](/jobs/spark-submit).
     """
 
     sql_task: VariableOrOptional[SqlTask] = None
@@ -307,7 +306,7 @@ class TaskDict(TypedDict, total=False):
 
     clean_rooms_notebook_task: VariableOrOptional[CleanRoomsNotebookTaskParam]
     """
-    The task runs a [clean rooms](https://docs.databricks.com/en/clean-rooms/index.html) notebook
+    The task runs a [clean rooms](https://docs.databricks.com/clean-rooms/index.html) notebook
     when the `clean_rooms_notebook_task` field is present.
     """
 
@@ -348,6 +347,13 @@ class TaskDict(TypedDict, total=False):
     An option to disable auto optimization in serverless
     """
 
+    disabled: VariableOrOptional[bool]
+    """
+    :meta private: [EXPERIMENTAL]
+    
+    An optional flag to disable the task. If set to true, the task will not run even if it is part of a job.
+    """
+
     email_notifications: VariableOrOptional[TaskEmailNotificationsParam]
     """
     An optional set of email addresses that is notified when runs of this task begin or complete as well as when this task is deleted. The default behavior is to not send any emails.
@@ -464,15 +470,7 @@ class TaskDict(TypedDict, total=False):
 
     spark_submit_task: VariableOrOptional[SparkSubmitTaskParam]
     """
-    (Legacy) The task runs the spark-submit script when the `spark_submit_task` field is present. This task can run only on new clusters and is not compatible with serverless compute.
-    
-    In the `new_cluster` specification, `libraries` and `spark_conf` are not supported. Instead, use `--jars` and `--py-files` to add Java and Python libraries and `--conf` to set the Spark configurations.
-    
-    `master`, `deploy-mode`, and `executor-cores` are automatically configured by Databricks; you _cannot_ specify them in parameters.
-    
-    By default, the Spark submit job uses all available memory (excluding reserved memory for Databricks services). You can set `--driver-memory`, and `--executor-memory` to a smaller value to leave some room for off-heap usage.
-    
-    The `--jars`, `--py-files`, `--files` arguments support DBFS and S3 paths.
+    [DEPRECATED] (Legacy) The task runs the spark-submit script when the spark_submit_task field is present. Databricks recommends using the spark_jar_task instead; see [Spark Submit task for jobs](/jobs/spark-submit).
     """
 
     sql_task: VariableOrOptional[SqlTaskParam]

{databricks_bundles-0.270.0 → databricks_bundles-0.272.0}/databricks/bundles/pipelines/__init__.py

@@ -51,6 +51,12 @@ __all__ = [
     "IngestionPipelineDefinitionTableSpecificConfigQueryBasedConnectorConfig",
     "IngestionPipelineDefinitionTableSpecificConfigQueryBasedConnectorConfigDict",
     "IngestionPipelineDefinitionTableSpecificConfigQueryBasedConnectorConfigParam",
+    "IngestionPipelineDefinitionWorkdayReportParameters",
+    "IngestionPipelineDefinitionWorkdayReportParametersDict",
+    "IngestionPipelineDefinitionWorkdayReportParametersParam",
+    "IngestionPipelineDefinitionWorkdayReportParametersQueryKeyValue",
+    "IngestionPipelineDefinitionWorkdayReportParametersQueryKeyValueDict",
+    "IngestionPipelineDefinitionWorkdayReportParametersQueryKeyValueParam",
     "IngestionSourceType",
     "IngestionSourceTypeParam",
     "InitScriptInfo",
@@ -230,6 +236,16 @@ from databricks.bundles.pipelines._models.ingestion_pipeline_definition_table_sp
     IngestionPipelineDefinitionTableSpecificConfigQueryBasedConnectorConfigDict,
     IngestionPipelineDefinitionTableSpecificConfigQueryBasedConnectorConfigParam,
 )
+from databricks.bundles.pipelines._models.ingestion_pipeline_definition_workday_report_parameters import (
+    IngestionPipelineDefinitionWorkdayReportParameters,
+    IngestionPipelineDefinitionWorkdayReportParametersDict,
+    IngestionPipelineDefinitionWorkdayReportParametersParam,
+)
+from databricks.bundles.pipelines._models.ingestion_pipeline_definition_workday_report_parameters_query_key_value import (
+    IngestionPipelineDefinitionWorkdayReportParametersQueryKeyValue,
+    IngestionPipelineDefinitionWorkdayReportParametersQueryKeyValueDict,
+    IngestionPipelineDefinitionWorkdayReportParametersQueryKeyValueParam,
+)
 from databricks.bundles.pipelines._models.ingestion_source_type import (
     IngestionSourceType,
     IngestionSourceTypeParam,

{databricks_bundles-0.270.0 → databricks_bundles-0.272.0}/databricks/bundles/pipelines/_models/ingestion_pipeline_definition.py

@@ -39,6 +39,15 @@ class IngestionPipelineDefinition:
     Immutable. Identifier for the gateway that is used by this ingestion pipeline to communicate with the source database. This is used with connectors to databases like SQL Server.
     """
 
+    netsuite_jar_path: VariableOrOptional[str] = None
+    """
+    :meta private: [EXPERIMENTAL]
+    
+    Netsuite only configuration. When the field is set for a netsuite connector,
+    the jar stored in the field will be validated and added to the classpath of
+    pipeline's cluster.
+    """
+
     objects: VariableOrList[IngestionConfig] = field(default_factory=list)
     """
     Required. Settings specifying tables to replicate and the destination for the replicated tables.
@@ -84,6 +93,15 @@ class IngestionPipelineDefinitionDict(TypedDict, total=False):
     Immutable. Identifier for the gateway that is used by this ingestion pipeline to communicate with the source database. This is used with connectors to databases like SQL Server.
     """
 
+    netsuite_jar_path: VariableOrOptional[str]
+    """
+    :meta private: [EXPERIMENTAL]
+    
+    Netsuite only configuration. When the field is set for a netsuite connector,
+    the jar stored in the field will be validated and added to the classpath of
+    pipeline's cluster.
+    """
+
     objects: VariableOrList[IngestionConfigParam]
     """
     Required. Settings specifying tables to replicate and the destination for the replicated tables.

databricks_bundles-0.272.0/databricks/bundles/pipelines/_models/ingestion_pipeline_definition_workday_report_parameters.py

@@ -0,0 +1,95 @@
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, TypedDict
+
+from databricks.bundles.core._transform import _transform
+from databricks.bundles.core._transform_to_json import _transform_to_json_value
+from databricks.bundles.core._variable import (
+    VariableOrDict,
+    VariableOrList,
+    VariableOrOptional,
+)
+from databricks.bundles.pipelines._models.ingestion_pipeline_definition_workday_report_parameters_query_key_value import (
+    IngestionPipelineDefinitionWorkdayReportParametersQueryKeyValue,
+    IngestionPipelineDefinitionWorkdayReportParametersQueryKeyValueParam,
+)
+
+if TYPE_CHECKING:
+    from typing_extensions import Self
+
+
+@dataclass(kw_only=True)
+class IngestionPipelineDefinitionWorkdayReportParameters:
+    """
+    :meta private: [EXPERIMENTAL]
+    """
+
+    incremental: VariableOrOptional[bool] = None
+    """
+    [DEPRECATED] (Optional) Marks the report as incremental.
+    This field is deprecated and should not be used. Use `parameters` instead. The incremental behavior is now
+    controlled by the `parameters` field.
+    """
+
+    parameters: VariableOrDict[str] = field(default_factory=dict)
+    """
+    Parameters for the Workday report. Each key represents the parameter name (e.g., "start_date", "end_date"),
+    and the corresponding value is a SQL-like expression used to compute the parameter value at runtime.
+    Example:
+    {
+    "start_date": "{ coalesce(current_offset(), date(\"2025-02-01\")) }",
+    "end_date": "{ current_date() - INTERVAL 1 DAY }"
+    }
+    """
+
+    report_parameters: VariableOrList[
+        IngestionPipelineDefinitionWorkdayReportParametersQueryKeyValue
+    ] = field(default_factory=list)
+    """
+    [DEPRECATED] (Optional) Additional custom parameters for Workday Report
+    This field is deprecated and should not be used. Use `parameters` instead.
+    """
+
+    @classmethod
+    def from_dict(
+        cls, value: "IngestionPipelineDefinitionWorkdayReportParametersDict"
+    ) -> "Self":
+        return _transform(cls, value)
+
+    def as_dict(self) -> "IngestionPipelineDefinitionWorkdayReportParametersDict":
+        return _transform_to_json_value(self)  # type:ignore
+
+
+class IngestionPipelineDefinitionWorkdayReportParametersDict(TypedDict, total=False):
+    """"""
+
+    incremental: VariableOrOptional[bool]
+    """
+    [DEPRECATED] (Optional) Marks the report as incremental.
+    This field is deprecated and should not be used. Use `parameters` instead. The incremental behavior is now
+    controlled by the `parameters` field.
+    """
+
+    parameters: VariableOrDict[str]
+    """
+    Parameters for the Workday report. Each key represents the parameter name (e.g., "start_date", "end_date"),
+    and the corresponding value is a SQL-like expression used to compute the parameter value at runtime.
+    Example:
+    {
+    "start_date": "{ coalesce(current_offset(), date(\"2025-02-01\")) }",
+    "end_date": "{ current_date() - INTERVAL 1 DAY }"
+    }
+    """
+
+    report_parameters: VariableOrList[
+        IngestionPipelineDefinitionWorkdayReportParametersQueryKeyValueParam
+    ]
+    """
+    [DEPRECATED] (Optional) Additional custom parameters for Workday Report
+    This field is deprecated and should not be used. Use `parameters` instead.
+    """
+
+
+IngestionPipelineDefinitionWorkdayReportParametersParam = (
+    IngestionPipelineDefinitionWorkdayReportParametersDict
+    | IngestionPipelineDefinitionWorkdayReportParameters
+)

databricks_bundles-0.272.0/databricks/bundles/pipelines/_models/ingestion_pipeline_definition_workday_report_parameters_query_key_value.py

@@ -0,0 +1,70 @@
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, TypedDict
+
+from databricks.bundles.core._transform import _transform
+from databricks.bundles.core._transform_to_json import _transform_to_json_value
+from databricks.bundles.core._variable import VariableOrOptional
+
+if TYPE_CHECKING:
+    from typing_extensions import Self
+
+
+@dataclass(kw_only=True)
+class IngestionPipelineDefinitionWorkdayReportParametersQueryKeyValue:
+    """
+    :meta private: [EXPERIMENTAL]
+
+    [DEPRECATED]
+    """
+
+    key: VariableOrOptional[str] = None
+    """
+    Key for the report parameter, can be a column name or other metadata
+    """
+
+    value: VariableOrOptional[str] = None
+    """
+    Value for the report parameter.
+    Possible values it can take are these sql functions:
+    1. coalesce(current_offset(), date("YYYY-MM-DD")) -> if current_offset() is null, then the passed date, else current_offset()
+    2. current_date()
+    3. date_sub(current_date(), x) -> subtract x (some non-negative integer) days from current date
+    """
+
+    @classmethod
+    def from_dict(
+        cls,
+        value: "IngestionPipelineDefinitionWorkdayReportParametersQueryKeyValueDict",
+    ) -> "Self":
+        return _transform(cls, value)
+
+    def as_dict(
+        self,
+    ) -> "IngestionPipelineDefinitionWorkdayReportParametersQueryKeyValueDict":
+        return _transform_to_json_value(self)  # type:ignore
+
+
+class IngestionPipelineDefinitionWorkdayReportParametersQueryKeyValueDict(
+    TypedDict, total=False
+):
+    """"""
+
+    key: VariableOrOptional[str]
+    """
+    Key for the report parameter, can be a column name or other metadata
+    """
+
+    value: VariableOrOptional[str]
+    """
+    Value for the report parameter.
+    Possible values it can take are these sql functions:
+    1. coalesce(current_offset(), date("YYYY-MM-DD")) -> if current_offset() is null, then the passed date, else current_offset()
+    2. current_date()
+    3. date_sub(current_date(), x) -> subtract x (some non-negative integer) days from current date
+    """
+
+
+IngestionPipelineDefinitionWorkdayReportParametersQueryKeyValueParam = (
+    IngestionPipelineDefinitionWorkdayReportParametersQueryKeyValueDict
+    | IngestionPipelineDefinitionWorkdayReportParametersQueryKeyValue
+)

{databricks_bundles-0.270.0 → databricks_bundles-0.272.0}/databricks/bundles/pipelines/_models/ingestion_source_type.py

@@ -21,6 +21,7 @@ class IngestionSourceType(Enum):
     DYNAMICS365 = "DYNAMICS365"
     CONFLUENCE = "CONFLUENCE"
     META_MARKETING = "META_MARKETING"
+    FOREIGN_CATALOG = "FOREIGN_CATALOG"
 
 
 IngestionSourceTypeParam = (
@@ -43,6 +44,7 @@ IngestionSourceTypeParam = (
         "DYNAMICS365",
         "CONFLUENCE",
         "META_MARKETING",
+        "FOREIGN_CATALOG",
     ]
     | IngestionSourceType
 )

{databricks_bundles-0.270.0 → databricks_bundles-0.272.0}/databricks/bundles/pipelines/_models/pipeline.py

@@ -184,9 +184,6 @@ class Pipeline(Resource):
     """
 
     run_as: VariableOrOptional[RunAs] = None
-    """
-    :meta private: [EXPERIMENTAL]
-    """
 
     schema: VariableOrOptional[str] = None
     """
@@ -347,9 +344,6 @@ class PipelineDict(TypedDict, total=False):
     """
 
     run_as: VariableOrOptional[RunAsParam]
-    """
-    :meta private: [EXPERIMENTAL]
-    """
 
     schema: VariableOrOptional[str]
     """

{databricks_bundles-0.270.0 → databricks_bundles-0.272.0}/databricks/bundles/pipelines/_models/run_as.py

@@ -12,8 +12,6 @@ if TYPE_CHECKING:
 @dataclass(kw_only=True)
 class RunAs:
     """
-    :meta private: [EXPERIMENTAL]
-
     Write-only setting, available only in Create/Update calls. Specifies the user or service principal that the pipeline runs as. If not specified, the pipeline runs as the user who created the pipeline.
 
     Only `user_name` or `service_principal_name` can be specified. If both are specified, an error is thrown.

{databricks_bundles-0.270.0 → databricks_bundles-0.272.0}/databricks/bundles/pipelines/_models/table_specific_config.py

@@ -8,6 +8,10 @@ from databricks.bundles.pipelines._models.ingestion_pipeline_definition_table_sp
     IngestionPipelineDefinitionTableSpecificConfigQueryBasedConnectorConfig,
     IngestionPipelineDefinitionTableSpecificConfigQueryBasedConnectorConfigParam,
 )
+from databricks.bundles.pipelines._models.ingestion_pipeline_definition_workday_report_parameters import (
+    IngestionPipelineDefinitionWorkdayReportParameters,
+    IngestionPipelineDefinitionWorkdayReportParametersParam,
+)
 from databricks.bundles.pipelines._models.table_specific_config_scd_type import (
     TableSpecificConfigScdType,
     TableSpecificConfigScdTypeParam,
@@ -71,6 +75,15 @@ class TableSpecificConfig:
     The column names specifying the logical order of events in the source data. Delta Live Tables uses this sequencing to handle change events that arrive out of order.
     """
 
+    workday_report_parameters: VariableOrOptional[
+        IngestionPipelineDefinitionWorkdayReportParameters
+    ] = None
+    """
+    :meta private: [EXPERIMENTAL]
+    
+    (Optional) Additional custom parameters for Workday Report
+    """
+
     @classmethod
     def from_dict(cls, value: "TableSpecificConfigDict") -> "Self":
         return _transform(cls, value)
@@ -132,5 +145,14 @@ class TableSpecificConfigDict(TypedDict, total=False):
     The column names specifying the logical order of events in the source data. Delta Live Tables uses this sequencing to handle change events that arrive out of order.
     """
 
+    workday_report_parameters: VariableOrOptional[
+        IngestionPipelineDefinitionWorkdayReportParametersParam
+    ]
+    """
+    :meta private: [EXPERIMENTAL]
+    
+    (Optional) Additional custom parameters for Workday Report
+    """
+
 
 TableSpecificConfigParam = TableSpecificConfigDict | TableSpecificConfig

databricks_bundles-0.272.0/databricks/bundles/version.py

@@ -0,0 +1 @@
+__version__ = "0.272.0"

{databricks_bundles-0.270.0 → databricks_bundles-0.272.0}/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "databricks-bundles"
 description = "Python support for Databricks Asset Bundles"
-version = "0.270.0"
+version = "0.272.0"
 
 authors = [
     { name = "Gleb Kanterov", email = "gleb.kanterov@databricks.com" },

databricks_bundles-0.270.0/databricks/bundles/version.py

@@ -1 +0,0 @@
-__version__ = "0.270.0"