databricks-bundles 0.265.0__py3-none-any.whl → 0.267.0__py3-none-any.whl

databricks/bundles/core/__init__.py

@@ -13,12 +13,13 @@ __all__ = [
     "VariableOrList",
     "VariableOrOptional",
     "job_mutator",
-    "pipeline_mutator",
     "load_resources_from_current_package_module",
     "load_resources_from_module",
     "load_resources_from_modules",
     "load_resources_from_package_module",
+    "pipeline_mutator",
     "variables",
+    "volume_mutator",
 ]
 
 from databricks.bundles.core._bundle import Bundle
@@ -39,6 +40,7 @@ from databricks.bundles.core._resource_mutator import (
     ResourceMutator,
     job_mutator,
     pipeline_mutator,
+    volume_mutator,
 )
 from databricks.bundles.core._resources import Resources
 from databricks.bundles.core._variable import (

databricks/bundles/core/_load.py

@@ -14,9 +14,9 @@ from databricks.bundles.core._resources import Resources
 
 __all__ = [
     "load_resources_from_current_package_module",
-    "load_resources_from_package_module",
-    "load_resources_from_modules",
     "load_resources_from_module",
+    "load_resources_from_modules",
+    "load_resources_from_package_module",
 ]
 
 """

databricks/bundles/core/_resource_mutator.py

@@ -8,6 +8,7 @@ from databricks.bundles.core._resource import Resource
 if TYPE_CHECKING:
     from databricks.bundles.jobs._models.job import Job
     from databricks.bundles.pipelines._models.pipeline import Pipeline
+    from databricks.bundles.volumes._models.volume import Volume
 
 _T = TypeVar("_T", bound=Resource)
 
@@ -127,3 +128,35 @@ def pipeline_mutator(function: Callable) -> ResourceMutator["Pipeline"]:
     from databricks.bundles.pipelines._models.pipeline import Pipeline
 
     return ResourceMutator(resource_type=Pipeline, function=function)
+
+
+@overload
+def volume_mutator(
+    function: Callable[[Bundle, "Volume"], "Volume"],
+) -> ResourceMutator["Volume"]: ...
+
+
+@overload
+def volume_mutator(
+    function: Callable[["Volume"], "Volume"],
+) -> ResourceMutator["Volume"]: ...
+
+
+def volume_mutator(function: Callable) -> ResourceMutator["Volume"]:
+    """
+    Decorator for defining a volume mutator. Function should return a new instance of the volume with the desired changes,
+    instead of mutating the input volume.
+
+    Example:
+
+    .. code-block:: python
+
+        @volume_mutator
+        def my_volume_mutator(bundle: Bundle, volume: Volume) -> Volume:
+            return replace(volume, name="my_volume")
+
+    :param function: Function that mutates a volume.
+    """
+    from databricks.bundles.volumes._models.volume import Volume
+
+    return ResourceMutator(resource_type=Volume, function=function)

databricks/bundles/core/_resource_type.py

@@ -2,6 +2,7 @@ from dataclasses import dataclass
 from typing import Type
 
 from databricks.bundles.core._resource import Resource
+from databricks.bundles.volumes._models.volume import Volume
 
 
 @dataclass(kw_only=True, frozen=True)
@@ -45,4 +46,9 @@ class _ResourceType:
                 plural_name="pipelines",
                 singular_name="pipeline",
             ),
+            _ResourceType(
+                resource_type=Volume,
+                plural_name="volumes",
+                singular_name="volume",
+            ),
         )

databricks/bundles/core/_resources.py

@@ -8,6 +8,7 @@ from databricks.bundles.core._transform import _transform
 if TYPE_CHECKING:
     from databricks.bundles.jobs._models.job import Job, JobParam
     from databricks.bundles.pipelines._models.pipeline import Pipeline, PipelineParam
+    from databricks.bundles.volumes._models.volume import Volume, VolumeParam
 
 __all__ = ["Resources"]
 
@@ -57,6 +58,7 @@ class Resources:
     def __init__(self):
         self._jobs = dict[str, "Job"]()
         self._pipelines = dict[str, "Pipeline"]()
+        self._volumes = dict[str, "Volume"]()
         self._locations = dict[tuple[str, ...], Location]()
         self._diagnostics = Diagnostics()
 
@@ -68,6 +70,10 @@ class Resources:
     def pipelines(self) -> dict[str, "Pipeline"]:
         return self._pipelines
 
+    @property
+    def volumes(self) -> dict[str, "Volume"]:
+        return self._volumes
+
     @property
     def diagnostics(self) -> Diagnostics:
         """
@@ -93,6 +99,7 @@ class Resources:
 
         from databricks.bundles.jobs import Job
         from databricks.bundles.pipelines import Pipeline
+        from databricks.bundles.volumes import Volume
 
         location = location or Location.from_stack_frame(depth=1)
 
@@ -101,6 +108,8 @@ class Resources:
                 self.add_job(resource_name, resource, location=location)
             case Pipeline():
                 self.add_pipeline(resource_name, resource, location=location)
+            case Volume():
+                self.add_volume(resource_name, resource, location=location)
             case _:
                 raise ValueError(f"Unsupported resource type: {type(resource)}")
 
@@ -168,6 +177,38 @@ class Resources:
 
             self._pipelines[resource_name] = pipeline
 
+    def add_volume(
+        self,
+        resource_name: str,
+        volume: "VolumeParam",
+        *,
+        location: Optional[Location] = None,
+    ) -> None:
+        """
+        Adds a volume to the collection of resources. Resource name must be unique across all volumes.
+
+        :param resource_name: unique identifier for the volume
+        :param volume: the volume to add, can be Volume or dict
+        :param location: optional location of the volume in the source code
+        """
+        from databricks.bundles.volumes import Volume
+
+        volume = _transform(Volume, volume)
+        path = ("resources", "volumes", resource_name)
+        location = location or Location.from_stack_frame(depth=1)
+
+        if self._volumes.get(resource_name):
+            self.add_diagnostic_error(
+                msg=f"Duplicate resource name '{resource_name}' for a volume. Resource names must be unique.",
+                location=location,
+                path=path,
+            )
+        else:
+            if location:
+                self.add_location(path, location)
+
+            self._volumes[resource_name] = volume
+
     def add_location(self, path: tuple[str, ...], location: Location) -> None:
         """
         Associate source code location with a path in the bundle configuration.
@@ -244,6 +285,9 @@ class Resources:
         for name, pipeline in other.pipelines.items():
             self.add_pipeline(name, pipeline)
 
+        for name, volume in other.volumes.items():
+            self.add_volume(name, volume)
+
         for path, location in other._locations.items():
             self.add_location(path, location)
 

databricks/bundles/core/_variable.py

@@ -11,9 +11,9 @@ from typing import (
 __all__ = [
     "Variable",
     "VariableOr",
-    "VariableOrOptional",
     "VariableOrDict",
     "VariableOrList",
+    "VariableOrOptional",
     "variables",
 ]
 

databricks/bundles/jobs/__init__.py

@@ -244,6 +244,8 @@ __all__ = [
     "TaskNotificationSettingsDict",
     "TaskNotificationSettingsParam",
     "TaskParam",
+    "TaskRetryMode",
+    "TaskRetryModeParam",
     "TriggerSettings",
     "TriggerSettingsDict",
     "TriggerSettingsParam",
@@ -648,6 +650,10 @@ from databricks.bundles.jobs._models.task_notification_settings import (
     TaskNotificationSettingsDict,
     TaskNotificationSettingsParam,
 )
+from databricks.bundles.jobs._models.task_retry_mode import (
+    TaskRetryMode,
+    TaskRetryModeParam,
+)
 from databricks.bundles.jobs._models.trigger_settings import (
     TriggerSettings,
     TriggerSettingsDict,

databricks/bundles/jobs/_models/continuous.py

@@ -5,6 +5,10 @@ from databricks.bundles.core._transform import _transform
 from databricks.bundles.core._transform_to_json import _transform_to_json_value
 from databricks.bundles.core._variable import VariableOrOptional
 from databricks.bundles.jobs._models.pause_status import PauseStatus, PauseStatusParam
+from databricks.bundles.jobs._models.task_retry_mode import (
+    TaskRetryMode,
+    TaskRetryModeParam,
+)
 
 if TYPE_CHECKING:
     from typing_extensions import Self
@@ -19,6 +23,11 @@ class Continuous:
     Indicate whether the continuous execution of the job is paused or not. Defaults to UNPAUSED.
     """
 
+    task_retry_mode: VariableOrOptional[TaskRetryMode] = None
+    """
+    Indicate whether the continuous job is applying task level retries or not. Defaults to NEVER.
+    """
+
     @classmethod
     def from_dict(cls, value: "ContinuousDict") -> "Self":
         return _transform(cls, value)
@@ -35,5 +44,10 @@ class ContinuousDict(TypedDict, total=False):
     Indicate whether the continuous execution of the job is paused or not. Defaults to UNPAUSED.
     """
 
+    task_retry_mode: VariableOrOptional[TaskRetryModeParam]
+    """
+    Indicate whether the continuous job is applying task level retries or not. Defaults to NEVER.
+    """
+
 
 ContinuousParam = ContinuousDict | Continuous

databricks/bundles/jobs/_models/environment.py

@@ -16,6 +16,11 @@ class Environment:
     In this minimal environment spec, only pip dependencies are supported.
     """
 
+    client: VariableOrOptional[str] = None
+    """
+    [DEPRECATED] Use `environment_version` instead.
+    """
+
     dependencies: VariableOrList[str] = field(default_factory=list)
     """
     List of pip dependencies, as supported by the version of pip in this environment.
@@ -46,6 +51,11 @@ class Environment:
 class EnvironmentDict(TypedDict, total=False):
     """"""
 
+    client: VariableOrOptional[str]
+    """
+    [DEPRECATED] Use `environment_version` instead.
+    """
+
     dependencies: VariableOrList[str]
     """
     List of pip dependencies, as supported by the version of pip in this environment.

databricks/bundles/jobs/_models/gcp_attributes.py

@@ -53,6 +53,13 @@ class GcpAttributes:
     for the supported number of local SSDs for each instance type.
     """
 
+    use_preemptible_executors: VariableOrOptional[bool] = None
+    """
+    [DEPRECATED] This field determines whether the spark executors will be scheduled to run on preemptible
+    VMs (when set to true) versus standard compute engine VMs (when set to false; default).
+    Note: Soon to be deprecated, use the 'availability' field instead.
+    """
+
     zone_id: VariableOrOptional[str] = None
     """
     Identifier for the availability zone in which the cluster resides.
@@ -108,6 +115,13 @@ class GcpAttributesDict(TypedDict, total=False):
     for the supported number of local SSDs for each instance type.
     """
 
+    use_preemptible_executors: VariableOrOptional[bool]
+    """
+    [DEPRECATED] This field determines whether the spark executors will be scheduled to run on preemptible
+    VMs (when set to true) versus standard compute engine VMs (when set to false; default).
+    Note: Soon to be deprecated, use the 'availability' field instead.
+    """
+
     zone_id: VariableOrOptional[str]
     """
     Identifier for the availability zone in which the cluster resides.

databricks/bundles/jobs/_models/init_script_info.py

@@ -8,6 +8,10 @@ from databricks.bundles.jobs._models.adlsgen2_info import (
     Adlsgen2Info,
     Adlsgen2InfoParam,
 )
+from databricks.bundles.jobs._models.dbfs_storage_info import (
+    DbfsStorageInfo,
+    DbfsStorageInfoParam,
+)
 from databricks.bundles.jobs._models.gcs_storage_info import (
     GcsStorageInfo,
     GcsStorageInfoParam,
@@ -45,6 +49,12 @@ class InitScriptInfo:
     Contains the Azure Data Lake Storage destination path
     """
 
+    dbfs: VariableOrOptional[DbfsStorageInfo] = None
+    """
+    [DEPRECATED] destination needs to be provided. e.g.
+    `{ "dbfs": { "destination" : "dbfs:/home/cluster_log" } }`
+    """
+
     file: VariableOrOptional[LocalFileInfo] = None
     """
     destination needs to be provided, e.g.
@@ -93,6 +103,12 @@ class InitScriptInfoDict(TypedDict, total=False):
     Contains the Azure Data Lake Storage destination path
     """
 
+    dbfs: VariableOrOptional[DbfsStorageInfoParam]
+    """
+    [DEPRECATED] destination needs to be provided. e.g.
+    `{ "dbfs": { "destination" : "dbfs:/home/cluster_log" } }`
+    """
+
     file: VariableOrOptional[LocalFileInfoParam]
     """
     destination needs to be provided, e.g.

databricks/bundles/jobs/_models/job_email_notifications.py

@@ -3,7 +3,7 @@ from typing import TYPE_CHECKING, TypedDict
 
 from databricks.bundles.core._transform import _transform
 from databricks.bundles.core._transform_to_json import _transform_to_json_value
-from databricks.bundles.core._variable import VariableOrList
+from databricks.bundles.core._variable import VariableOrList, VariableOrOptional
 
 if TYPE_CHECKING:
     from typing_extensions import Self
@@ -13,6 +13,12 @@ if TYPE_CHECKING:
 class JobEmailNotifications:
     """"""
 
+    no_alert_for_skipped_runs: VariableOrOptional[bool] = None
+    """
+    [DEPRECATED] If true, do not send email to recipients specified in `on_failure` if the run is skipped.
+    This field is `deprecated`. Please use the `notification_settings.no_alert_for_skipped_runs` field.
+    """
+
     on_duration_warning_threshold_exceeded: VariableOrList[str] = field(
         default_factory=list
     )
@@ -53,6 +59,12 @@ class JobEmailNotifications:
 class JobEmailNotificationsDict(TypedDict, total=False):
     """"""
 
+    no_alert_for_skipped_runs: VariableOrOptional[bool]
+    """
+    [DEPRECATED] If true, do not send email to recipients specified in `on_failure` if the run is skipped.
+    This field is `deprecated`. Please use the `notification_settings.no_alert_for_skipped_runs` field.
+    """
+
     on_duration_warning_threshold_exceeded: VariableOrList[str]
     """
     A list of email addresses to be notified when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. If no rule for the `RUN_DURATION_SECONDS` metric is specified in the `health` field for the job, notifications are not sent.

databricks/bundles/jobs/_models/library.py

@@ -30,6 +30,11 @@ class Library:
     Specification of a CRAN library to be installed as part of the library
     """
 
+    egg: VariableOrOptional[str] = None
+    """
+    [DEPRECATED] Deprecated. URI of the egg library to install. Installing Python egg files is deprecated and is not supported in Databricks Runtime 14.0 and above.
+    """
+
     jar: VariableOrOptional[str] = None
     """
     URI of the JAR library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.
@@ -82,6 +87,11 @@ class LibraryDict(TypedDict, total=False):
     Specification of a CRAN library to be installed as part of the library
     """
 
+    egg: VariableOrOptional[str]
+    """
+    [DEPRECATED] Deprecated. URI of the egg library to install. Installing Python egg files is deprecated and is not supported in Databricks Runtime 14.0 and above.
+    """
+
     jar: VariableOrOptional[str]
     """
     URI of the JAR library to install. Supported URIs include Workspace paths, Unity Catalog Volumes paths, and S3 URIs.

databricks/bundles/jobs/_models/spark_jar_task.py

@@ -3,7 +3,11 @@ from typing import TYPE_CHECKING, TypedDict
 
 from databricks.bundles.core._transform import _transform
 from databricks.bundles.core._transform_to_json import _transform_to_json_value
-from databricks.bundles.core._variable import VariableOr, VariableOrList
+from databricks.bundles.core._variable import (
+    VariableOr,
+    VariableOrList,
+    VariableOrOptional,
+)
 
 if TYPE_CHECKING:
     from typing_extensions import Self
@@ -20,6 +24,11 @@ class SparkJarTask:
     The code must use `SparkContext.getOrCreate` to obtain a Spark context; otherwise, runs of the job fail.
     """
 
+    jar_uri: VariableOrOptional[str] = None
+    """
+    [DEPRECATED] Deprecated since 04/2016. Provide a `jar` through the `libraries` field instead. For an example, see :method:jobs/create.
+    """
+
     parameters: VariableOrList[str] = field(default_factory=list)
     """
     Parameters passed to the main method.
@@ -27,6 +36,11 @@ class SparkJarTask:
     Use [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.
     """
 
+    run_as_repl: VariableOrOptional[bool] = None
+    """
+    [DEPRECATED] Deprecated. A value of `false` is no longer supported.
+    """
+
     @classmethod
     def from_dict(cls, value: "SparkJarTaskDict") -> "Self":
         return _transform(cls, value)
@@ -45,6 +59,11 @@ class SparkJarTaskDict(TypedDict, total=False):
     The code must use `SparkContext.getOrCreate` to obtain a Spark context; otherwise, runs of the job fail.
     """
 
+    jar_uri: VariableOrOptional[str]
+    """
+    [DEPRECATED] Deprecated since 04/2016. Provide a `jar` through the `libraries` field instead. For an example, see :method:jobs/create.
+    """
+
     parameters: VariableOrList[str]
     """
     Parameters passed to the main method.
@@ -52,5 +71,10 @@ class SparkJarTaskDict(TypedDict, total=False):
     Use [Task parameter variables](https://docs.databricks.com/jobs.html#parameter-variables) to set parameters containing information about job runs.
     """
 
+    run_as_repl: VariableOrOptional[bool]
+    """
+    [DEPRECATED] Deprecated. A value of `false` is no longer supported.
+    """
+
 
 SparkJarTaskParam = SparkJarTaskDict | SparkJarTask

databricks/bundles/jobs/_models/task_email_notifications.py

@@ -3,7 +3,7 @@ from typing import TYPE_CHECKING, TypedDict
 
 from databricks.bundles.core._transform import _transform
 from databricks.bundles.core._transform_to_json import _transform_to_json_value
-from databricks.bundles.core._variable import VariableOrList
+from databricks.bundles.core._variable import VariableOrList, VariableOrOptional
 
 if TYPE_CHECKING:
     from typing_extensions import Self
@@ -13,6 +13,12 @@ if TYPE_CHECKING:
 class TaskEmailNotifications:
     """"""
 
+    no_alert_for_skipped_runs: VariableOrOptional[bool] = None
+    """
+    [DEPRECATED] If true, do not send email to recipients specified in `on_failure` if the run is skipped.
+    This field is `deprecated`. Please use the `notification_settings.no_alert_for_skipped_runs` field.
+    """
+
     on_duration_warning_threshold_exceeded: VariableOrList[str] = field(
         default_factory=list
     )
@@ -53,6 +59,12 @@ class TaskEmailNotifications:
 class TaskEmailNotificationsDict(TypedDict, total=False):
     """"""
 
+    no_alert_for_skipped_runs: VariableOrOptional[bool]
+    """
+    [DEPRECATED] If true, do not send email to recipients specified in `on_failure` if the run is skipped.
+    This field is `deprecated`. Please use the `notification_settings.no_alert_for_skipped_runs` field.
+    """
+
     on_duration_warning_threshold_exceeded: VariableOrList[str]
     """
     A list of email addresses to be notified when the duration of a run exceeds the threshold specified for the `RUN_DURATION_SECONDS` metric in the `health` field. If no rule for the `RUN_DURATION_SECONDS` metric is specified in the `health` field for the job, notifications are not sent.

databricks/bundles/jobs/_models/task_retry_mode.py

@@ -0,0 +1,17 @@
+from enum import Enum
+from typing import Literal
+
+
+class TaskRetryMode(Enum):
+    """
+    task retry mode of the continuous job
+    * NEVER: The failed task will not be retried.
+    * ON_FAILURE: Retry a failed task if at least one other task in the job is still running its first attempt.
+    When this condition is no longer met or the retry limit is reached, the job run is cancelled and a new run is started.
+    """
+
+    NEVER = "NEVER"
+    ON_FAILURE = "ON_FAILURE"
+
+
+TaskRetryModeParam = Literal["NEVER", "ON_FAILURE"] | TaskRetryMode

databricks/bundles/pipelines/__init__.py

@@ -96,6 +96,12 @@ __all__ = [
     "PipelinesEnvironment",
     "PipelinesEnvironmentDict",
     "PipelinesEnvironmentParam",
+    "PostgresCatalogConfig",
+    "PostgresCatalogConfigDict",
+    "PostgresCatalogConfigParam",
+    "PostgresSlotConfig",
+    "PostgresSlotConfigDict",
+    "PostgresSlotConfigParam",
     "ReportSpec",
     "ReportSpecDict",
     "ReportSpecParam",
@@ -111,6 +117,12 @@ __all__ = [
     "SchemaSpec",
     "SchemaSpecDict",
     "SchemaSpecParam",
+    "SourceCatalogConfig",
+    "SourceCatalogConfigDict",
+    "SourceCatalogConfigParam",
+    "SourceConfig",
+    "SourceConfigDict",
+    "SourceConfigParam",
     "TableSpec",
     "TableSpecDict",
     "TableSpecParam",
@@ -292,6 +304,16 @@ from databricks.bundles.pipelines._models.pipelines_environment import (
     PipelinesEnvironmentDict,
     PipelinesEnvironmentParam,
 )
+from databricks.bundles.pipelines._models.postgres_catalog_config import (
+    PostgresCatalogConfig,
+    PostgresCatalogConfigDict,
+    PostgresCatalogConfigParam,
+)
+from databricks.bundles.pipelines._models.postgres_slot_config import (
+    PostgresSlotConfig,
+    PostgresSlotConfigDict,
+    PostgresSlotConfigParam,
+)
 from databricks.bundles.pipelines._models.report_spec import (
     ReportSpec,
     ReportSpecDict,
@@ -313,6 +335,16 @@ from databricks.bundles.pipelines._models.schema_spec import (
     SchemaSpecDict,
     SchemaSpecParam,
 )
+from databricks.bundles.pipelines._models.source_catalog_config import (
+    SourceCatalogConfig,
+    SourceCatalogConfigDict,
+    SourceCatalogConfigParam,
+)
+from databricks.bundles.pipelines._models.source_config import (
+    SourceConfig,
+    SourceConfigDict,
+    SourceConfigParam,
+)
 from databricks.bundles.pipelines._models.table_spec import (
     TableSpec,
     TableSpecDict,

databricks/bundles/pipelines/_models/gcp_attributes.py

@@ -53,6 +53,13 @@ class GcpAttributes:
     for the supported number of local SSDs for each instance type.
     """
 
+    use_preemptible_executors: VariableOrOptional[bool] = None
+    """
+    [DEPRECATED] This field determines whether the spark executors will be scheduled to run on preemptible
+    VMs (when set to true) versus standard compute engine VMs (when set to false; default).
+    Note: Soon to be deprecated, use the 'availability' field instead.
+    """
+
     zone_id: VariableOrOptional[str] = None
     """
     Identifier for the availability zone in which the cluster resides.
@@ -108,6 +115,13 @@ class GcpAttributesDict(TypedDict, total=False):
     for the supported number of local SSDs for each instance type.
     """
 
+    use_preemptible_executors: VariableOrOptional[bool]
+    """
+    [DEPRECATED] This field determines whether the spark executors will be scheduled to run on preemptible
+    VMs (when set to true) versus standard compute engine VMs (when set to false; default).
+    Note: Soon to be deprecated, use the 'availability' field instead.
+    """
+
     zone_id: VariableOrOptional[str]
     """
     Identifier for the availability zone in which the cluster resides.

databricks/bundles/pipelines/_models/ingestion_gateway_pipeline_definition.py

@@ -30,6 +30,11 @@ class IngestionGatewayPipelineDefinition:
     Required, Immutable. The name of the schema for the gateway pipelines's storage location.
     """
 
+    connection_id: VariableOrOptional[str] = None
+    """
+    [DEPRECATED] [Deprecated, use connection_name instead] Immutable. The Unity Catalog connection that this gateway pipeline uses to communicate with the source.
+    """
+
     gateway_storage_name: VariableOrOptional[str] = None
     """
     Optional. The Unity Catalog-compatible name for the gateway storage location.
@@ -63,6 +68,11 @@ class IngestionGatewayPipelineDefinitionDict(TypedDict, total=False):
     Required, Immutable. The name of the schema for the gateway pipelines's storage location.
     """
 
+    connection_id: VariableOrOptional[str]
+    """
+    [DEPRECATED] [Deprecated, use connection_name instead] Immutable. The Unity Catalog connection that this gateway pipeline uses to communicate with the source.
+    """
+
     gateway_storage_name: VariableOrOptional[str]
     """
     Optional. The Unity Catalog-compatible name for the gateway storage location.

databricks/bundles/pipelines/_models/ingestion_pipeline_definition.py

@@ -12,6 +12,10 @@ from databricks.bundles.pipelines._models.ingestion_source_type import (
     IngestionSourceType,
     IngestionSourceTypeParam,
 )
+from databricks.bundles.pipelines._models.source_config import (
+    SourceConfig,
+    SourceConfigParam,
+)
 from databricks.bundles.pipelines._models.table_specific_config import (
     TableSpecificConfig,
     TableSpecificConfigParam,
@@ -40,6 +44,13 @@ class IngestionPipelineDefinition:
     Required. Settings specifying tables to replicate and the destination for the replicated tables.
     """
 
+    source_configurations: VariableOrList[SourceConfig] = field(default_factory=list)
+    """
+    :meta private: [EXPERIMENTAL]
+    
+    Top-level source configurations
+    """
+
     source_type: VariableOrOptional[IngestionSourceType] = None
     """
     The type of the foreign source.
@@ -78,6 +89,13 @@ class IngestionPipelineDefinitionDict(TypedDict, total=False):
     Required. Settings specifying tables to replicate and the destination for the replicated tables.
     """
 
+    source_configurations: VariableOrList[SourceConfigParam]
+    """
+    :meta private: [EXPERIMENTAL]
+    
+    Top-level source configurations
+    """
+
     source_type: VariableOrOptional[IngestionSourceTypeParam]
     """
     The type of the foreign source.