databricks-sdk 0.25.1__py3-none-any.whl → 0.27.0__py3-none-any.whl

databricks/sdk/service/pipelines.py

@@ -43,6 +43,9 @@ class CreatePipeline:
     continuous: Optional[bool] = None
     """Whether the pipeline is continuous or triggered. This replaces `trigger`."""
 
+    deployment: Optional[PipelineDeployment] = None
+    """Deployment type of this pipeline."""
+
     development: Optional[bool] = None
     """Whether the pipeline is in Development mode. Defaults to false."""
 
@@ -57,6 +60,10 @@ class CreatePipeline:
     id: Optional[str] = None
     """Unique identifier for this pipeline."""
 
+    ingestion_definition: Optional[ManagedIngestionPipelineDefinition] = None
+    """The configuration for a managed ingestion pipeline. These settings cannot be used with the
+    'libraries', 'target' or 'catalog' settings."""
+
     libraries: Optional[List[PipelineLibrary]] = None
     """Libraries or code needed by this deployment."""
 
@@ -92,11 +99,13 @@ class CreatePipeline:
         if self.clusters: body['clusters'] = [v.as_dict() for v in self.clusters]
         if self.configuration: body['configuration'] = self.configuration
         if self.continuous is not None: body['continuous'] = self.continuous
+        if self.deployment: body['deployment'] = self.deployment.as_dict()
         if self.development is not None: body['development'] = self.development
         if self.dry_run is not None: body['dry_run'] = self.dry_run
         if self.edition is not None: body['edition'] = self.edition
         if self.filters: body['filters'] = self.filters.as_dict()
         if self.id is not None: body['id'] = self.id
+        if self.ingestion_definition: body['ingestion_definition'] = self.ingestion_definition.as_dict()
         if self.libraries: body['libraries'] = [v.as_dict() for v in self.libraries]
         if self.name is not None: body['name'] = self.name
         if self.notifications: body['notifications'] = [v.as_dict() for v in self.notifications]
@@ -116,11 +125,14 @@ class CreatePipeline:
                    clusters=_repeated_dict(d, 'clusters', PipelineCluster),
                    configuration=d.get('configuration', None),
                    continuous=d.get('continuous', None),
+                   deployment=_from_dict(d, 'deployment', PipelineDeployment),
                    development=d.get('development', None),
                    dry_run=d.get('dry_run', None),
                    edition=d.get('edition', None),
                    filters=_from_dict(d, 'filters', Filters),
                    id=d.get('id', None),
+                   ingestion_definition=_from_dict(d, 'ingestion_definition',
+                                                   ManagedIngestionPipelineDefinition),
                    libraries=_repeated_dict(d, 'libraries', PipelineLibrary),
                    name=d.get('name', None),
                    notifications=_repeated_dict(d, 'notifications', Notifications),
@@ -208,6 +220,13 @@ class DeletePipelineResponse:
         return cls()
 
 
+class DeploymentKind(Enum):
+    """The deployment method that manages the pipeline: - BUNDLE: The pipeline is managed by a
+    Databricks Asset Bundle."""
+
+    BUNDLE = 'BUNDLE'
+
+
 @dataclass
 class EditPipeline:
     allow_duplicate_names: Optional[bool] = None
@@ -231,6 +250,9 @@ class EditPipeline:
     continuous: Optional[bool] = None
     """Whether the pipeline is continuous or triggered. This replaces `trigger`."""
 
+    deployment: Optional[PipelineDeployment] = None
+    """Deployment type of this pipeline."""
+
     development: Optional[bool] = None
     """Whether the pipeline is in Development mode. Defaults to false."""
 
@@ -247,6 +269,10 @@ class EditPipeline:
     id: Optional[str] = None
     """Unique identifier for this pipeline."""
 
+    ingestion_definition: Optional[ManagedIngestionPipelineDefinition] = None
+    """The configuration for a managed ingestion pipeline. These settings cannot be used with the
+    'libraries', 'target' or 'catalog' settings."""
+
     libraries: Optional[List[PipelineLibrary]] = None
     """Libraries or code needed by this deployment."""
 
@@ -285,12 +311,14 @@ class EditPipeline:
         if self.clusters: body['clusters'] = [v.as_dict() for v in self.clusters]
         if self.configuration: body['configuration'] = self.configuration
         if self.continuous is not None: body['continuous'] = self.continuous
+        if self.deployment: body['deployment'] = self.deployment.as_dict()
         if self.development is not None: body['development'] = self.development
         if self.edition is not None: body['edition'] = self.edition
         if self.expected_last_modified is not None:
             body['expected_last_modified'] = self.expected_last_modified
         if self.filters: body['filters'] = self.filters.as_dict()
         if self.id is not None: body['id'] = self.id
+        if self.ingestion_definition: body['ingestion_definition'] = self.ingestion_definition.as_dict()
         if self.libraries: body['libraries'] = [v.as_dict() for v in self.libraries]
         if self.name is not None: body['name'] = self.name
         if self.notifications: body['notifications'] = [v.as_dict() for v in self.notifications]
@@ -311,11 +339,14 @@ class EditPipeline:
                    clusters=_repeated_dict(d, 'clusters', PipelineCluster),
                    configuration=d.get('configuration', None),
                    continuous=d.get('continuous', None),
+                   deployment=_from_dict(d, 'deployment', PipelineDeployment),
                    development=d.get('development', None),
                    edition=d.get('edition', None),
                    expected_last_modified=d.get('expected_last_modified', None),
                    filters=_from_dict(d, 'filters', Filters),
                    id=d.get('id', None),
+                   ingestion_definition=_from_dict(d, 'ingestion_definition',
+                                                   ManagedIngestionPipelineDefinition),
                    libraries=_repeated_dict(d, 'libraries', PipelineLibrary),
                    name=d.get('name', None),
                    notifications=_repeated_dict(d, 'notifications', Notifications),
@@ -518,6 +549,27 @@ class GetUpdateResponse:
         return cls(update=_from_dict(d, 'update', UpdateInfo))
 
 
+@dataclass
+class IngestionConfig:
+    schema: Optional[SchemaSpec] = None
+    """Select tables from a specific source schema."""
+
+    table: Optional[TableSpec] = None
+    """Select tables from a specific source table."""
+
+    def as_dict(self) -> dict:
+        """Serializes the IngestionConfig into a dictionary suitable for use as a JSON request body."""
+        body = {}
+        if self.schema: body['schema'] = self.schema.as_dict()
+        if self.table: body['table'] = self.table.as_dict()
+        return body
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, any]) -> IngestionConfig:
+        """Deserializes the IngestionConfig from a dictionary."""
+        return cls(schema=_from_dict(d, 'schema', SchemaSpec), table=_from_dict(d, 'table', TableSpec))
+
+
 @dataclass
 class ListPipelineEventsResponse:
     events: Optional[List[PipelineEvent]] = None
@@ -594,6 +646,35 @@ class ListUpdatesResponse:
                    updates=_repeated_dict(d, 'updates', UpdateInfo))
 
 
+@dataclass
+class ManagedIngestionPipelineDefinition:
+    connection_name: Optional[str] = None
+    """Immutable. The Unity Catalog connection this ingestion pipeline uses to communicate with the
+    source. Specify either ingestion_gateway_id or connection_name."""
+
+    ingestion_gateway_id: Optional[str] = None
+    """Immutable. Identifier for the ingestion gateway used by this ingestion pipeline to communicate
+    with the source. Specify either ingestion_gateway_id or connection_name."""
+
+    objects: Optional[List[IngestionConfig]] = None
+    """Required. Settings specifying tables to replicate and the destination for the replicated tables."""
+
+    def as_dict(self) -> dict:
+        """Serializes the ManagedIngestionPipelineDefinition into a dictionary suitable for use as a JSON request body."""
+        body = {}
+        if self.connection_name is not None: body['connection_name'] = self.connection_name
+        if self.ingestion_gateway_id is not None: body['ingestion_gateway_id'] = self.ingestion_gateway_id
+        if self.objects: body['objects'] = [v.as_dict() for v in self.objects]
+        return body
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, any]) -> ManagedIngestionPipelineDefinition:
+        """Deserializes the ManagedIngestionPipelineDefinition from a dictionary."""
+        return cls(connection_name=d.get('connection_name', None),
+                   ingestion_gateway_id=d.get('ingestion_gateway_id', None),
+                   objects=_repeated_dict(d, 'objects', IngestionConfig))
+
+
 @dataclass
 class ManualTrigger:
 
@@ -1017,6 +1098,28 @@ class PipelineClusterAutoscaleMode(Enum):
     LEGACY = 'LEGACY'
 
 
+@dataclass
+class PipelineDeployment:
+    kind: Optional[DeploymentKind] = None
+    """The deployment method that manages the pipeline."""
+
+    metadata_file_path: Optional[str] = None
+    """The path to the file containing metadata about the deployment."""
+
+    def as_dict(self) -> dict:
+        """Serializes the PipelineDeployment into a dictionary suitable for use as a JSON request body."""
+        body = {}
+        if self.kind is not None: body['kind'] = self.kind.value
+        if self.metadata_file_path is not None: body['metadata_file_path'] = self.metadata_file_path
+        return body
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, any]) -> PipelineDeployment:
+        """Deserializes the PipelineDeployment from a dictionary."""
+        return cls(kind=_enum(d, 'kind', DeploymentKind),
+                   metadata_file_path=d.get('metadata_file_path', None))
+
+
 @dataclass
 class PipelineEvent:
     error: Optional[ErrorDetail] = None
@@ -1229,6 +1332,9 @@ class PipelineSpec:
     continuous: Optional[bool] = None
     """Whether the pipeline is continuous or triggered. This replaces `trigger`."""
 
+    deployment: Optional[PipelineDeployment] = None
+    """Deployment type of this pipeline."""
+
     development: Optional[bool] = None
     """Whether the pipeline is in Development mode. Defaults to false."""
 
@@ -1241,6 +1347,10 @@ class PipelineSpec:
     id: Optional[str] = None
     """Unique identifier for this pipeline."""
 
+    ingestion_definition: Optional[ManagedIngestionPipelineDefinition] = None
+    """The configuration for a managed ingestion pipeline. These settings cannot be used with the
+    'libraries', 'target' or 'catalog' settings."""
+
     libraries: Optional[List[PipelineLibrary]] = None
     """Libraries or code needed by this deployment."""
 
@@ -1275,10 +1385,12 @@ class PipelineSpec:
         if self.clusters: body['clusters'] = [v.as_dict() for v in self.clusters]
         if self.configuration: body['configuration'] = self.configuration
         if self.continuous is not None: body['continuous'] = self.continuous
+        if self.deployment: body['deployment'] = self.deployment.as_dict()
         if self.development is not None: body['development'] = self.development
         if self.edition is not None: body['edition'] = self.edition
         if self.filters: body['filters'] = self.filters.as_dict()
         if self.id is not None: body['id'] = self.id
+        if self.ingestion_definition: body['ingestion_definition'] = self.ingestion_definition.as_dict()
         if self.libraries: body['libraries'] = [v.as_dict() for v in self.libraries]
         if self.name is not None: body['name'] = self.name
         if self.notifications: body['notifications'] = [v.as_dict() for v in self.notifications]
@@ -1297,10 +1409,13 @@ class PipelineSpec:
                    clusters=_repeated_dict(d, 'clusters', PipelineCluster),
                    configuration=d.get('configuration', None),
                    continuous=d.get('continuous', None),
+                   deployment=_from_dict(d, 'deployment', PipelineDeployment),
                    development=d.get('development', None),
                    edition=d.get('edition', None),
                    filters=_from_dict(d, 'filters', Filters),
                    id=d.get('id', None),
+                   ingestion_definition=_from_dict(d, 'ingestion_definition',
+                                                   ManagedIngestionPipelineDefinition),
                    libraries=_repeated_dict(d, 'libraries', PipelineLibrary),
                    name=d.get('name', None),
                    notifications=_repeated_dict(d, 'notifications', Notifications),
@@ -1392,6 +1507,40 @@ class PipelineTrigger:
         return cls(cron=_from_dict(d, 'cron', CronTrigger), manual=_from_dict(d, 'manual', ManualTrigger))
 
 
+@dataclass
+class SchemaSpec:
+    destination_catalog: Optional[str] = None
+    """Required. Destination catalog to store tables."""
+
+    destination_schema: Optional[str] = None
+    """Required. Destination schema to store tables in. Tables with the same name as the source tables
+    are created in this destination schema. The pipeline fails If a table with the same name already
+    exists."""
+
+    source_catalog: Optional[str] = None
+    """The source catalog name. Might be optional depending on the type of source."""
+
+    source_schema: Optional[str] = None
+    """Required. Schema name in the source database."""
+
+    def as_dict(self) -> dict:
+        """Serializes the SchemaSpec into a dictionary suitable for use as a JSON request body."""
+        body = {}
+        if self.destination_catalog is not None: body['destination_catalog'] = self.destination_catalog
+        if self.destination_schema is not None: body['destination_schema'] = self.destination_schema
+        if self.source_catalog is not None: body['source_catalog'] = self.source_catalog
+        if self.source_schema is not None: body['source_schema'] = self.source_schema
+        return body
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, any]) -> SchemaSpec:
+        """Deserializes the SchemaSpec from a dictionary."""
+        return cls(destination_catalog=d.get('destination_catalog', None),
+                   destination_schema=d.get('destination_schema', None),
+                   source_catalog=d.get('source_catalog', None),
+                   source_schema=d.get('source_schema', None))
+
+
 @dataclass
 class Sequencing:
     control_plane_seq_no: Optional[int] = None
@@ -1559,6 +1708,49 @@ class StopPipelineResponse:
         return cls()
 
 
+@dataclass
+class TableSpec:
+    destination_catalog: Optional[str] = None
+    """Required. Destination catalog to store table."""
+
+    destination_schema: Optional[str] = None
+    """Required. Destination schema to store table."""
+
+    destination_table: Optional[str] = None
+    """Optional. Destination table name. The pipeline fails If a table with that name already exists.
+    If not set, the source table name is used."""
+
+    source_catalog: Optional[str] = None
+    """Source catalog name. Might be optional depending on the type of source."""
+
+    source_schema: Optional[str] = None
+    """Schema name in the source database. Might be optional depending on the type of source."""
+
+    source_table: Optional[str] = None
+    """Required. Table name in the source database."""
+
+    def as_dict(self) -> dict:
+        """Serializes the TableSpec into a dictionary suitable for use as a JSON request body."""
+        body = {}
+        if self.destination_catalog is not None: body['destination_catalog'] = self.destination_catalog
+        if self.destination_schema is not None: body['destination_schema'] = self.destination_schema
+        if self.destination_table is not None: body['destination_table'] = self.destination_table
+        if self.source_catalog is not None: body['source_catalog'] = self.source_catalog
+        if self.source_schema is not None: body['source_schema'] = self.source_schema
+        if self.source_table is not None: body['source_table'] = self.source_table
+        return body
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, any]) -> TableSpec:
+        """Deserializes the TableSpec from a dictionary."""
+        return cls(destination_catalog=d.get('destination_catalog', None),
+                   destination_schema=d.get('destination_schema', None),
+                   destination_table=d.get('destination_table', None),
+                   source_catalog=d.get('source_catalog', None),
+                   source_schema=d.get('source_schema', None),
+                   source_table=d.get('source_table', None))
+
+
 @dataclass
 class UpdateInfo:
     cause: Optional[UpdateInfoCause] = None
@@ -1784,11 +1976,13 @@ class PipelinesAPI:
                clusters: Optional[List[PipelineCluster]] = None,
                configuration: Optional[Dict[str, str]] = None,
                continuous: Optional[bool] = None,
+               deployment: Optional[PipelineDeployment] = None,
                development: Optional[bool] = None,
                dry_run: Optional[bool] = None,
                edition: Optional[str] = None,
                filters: Optional[Filters] = None,
                id: Optional[str] = None,
+               ingestion_definition: Optional[ManagedIngestionPipelineDefinition] = None,
                libraries: Optional[List[PipelineLibrary]] = None,
                name: Optional[str] = None,
                notifications: Optional[List[Notifications]] = None,
@@ -1816,6 +2010,8 @@ class PipelinesAPI:
           String-String configuration for this pipeline execution.
         :param continuous: bool (optional)
           Whether the pipeline is continuous or triggered. This replaces `trigger`.
+        :param deployment: :class:`PipelineDeployment` (optional)
+          Deployment type of this pipeline.
         :param development: bool (optional)
           Whether the pipeline is in Development mode. Defaults to false.
         :param dry_run: bool (optional)
@@ -1825,6 +2021,9 @@ class PipelinesAPI:
           Filters on which Pipeline packages to include in the deployed graph.
         :param id: str (optional)
           Unique identifier for this pipeline.
+        :param ingestion_definition: :class:`ManagedIngestionPipelineDefinition` (optional)
+          The configuration for a managed ingestion pipeline. These settings cannot be used with the
+          'libraries', 'target' or 'catalog' settings.
         :param libraries: List[:class:`PipelineLibrary`] (optional)
           Libraries or code needed by this deployment.
         :param name: str (optional)
@@ -1852,11 +2051,13 @@ class PipelinesAPI:
         if clusters is not None: body['clusters'] = [v.as_dict() for v in clusters]
         if configuration is not None: body['configuration'] = configuration
         if continuous is not None: body['continuous'] = continuous
+        if deployment is not None: body['deployment'] = deployment.as_dict()
         if development is not None: body['development'] = development
         if dry_run is not None: body['dry_run'] = dry_run
         if edition is not None: body['edition'] = edition
         if filters is not None: body['filters'] = filters.as_dict()
         if id is not None: body['id'] = id
+        if ingestion_definition is not None: body['ingestion_definition'] = ingestion_definition.as_dict()
         if libraries is not None: body['libraries'] = [v.as_dict() for v in libraries]
         if name is not None: body['name'] = name
         if notifications is not None: body['notifications'] = [v.as_dict() for v in notifications]
@@ -2179,11 +2380,13 @@ class PipelinesAPI:
                clusters: Optional[List[PipelineCluster]] = None,
                configuration: Optional[Dict[str, str]] = None,
                continuous: Optional[bool] = None,
+               deployment: Optional[PipelineDeployment] = None,
                development: Optional[bool] = None,
                edition: Optional[str] = None,
                expected_last_modified: Optional[int] = None,
                filters: Optional[Filters] = None,
                id: Optional[str] = None,
+               ingestion_definition: Optional[ManagedIngestionPipelineDefinition] = None,
                libraries: Optional[List[PipelineLibrary]] = None,
                name: Optional[str] = None,
                notifications: Optional[List[Notifications]] = None,
@@ -2212,6 +2415,8 @@ class PipelinesAPI:
           String-String configuration for this pipeline execution.
         :param continuous: bool (optional)
           Whether the pipeline is continuous or triggered. This replaces `trigger`.
+        :param deployment: :class:`PipelineDeployment` (optional)
+          Deployment type of this pipeline.
         :param development: bool (optional)
           Whether the pipeline is in Development mode. Defaults to false.
         :param edition: str (optional)
@@ -2223,6 +2428,9 @@ class PipelinesAPI:
           Filters on which Pipeline packages to include in the deployed graph.
         :param id: str (optional)
           Unique identifier for this pipeline.
+        :param ingestion_definition: :class:`ManagedIngestionPipelineDefinition` (optional)
+          The configuration for a managed ingestion pipeline. These settings cannot be used with the
+          'libraries', 'target' or 'catalog' settings.
         :param libraries: List[:class:`PipelineLibrary`] (optional)
           Libraries or code needed by this deployment.
         :param name: str (optional)
@@ -2250,11 +2458,13 @@ class PipelinesAPI:
         if clusters is not None: body['clusters'] = [v.as_dict() for v in clusters]
         if configuration is not None: body['configuration'] = configuration
         if continuous is not None: body['continuous'] = continuous
+        if deployment is not None: body['deployment'] = deployment.as_dict()
         if development is not None: body['development'] = development
         if edition is not None: body['edition'] = edition
         if expected_last_modified is not None: body['expected_last_modified'] = expected_last_modified
         if filters is not None: body['filters'] = filters.as_dict()
         if id is not None: body['id'] = id
+        if ingestion_definition is not None: body['ingestion_definition'] = ingestion_definition.as_dict()
         if libraries is not None: body['libraries'] = [v.as_dict() for v in libraries]
         if name is not None: body['name'] = name
         if notifications is not None: body['notifications'] = [v.as_dict() for v in notifications]