databricks-sdk 0.26.0__py3-none-any.whl → 0.27.0__py3-none-any.whl

databricks/sdk/service/pipelines.py

@@ -60,6 +60,10 @@ class CreatePipeline:
     id: Optional[str] = None
     """Unique identifier for this pipeline."""
 
+    ingestion_definition: Optional[ManagedIngestionPipelineDefinition] = None
+    """The configuration for a managed ingestion pipeline. These settings cannot be used with the
+    'libraries', 'target' or 'catalog' settings."""
+
     libraries: Optional[List[PipelineLibrary]] = None
     """Libraries or code needed by this deployment."""
 
@@ -101,6 +105,7 @@ class CreatePipeline:
         if self.edition is not None: body['edition'] = self.edition
         if self.filters: body['filters'] = self.filters.as_dict()
         if self.id is not None: body['id'] = self.id
+        if self.ingestion_definition: body['ingestion_definition'] = self.ingestion_definition.as_dict()
         if self.libraries: body['libraries'] = [v.as_dict() for v in self.libraries]
         if self.name is not None: body['name'] = self.name
         if self.notifications: body['notifications'] = [v.as_dict() for v in self.notifications]
@@ -126,6 +131,8 @@ class CreatePipeline:
                    edition=d.get('edition', None),
                    filters=_from_dict(d, 'filters', Filters),
                    id=d.get('id', None),
+                   ingestion_definition=_from_dict(d, 'ingestion_definition',
+                                                   ManagedIngestionPipelineDefinition),
                    libraries=_repeated_dict(d, 'libraries', PipelineLibrary),
                    name=d.get('name', None),
                    notifications=_repeated_dict(d, 'notifications', Notifications),
@@ -262,6 +269,10 @@ class EditPipeline:
     id: Optional[str] = None
     """Unique identifier for this pipeline."""
 
+    ingestion_definition: Optional[ManagedIngestionPipelineDefinition] = None
+    """The configuration for a managed ingestion pipeline. These settings cannot be used with the
+    'libraries', 'target' or 'catalog' settings."""
+
     libraries: Optional[List[PipelineLibrary]] = None
     """Libraries or code needed by this deployment."""
 
@@ -307,6 +318,7 @@ class EditPipeline:
             body['expected_last_modified'] = self.expected_last_modified
         if self.filters: body['filters'] = self.filters.as_dict()
         if self.id is not None: body['id'] = self.id
+        if self.ingestion_definition: body['ingestion_definition'] = self.ingestion_definition.as_dict()
         if self.libraries: body['libraries'] = [v.as_dict() for v in self.libraries]
         if self.name is not None: body['name'] = self.name
         if self.notifications: body['notifications'] = [v.as_dict() for v in self.notifications]
@@ -333,6 +345,8 @@ class EditPipeline:
                    expected_last_modified=d.get('expected_last_modified', None),
                    filters=_from_dict(d, 'filters', Filters),
                    id=d.get('id', None),
+                   ingestion_definition=_from_dict(d, 'ingestion_definition',
+                                                   ManagedIngestionPipelineDefinition),
                    libraries=_repeated_dict(d, 'libraries', PipelineLibrary),
                    name=d.get('name', None),
                    notifications=_repeated_dict(d, 'notifications', Notifications),
@@ -535,6 +549,27 @@ class GetUpdateResponse:
         return cls(update=_from_dict(d, 'update', UpdateInfo))
 
 
+@dataclass
+class IngestionConfig:
+    schema: Optional[SchemaSpec] = None
+    """Select tables from a specific source schema."""
+
+    table: Optional[TableSpec] = None
+    """Select tables from a specific source table."""
+
+    def as_dict(self) -> dict:
+        """Serializes the IngestionConfig into a dictionary suitable for use as a JSON request body."""
+        body = {}
+        if self.schema: body['schema'] = self.schema.as_dict()
+        if self.table: body['table'] = self.table.as_dict()
+        return body
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, any]) -> IngestionConfig:
+        """Deserializes the IngestionConfig from a dictionary."""
+        return cls(schema=_from_dict(d, 'schema', SchemaSpec), table=_from_dict(d, 'table', TableSpec))
+
+
 @dataclass
 class ListPipelineEventsResponse:
     events: Optional[List[PipelineEvent]] = None
@@ -611,6 +646,35 @@ class ListUpdatesResponse:
                    updates=_repeated_dict(d, 'updates', UpdateInfo))
 
 
+@dataclass
+class ManagedIngestionPipelineDefinition:
+    connection_name: Optional[str] = None
+    """Immutable. The Unity Catalog connection this ingestion pipeline uses to communicate with the
+    source. Specify either ingestion_gateway_id or connection_name."""
+
+    ingestion_gateway_id: Optional[str] = None
+    """Immutable. Identifier for the ingestion gateway used by this ingestion pipeline to communicate
+    with the source. Specify either ingestion_gateway_id or connection_name."""
+
+    objects: Optional[List[IngestionConfig]] = None
+    """Required. Settings specifying tables to replicate and the destination for the replicated tables."""
+
+    def as_dict(self) -> dict:
+        """Serializes the ManagedIngestionPipelineDefinition into a dictionary suitable for use as a JSON request body."""
+        body = {}
+        if self.connection_name is not None: body['connection_name'] = self.connection_name
+        if self.ingestion_gateway_id is not None: body['ingestion_gateway_id'] = self.ingestion_gateway_id
+        if self.objects: body['objects'] = [v.as_dict() for v in self.objects]
+        return body
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, any]) -> ManagedIngestionPipelineDefinition:
+        """Deserializes the ManagedIngestionPipelineDefinition from a dictionary."""
+        return cls(connection_name=d.get('connection_name', None),
+                   ingestion_gateway_id=d.get('ingestion_gateway_id', None),
+                   objects=_repeated_dict(d, 'objects', IngestionConfig))
+
+
 @dataclass
 class ManualTrigger:
 
@@ -1283,6 +1347,10 @@ class PipelineSpec:
     id: Optional[str] = None
     """Unique identifier for this pipeline."""
 
+    ingestion_definition: Optional[ManagedIngestionPipelineDefinition] = None
+    """The configuration for a managed ingestion pipeline. These settings cannot be used with the
+    'libraries', 'target' or 'catalog' settings."""
+
     libraries: Optional[List[PipelineLibrary]] = None
     """Libraries or code needed by this deployment."""
 
@@ -1322,6 +1390,7 @@ class PipelineSpec:
         if self.edition is not None: body['edition'] = self.edition
         if self.filters: body['filters'] = self.filters.as_dict()
         if self.id is not None: body['id'] = self.id
+        if self.ingestion_definition: body['ingestion_definition'] = self.ingestion_definition.as_dict()
         if self.libraries: body['libraries'] = [v.as_dict() for v in self.libraries]
         if self.name is not None: body['name'] = self.name
         if self.notifications: body['notifications'] = [v.as_dict() for v in self.notifications]
@@ -1345,6 +1414,8 @@ class PipelineSpec:
                    edition=d.get('edition', None),
                    filters=_from_dict(d, 'filters', Filters),
                    id=d.get('id', None),
+                   ingestion_definition=_from_dict(d, 'ingestion_definition',
+                                                   ManagedIngestionPipelineDefinition),
                    libraries=_repeated_dict(d, 'libraries', PipelineLibrary),
                    name=d.get('name', None),
                    notifications=_repeated_dict(d, 'notifications', Notifications),
@@ -1436,6 +1507,40 @@ class PipelineTrigger:
         return cls(cron=_from_dict(d, 'cron', CronTrigger), manual=_from_dict(d, 'manual', ManualTrigger))
 
 
+@dataclass
+class SchemaSpec:
+    destination_catalog: Optional[str] = None
+    """Required. Destination catalog to store tables."""
+
+    destination_schema: Optional[str] = None
+    """Required. Destination schema to store tables in. Tables with the same name as the source tables
+    are created in this destination schema. The pipeline fails If a table with the same name already
+    exists."""
+
+    source_catalog: Optional[str] = None
+    """The source catalog name. Might be optional depending on the type of source."""
+
+    source_schema: Optional[str] = None
+    """Required. Schema name in the source database."""
+
+    def as_dict(self) -> dict:
+        """Serializes the SchemaSpec into a dictionary suitable for use as a JSON request body."""
+        body = {}
+        if self.destination_catalog is not None: body['destination_catalog'] = self.destination_catalog
+        if self.destination_schema is not None: body['destination_schema'] = self.destination_schema
+        if self.source_catalog is not None: body['source_catalog'] = self.source_catalog
+        if self.source_schema is not None: body['source_schema'] = self.source_schema
+        return body
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, any]) -> SchemaSpec:
+        """Deserializes the SchemaSpec from a dictionary."""
+        return cls(destination_catalog=d.get('destination_catalog', None),
+                   destination_schema=d.get('destination_schema', None),
+                   source_catalog=d.get('source_catalog', None),
+                   source_schema=d.get('source_schema', None))
+
+
 @dataclass
 class Sequencing:
     control_plane_seq_no: Optional[int] = None
@@ -1603,6 +1708,49 @@ class StopPipelineResponse:
         return cls()
 
 
+@dataclass
+class TableSpec:
+    destination_catalog: Optional[str] = None
+    """Required. Destination catalog to store table."""
+
+    destination_schema: Optional[str] = None
+    """Required. Destination schema to store table."""
+
+    destination_table: Optional[str] = None
+    """Optional. Destination table name. The pipeline fails If a table with that name already exists.
+    If not set, the source table name is used."""
+
+    source_catalog: Optional[str] = None
+    """Source catalog name. Might be optional depending on the type of source."""
+
+    source_schema: Optional[str] = None
+    """Schema name in the source database. Might be optional depending on the type of source."""
+
+    source_table: Optional[str] = None
+    """Required. Table name in the source database."""
+
+    def as_dict(self) -> dict:
+        """Serializes the TableSpec into a dictionary suitable for use as a JSON request body."""
+        body = {}
+        if self.destination_catalog is not None: body['destination_catalog'] = self.destination_catalog
+        if self.destination_schema is not None: body['destination_schema'] = self.destination_schema
+        if self.destination_table is not None: body['destination_table'] = self.destination_table
+        if self.source_catalog is not None: body['source_catalog'] = self.source_catalog
+        if self.source_schema is not None: body['source_schema'] = self.source_schema
+        if self.source_table is not None: body['source_table'] = self.source_table
+        return body
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, any]) -> TableSpec:
+        """Deserializes the TableSpec from a dictionary."""
+        return cls(destination_catalog=d.get('destination_catalog', None),
+                   destination_schema=d.get('destination_schema', None),
+                   destination_table=d.get('destination_table', None),
+                   source_catalog=d.get('source_catalog', None),
+                   source_schema=d.get('source_schema', None),
+                   source_table=d.get('source_table', None))
+
+
 @dataclass
 class UpdateInfo:
     cause: Optional[UpdateInfoCause] = None
@@ -1834,6 +1982,7 @@ class PipelinesAPI:
                edition: Optional[str] = None,
                filters: Optional[Filters] = None,
                id: Optional[str] = None,
+               ingestion_definition: Optional[ManagedIngestionPipelineDefinition] = None,
                libraries: Optional[List[PipelineLibrary]] = None,
                name: Optional[str] = None,
                notifications: Optional[List[Notifications]] = None,
@@ -1872,6 +2021,9 @@ class PipelinesAPI:
           Filters on which Pipeline packages to include in the deployed graph.
         :param id: str (optional)
           Unique identifier for this pipeline.
+        :param ingestion_definition: :class:`ManagedIngestionPipelineDefinition` (optional)
+          The configuration for a managed ingestion pipeline. These settings cannot be used with the
+          'libraries', 'target' or 'catalog' settings.
         :param libraries: List[:class:`PipelineLibrary`] (optional)
           Libraries or code needed by this deployment.
         :param name: str (optional)
@@ -1905,6 +2057,7 @@ class PipelinesAPI:
         if edition is not None: body['edition'] = edition
         if filters is not None: body['filters'] = filters.as_dict()
         if id is not None: body['id'] = id
+        if ingestion_definition is not None: body['ingestion_definition'] = ingestion_definition.as_dict()
         if libraries is not None: body['libraries'] = [v.as_dict() for v in libraries]
         if name is not None: body['name'] = name
         if notifications is not None: body['notifications'] = [v.as_dict() for v in notifications]
@@ -2233,6 +2386,7 @@ class PipelinesAPI:
                expected_last_modified: Optional[int] = None,
                filters: Optional[Filters] = None,
                id: Optional[str] = None,
+               ingestion_definition: Optional[ManagedIngestionPipelineDefinition] = None,
                libraries: Optional[List[PipelineLibrary]] = None,
                name: Optional[str] = None,
                notifications: Optional[List[Notifications]] = None,
@@ -2274,6 +2428,9 @@ class PipelinesAPI:
           Filters on which Pipeline packages to include in the deployed graph.
         :param id: str (optional)
           Unique identifier for this pipeline.
+        :param ingestion_definition: :class:`ManagedIngestionPipelineDefinition` (optional)
+          The configuration for a managed ingestion pipeline. These settings cannot be used with the
+          'libraries', 'target' or 'catalog' settings.
         :param libraries: List[:class:`PipelineLibrary`] (optional)
           Libraries or code needed by this deployment.
         :param name: str (optional)
@@ -2307,6 +2464,7 @@ class PipelinesAPI:
         if expected_last_modified is not None: body['expected_last_modified'] = expected_last_modified
         if filters is not None: body['filters'] = filters.as_dict()
         if id is not None: body['id'] = id
+        if ingestion_definition is not None: body['ingestion_definition'] = ingestion_definition.as_dict()
         if libraries is not None: body['libraries'] = [v.as_dict() for v in libraries]
         if name is not None: body['name'] = name
         if notifications is not None: body['notifications'] = [v.as_dict() for v in notifications]