databricks-sdk 0.56.0__py3-none-any.whl → 0.58.0__py3-none-any.whl

databricks/sdk/service/pipelines.py

@@ -58,6 +58,9 @@ class CreatePipeline:
     edition: Optional[str] = None
     """Pipeline product edition."""
 
+    environment: Optional[PipelinesEnvironment] = None
+    """Environment specification for this pipeline used to install dependencies."""
+
     event_log: Optional[EventLogSpec] = None
     """Event log configuration for this pipeline"""
 
@@ -95,12 +98,6 @@ class CreatePipeline:
     pipeline execution."""
 
     run_as: Optional[RunAs] = None
-    """Write-only setting, available only in Create/Update calls. Specifies the user or service
-    principal that the pipeline runs as. If not specified, the pipeline runs as the user who created
-    the pipeline.
-    
-    Only `user_name` or `service_principal_name` can be specified. If both are specified, an error
-    is thrown."""
 
     schema: Optional[str] = None
     """The default schema (database) where tables are read from or published to."""
@@ -149,6 +146,8 @@ class CreatePipeline:
             body["dry_run"] = self.dry_run
         if self.edition is not None:
             body["edition"] = self.edition
+        if self.environment:
+            body["environment"] = self.environment.as_dict()
         if self.event_log:
             body["event_log"] = self.event_log.as_dict()
         if self.filters:
@@ -212,6 +211,8 @@ class CreatePipeline:
             body["dry_run"] = self.dry_run
         if self.edition is not None:
             body["edition"] = self.edition
+        if self.environment:
+            body["environment"] = self.environment
         if self.event_log:
             body["event_log"] = self.event_log
         if self.filters:
@@ -265,6 +266,7 @@ class CreatePipeline:
             development=d.get("development", None),
             dry_run=d.get("dry_run", None),
             edition=d.get("edition", None),
+            environment=_from_dict(d, "environment", PipelinesEnvironment),
             event_log=_from_dict(d, "event_log", EventLogSpec),
             filters=_from_dict(d, "filters", Filters),
             gateway_definition=_from_dict(d, "gateway_definition", IngestionGatewayPipelineDefinition),
@@ -455,6 +457,9 @@ class EditPipeline:
     edition: Optional[str] = None
     """Pipeline product edition."""
 
+    environment: Optional[PipelinesEnvironment] = None
+    """Environment specification for this pipeline used to install dependencies."""
+
     event_log: Optional[EventLogSpec] = None
     """Event log configuration for this pipeline"""
 
@@ -499,12 +504,6 @@ class EditPipeline:
     pipeline execution."""
 
     run_as: Optional[RunAs] = None
-    """Write-only setting, available only in Create/Update calls. Specifies the user or service
-    principal that the pipeline runs as. If not specified, the pipeline runs as the user who created
-    the pipeline.
-    
-    Only `user_name` or `service_principal_name` can be specified. If both are specified, an error
-    is thrown."""
 
     schema: Optional[str] = None
     """The default schema (database) where tables are read from or published to."""
@@ -551,6 +550,8 @@ class EditPipeline:
             body["development"] = self.development
         if self.edition is not None:
             body["edition"] = self.edition
+        if self.environment:
+            body["environment"] = self.environment.as_dict()
         if self.event_log:
             body["event_log"] = self.event_log.as_dict()
         if self.expected_last_modified is not None:
@@ -616,6 +617,8 @@ class EditPipeline:
             body["development"] = self.development
         if self.edition is not None:
             body["edition"] = self.edition
+        if self.environment:
+            body["environment"] = self.environment
         if self.event_log:
             body["event_log"] = self.event_log
         if self.expected_last_modified is not None:
@@ -672,6 +675,7 @@ class EditPipeline:
             deployment=_from_dict(d, "deployment", PipelineDeployment),
             development=d.get("development", None),
             edition=d.get("edition", None),
+            environment=_from_dict(d, "environment", PipelinesEnvironment),
             event_log=_from_dict(d, "event_log", EventLogSpec),
             expected_last_modified=d.get("expected_last_modified", None),
             filters=_from_dict(d, "filters", Filters),
@@ -906,6 +910,11 @@ class GetPipelineResponse:
     pipeline_id: Optional[str] = None
     """The ID of the pipeline."""
 
+    run_as: Optional[RunAs] = None
+    """The user or service principal that the pipeline runs as, if specified in the request. This field
+    indicates the explicit configuration of `run_as` for the pipeline. To find the value in all
+    cases, explicit or implicit, use `run_as_user_name`."""
+
     run_as_user_name: Optional[str] = None
     """Username of the user that the pipeline will run on behalf of."""
 
@@ -936,6 +945,8 @@ class GetPipelineResponse:
             body["name"] = self.name
         if self.pipeline_id is not None:
             body["pipeline_id"] = self.pipeline_id
+        if self.run_as:
+            body["run_as"] = self.run_as.as_dict()
         if self.run_as_user_name is not None:
             body["run_as_user_name"] = self.run_as_user_name
         if self.spec:
@@ -965,6 +976,8 @@ class GetPipelineResponse:
             body["name"] = self.name
         if self.pipeline_id is not None:
             body["pipeline_id"] = self.pipeline_id
+        if self.run_as:
+            body["run_as"] = self.run_as
         if self.run_as_user_name is not None:
             body["run_as_user_name"] = self.run_as_user_name
         if self.spec:
@@ -986,6 +999,7 @@ class GetPipelineResponse:
             latest_updates=_repeated_dict(d, "latest_updates", UpdateStateInfo),
             name=d.get("name", None),
             pipeline_id=d.get("pipeline_id", None),
+            run_as=_from_dict(d, "run_as", RunAs),
             run_as_user_name=d.get("run_as_user_name", None),
             spec=_from_dict(d, "spec", PipelineSpec),
             state=_enum(d, "state", PipelineState),
@@ -1195,6 +1209,7 @@ class IngestionPipelineDefinition:
 
 class IngestionSourceType(Enum):
 
+    BIGQUERY = "BIGQUERY"
     DYNAMICS365 = "DYNAMICS365"
     GA4_RAW_DATA = "GA4_RAW_DATA"
     MANAGED_POSTGRESQL = "MANAGED_POSTGRESQL"
@@ -1605,7 +1620,6 @@ class PipelineAccessControlRequest:
     """name of the group"""
 
     permission_level: Optional[PipelinePermissionLevel] = None
-    """Permission level"""
 
     service_principal_name: Optional[str] = None
     """application ID of a service principal"""
@@ -2179,7 +2193,6 @@ class PipelinePermission:
     inherited_from_object: Optional[List[str]] = None
 
     permission_level: Optional[PipelinePermissionLevel] = None
-    """Permission level"""
 
     def as_dict(self) -> dict:
         """Serializes the PipelinePermission into a dictionary suitable for use as a JSON request body."""
@@ -2267,7 +2280,6 @@ class PipelinePermissionsDescription:
     description: Optional[str] = None
 
     permission_level: Optional[PipelinePermissionLevel] = None
-    """Permission level"""
 
     def as_dict(self) -> dict:
         """Serializes the PipelinePermissionsDescription into a dictionary suitable for use as a JSON request body."""
@@ -2362,6 +2374,9 @@ class PipelineSpec:
     edition: Optional[str] = None
     """Pipeline product edition."""
 
+    environment: Optional[PipelinesEnvironment] = None
+    """Environment specification for this pipeline used to install dependencies."""
+
     event_log: Optional[EventLogSpec] = None
     """Event log configuration for this pipeline"""
 
@@ -2441,6 +2456,8 @@ class PipelineSpec:
             body["development"] = self.development
         if self.edition is not None:
             body["edition"] = self.edition
+        if self.environment:
+            body["environment"] = self.environment.as_dict()
         if self.event_log:
             body["event_log"] = self.event_log.as_dict()
         if self.filters:
@@ -2498,6 +2515,8 @@ class PipelineSpec:
             body["development"] = self.development
         if self.edition is not None:
             body["edition"] = self.edition
+        if self.environment:
+            body["environment"] = self.environment
         if self.event_log:
             body["event_log"] = self.event_log
         if self.filters:
@@ -2547,6 +2566,7 @@ class PipelineSpec:
             deployment=_from_dict(d, "deployment", PipelineDeployment),
             development=d.get("development", None),
             edition=d.get("edition", None),
+            environment=_from_dict(d, "environment", PipelinesEnvironment),
             event_log=_from_dict(d, "event_log", EventLogSpec),
             filters=_from_dict(d, "filters", Filters),
             gateway_definition=_from_dict(d, "gateway_definition", IngestionGatewayPipelineDefinition),
@@ -2606,7 +2626,6 @@ class PipelineStateInfo:
     owner."""
 
     state: Optional[PipelineState] = None
-    """The pipeline state."""
 
     def as_dict(self) -> dict:
         """Serializes the PipelineStateInfo into a dictionary suitable for use as a JSON request body."""
@@ -2702,6 +2721,39 @@ class PipelineTrigger:
         return cls(cron=_from_dict(d, "cron", CronTrigger), manual=_from_dict(d, "manual", ManualTrigger))
 
 
+@dataclass
+class PipelinesEnvironment:
+    """The environment entity used to preserve serverless environment side panel, jobs' environment for
+    non-notebook task, and DLT's environment for classic and serverless pipelines. In this minimal
+    environment spec, only pip dependencies are supported."""
+
+    dependencies: Optional[List[str]] = None
+    """List of pip dependencies, as supported by the version of pip in this environment. Each
+    dependency is a pip requirement file line
+    https://pip.pypa.io/en/stable/reference/requirements-file-format/ Allowed dependency could be
+    <requirement specifier>, <archive url/path>, <local project path>(WSFS or Volumes in
+    Databricks), <vcs project url>"""
+
+    def as_dict(self) -> dict:
+        """Serializes the PipelinesEnvironment into a dictionary suitable for use as a JSON request body."""
+        body = {}
+        if self.dependencies:
+            body["dependencies"] = [v for v in self.dependencies]
+        return body
+
+    def as_shallow_dict(self) -> dict:
+        """Serializes the PipelinesEnvironment into a shallow dictionary of its immediate attributes."""
+        body = {}
+        if self.dependencies:
+            body["dependencies"] = self.dependencies
+        return body
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, Any]) -> PipelinesEnvironment:
+        """Deserializes the PipelinesEnvironment from a dictionary."""
+        return cls(dependencies=d.get("dependencies", None))
+
+
 @dataclass
 class ReportSpec:
     source_url: str
@@ -2915,7 +2967,7 @@ class SchemaSpec:
 @dataclass
 class Sequencing:
     control_plane_seq_no: Optional[int] = None
-    """A sequence number, unique and increasing within the control plane."""
+    """A sequence number, unique and increasing per pipeline."""
 
     data_plane_id: Optional[DataPlaneId] = None
     """the ID assigned by the data plane."""
@@ -3044,7 +3096,6 @@ class StackFrame:
 @dataclass
 class StartUpdate:
     cause: Optional[StartUpdateCause] = None
-    """What triggered this update."""
 
     full_refresh: Optional[bool] = None
     """If true, this update will reset all tables before running."""
@@ -3321,6 +3372,7 @@ class TableSpecificConfig:
 class TableSpecificConfigScdType(Enum):
     """The SCD type to use to ingest the table."""
 
+    APPEND_ONLY = "APPEND_ONLY"
     SCD_TYPE_1 = "SCD_TYPE_1"
     SCD_TYPE_2 = "SCD_TYPE_2"
 
@@ -3471,7 +3523,6 @@ class UpdateStateInfo:
     creation_time: Optional[str] = None
 
     state: Optional[UpdateStateInfoState] = None
-    """The update state."""
 
     update_id: Optional[str] = None
 
@@ -3584,6 +3635,7 @@ class PipelinesAPI:
         development: Optional[bool] = None,
         dry_run: Optional[bool] = None,
         edition: Optional[str] = None,
+        environment: Optional[PipelinesEnvironment] = None,
         event_log: Optional[EventLogSpec] = None,
         filters: Optional[Filters] = None,
         gateway_definition: Optional[IngestionGatewayPipelineDefinition] = None,
@@ -3603,9 +3655,7 @@ class PipelinesAPI:
         target: Optional[str] = None,
         trigger: Optional[PipelineTrigger] = None,
     ) -> CreatePipelineResponse:
-        """Create a pipeline.
-
-        Creates a new data processing pipeline based on the requested configuration. If successful, this
+        """Creates a new data processing pipeline based on the requested configuration. If successful, this
         method returns the ID of the new pipeline.
 
         :param allow_duplicate_names: bool (optional)
@@ -3631,6 +3681,8 @@ class PipelinesAPI:
         :param dry_run: bool (optional)
         :param edition: str (optional)
           Pipeline product edition.
+        :param environment: :class:`PipelinesEnvironment` (optional)
+          Environment specification for this pipeline used to install dependencies.
         :param event_log: :class:`EventLogSpec` (optional)
           Event log configuration for this pipeline
         :param filters: :class:`Filters` (optional)
@@ -3657,11 +3709,6 @@ class PipelinesAPI:
           Databricks user interface and it is added to sys.path when executing Python sources during pipeline
           execution.
         :param run_as: :class:`RunAs` (optional)
-          Write-only setting, available only in Create/Update calls. Specifies the user or service principal
-          that the pipeline runs as. If not specified, the pipeline runs as the user who created the pipeline.
-
-          Only `user_name` or `service_principal_name` can be specified. If both are specified, an error is
-          thrown.
         :param schema: str (optional)
           The default schema (database) where tables are read from or published to.
         :param serverless: bool (optional)
@@ -3703,6 +3750,8 @@ class PipelinesAPI:
             body["dry_run"] = dry_run
         if edition is not None:
             body["edition"] = edition
+        if environment is not None:
+            body["environment"] = environment.as_dict()
         if event_log is not None:
             body["event_log"] = event_log.as_dict()
         if filters is not None:
@@ -3748,9 +3797,7 @@ class PipelinesAPI:
         return CreatePipelineResponse.from_dict(res)
 
     def delete(self, pipeline_id: str):
-        """Delete a pipeline.
-
-        Deletes a pipeline. Deleting a pipeline is a permanent action that stops and removes the pipeline and
+        """Deletes a pipeline. Deleting a pipeline is a permanent action that stops and removes the pipeline and
         its tables. You cannot undo this action.
 
         :param pipeline_id: str
@@ -3780,9 +3827,7 @@ class PipelinesAPI:
         return GetPipelineResponse.from_dict(res)
 
     def get_permission_levels(self, pipeline_id: str) -> GetPipelinePermissionLevelsResponse:
-        """Get pipeline permission levels.
-
-        Gets the permission levels that a user can have on an object.
+        """Gets the permission levels that a user can have on an object.
 
         :param pipeline_id: str
           The pipeline for which to get or manage permissions.
@@ -3798,9 +3843,7 @@ class PipelinesAPI:
         return GetPipelinePermissionLevelsResponse.from_dict(res)
 
     def get_permissions(self, pipeline_id: str) -> PipelinePermissions:
-        """Get pipeline permissions.
-
-        Gets the permissions of a pipeline. Pipelines can inherit permissions from their root object.
+        """Gets the permissions of a pipeline. Pipelines can inherit permissions from their root object.
 
         :param pipeline_id: str
           The pipeline for which to get or manage permissions.
@@ -3816,9 +3859,7 @@ class PipelinesAPI:
         return PipelinePermissions.from_dict(res)
 
     def get_update(self, pipeline_id: str, update_id: str) -> GetUpdateResponse:
-        """Get a pipeline update.
-
-        Gets an update from an active pipeline.
+        """Gets an update from an active pipeline.
 
         :param pipeline_id: str
           The ID of the pipeline.
@@ -3844,9 +3885,7 @@ class PipelinesAPI:
         order_by: Optional[List[str]] = None,
         page_token: Optional[str] = None,
     ) -> Iterator[PipelineEvent]:
-        """List pipeline events.
-
-        Retrieves events for a pipeline.
+        """Retrieves events for a pipeline.
 
         :param pipeline_id: str
           The pipeline to return events for.
@@ -3902,9 +3941,7 @@ class PipelinesAPI:
         order_by: Optional[List[str]] = None,
         page_token: Optional[str] = None,
     ) -> Iterator[PipelineStateInfo]:
-        """List pipelines.
-
-        Lists pipelines defined in the Delta Live Tables system.
+        """Lists pipelines defined in the Delta Live Tables system.
 
         :param filter: str (optional)
           Select a subset of results based on the specified criteria. The supported filters are:
@@ -3958,9 +3995,7 @@ class PipelinesAPI:
         page_token: Optional[str] = None,
         until_update_id: Optional[str] = None,
     ) -> ListUpdatesResponse:
-        """List pipeline updates.
-
-        List updates for an active pipeline.
+        """List updates for an active pipeline.
 
         :param pipeline_id: str
           The pipeline to return updates for.
@@ -3991,9 +4026,7 @@ class PipelinesAPI:
     def set_permissions(
         self, pipeline_id: str, *, access_control_list: Optional[List[PipelineAccessControlRequest]] = None
     ) -> PipelinePermissions:
-        """Set pipeline permissions.
-
-        Sets permissions on an object, replacing existing permissions if they exist. Deletes all direct
+        """Sets permissions on an object, replacing existing permissions if they exist. Deletes all direct
         permissions if none are specified. Objects can inherit permissions from their root object.
 
         :param pipeline_id: str
@@ -4023,14 +4056,11 @@ class PipelinesAPI:
         refresh_selection: Optional[List[str]] = None,
         validate_only: Optional[bool] = None,
     ) -> StartUpdateResponse:
-        """Start a pipeline.
-
-        Starts a new update for the pipeline. If there is already an active update for the pipeline, the
+        """Starts a new update for the pipeline. If there is already an active update for the pipeline, the
         request will fail and the active update will remain running.
 
         :param pipeline_id: str
         :param cause: :class:`StartUpdateCause` (optional)
-          What triggered this update.
         :param full_refresh: bool (optional)
           If true, this update will reset all tables before running.
         :param full_refresh_selection: List[str] (optional)
@@ -4067,9 +4097,7 @@ class PipelinesAPI:
         return StartUpdateResponse.from_dict(res)
 
     def stop(self, pipeline_id: str) -> Wait[GetPipelineResponse]:
-        """Stop a pipeline.
-
-        Stops the pipeline by canceling the active update. If there is no active update for the pipeline, this
+        """Stops the pipeline by canceling the active update. If there is no active update for the pipeline, this
         request is a no-op.
 
         :param pipeline_id: str
@@ -4105,6 +4133,7 @@ class PipelinesAPI:
         deployment: Optional[PipelineDeployment] = None,
         development: Optional[bool] = None,
         edition: Optional[str] = None,
+        environment: Optional[PipelinesEnvironment] = None,
         event_log: Optional[EventLogSpec] = None,
         expected_last_modified: Optional[int] = None,
         filters: Optional[Filters] = None,
@@ -4125,9 +4154,7 @@ class PipelinesAPI:
         target: Optional[str] = None,
         trigger: Optional[PipelineTrigger] = None,
     ):
-        """Edit a pipeline.
-
-        Updates a pipeline with the supplied configuration.
+        """Updates a pipeline with the supplied configuration.
 
         :param pipeline_id: str
           Unique identifier for this pipeline.
@@ -4153,6 +4180,8 @@ class PipelinesAPI:
           Whether the pipeline is in Development mode. Defaults to false.
         :param edition: str (optional)
           Pipeline product edition.
+        :param environment: :class:`PipelinesEnvironment` (optional)
+          Environment specification for this pipeline used to install dependencies.
         :param event_log: :class:`EventLogSpec` (optional)
           Event log configuration for this pipeline
         :param expected_last_modified: int (optional)
@@ -4182,11 +4211,6 @@ class PipelinesAPI:
           Databricks user interface and it is added to sys.path when executing Python sources during pipeline
           execution.
         :param run_as: :class:`RunAs` (optional)
-          Write-only setting, available only in Create/Update calls. Specifies the user or service principal
-          that the pipeline runs as. If not specified, the pipeline runs as the user who created the pipeline.
-
-          Only `user_name` or `service_principal_name` can be specified. If both are specified, an error is
-          thrown.
         :param schema: str (optional)
           The default schema (database) where tables are read from or published to.
         :param serverless: bool (optional)
@@ -4226,6 +4250,8 @@ class PipelinesAPI:
             body["development"] = development
         if edition is not None:
             body["edition"] = edition
+        if environment is not None:
+            body["environment"] = environment.as_dict()
         if event_log is not None:
             body["event_log"] = event_log.as_dict()
         if expected_last_modified is not None:
@@ -4274,9 +4300,7 @@ class PipelinesAPI:
     def update_permissions(
         self, pipeline_id: str, *, access_control_list: Optional[List[PipelineAccessControlRequest]] = None
     ) -> PipelinePermissions:
-        """Update pipeline permissions.
-
-        Updates the permissions on a pipeline. Pipelines can inherit permissions from their root object.
+        """Updates the permissions on a pipeline. Pipelines can inherit permissions from their root object.
 
         :param pipeline_id: str
           The pipeline for which to get or manage permissions.