databricks-sdk 0.28.0__py3-none-any.whl → 0.30.0__py3-none-any.whl

databricks/sdk/service/jobs.py

@@ -15,7 +15,7 @@ from ._internal import Wait, _enum, _from_dict, _repeated_dict
 
 _LOG = logging.getLogger('databricks.sdk')
 
-from databricks.sdk.service import compute, iam
+from databricks.sdk.service import compute
 
 # all definitions in this file are in alphabetical order
 
@@ -469,7 +469,7 @@ class Continuous:
 
 @dataclass
 class CreateJob:
-    access_control_list: Optional[List[iam.AccessControlRequest]] = None
+    access_control_list: Optional[List[JobAccessControlRequest]] = None
     """List of permissions to set on the job."""
 
     continuous: Optional[Continuous] = None
@@ -480,7 +480,7 @@ class CreateJob:
     """Deployment information for jobs managed by external sources."""
 
     description: Optional[str] = None
-    """An optional description for the job. The maximum length is 1024 characters in UTF-8 encoding."""
+    """An optional description for the job. The maximum length is 27700 characters in UTF-8 encoding."""
 
     edit_mode: Optional[JobEditMode] = None
     """Edit mode of the job.
@@ -603,7 +603,7 @@ class CreateJob:
     @classmethod
     def from_dict(cls, d: Dict[str, any]) -> CreateJob:
         """Deserializes the CreateJob from a dictionary."""
-        return cls(access_control_list=_repeated_dict(d, 'access_control_list', iam.AccessControlRequest),
+        return cls(access_control_list=_repeated_dict(d, 'access_control_list', JobAccessControlRequest),
                    continuous=_from_dict(d, 'continuous', Continuous),
                    deployment=_from_dict(d, 'deployment', JobDeployment),
                    description=d.get('description', None),
@@ -940,17 +940,23 @@ class ForEachTaskErrorMessageStats:
     error_message: Optional[str] = None
     """Describes the error message occured during the iterations."""
 
+    termination_category: Optional[str] = None
+    """Describes the termination reason for the error message."""
+
     def as_dict(self) -> dict:
         """Serializes the ForEachTaskErrorMessageStats into a dictionary suitable for use as a JSON request body."""
         body = {}
         if self.count is not None: body['count'] = self.count
         if self.error_message is not None: body['error_message'] = self.error_message
+        if self.termination_category is not None: body['termination_category'] = self.termination_category
         return body
 
     @classmethod
     def from_dict(cls, d: Dict[str, any]) -> ForEachTaskErrorMessageStats:
         """Deserializes the ForEachTaskErrorMessageStats from a dictionary."""
-        return cls(count=d.get('count', None), error_message=d.get('error_message', None))
+        return cls(count=d.get('count', None),
+                   error_message=d.get('error_message', None),
+                   termination_category=d.get('termination_category', None))
 
 
 @dataclass
@@ -1315,6 +1321,13 @@ class JobEmailNotifications:
     """A list of email addresses to be notified when a run begins. If not specified on job creation,
     reset, or update, the list is empty, and notifications are not sent."""
 
+    on_streaming_backlog_exceeded: Optional[List[str]] = None
+    """A list of email addresses to notify when any streaming backlog thresholds are exceeded for any
+    stream. Streaming backlog thresholds can be set in the `health` field using the following
+    metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or
+    `STREAMING_BACKLOG_FILES`. Alerting is based on the 10-minute average of these metrics. If the
+    issue persists, notifications are resent every 30 minutes."""
+
     on_success: Optional[List[str]] = None
     """A list of email addresses to be notified when a run successfully completes. A run is considered
     to have completed successfully if it ends with a `TERMINATED` `life_cycle_state` and a `SUCCESS`
@@ -1332,6 +1345,8 @@ class JobEmailNotifications:
             ]
         if self.on_failure: body['on_failure'] = [v for v in self.on_failure]
         if self.on_start: body['on_start'] = [v for v in self.on_start]
+        if self.on_streaming_backlog_exceeded:
+            body['on_streaming_backlog_exceeded'] = [v for v in self.on_streaming_backlog_exceeded]
         if self.on_success: body['on_success'] = [v for v in self.on_success]
         return body
 
@@ -1343,6 +1358,7 @@ class JobEmailNotifications:
                                                                 None),
                    on_failure=d.get('on_failure', None),
                    on_start=d.get('on_start', None),
+                   on_streaming_backlog_exceeded=d.get('on_streaming_backlog_exceeded', None),
                    on_success=d.get('on_success', None))
 
 
@@ -1352,9 +1368,8 @@ class JobEnvironment:
     """The key of an environment. It has to be unique within a job."""
 
     spec: Optional[compute.Environment] = None
-    """The a environment entity used to preserve serverless environment side panel and jobs'
-    environment for non-notebook task. In this minimal environment spec, only pip dependencies are
-    supported. Next ID: 5"""
+    """The environment entity used to preserve serverless environment side panel and jobs' environment
+    for non-notebook task. In this minimal environment spec, only pip dependencies are supported."""
 
     def as_dict(self) -> dict:
         """Serializes the JobEnvironment into a dictionary suitable for use as a JSON request body."""
@@ -1586,7 +1601,7 @@ class JobSettings:
     """Deployment information for jobs managed by external sources."""
 
     description: Optional[str] = None
-    """An optional description for the job. The maximum length is 1024 characters in UTF-8 encoding."""
+    """An optional description for the job. The maximum length is 27700 characters in UTF-8 encoding."""
 
     edit_mode: Optional[JobEditMode] = None
     """Edit mode of the job.
@@ -1783,9 +1798,21 @@ class JobSourceDirtyState(Enum):
 
 
 class JobsHealthMetric(Enum):
-    """Specifies the health metric that is being evaluated for a particular health rule."""
+    """Specifies the health metric that is being evaluated for a particular health rule.
+    
+    * `RUN_DURATION_SECONDS`: Expected total time for a run in seconds. * `STREAMING_BACKLOG_BYTES`:
+    An estimate of the maximum bytes of data waiting to be consumed across all streams. This metric
+    is in Private Preview. * `STREAMING_BACKLOG_RECORDS`: An estimate of the maximum offset lag
+    across all streams. This metric is in Private Preview. * `STREAMING_BACKLOG_SECONDS`: An
+    estimate of the maximum consumer delay across all streams. This metric is in Private Preview. *
+    `STREAMING_BACKLOG_FILES`: An estimate of the maximum number of outstanding files across all
+    streams. This metric is in Private Preview."""
 
     RUN_DURATION_SECONDS = 'RUN_DURATION_SECONDS'
+    STREAMING_BACKLOG_BYTES = 'STREAMING_BACKLOG_BYTES'
+    STREAMING_BACKLOG_FILES = 'STREAMING_BACKLOG_FILES'
+    STREAMING_BACKLOG_RECORDS = 'STREAMING_BACKLOG_RECORDS'
+    STREAMING_BACKLOG_SECONDS = 'STREAMING_BACKLOG_SECONDS'
 
 
 class JobsHealthOperator(Enum):
@@ -1797,7 +1824,15 @@ class JobsHealthOperator(Enum):
 @dataclass
 class JobsHealthRule:
     metric: JobsHealthMetric
-    """Specifies the health metric that is being evaluated for a particular health rule."""
+    """Specifies the health metric that is being evaluated for a particular health rule.
+    
+    * `RUN_DURATION_SECONDS`: Expected total time for a run in seconds. * `STREAMING_BACKLOG_BYTES`:
+    An estimate of the maximum bytes of data waiting to be consumed across all streams. This metric
+    is in Private Preview. * `STREAMING_BACKLOG_RECORDS`: An estimate of the maximum offset lag
+    across all streams. This metric is in Private Preview. * `STREAMING_BACKLOG_SECONDS`: An
+    estimate of the maximum consumer delay across all streams. This metric is in Private Preview. *
+    `STREAMING_BACKLOG_FILES`: An estimate of the maximum number of outstanding files across all
+    streams. This metric is in Private Preview."""
 
     op: JobsHealthOperator
     """Specifies the operator used to compare the health metric value with the specified threshold."""
@@ -1994,6 +2029,35 @@ class PauseStatus(Enum):
     UNPAUSED = 'UNPAUSED'
 
 
+@dataclass
+class PeriodicTriggerConfiguration:
+    interval: int
+    """The interval at which the trigger should run."""
+
+    unit: PeriodicTriggerConfigurationTimeUnit
+    """The unit of time for the interval."""
+
+    def as_dict(self) -> dict:
+        """Serializes the PeriodicTriggerConfiguration into a dictionary suitable for use as a JSON request body."""
+        body = {}
+        if self.interval is not None: body['interval'] = self.interval
+        if self.unit is not None: body['unit'] = self.unit.value
+        return body
+
+    @classmethod
+    def from_dict(cls, d: Dict[str, any]) -> PeriodicTriggerConfiguration:
+        """Deserializes the PeriodicTriggerConfiguration from a dictionary."""
+        return cls(interval=d.get('interval', None),
+                   unit=_enum(d, 'unit', PeriodicTriggerConfigurationTimeUnit))
+
+
+class PeriodicTriggerConfigurationTimeUnit(Enum):
+
+    DAYS = 'DAYS'
+    HOURS = 'HOURS'
+    WEEKS = 'WEEKS'
+
+
 @dataclass
 class PipelineParams:
     full_refresh: Optional[bool] = None
@@ -2448,7 +2512,6 @@ class ResolvedStringParamsValues:
 
 @dataclass
 class ResolvedValues:
-
     condition_task: Optional[ResolvedConditionTaskValues] = None
 
     dbt_task: Optional[ResolvedDbtTaskValues] = None
@@ -2551,6 +2614,9 @@ class Run:
     Note: dbt and SQL File tasks support only version-controlled sources. If dbt or SQL File tasks
     are used, `git_source` must be defined on the job."""
 
+    iterations: Optional[List[RunTask]] = None
+    """Only populated by for-each iterations. The parent for-each task is located in tasks array."""
+
     job_clusters: Optional[List[JobCluster]] = None
     """A list of job cluster specifications that can be shared and reused by tasks of this job.
     Libraries cannot be declared in a shared job cluster. You must declare dependent libraries in
@@ -2562,6 +2628,9 @@ class Run:
     job_parameters: Optional[List[JobParameter]] = None
     """Job-level parameters used in the run"""
 
+    next_page_token: Optional[str] = None
+    """A token that can be used to list the next page of sub-resources."""
+
     number_in_job: Optional[int] = None
     """A unique identifier for this job run. This is set to the same value as `run_id`."""
 
@@ -2572,6 +2641,9 @@ class Run:
     overriding_parameters: Optional[RunParameters] = None
     """The parameters used for this run."""
 
+    prev_page_token: Optional[str] = None
+    """A token that can be used to list the previous page of sub-resources."""
+
     queue_duration: Optional[int] = None
     """The time in milliseconds that the run has spent in the queue."""
 
@@ -2645,13 +2717,16 @@ class Run:
         if self.end_time is not None: body['end_time'] = self.end_time
         if self.execution_duration is not None: body['execution_duration'] = self.execution_duration
         if self.git_source: body['git_source'] = self.git_source.as_dict()
+        if self.iterations: body['iterations'] = [v.as_dict() for v in self.iterations]
         if self.job_clusters: body['job_clusters'] = [v.as_dict() for v in self.job_clusters]
         if self.job_id is not None: body['job_id'] = self.job_id
         if self.job_parameters: body['job_parameters'] = [v.as_dict() for v in self.job_parameters]
+        if self.next_page_token is not None: body['next_page_token'] = self.next_page_token
         if self.number_in_job is not None: body['number_in_job'] = self.number_in_job
         if self.original_attempt_run_id is not None:
             body['original_attempt_run_id'] = self.original_attempt_run_id
         if self.overriding_parameters: body['overriding_parameters'] = self.overriding_parameters.as_dict()
+        if self.prev_page_token is not None: body['prev_page_token'] = self.prev_page_token
         if self.queue_duration is not None: body['queue_duration'] = self.queue_duration
         if self.repair_history: body['repair_history'] = [v.as_dict() for v in self.repair_history]
         if self.run_duration is not None: body['run_duration'] = self.run_duration
@@ -2680,12 +2755,15 @@ class Run:
                    end_time=d.get('end_time', None),
                    execution_duration=d.get('execution_duration', None),
                    git_source=_from_dict(d, 'git_source', GitSource),
+                   iterations=_repeated_dict(d, 'iterations', RunTask),
                    job_clusters=_repeated_dict(d, 'job_clusters', JobCluster),
                    job_id=d.get('job_id', None),
                    job_parameters=_repeated_dict(d, 'job_parameters', JobParameter),
+                   next_page_token=d.get('next_page_token', None),
                    number_in_job=d.get('number_in_job', None),
                    original_attempt_run_id=d.get('original_attempt_run_id', None),
                    overriding_parameters=_from_dict(d, 'overriding_parameters', RunParameters),
+                   prev_page_token=d.get('prev_page_token', None),
                    queue_duration=d.get('queue_duration', None),
                    repair_history=_repeated_dict(d, 'repair_history', RepairHistoryItem),
                    run_duration=d.get('run_duration', None),
@@ -3391,6 +3469,10 @@ class RunTask:
     """The time at which this run ended in epoch milliseconds (milliseconds since 1/1/1970 UTC). This
     field is set to 0 if the job is still running."""
 
+    environment_key: Optional[str] = None
+    """The key that references an environment spec in a job. This field is required for Python script,
+    Python wheel and dbt tasks when using serverless compute."""
+
     execution_duration: Optional[int] = None
     """The time in milliseconds it took to execute the commands in the JAR or notebook until they
     completed, failed, timed out, were cancelled, or encountered an unexpected error. The duration
@@ -3522,6 +3604,7 @@ class RunTask:
         if self.description is not None: body['description'] = self.description
         if self.email_notifications: body['email_notifications'] = self.email_notifications.as_dict()
         if self.end_time is not None: body['end_time'] = self.end_time
+        if self.environment_key is not None: body['environment_key'] = self.environment_key
         if self.execution_duration is not None: body['execution_duration'] = self.execution_duration
         if self.existing_cluster_id is not None: body['existing_cluster_id'] = self.existing_cluster_id
         if self.for_each_task: body['for_each_task'] = self.for_each_task.as_dict()
@@ -3564,6 +3647,7 @@ class RunTask:
                    description=d.get('description', None),
                    email_notifications=_from_dict(d, 'email_notifications', JobEmailNotifications),
                    end_time=d.get('end_time', None),
+                   environment_key=d.get('environment_key', None),
                    execution_duration=d.get('execution_duration', None),
                    existing_cluster_id=d.get('existing_cluster_id', None),
                    for_each_task=_from_dict(d, 'for_each_task', RunForEachTask),
@@ -4118,21 +4202,15 @@ class SqlTaskSubscription:
 
 @dataclass
 class SubmitRun:
-    access_control_list: Optional[List[iam.AccessControlRequest]] = None
+    access_control_list: Optional[List[JobAccessControlRequest]] = None
     """List of permissions to set on the job."""
 
-    condition_task: Optional[ConditionTask] = None
-    """If condition_task, specifies a condition with an outcome that can be used to control the
-    execution of other tasks. Does not require a cluster to execute and does not support retries or
-    notifications."""
-
-    dbt_task: Optional[DbtTask] = None
-    """If dbt_task, indicates that this must execute a dbt task. It requires both Databricks SQL and
-    the ability to use a serverless or a pro SQL warehouse."""
-
     email_notifications: Optional[JobEmailNotifications] = None
     """An optional set of email addresses notified when the run begins or completes."""
 
+    environments: Optional[List[JobEnvironment]] = None
+    """A list of task execution environment specifications that can be referenced by tasks of this run."""
+
     git_source: Optional[GitSource] = None
     """An optional specification for a remote Git repository containing the source code used by tasks.
     Version-controlled source code is supported by notebook, dbt, Python script, and SQL File tasks.
@@ -4160,20 +4238,10 @@ class SubmitRun:
     
     [How to ensure idempotency for jobs]: https://kb.databricks.com/jobs/jobs-idempotency.html"""
 
-    notebook_task: Optional[NotebookTask] = None
-    """If notebook_task, indicates that this task must run a notebook. This field may not be specified
-    in conjunction with spark_jar_task."""
-
     notification_settings: Optional[JobNotificationSettings] = None
     """Optional notification settings that are used when sending notifications to each of the
     `email_notifications` and `webhook_notifications` for this run."""
 
-    pipeline_task: Optional[PipelineTask] = None
-    """If pipeline_task, indicates that this task must execute a Pipeline."""
-
-    python_wheel_task: Optional[PythonWheelTask] = None
-    """If python_wheel_task, indicates that this job must execute a PythonWheel."""
-
     queue: Optional[QueueSettings] = None
     """The queue settings of the one-time run."""
 
@@ -4181,38 +4249,9 @@ class SubmitRun:
     """Specifies the user or service principal that the job runs as. If not specified, the job runs as
     the user who submits the request."""
 
-    run_job_task: Optional[RunJobTask] = None
-    """If run_job_task, indicates that this task must execute another job."""
-
     run_name: Optional[str] = None
     """An optional name for the run. The default value is `Untitled`."""
 
-    spark_jar_task: Optional[SparkJarTask] = None
-    """If spark_jar_task, indicates that this task must run a JAR."""
-
-    spark_python_task: Optional[SparkPythonTask] = None
-    """If spark_python_task, indicates that this task must run a Python file."""
-
-    spark_submit_task: Optional[SparkSubmitTask] = None
-    """If `spark_submit_task`, indicates that this task must be launched by the spark submit script.
-    This task can run only on new clusters.
-    
-    In the `new_cluster` specification, `libraries` and `spark_conf` are not supported. Instead, use
-    `--jars` and `--py-files` to add Java and Python libraries and `--conf` to set the Spark
-    configurations.
-    
-    `master`, `deploy-mode`, and `executor-cores` are automatically configured by Databricks; you
-    _cannot_ specify them in parameters.
-    
-    By default, the Spark submit job uses all available memory (excluding reserved memory for
-    Databricks services). You can set `--driver-memory`, and `--executor-memory` to a smaller value
-    to leave some room for off-heap usage.
-    
-    The `--jars`, `--py-files`, `--files` arguments support DBFS and S3 paths."""
-
-    sql_task: Optional[SqlTask] = None
-    """If sql_task, indicates that this job must execute a SQL task."""
-
     tasks: Optional[List[SubmitTask]] = None
 
     timeout_seconds: Optional[int] = None
@@ -4226,24 +4265,15 @@ class SubmitRun:
         body = {}
         if self.access_control_list:
             body['access_control_list'] = [v.as_dict() for v in self.access_control_list]
-        if self.condition_task: body['condition_task'] = self.condition_task.as_dict()
-        if self.dbt_task: body['dbt_task'] = self.dbt_task.as_dict()
         if self.email_notifications: body['email_notifications'] = self.email_notifications.as_dict()
+        if self.environments: body['environments'] = [v.as_dict() for v in self.environments]
         if self.git_source: body['git_source'] = self.git_source.as_dict()
         if self.health: body['health'] = self.health.as_dict()
         if self.idempotency_token is not None: body['idempotency_token'] = self.idempotency_token
-        if self.notebook_task: body['notebook_task'] = self.notebook_task.as_dict()
         if self.notification_settings: body['notification_settings'] = self.notification_settings.as_dict()
-        if self.pipeline_task: body['pipeline_task'] = self.pipeline_task.as_dict()
-        if self.python_wheel_task: body['python_wheel_task'] = self.python_wheel_task.as_dict()
         if self.queue: body['queue'] = self.queue.as_dict()
         if self.run_as: body['run_as'] = self.run_as.as_dict()
-        if self.run_job_task: body['run_job_task'] = self.run_job_task.as_dict()
         if self.run_name is not None: body['run_name'] = self.run_name
-        if self.spark_jar_task: body['spark_jar_task'] = self.spark_jar_task.as_dict()
-        if self.spark_python_task: body['spark_python_task'] = self.spark_python_task.as_dict()
-        if self.spark_submit_task: body['spark_submit_task'] = self.spark_submit_task.as_dict()
-        if self.sql_task: body['sql_task'] = self.sql_task.as_dict()
         if self.tasks: body['tasks'] = [v.as_dict() for v in self.tasks]
         if self.timeout_seconds is not None: body['timeout_seconds'] = self.timeout_seconds
         if self.webhook_notifications: body['webhook_notifications'] = self.webhook_notifications.as_dict()
@@ -4252,25 +4282,16 @@ class SubmitRun:
     @classmethod
     def from_dict(cls, d: Dict[str, any]) -> SubmitRun:
         """Deserializes the SubmitRun from a dictionary."""
-        return cls(access_control_list=_repeated_dict(d, 'access_control_list', iam.AccessControlRequest),
-                   condition_task=_from_dict(d, 'condition_task', ConditionTask),
-                   dbt_task=_from_dict(d, 'dbt_task', DbtTask),
+        return cls(access_control_list=_repeated_dict(d, 'access_control_list', JobAccessControlRequest),
                    email_notifications=_from_dict(d, 'email_notifications', JobEmailNotifications),
+                   environments=_repeated_dict(d, 'environments', JobEnvironment),
                    git_source=_from_dict(d, 'git_source', GitSource),
                    health=_from_dict(d, 'health', JobsHealthRules),
                    idempotency_token=d.get('idempotency_token', None),
-                   notebook_task=_from_dict(d, 'notebook_task', NotebookTask),
                    notification_settings=_from_dict(d, 'notification_settings', JobNotificationSettings),
-                   pipeline_task=_from_dict(d, 'pipeline_task', PipelineTask),
-                   python_wheel_task=_from_dict(d, 'python_wheel_task', PythonWheelTask),
                    queue=_from_dict(d, 'queue', QueueSettings),
                    run_as=_from_dict(d, 'run_as', JobRunAs),
-                   run_job_task=_from_dict(d, 'run_job_task', RunJobTask),
                    run_name=d.get('run_name', None),
-                   spark_jar_task=_from_dict(d, 'spark_jar_task', SparkJarTask),
-                   spark_python_task=_from_dict(d, 'spark_python_task', SparkPythonTask),
-                   spark_submit_task=_from_dict(d, 'spark_submit_task', SparkSubmitTask),
-                   sql_task=_from_dict(d, 'sql_task', SqlTask),
                    tasks=_repeated_dict(d, 'tasks', SubmitTask),
                    timeout_seconds=d.get('timeout_seconds', None),
                    webhook_notifications=_from_dict(d, 'webhook_notifications', WebhookNotifications))
@@ -4307,6 +4328,10 @@ class SubmitTask:
     execution of other tasks. Does not require a cluster to execute and does not support retries or
     notifications."""
 
+    dbt_task: Optional[DbtTask] = None
+    """If dbt_task, indicates that this must execute a dbt task. It requires both Databricks SQL and
+    the ability to use a serverless or a pro SQL warehouse."""
+
     depends_on: Optional[List[TaskDependency]] = None
     """An optional array of objects specifying the dependency graph of the task. All tasks specified in
     this field must complete successfully before executing this task. The key is `task_key`, and the
@@ -4319,6 +4344,10 @@ class SubmitTask:
     """An optional set of email addresses notified when the task run begins or completes. The default
     behavior is to not send any emails."""
 
+    environment_key: Optional[str] = None
+    """The key that references an environment spec in a job. This field is required for Python script,
+    Python wheel and dbt tasks when using serverless compute."""
+
     existing_cluster_id: Optional[str] = None
     """If existing_cluster_id, the ID of an existing cluster that is used for all runs. When running
     jobs or tasks on an existing cluster, you may need to manually restart the cluster if it stops
@@ -4397,9 +4426,11 @@ class SubmitTask:
         """Serializes the SubmitTask into a dictionary suitable for use as a JSON request body."""
         body = {}
         if self.condition_task: body['condition_task'] = self.condition_task.as_dict()
+        if self.dbt_task: body['dbt_task'] = self.dbt_task.as_dict()
         if self.depends_on: body['depends_on'] = [v.as_dict() for v in self.depends_on]
         if self.description is not None: body['description'] = self.description
         if self.email_notifications: body['email_notifications'] = self.email_notifications.as_dict()
+        if self.environment_key is not None: body['environment_key'] = self.environment_key
         if self.existing_cluster_id is not None: body['existing_cluster_id'] = self.existing_cluster_id
         if self.for_each_task: body['for_each_task'] = self.for_each_task.as_dict()
         if self.health: body['health'] = self.health.as_dict()
@@ -4424,9 +4455,11 @@ class SubmitTask:
     def from_dict(cls, d: Dict[str, any]) -> SubmitTask:
         """Deserializes the SubmitTask from a dictionary."""
         return cls(condition_task=_from_dict(d, 'condition_task', ConditionTask),
+                   dbt_task=_from_dict(d, 'dbt_task', DbtTask),
                    depends_on=_repeated_dict(d, 'depends_on', TaskDependency),
                    description=d.get('description', None),
                    email_notifications=_from_dict(d, 'email_notifications', JobEmailNotifications),
+                   environment_key=d.get('environment_key', None),
                    existing_cluster_id=d.get('existing_cluster_id', None),
                    for_each_task=_from_dict(d, 'for_each_task', ForEachTask),
                    health=_from_dict(d, 'health', JobsHealthRules),
@@ -4729,6 +4762,13 @@ class TaskEmailNotifications:
     """A list of email addresses to be notified when a run begins. If not specified on job creation,
     reset, or update, the list is empty, and notifications are not sent."""
 
+    on_streaming_backlog_exceeded: Optional[List[str]] = None
+    """A list of email addresses to notify when any streaming backlog thresholds are exceeded for any
+    stream. Streaming backlog thresholds can be set in the `health` field using the following
+    metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`, `STREAMING_BACKLOG_SECONDS`, or
+    `STREAMING_BACKLOG_FILES`. Alerting is based on the 10-minute average of these metrics. If the
+    issue persists, notifications are resent every 30 minutes."""
+
     on_success: Optional[List[str]] = None
     """A list of email addresses to be notified when a run successfully completes. A run is considered
     to have completed successfully if it ends with a `TERMINATED` `life_cycle_state` and a `SUCCESS`
@@ -4746,6 +4786,8 @@ class TaskEmailNotifications:
             ]
         if self.on_failure: body['on_failure'] = [v for v in self.on_failure]
         if self.on_start: body['on_start'] = [v for v in self.on_start]
+        if self.on_streaming_backlog_exceeded:
+            body['on_streaming_backlog_exceeded'] = [v for v in self.on_streaming_backlog_exceeded]
         if self.on_success: body['on_success'] = [v for v in self.on_success]
         return body
 
@@ -4757,6 +4799,7 @@ class TaskEmailNotifications:
                                                                 None),
                    on_failure=d.get('on_failure', None),
                    on_start=d.get('on_start', None),
+                   on_streaming_backlog_exceeded=d.get('on_streaming_backlog_exceeded', None),
                    on_success=d.get('on_success', None))
 
 
@@ -4820,6 +4863,9 @@ class TriggerSettings:
     pause_status: Optional[PauseStatus] = None
     """Whether this trigger is paused or not."""
 
+    periodic: Optional[PeriodicTriggerConfiguration] = None
+    """Periodic trigger settings."""
+
     table: Optional[TableUpdateTriggerConfiguration] = None
     """Old table trigger settings name. Deprecated in favor of `table_update`."""
 
@@ -4830,6 +4876,7 @@ class TriggerSettings:
         body = {}
         if self.file_arrival: body['file_arrival'] = self.file_arrival.as_dict()
         if self.pause_status is not None: body['pause_status'] = self.pause_status.value
+        if self.periodic: body['periodic'] = self.periodic.as_dict()
         if self.table: body['table'] = self.table.as_dict()
         if self.table_update: body['table_update'] = self.table_update.as_dict()
         return body
@@ -4839,6 +4886,7 @@ class TriggerSettings:
         """Deserializes the TriggerSettings from a dictionary."""
         return cls(file_arrival=_from_dict(d, 'file_arrival', FileArrivalTriggerConfiguration),
                    pause_status=_enum(d, 'pause_status', PauseStatus),
+                   periodic=_from_dict(d, 'periodic', PeriodicTriggerConfiguration),
                    table=_from_dict(d, 'table', TableUpdateTriggerConfiguration),
                    table_update=_from_dict(d, 'table_update', TableUpdateTriggerConfiguration))
 
@@ -4986,6 +5034,14 @@ class WebhookNotifications:
     """An optional list of system notification IDs to call when the run starts. A maximum of 3
     destinations can be specified for the `on_start` property."""
 
+    on_streaming_backlog_exceeded: Optional[List[Webhook]] = None
+    """An optional list of system notification IDs to call when any streaming backlog thresholds are
+    exceeded for any stream. Streaming backlog thresholds can be set in the `health` field using the
+    following metrics: `STREAMING_BACKLOG_BYTES`, `STREAMING_BACKLOG_RECORDS`,
+    `STREAMING_BACKLOG_SECONDS`, or `STREAMING_BACKLOG_FILES`. Alerting is based on the 10-minute
+    average of these metrics. If the issue persists, notifications are resent every 30 minutes. A
+    maximum of 3 destinations can be specified for the `on_streaming_backlog_exceeded` property."""
+
     on_success: Optional[List[Webhook]] = None
     """An optional list of system notification IDs to call when the run completes successfully. A
     maximum of 3 destinations can be specified for the `on_success` property."""
@@ -4999,6 +5055,8 @@ class WebhookNotifications:
             ]
         if self.on_failure: body['on_failure'] = [v.as_dict() for v in self.on_failure]
         if self.on_start: body['on_start'] = [v.as_dict() for v in self.on_start]
+        if self.on_streaming_backlog_exceeded:
+            body['on_streaming_backlog_exceeded'] = [v.as_dict() for v in self.on_streaming_backlog_exceeded]
         if self.on_success: body['on_success'] = [v.as_dict() for v in self.on_success]
         return body
 
@@ -5009,6 +5067,7 @@ class WebhookNotifications:
             d, 'on_duration_warning_threshold_exceeded', Webhook),
                    on_failure=_repeated_dict(d, 'on_failure', Webhook),
                    on_start=_repeated_dict(d, 'on_start', Webhook),
+                   on_streaming_backlog_exceeded=_repeated_dict(d, 'on_streaming_backlog_exceeded', Webhook),
                    on_success=_repeated_dict(d, 'on_success', Webhook))
 
 
@@ -5112,7 +5171,7 @@ class JobsAPI:
 
     def create(self,
                *,
-               access_control_list: Optional[List[iam.AccessControlRequest]] = None,
+               access_control_list: Optional[List[JobAccessControlRequest]] = None,
                continuous: Optional[Continuous] = None,
                deployment: Optional[JobDeployment] = None,
                description: Optional[str] = None,
@@ -5139,7 +5198,7 @@ class JobsAPI:
         
         Create a new job.
         
-        :param access_control_list: List[:class:`AccessControlRequest`] (optional)
+        :param access_control_list: List[:class:`JobAccessControlRequest`] (optional)
           List of permissions to set on the job.
         :param continuous: :class:`Continuous` (optional)
           An optional continuous property for this job. The continuous property will ensure that there is
@@ -5147,7 +5206,7 @@ class JobsAPI:
         :param deployment: :class:`JobDeployment` (optional)
           Deployment information for jobs managed by external sources.
         :param description: str (optional)
-          An optional description for the job. The maximum length is 1024 characters in UTF-8 encoding.
+          An optional description for the job. The maximum length is 27700 characters in UTF-8 encoding.
         :param edit_mode: :class:`JobEditMode` (optional)
           Edit mode of the job.
           
@@ -5357,7 +5416,8 @@ class JobsAPI:
                 run_id: int,
                 *,
                 include_history: Optional[bool] = None,
-                include_resolved_values: Optional[bool] = None) -> Run:
+                include_resolved_values: Optional[bool] = None,
+                page_token: Optional[str] = None) -> Run:
         """Get a single job run.
         
         Retrieve the metadata of a run.
@@ -5368,6 +5428,9 @@ class JobsAPI:
           Whether to include the repair history in the response.
         :param include_resolved_values: bool (optional)
           Whether to include resolved parameter values in the response.
+        :param page_token: str (optional)
+          To list the next page or the previous page of job tasks, set this field to the value of the
+          `next_page_token` or `prev_page_token` returned in the GetJob response.
         
         :returns: :class:`Run`
         """
@@ -5375,6 +5438,7 @@ class JobsAPI:
         query = {}
         if include_history is not None: query['include_history'] = include_history
         if include_resolved_values is not None: query['include_resolved_values'] = include_resolved_values
+        if page_token is not None: query['page_token'] = page_token
         if run_id is not None: query['run_id'] = run_id
         headers = {'Accept': 'application/json', }
 
@@ -5882,25 +5946,16 @@ class JobsAPI:
 
     def submit(self,
                *,
-               access_control_list: Optional[List[iam.AccessControlRequest]] = None,
-               condition_task: Optional[ConditionTask] = None,
-               dbt_task: Optional[DbtTask] = None,
+               access_control_list: Optional[List[JobAccessControlRequest]] = None,
                email_notifications: Optional[JobEmailNotifications] = None,
+               environments: Optional[List[JobEnvironment]] = None,
                git_source: Optional[GitSource] = None,
                health: Optional[JobsHealthRules] = None,
                idempotency_token: Optional[str] = None,
-               notebook_task: Optional[NotebookTask] = None,
                notification_settings: Optional[JobNotificationSettings] = None,
-               pipeline_task: Optional[PipelineTask] = None,
-               python_wheel_task: Optional[PythonWheelTask] = None,
                queue: Optional[QueueSettings] = None,
                run_as: Optional[JobRunAs] = None,
-               run_job_task: Optional[RunJobTask] = None,
                run_name: Optional[str] = None,
-               spark_jar_task: Optional[SparkJarTask] = None,
-               spark_python_task: Optional[SparkPythonTask] = None,
-               spark_submit_task: Optional[SparkSubmitTask] = None,
-               sql_task: Optional[SqlTask] = None,
                tasks: Optional[List[SubmitTask]] = None,
                timeout_seconds: Optional[int] = None,
                webhook_notifications: Optional[WebhookNotifications] = None) -> Wait[Run]:
@@ -5910,16 +5965,12 @@ class JobsAPI:
         Runs submitted using this endpoint don’t display in the UI. Use the `jobs/runs/get` API to check the
         run state after the job is submitted.
         
-        :param access_control_list: List[:class:`AccessControlRequest`] (optional)
+        :param access_control_list: List[:class:`JobAccessControlRequest`] (optional)
           List of permissions to set on the job.
-        :param condition_task: :class:`ConditionTask` (optional)
-          If condition_task, specifies a condition with an outcome that can be used to control the execution
-          of other tasks. Does not require a cluster to execute and does not support retries or notifications.
-        :param dbt_task: :class:`DbtTask` (optional)
-          If dbt_task, indicates that this must execute a dbt task. It requires both Databricks SQL and the
-          ability to use a serverless or a pro SQL warehouse.
         :param email_notifications: :class:`JobEmailNotifications` (optional)
           An optional set of email addresses notified when the run begins or completes.
+        :param environments: List[:class:`JobEnvironment`] (optional)
+          A list of task execution environment specifications that can be referenced by tasks of this run.
         :param git_source: :class:`GitSource` (optional)
           An optional specification for a remote Git repository containing the source code used by tasks.
           Version-controlled source code is supported by notebook, dbt, Python script, and SQL File tasks.
@@ -5944,47 +5995,16 @@ class JobsAPI:
           For more information, see [How to ensure idempotency for jobs].
           
           [How to ensure idempotency for jobs]: https://kb.databricks.com/jobs/jobs-idempotency.html
-        :param notebook_task: :class:`NotebookTask` (optional)
-          If notebook_task, indicates that this task must run a notebook. This field may not be specified in
-          conjunction with spark_jar_task.
         :param notification_settings: :class:`JobNotificationSettings` (optional)
           Optional notification settings that are used when sending notifications to each of the
           `email_notifications` and `webhook_notifications` for this run.
-        :param pipeline_task: :class:`PipelineTask` (optional)
-          If pipeline_task, indicates that this task must execute a Pipeline.
-        :param python_wheel_task: :class:`PythonWheelTask` (optional)
-          If python_wheel_task, indicates that this job must execute a PythonWheel.
         :param queue: :class:`QueueSettings` (optional)
           The queue settings of the one-time run.
         :param run_as: :class:`JobRunAs` (optional)
           Specifies the user or service principal that the job runs as. If not specified, the job runs as the
           user who submits the request.
-        :param run_job_task: :class:`RunJobTask` (optional)
-          If run_job_task, indicates that this task must execute another job.
         :param run_name: str (optional)
           An optional name for the run. The default value is `Untitled`.
-        :param spark_jar_task: :class:`SparkJarTask` (optional)
-          If spark_jar_task, indicates that this task must run a JAR.
-        :param spark_python_task: :class:`SparkPythonTask` (optional)
-          If spark_python_task, indicates that this task must run a Python file.
-        :param spark_submit_task: :class:`SparkSubmitTask` (optional)
-          If `spark_submit_task`, indicates that this task must be launched by the spark submit script. This
-          task can run only on new clusters.
-          
-          In the `new_cluster` specification, `libraries` and `spark_conf` are not supported. Instead, use
-          `--jars` and `--py-files` to add Java and Python libraries and `--conf` to set the Spark
-          configurations.
-          
-          `master`, `deploy-mode`, and `executor-cores` are automatically configured by Databricks; you
-          _cannot_ specify them in parameters.
-          
-          By default, the Spark submit job uses all available memory (excluding reserved memory for Databricks
-          services). You can set `--driver-memory`, and `--executor-memory` to a smaller value to leave some
-          room for off-heap usage.
-          
-          The `--jars`, `--py-files`, `--files` arguments support DBFS and S3 paths.
-        :param sql_task: :class:`SqlTask` (optional)
-          If sql_task, indicates that this job must execute a SQL task.
         :param tasks: List[:class:`SubmitTask`] (optional)
         :param timeout_seconds: int (optional)
           An optional timeout applied to each run of this job. A value of `0` means no timeout.
@@ -5998,24 +6018,15 @@ class JobsAPI:
         body = {}
         if access_control_list is not None:
             body['access_control_list'] = [v.as_dict() for v in access_control_list]
-        if condition_task is not None: body['condition_task'] = condition_task.as_dict()
-        if dbt_task is not None: body['dbt_task'] = dbt_task.as_dict()
         if email_notifications is not None: body['email_notifications'] = email_notifications.as_dict()
+        if environments is not None: body['environments'] = [v.as_dict() for v in environments]
         if git_source is not None: body['git_source'] = git_source.as_dict()
         if health is not None: body['health'] = health.as_dict()
         if idempotency_token is not None: body['idempotency_token'] = idempotency_token
-        if notebook_task is not None: body['notebook_task'] = notebook_task.as_dict()
         if notification_settings is not None: body['notification_settings'] = notification_settings.as_dict()
-        if pipeline_task is not None: body['pipeline_task'] = pipeline_task.as_dict()
-        if python_wheel_task is not None: body['python_wheel_task'] = python_wheel_task.as_dict()
         if queue is not None: body['queue'] = queue.as_dict()
         if run_as is not None: body['run_as'] = run_as.as_dict()
-        if run_job_task is not None: body['run_job_task'] = run_job_task.as_dict()
         if run_name is not None: body['run_name'] = run_name
-        if spark_jar_task is not None: body['spark_jar_task'] = spark_jar_task.as_dict()
-        if spark_python_task is not None: body['spark_python_task'] = spark_python_task.as_dict()
-        if spark_submit_task is not None: body['spark_submit_task'] = spark_submit_task.as_dict()
-        if sql_task is not None: body['sql_task'] = sql_task.as_dict()
         if tasks is not None: body['tasks'] = [v.as_dict() for v in tasks]
         if timeout_seconds is not None: body['timeout_seconds'] = timeout_seconds
         if webhook_notifications is not None: body['webhook_notifications'] = webhook_notifications.as_dict()
@@ -6029,48 +6040,30 @@ class JobsAPI:
     def submit_and_wait(
         self,
         *,
-        access_control_list: Optional[List[iam.AccessControlRequest]] = None,
-        condition_task: Optional[ConditionTask] = None,
-        dbt_task: Optional[DbtTask] = None,
+        access_control_list: Optional[List[JobAccessControlRequest]] = None,
         email_notifications: Optional[JobEmailNotifications] = None,
+        environments: Optional[List[JobEnvironment]] = None,
         git_source: Optional[GitSource] = None,
         health: Optional[JobsHealthRules] = None,
         idempotency_token: Optional[str] = None,
-        notebook_task: Optional[NotebookTask] = None,
         notification_settings: Optional[JobNotificationSettings] = None,
-        pipeline_task: Optional[PipelineTask] = None,
-        python_wheel_task: Optional[PythonWheelTask] = None,
         queue: Optional[QueueSettings] = None,
         run_as: Optional[JobRunAs] = None,
-        run_job_task: Optional[RunJobTask] = None,
         run_name: Optional[str] = None,
-        spark_jar_task: Optional[SparkJarTask] = None,
-        spark_python_task: Optional[SparkPythonTask] = None,
-        spark_submit_task: Optional[SparkSubmitTask] = None,
-        sql_task: Optional[SqlTask] = None,
         tasks: Optional[List[SubmitTask]] = None,
         timeout_seconds: Optional[int] = None,
         webhook_notifications: Optional[WebhookNotifications] = None,
         timeout=timedelta(minutes=20)) -> Run:
         return self.submit(access_control_list=access_control_list,
-                           condition_task=condition_task,
-                           dbt_task=dbt_task,
                            email_notifications=email_notifications,
+                           environments=environments,
                            git_source=git_source,
                            health=health,
                            idempotency_token=idempotency_token,
-                           notebook_task=notebook_task,
                            notification_settings=notification_settings,
-                           pipeline_task=pipeline_task,
-                           python_wheel_task=python_wheel_task,
                            queue=queue,
                            run_as=run_as,
-                           run_job_task=run_job_task,
                            run_name=run_name,
-                           spark_jar_task=spark_jar_task,
-                           spark_python_task=spark_python_task,
-                           spark_submit_task=spark_submit_task,
-                           sql_task=sql_task,
                            tasks=tasks,
                            timeout_seconds=timeout_seconds,
                            webhook_notifications=webhook_notifications).result(timeout=timeout)