pulumi-databricks 1.74.0a1753335781__py3-none-any.whl → 1.77.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pulumi_databricks/__init__.py +185 -12
- pulumi_databricks/_inputs.py +30467 -20225
- pulumi_databricks/access_control_rule_set.py +28 -29
- pulumi_databricks/account_federation_policy.py +473 -0
- pulumi_databricks/account_network_policy.py +74 -43
- pulumi_databricks/account_setting_v2.py +759 -0
- pulumi_databricks/aibi_dashboard_embedding_access_policy_setting.py +30 -31
- pulumi_databricks/aibi_dashboard_embedding_approved_domains_setting.py +30 -31
- pulumi_databricks/alert.py +185 -186
- pulumi_databricks/alert_v2.py +369 -211
- pulumi_databricks/app.py +315 -209
- pulumi_databricks/apps_settings_custom_template.py +531 -0
- pulumi_databricks/artifact_allowlist.py +72 -73
- pulumi_databricks/automatic_cluster_update_workspace_setting.py +30 -31
- pulumi_databricks/budget.py +84 -85
- pulumi_databricks/budget_policy.py +75 -53
- pulumi_databricks/catalog.py +544 -225
- pulumi_databricks/catalog_workspace_binding.py +82 -83
- pulumi_databricks/cluster.py +561 -683
- pulumi_databricks/cluster_policy.py +115 -116
- pulumi_databricks/compliance_security_profile_workspace_setting.py +30 -31
- pulumi_databricks/config/__init__.py +1 -1
- pulumi_databricks/config/__init__.pyi +3 -2
- pulumi_databricks/config/vars.py +40 -37
- pulumi_databricks/connection.py +346 -211
- pulumi_databricks/credential.py +262 -263
- pulumi_databricks/custom_app_integration.py +175 -176
- pulumi_databricks/dashboard.py +207 -208
- pulumi_databricks/data_quality_monitor.py +453 -0
- pulumi_databricks/data_quality_refresh.py +492 -0
- pulumi_databricks/database_database_catalog.py +437 -0
- pulumi_databricks/database_instance.py +938 -137
- pulumi_databricks/database_synced_database_table.py +589 -0
- pulumi_databricks/dbfs_file.py +87 -84
- pulumi_databricks/default_namespace_setting.py +30 -31
- pulumi_databricks/directory.py +61 -62
- pulumi_databricks/disable_legacy_access_setting.py +109 -33
- pulumi_databricks/disable_legacy_dbfs_setting.py +58 -39
- pulumi_databricks/disable_legacy_features_setting.py +62 -37
- pulumi_databricks/enhanced_security_monitoring_workspace_setting.py +30 -31
- pulumi_databricks/entitlements.py +148 -149
- pulumi_databricks/entity_tag_assignment.py +409 -0
- pulumi_databricks/external_location.py +276 -290
- pulumi_databricks/external_metadata.py +726 -0
- pulumi_databricks/feature_engineering_feature.py +480 -0
- pulumi_databricks/feature_engineering_materialized_feature.py +397 -0
- pulumi_databricks/file.py +96 -97
- pulumi_databricks/get_account_federation_policies.py +127 -0
- pulumi_databricks/get_account_federation_policy.py +214 -0
- pulumi_databricks/get_account_network_policies.py +36 -7
- pulumi_databricks/get_account_network_policy.py +33 -29
- pulumi_databricks/get_account_setting_v2.py +331 -0
- pulumi_databricks/get_alert_v2.py +100 -84
- pulumi_databricks/get_alerts_v2.py +35 -17
- pulumi_databricks/get_app.py +31 -20
- pulumi_databricks/get_apps.py +26 -15
- pulumi_databricks/get_apps_settings_custom_template.py +207 -0
- pulumi_databricks/get_apps_settings_custom_templates.py +133 -0
- pulumi_databricks/get_aws_assume_role_policy.py +27 -28
- pulumi_databricks/get_aws_bucket_policy.py +39 -40
- pulumi_databricks/get_aws_cross_account_policy.py +47 -48
- pulumi_databricks/get_aws_unity_catalog_assume_role_policy.py +35 -36
- pulumi_databricks/get_aws_unity_catalog_policy.py +35 -36
- pulumi_databricks/get_budget_policies.py +74 -11
- pulumi_databricks/get_budget_policy.py +43 -39
- pulumi_databricks/get_catalog.py +35 -19
- pulumi_databricks/get_catalogs.py +32 -14
- pulumi_databricks/get_cluster.py +41 -25
- pulumi_databricks/get_cluster_policy.py +72 -54
- pulumi_databricks/get_clusters.py +41 -25
- pulumi_databricks/get_current_config.py +23 -24
- pulumi_databricks/get_current_metastore.py +29 -13
- pulumi_databricks/get_current_user.py +17 -18
- pulumi_databricks/get_dashboards.py +34 -15
- pulumi_databricks/get_data_quality_monitor.py +210 -0
- pulumi_databricks/get_data_quality_monitors.py +143 -0
- pulumi_databricks/get_data_quality_refresh.py +270 -0
- pulumi_databricks/get_data_quality_refreshes.py +207 -0
- pulumi_databricks/get_database_database_catalog.py +176 -0
- pulumi_databricks/get_database_database_catalogs.py +120 -0
- pulumi_databricks/get_database_instance.py +274 -50
- pulumi_databricks/get_database_instances.py +35 -11
- pulumi_databricks/get_database_synced_database_table.py +225 -0
- pulumi_databricks/get_database_synced_database_tables.py +120 -0
- pulumi_databricks/get_dbfs_file.py +19 -20
- pulumi_databricks/get_dbfs_file_paths.py +16 -17
- pulumi_databricks/get_directory.py +43 -25
- pulumi_databricks/get_entity_tag_assignment.py +202 -0
- pulumi_databricks/get_entity_tag_assignments.py +187 -0
- pulumi_databricks/get_external_location.py +35 -19
- pulumi_databricks/get_external_locations.py +32 -14
- pulumi_databricks/get_external_metadata.py +292 -0
- pulumi_databricks/get_external_metadatas.py +135 -0
- pulumi_databricks/get_feature_engineering_feature.py +179 -0
- pulumi_databricks/get_feature_engineering_features.py +103 -0
- pulumi_databricks/get_feature_engineering_materialized_feature.py +180 -0
- pulumi_databricks/get_feature_engineering_materialized_features.py +123 -0
- pulumi_databricks/get_functions.py +38 -24
- pulumi_databricks/get_group.py +85 -86
- pulumi_databricks/get_instance_pool.py +10 -11
- pulumi_databricks/get_instance_profiles.py +25 -9
- pulumi_databricks/get_job.py +22 -23
- pulumi_databricks/get_jobs.py +44 -26
- pulumi_databricks/get_materialized_features_feature_tag.py +102 -0
- pulumi_databricks/get_materialized_features_feature_tags.py +133 -0
- pulumi_databricks/get_metastore.py +28 -29
- pulumi_databricks/get_metastores.py +9 -10
- pulumi_databricks/get_mlflow_experiment.py +62 -46
- pulumi_databricks/get_mlflow_model.py +47 -31
- pulumi_databricks/get_mlflow_models.py +32 -14
- pulumi_databricks/get_mws_credentials.py +9 -10
- pulumi_databricks/get_mws_network_connectivity_config.py +40 -41
- pulumi_databricks/get_mws_network_connectivity_configs.py +15 -16
- pulumi_databricks/get_mws_workspaces.py +5 -6
- pulumi_databricks/get_node_type.py +123 -87
- pulumi_databricks/get_notebook.py +37 -38
- pulumi_databricks/get_notebook_paths.py +16 -17
- pulumi_databricks/get_notification_destinations.py +33 -18
- pulumi_databricks/get_online_store.py +36 -27
- pulumi_databricks/get_online_stores.py +33 -13
- pulumi_databricks/get_pipelines.py +49 -31
- pulumi_databricks/get_policy_info.py +338 -0
- pulumi_databricks/get_policy_infos.py +169 -0
- pulumi_databricks/get_quality_monitor_v2.py +21 -18
- pulumi_databricks/get_quality_monitors_v2.py +26 -8
- pulumi_databricks/get_registered_model.py +41 -27
- pulumi_databricks/get_registered_model_versions.py +29 -15
- pulumi_databricks/get_rfa_access_request_destinations.py +126 -0
- pulumi_databricks/get_schema.py +32 -16
- pulumi_databricks/get_schemas.py +38 -20
- pulumi_databricks/get_service_principal.py +80 -58
- pulumi_databricks/get_service_principal_federation_policies.py +151 -0
- pulumi_databricks/get_service_principal_federation_policy.py +220 -0
- pulumi_databricks/get_service_principals.py +16 -17
- pulumi_databricks/get_serving_endpoints.py +23 -9
- pulumi_databricks/get_share.py +130 -32
- pulumi_databricks/get_shares.py +29 -11
- pulumi_databricks/get_spark_version.py +87 -69
- pulumi_databricks/get_sql_warehouse.py +124 -110
- pulumi_databricks/get_sql_warehouses.py +37 -19
- pulumi_databricks/get_storage_credential.py +32 -16
- pulumi_databricks/get_storage_credentials.py +32 -14
- pulumi_databricks/get_table.py +30 -14
- pulumi_databricks/get_tables.py +41 -23
- pulumi_databricks/get_tag_policies.py +139 -0
- pulumi_databricks/get_tag_policy.py +175 -0
- pulumi_databricks/get_user.py +33 -34
- pulumi_databricks/get_views.py +41 -23
- pulumi_databricks/get_volume.py +32 -16
- pulumi_databricks/get_volumes.py +41 -23
- pulumi_databricks/get_workspace_network_option.py +45 -26
- pulumi_databricks/get_workspace_setting_v2.py +331 -0
- pulumi_databricks/get_zones.py +39 -21
- pulumi_databricks/git_credential.py +242 -76
- pulumi_databricks/global_init_script.py +99 -100
- pulumi_databricks/grant.py +215 -212
- pulumi_databricks/grants.py +200 -211
- pulumi_databricks/group.py +176 -177
- pulumi_databricks/group_instance_profile.py +37 -38
- pulumi_databricks/group_member.py +37 -38
- pulumi_databricks/group_role.py +37 -38
- pulumi_databricks/instance_pool.py +168 -169
- pulumi_databricks/instance_profile.py +69 -70
- pulumi_databricks/ip_access_list.py +71 -72
- pulumi_databricks/job.py +426 -346
- pulumi_databricks/lakehouse_monitor.py +199 -200
- pulumi_databricks/library.py +264 -99
- pulumi_databricks/materialized_features_feature_tag.py +213 -0
- pulumi_databricks/metastore.py +254 -255
- pulumi_databricks/metastore_assignment.py +52 -53
- pulumi_databricks/metastore_data_access.py +153 -154
- pulumi_databricks/metastore_provider.py +69 -70
- pulumi_databricks/mlflow_experiment.py +108 -109
- pulumi_databricks/mlflow_model.py +45 -46
- pulumi_databricks/mlflow_webhook.py +75 -76
- pulumi_databricks/model_serving.py +195 -74
- pulumi_databricks/model_serving_provisioned_throughput.py +100 -54
- pulumi_databricks/mount.py +103 -104
- pulumi_databricks/mws_credentials.py +99 -100
- pulumi_databricks/mws_customer_managed_keys.py +75 -76
- pulumi_databricks/mws_log_delivery.py +188 -189
- pulumi_databricks/mws_ncc_binding.py +35 -36
- pulumi_databricks/mws_ncc_private_endpoint_rule.py +312 -289
- pulumi_databricks/mws_network_connectivity_config.py +136 -98
- pulumi_databricks/mws_networks.py +159 -160
- pulumi_databricks/mws_permission_assignment.py +70 -71
- pulumi_databricks/mws_private_access_settings.py +116 -117
- pulumi_databricks/mws_storage_configurations.py +74 -75
- pulumi_databricks/mws_vpc_endpoint.py +149 -150
- pulumi_databricks/mws_workspaces.py +464 -418
- pulumi_databricks/notebook.py +143 -144
- pulumi_databricks/notification_destination.py +38 -39
- pulumi_databricks/obo_token.py +63 -64
- pulumi_databricks/online_store.py +121 -70
- pulumi_databricks/online_table.py +41 -42
- pulumi_databricks/outputs.py +47821 -30802
- pulumi_databricks/permission_assignment.py +298 -57
- pulumi_databricks/permissions.py +523 -362
- pulumi_databricks/pipeline.py +515 -480
- pulumi_databricks/policy_info.py +916 -0
- pulumi_databricks/provider.py +283 -269
- pulumi_databricks/pulumi-plugin.json +1 -1
- pulumi_databricks/quality_monitor.py +263 -217
- pulumi_databricks/quality_monitor_v2.py +49 -46
- pulumi_databricks/query.py +227 -228
- pulumi_databricks/recipient.py +208 -209
- pulumi_databricks/registered_model.py +393 -122
- pulumi_databricks/repo.py +117 -118
- pulumi_databricks/restrict_workspace_admins_setting.py +30 -31
- pulumi_databricks/rfa_access_request_destinations.py +286 -0
- pulumi_databricks/schema.py +159 -160
- pulumi_databricks/secret.py +72 -73
- pulumi_databricks/secret_acl.py +52 -53
- pulumi_databricks/secret_scope.py +55 -56
- pulumi_databricks/service_principal.py +279 -278
- pulumi_databricks/service_principal_federation_policy.py +469 -0
- pulumi_databricks/service_principal_role.py +35 -36
- pulumi_databricks/service_principal_secret.py +156 -157
- pulumi_databricks/share.py +186 -187
- pulumi_databricks/sql_alert.py +98 -99
- pulumi_databricks/sql_dashboard.py +94 -95
- pulumi_databricks/sql_endpoint.py +298 -266
- pulumi_databricks/sql_global_config.py +103 -104
- pulumi_databricks/sql_permissions.py +121 -122
- pulumi_databricks/sql_query.py +150 -151
- pulumi_databricks/sql_table.py +310 -283
- pulumi_databricks/sql_visualization.py +92 -93
- pulumi_databricks/sql_widget.py +91 -88
- pulumi_databricks/storage_credential.py +236 -184
- pulumi_databricks/system_schema.py +50 -51
- pulumi_databricks/table.py +147 -148
- pulumi_databricks/tag_policy.py +357 -0
- pulumi_databricks/token.py +83 -84
- pulumi_databricks/user.py +284 -285
- pulumi_databricks/user_instance_profile.py +35 -36
- pulumi_databricks/user_role.py +35 -36
- pulumi_databricks/vector_search_endpoint.py +117 -118
- pulumi_databricks/vector_search_index.py +86 -87
- pulumi_databricks/volume.py +129 -130
- pulumi_databricks/workspace_binding.py +82 -131
- pulumi_databricks/workspace_conf.py +18 -19
- pulumi_databricks/workspace_file.py +100 -101
- pulumi_databricks/workspace_network_option.py +89 -40
- pulumi_databricks/workspace_setting_v2.py +759 -0
- {pulumi_databricks-1.74.0a1753335781.dist-info → pulumi_databricks-1.77.0.dist-info}/METADATA +1 -1
- pulumi_databricks-1.77.0.dist-info/RECORD +250 -0
- pulumi_databricks/get_recipient_federation_policies.py +0 -83
- pulumi_databricks/get_recipient_federation_policy.py +0 -166
- pulumi_databricks/recipient_federation_policy.py +0 -347
- pulumi_databricks-1.74.0a1753335781.dist-info/RECORD +0 -205
- {pulumi_databricks-1.74.0a1753335781.dist-info → pulumi_databricks-1.77.0.dist-info}/WHEEL +0 -0
- {pulumi_databricks-1.74.0a1753335781.dist-info → pulumi_databricks-1.77.0.dist-info}/top_level.txt +0 -0
pulumi_databricks/pipeline.py
CHANGED
|
@@ -2,8 +2,7 @@
|
|
|
2
2
|
# *** WARNING: this file was generated by pulumi-language-python. ***
|
|
3
3
|
# *** Do not edit by hand unless you're certain you know what you are doing! ***
|
|
4
4
|
|
|
5
|
-
import builtins
|
|
6
|
-
import copy
|
|
5
|
+
import builtins as _builtins
|
|
7
6
|
import warnings
|
|
8
7
|
import sys
|
|
9
8
|
import pulumi
|
|
@@ -22,69 +21,70 @@ __all__ = ['PipelineArgs', 'Pipeline']
|
|
|
22
21
|
@pulumi.input_type
|
|
23
22
|
class PipelineArgs:
|
|
24
23
|
def __init__(__self__, *,
|
|
25
|
-
allow_duplicate_names: Optional[pulumi.Input[
|
|
26
|
-
budget_policy_id: Optional[pulumi.Input[
|
|
27
|
-
catalog: Optional[pulumi.Input[
|
|
28
|
-
cause: Optional[pulumi.Input[
|
|
29
|
-
channel: Optional[pulumi.Input[
|
|
30
|
-
cluster_id: Optional[pulumi.Input[
|
|
24
|
+
allow_duplicate_names: Optional[pulumi.Input[_builtins.bool]] = None,
|
|
25
|
+
budget_policy_id: Optional[pulumi.Input[_builtins.str]] = None,
|
|
26
|
+
catalog: Optional[pulumi.Input[_builtins.str]] = None,
|
|
27
|
+
cause: Optional[pulumi.Input[_builtins.str]] = None,
|
|
28
|
+
channel: Optional[pulumi.Input[_builtins.str]] = None,
|
|
29
|
+
cluster_id: Optional[pulumi.Input[_builtins.str]] = None,
|
|
31
30
|
clusters: Optional[pulumi.Input[Sequence[pulumi.Input['PipelineClusterArgs']]]] = None,
|
|
32
|
-
configuration: Optional[pulumi.Input[Mapping[str, pulumi.Input[
|
|
33
|
-
continuous: Optional[pulumi.Input[
|
|
34
|
-
creator_user_name: Optional[pulumi.Input[
|
|
31
|
+
configuration: Optional[pulumi.Input[Mapping[str, pulumi.Input[_builtins.str]]]] = None,
|
|
32
|
+
continuous: Optional[pulumi.Input[_builtins.bool]] = None,
|
|
33
|
+
creator_user_name: Optional[pulumi.Input[_builtins.str]] = None,
|
|
35
34
|
deployment: Optional[pulumi.Input['PipelineDeploymentArgs']] = None,
|
|
36
|
-
development: Optional[pulumi.Input[
|
|
37
|
-
edition: Optional[pulumi.Input[
|
|
35
|
+
development: Optional[pulumi.Input[_builtins.bool]] = None,
|
|
36
|
+
edition: Optional[pulumi.Input[_builtins.str]] = None,
|
|
38
37
|
environment: Optional[pulumi.Input['PipelineEnvironmentArgs']] = None,
|
|
39
38
|
event_log: Optional[pulumi.Input['PipelineEventLogArgs']] = None,
|
|
40
|
-
expected_last_modified: Optional[pulumi.Input[
|
|
39
|
+
expected_last_modified: Optional[pulumi.Input[_builtins.int]] = None,
|
|
41
40
|
filters: Optional[pulumi.Input['PipelineFiltersArgs']] = None,
|
|
42
41
|
gateway_definition: Optional[pulumi.Input['PipelineGatewayDefinitionArgs']] = None,
|
|
43
|
-
health: Optional[pulumi.Input[
|
|
42
|
+
health: Optional[pulumi.Input[_builtins.str]] = None,
|
|
44
43
|
ingestion_definition: Optional[pulumi.Input['PipelineIngestionDefinitionArgs']] = None,
|
|
45
|
-
last_modified: Optional[pulumi.Input[
|
|
44
|
+
last_modified: Optional[pulumi.Input[_builtins.int]] = None,
|
|
46
45
|
latest_updates: Optional[pulumi.Input[Sequence[pulumi.Input['PipelineLatestUpdateArgs']]]] = None,
|
|
47
46
|
libraries: Optional[pulumi.Input[Sequence[pulumi.Input['PipelineLibraryArgs']]]] = None,
|
|
48
|
-
name: Optional[pulumi.Input[
|
|
47
|
+
name: Optional[pulumi.Input[_builtins.str]] = None,
|
|
49
48
|
notifications: Optional[pulumi.Input[Sequence[pulumi.Input['PipelineNotificationArgs']]]] = None,
|
|
50
|
-
photon: Optional[pulumi.Input[
|
|
49
|
+
photon: Optional[pulumi.Input[_builtins.bool]] = None,
|
|
51
50
|
restart_window: Optional[pulumi.Input['PipelineRestartWindowArgs']] = None,
|
|
52
|
-
root_path: Optional[pulumi.Input[
|
|
51
|
+
root_path: Optional[pulumi.Input[_builtins.str]] = None,
|
|
53
52
|
run_as: Optional[pulumi.Input['PipelineRunAsArgs']] = None,
|
|
54
|
-
run_as_user_name: Optional[pulumi.Input[
|
|
55
|
-
schema: Optional[pulumi.Input[
|
|
56
|
-
serverless: Optional[pulumi.Input[
|
|
57
|
-
state: Optional[pulumi.Input[
|
|
58
|
-
storage: Optional[pulumi.Input[
|
|
59
|
-
tags: Optional[pulumi.Input[Mapping[str, pulumi.Input[
|
|
60
|
-
target: Optional[pulumi.Input[
|
|
53
|
+
run_as_user_name: Optional[pulumi.Input[_builtins.str]] = None,
|
|
54
|
+
schema: Optional[pulumi.Input[_builtins.str]] = None,
|
|
55
|
+
serverless: Optional[pulumi.Input[_builtins.bool]] = None,
|
|
56
|
+
state: Optional[pulumi.Input[_builtins.str]] = None,
|
|
57
|
+
storage: Optional[pulumi.Input[_builtins.str]] = None,
|
|
58
|
+
tags: Optional[pulumi.Input[Mapping[str, pulumi.Input[_builtins.str]]]] = None,
|
|
59
|
+
target: Optional[pulumi.Input[_builtins.str]] = None,
|
|
61
60
|
trigger: Optional[pulumi.Input['PipelineTriggerArgs']] = None,
|
|
62
|
-
url: Optional[pulumi.Input[
|
|
61
|
+
url: Optional[pulumi.Input[_builtins.str]] = None,
|
|
62
|
+
usage_policy_id: Optional[pulumi.Input[_builtins.str]] = None):
|
|
63
63
|
"""
|
|
64
64
|
The set of arguments for constructing a Pipeline resource.
|
|
65
|
-
:param pulumi.Input[
|
|
66
|
-
:param pulumi.Input[
|
|
67
|
-
:param pulumi.Input[
|
|
68
|
-
:param pulumi.Input[
|
|
69
|
-
:param pulumi.Input[Sequence[pulumi.Input['PipelineClusterArgs']]] clusters: blocks - Clusters to run the pipeline. If none is specified, pipelines will automatically select a default cluster configuration for the pipeline. *Please note that
|
|
70
|
-
:param pulumi.Input[Mapping[str, pulumi.Input[
|
|
71
|
-
:param pulumi.Input[
|
|
65
|
+
:param pulumi.Input[_builtins.bool] allow_duplicate_names: Optional boolean flag. If false, deployment will fail if name conflicts with that of another pipeline. default is `false`.
|
|
66
|
+
:param pulumi.Input[_builtins.str] budget_policy_id: optional string specifying ID of the budget policy for this Lakeflow Declarative Pipeline.
|
|
67
|
+
:param pulumi.Input[_builtins.str] catalog: The name of catalog in Unity Catalog. *Change of this parameter forces recreation of the pipeline.* (Conflicts with `storage`).
|
|
68
|
+
:param pulumi.Input[_builtins.str] channel: optional name of the release channel for Spark version used by Lakeflow Declarative Pipeline. Supported values are: `CURRENT` (default) and `PREVIEW`.
|
|
69
|
+
:param pulumi.Input[Sequence[pulumi.Input['PipelineClusterArgs']]] clusters: blocks - Clusters to run the pipeline. If none is specified, pipelines will automatically select a default cluster configuration for the pipeline. *Please note that Lakeflow Declarative Pipeline clusters are supporting only subset of attributes as described in [documentation](https://docs.databricks.com/api/workspace/pipelines/create#clusters).* Also, note that `autoscale` block is extended with the `mode` parameter that controls the autoscaling algorithm (possible values are `ENHANCED` for new, enhanced autoscaling algorithm, or `LEGACY` for old algorithm).
|
|
70
|
+
:param pulumi.Input[Mapping[str, pulumi.Input[_builtins.str]]] configuration: An optional list of values to apply to the entire pipeline. Elements must be formatted as key:value pairs.
|
|
71
|
+
:param pulumi.Input[_builtins.bool] continuous: A flag indicating whether to run the pipeline continuously. The default value is `false`.
|
|
72
72
|
:param pulumi.Input['PipelineDeploymentArgs'] deployment: Deployment type of this pipeline. Supports following attributes:
|
|
73
|
-
:param pulumi.Input[
|
|
74
|
-
:param pulumi.Input[
|
|
75
|
-
:param pulumi.Input['PipelineEventLogArgs'] event_log: an optional block specifying a table where
|
|
73
|
+
:param pulumi.Input[_builtins.bool] development: A flag indicating whether to run the pipeline in development mode. The default value is `false`.
|
|
74
|
+
:param pulumi.Input[_builtins.str] edition: optional name of the [product edition](https://docs.databricks.com/aws/en/dlt/configure-pipeline#choose-a-product-edition). Supported values are: `CORE`, `PRO`, `ADVANCED` (default). Not required when `serverless` is set to `true`.
|
|
75
|
+
:param pulumi.Input['PipelineEventLogArgs'] event_log: an optional block specifying a table where LDP Event Log will be stored. Consists of the following fields:
|
|
76
76
|
:param pulumi.Input['PipelineFiltersArgs'] filters: Filters on which Pipeline packages to include in the deployed graph. This block consists of following attributes:
|
|
77
77
|
:param pulumi.Input['PipelineGatewayDefinitionArgs'] gateway_definition: The definition of a gateway pipeline to support CDC. Consists of following attributes:
|
|
78
78
|
:param pulumi.Input[Sequence[pulumi.Input['PipelineLibraryArgs']]] libraries: blocks - Specifies pipeline code.
|
|
79
|
-
:param pulumi.Input[
|
|
80
|
-
:param pulumi.Input[
|
|
81
|
-
:param pulumi.Input[
|
|
82
|
-
:param pulumi.Input[
|
|
83
|
-
:param pulumi.Input[
|
|
84
|
-
:param pulumi.Input[
|
|
85
|
-
:param pulumi.Input[Mapping[str, pulumi.Input[
|
|
86
|
-
:param pulumi.Input[
|
|
87
|
-
:param pulumi.Input[
|
|
79
|
+
:param pulumi.Input[_builtins.str] name: A user-friendly name for this pipeline. The name can be used to identify pipeline jobs in the UI.
|
|
80
|
+
:param pulumi.Input[_builtins.bool] photon: A flag indicating whether to use Photon engine. The default value is `false`.
|
|
81
|
+
:param pulumi.Input[_builtins.str] root_path: An optional string specifying the root path for this pipeline. This is used as the root directory when editing the pipeline in the Databricks user interface and it is added to `sys.path` when executing Python sources during pipeline execution.
|
|
82
|
+
:param pulumi.Input[_builtins.str] schema: The default schema (database) where tables are read from or published to. The presence of this attribute implies that the pipeline is in direct publishing mode.
|
|
83
|
+
:param pulumi.Input[_builtins.bool] serverless: An optional flag indicating if serverless compute should be used for this Lakeflow Declarative Pipeline. Requires `catalog` to be set, as it could be used only with Unity Catalog.
|
|
84
|
+
:param pulumi.Input[_builtins.str] storage: A location on cloud storage where output data and metadata required for pipeline execution are stored. By default, tables are stored in a subdirectory of this location. *Change of this parameter forces recreation of the pipeline.* (Conflicts with `catalog`).
|
|
85
|
+
:param pulumi.Input[Mapping[str, pulumi.Input[_builtins.str]]] tags: A map of tags associated with the pipeline. These are forwarded to the cluster as cluster tags, and are therefore subject to the same limitations. A maximum of 25 tags can be added to the pipeline.
|
|
86
|
+
:param pulumi.Input[_builtins.str] target: The name of a database (in either the Hive metastore or in a UC catalog) for persisting pipeline output data. Configuring the target setting allows you to view and query the pipeline output data from the Databricks UI.
|
|
87
|
+
:param pulumi.Input[_builtins.str] url: URL of the Lakeflow Declarative Pipeline on the given workspace.
|
|
88
88
|
"""
|
|
89
89
|
if allow_duplicate_names is not None:
|
|
90
90
|
pulumi.set(__self__, "allow_duplicate_names", allow_duplicate_names)
|
|
@@ -162,78 +162,80 @@ class PipelineArgs:
|
|
|
162
162
|
pulumi.set(__self__, "trigger", trigger)
|
|
163
163
|
if url is not None:
|
|
164
164
|
pulumi.set(__self__, "url", url)
|
|
165
|
+
if usage_policy_id is not None:
|
|
166
|
+
pulumi.set(__self__, "usage_policy_id", usage_policy_id)
|
|
165
167
|
|
|
166
|
-
@property
|
|
168
|
+
@_builtins.property
|
|
167
169
|
@pulumi.getter(name="allowDuplicateNames")
|
|
168
|
-
def allow_duplicate_names(self) -> Optional[pulumi.Input[
|
|
170
|
+
def allow_duplicate_names(self) -> Optional[pulumi.Input[_builtins.bool]]:
|
|
169
171
|
"""
|
|
170
172
|
Optional boolean flag. If false, deployment will fail if name conflicts with that of another pipeline. default is `false`.
|
|
171
173
|
"""
|
|
172
174
|
return pulumi.get(self, "allow_duplicate_names")
|
|
173
175
|
|
|
174
176
|
@allow_duplicate_names.setter
|
|
175
|
-
def allow_duplicate_names(self, value: Optional[pulumi.Input[
|
|
177
|
+
def allow_duplicate_names(self, value: Optional[pulumi.Input[_builtins.bool]]):
|
|
176
178
|
pulumi.set(self, "allow_duplicate_names", value)
|
|
177
179
|
|
|
178
|
-
@property
|
|
180
|
+
@_builtins.property
|
|
179
181
|
@pulumi.getter(name="budgetPolicyId")
|
|
180
|
-
def budget_policy_id(self) -> Optional[pulumi.Input[
|
|
182
|
+
def budget_policy_id(self) -> Optional[pulumi.Input[_builtins.str]]:
|
|
181
183
|
"""
|
|
182
|
-
optional string specifying ID of the budget policy for this
|
|
184
|
+
optional string specifying ID of the budget policy for this Lakeflow Declarative Pipeline.
|
|
183
185
|
"""
|
|
184
186
|
return pulumi.get(self, "budget_policy_id")
|
|
185
187
|
|
|
186
188
|
@budget_policy_id.setter
|
|
187
|
-
def budget_policy_id(self, value: Optional[pulumi.Input[
|
|
189
|
+
def budget_policy_id(self, value: Optional[pulumi.Input[_builtins.str]]):
|
|
188
190
|
pulumi.set(self, "budget_policy_id", value)
|
|
189
191
|
|
|
190
|
-
@property
|
|
192
|
+
@_builtins.property
|
|
191
193
|
@pulumi.getter
|
|
192
|
-
def catalog(self) -> Optional[pulumi.Input[
|
|
194
|
+
def catalog(self) -> Optional[pulumi.Input[_builtins.str]]:
|
|
193
195
|
"""
|
|
194
196
|
The name of catalog in Unity Catalog. *Change of this parameter forces recreation of the pipeline.* (Conflicts with `storage`).
|
|
195
197
|
"""
|
|
196
198
|
return pulumi.get(self, "catalog")
|
|
197
199
|
|
|
198
200
|
@catalog.setter
|
|
199
|
-
def catalog(self, value: Optional[pulumi.Input[
|
|
201
|
+
def catalog(self, value: Optional[pulumi.Input[_builtins.str]]):
|
|
200
202
|
pulumi.set(self, "catalog", value)
|
|
201
203
|
|
|
202
|
-
@property
|
|
204
|
+
@_builtins.property
|
|
203
205
|
@pulumi.getter
|
|
204
|
-
def cause(self) -> Optional[pulumi.Input[
|
|
206
|
+
def cause(self) -> Optional[pulumi.Input[_builtins.str]]:
|
|
205
207
|
return pulumi.get(self, "cause")
|
|
206
208
|
|
|
207
209
|
@cause.setter
|
|
208
|
-
def cause(self, value: Optional[pulumi.Input[
|
|
210
|
+
def cause(self, value: Optional[pulumi.Input[_builtins.str]]):
|
|
209
211
|
pulumi.set(self, "cause", value)
|
|
210
212
|
|
|
211
|
-
@property
|
|
213
|
+
@_builtins.property
|
|
212
214
|
@pulumi.getter
|
|
213
|
-
def channel(self) -> Optional[pulumi.Input[
|
|
215
|
+
def channel(self) -> Optional[pulumi.Input[_builtins.str]]:
|
|
214
216
|
"""
|
|
215
|
-
optional name of the release channel for Spark version used by
|
|
217
|
+
optional name of the release channel for Spark version used by Lakeflow Declarative Pipeline. Supported values are: `CURRENT` (default) and `PREVIEW`.
|
|
216
218
|
"""
|
|
217
219
|
return pulumi.get(self, "channel")
|
|
218
220
|
|
|
219
221
|
@channel.setter
|
|
220
|
-
def channel(self, value: Optional[pulumi.Input[
|
|
222
|
+
def channel(self, value: Optional[pulumi.Input[_builtins.str]]):
|
|
221
223
|
pulumi.set(self, "channel", value)
|
|
222
224
|
|
|
223
|
-
@property
|
|
225
|
+
@_builtins.property
|
|
224
226
|
@pulumi.getter(name="clusterId")
|
|
225
|
-
def cluster_id(self) -> Optional[pulumi.Input[
|
|
227
|
+
def cluster_id(self) -> Optional[pulumi.Input[_builtins.str]]:
|
|
226
228
|
return pulumi.get(self, "cluster_id")
|
|
227
229
|
|
|
228
230
|
@cluster_id.setter
|
|
229
|
-
def cluster_id(self, value: Optional[pulumi.Input[
|
|
231
|
+
def cluster_id(self, value: Optional[pulumi.Input[_builtins.str]]):
|
|
230
232
|
pulumi.set(self, "cluster_id", value)
|
|
231
233
|
|
|
232
|
-
@property
|
|
234
|
+
@_builtins.property
|
|
233
235
|
@pulumi.getter
|
|
234
236
|
def clusters(self) -> Optional[pulumi.Input[Sequence[pulumi.Input['PipelineClusterArgs']]]]:
|
|
235
237
|
"""
|
|
236
|
-
blocks - Clusters to run the pipeline. If none is specified, pipelines will automatically select a default cluster configuration for the pipeline. *Please note that
|
|
238
|
+
blocks - Clusters to run the pipeline. If none is specified, pipelines will automatically select a default cluster configuration for the pipeline. *Please note that Lakeflow Declarative Pipeline clusters are supporting only subset of attributes as described in [documentation](https://docs.databricks.com/api/workspace/pipelines/create#clusters).* Also, note that `autoscale` block is extended with the `mode` parameter that controls the autoscaling algorithm (possible values are `ENHANCED` for new, enhanced autoscaling algorithm, or `LEGACY` for old algorithm).
|
|
237
239
|
"""
|
|
238
240
|
return pulumi.get(self, "clusters")
|
|
239
241
|
|
|
@@ -241,40 +243,40 @@ class PipelineArgs:
|
|
|
241
243
|
def clusters(self, value: Optional[pulumi.Input[Sequence[pulumi.Input['PipelineClusterArgs']]]]):
|
|
242
244
|
pulumi.set(self, "clusters", value)
|
|
243
245
|
|
|
244
|
-
@property
|
|
246
|
+
@_builtins.property
|
|
245
247
|
@pulumi.getter
|
|
246
|
-
def configuration(self) -> Optional[pulumi.Input[Mapping[str, pulumi.Input[
|
|
248
|
+
def configuration(self) -> Optional[pulumi.Input[Mapping[str, pulumi.Input[_builtins.str]]]]:
|
|
247
249
|
"""
|
|
248
250
|
An optional list of values to apply to the entire pipeline. Elements must be formatted as key:value pairs.
|
|
249
251
|
"""
|
|
250
252
|
return pulumi.get(self, "configuration")
|
|
251
253
|
|
|
252
254
|
@configuration.setter
|
|
253
|
-
def configuration(self, value: Optional[pulumi.Input[Mapping[str, pulumi.Input[
|
|
255
|
+
def configuration(self, value: Optional[pulumi.Input[Mapping[str, pulumi.Input[_builtins.str]]]]):
|
|
254
256
|
pulumi.set(self, "configuration", value)
|
|
255
257
|
|
|
256
|
-
@property
|
|
258
|
+
@_builtins.property
|
|
257
259
|
@pulumi.getter
|
|
258
|
-
def continuous(self) -> Optional[pulumi.Input[
|
|
260
|
+
def continuous(self) -> Optional[pulumi.Input[_builtins.bool]]:
|
|
259
261
|
"""
|
|
260
262
|
A flag indicating whether to run the pipeline continuously. The default value is `false`.
|
|
261
263
|
"""
|
|
262
264
|
return pulumi.get(self, "continuous")
|
|
263
265
|
|
|
264
266
|
@continuous.setter
|
|
265
|
-
def continuous(self, value: Optional[pulumi.Input[
|
|
267
|
+
def continuous(self, value: Optional[pulumi.Input[_builtins.bool]]):
|
|
266
268
|
pulumi.set(self, "continuous", value)
|
|
267
269
|
|
|
268
|
-
@property
|
|
270
|
+
@_builtins.property
|
|
269
271
|
@pulumi.getter(name="creatorUserName")
|
|
270
|
-
def creator_user_name(self) -> Optional[pulumi.Input[
|
|
272
|
+
def creator_user_name(self) -> Optional[pulumi.Input[_builtins.str]]:
|
|
271
273
|
return pulumi.get(self, "creator_user_name")
|
|
272
274
|
|
|
273
275
|
@creator_user_name.setter
|
|
274
|
-
def creator_user_name(self, value: Optional[pulumi.Input[
|
|
276
|
+
def creator_user_name(self, value: Optional[pulumi.Input[_builtins.str]]):
|
|
275
277
|
pulumi.set(self, "creator_user_name", value)
|
|
276
278
|
|
|
277
|
-
@property
|
|
279
|
+
@_builtins.property
|
|
278
280
|
@pulumi.getter
|
|
279
281
|
def deployment(self) -> Optional[pulumi.Input['PipelineDeploymentArgs']]:
|
|
280
282
|
"""
|
|
@@ -286,31 +288,31 @@ class PipelineArgs:
|
|
|
286
288
|
def deployment(self, value: Optional[pulumi.Input['PipelineDeploymentArgs']]):
|
|
287
289
|
pulumi.set(self, "deployment", value)
|
|
288
290
|
|
|
289
|
-
@property
|
|
291
|
+
@_builtins.property
|
|
290
292
|
@pulumi.getter
|
|
291
|
-
def development(self) -> Optional[pulumi.Input[
|
|
293
|
+
def development(self) -> Optional[pulumi.Input[_builtins.bool]]:
|
|
292
294
|
"""
|
|
293
295
|
A flag indicating whether to run the pipeline in development mode. The default value is `false`.
|
|
294
296
|
"""
|
|
295
297
|
return pulumi.get(self, "development")
|
|
296
298
|
|
|
297
299
|
@development.setter
|
|
298
|
-
def development(self, value: Optional[pulumi.Input[
|
|
300
|
+
def development(self, value: Optional[pulumi.Input[_builtins.bool]]):
|
|
299
301
|
pulumi.set(self, "development", value)
|
|
300
302
|
|
|
301
|
-
@property
|
|
303
|
+
@_builtins.property
|
|
302
304
|
@pulumi.getter
|
|
303
|
-
def edition(self) -> Optional[pulumi.Input[
|
|
305
|
+
def edition(self) -> Optional[pulumi.Input[_builtins.str]]:
|
|
304
306
|
"""
|
|
305
307
|
optional name of the [product edition](https://docs.databricks.com/aws/en/dlt/configure-pipeline#choose-a-product-edition). Supported values are: `CORE`, `PRO`, `ADVANCED` (default). Not required when `serverless` is set to `true`.
|
|
306
308
|
"""
|
|
307
309
|
return pulumi.get(self, "edition")
|
|
308
310
|
|
|
309
311
|
@edition.setter
|
|
310
|
-
def edition(self, value: Optional[pulumi.Input[
|
|
312
|
+
def edition(self, value: Optional[pulumi.Input[_builtins.str]]):
|
|
311
313
|
pulumi.set(self, "edition", value)
|
|
312
314
|
|
|
313
|
-
@property
|
|
315
|
+
@_builtins.property
|
|
314
316
|
@pulumi.getter
|
|
315
317
|
def environment(self) -> Optional[pulumi.Input['PipelineEnvironmentArgs']]:
|
|
316
318
|
return pulumi.get(self, "environment")
|
|
@@ -319,11 +321,11 @@ class PipelineArgs:
|
|
|
319
321
|
def environment(self, value: Optional[pulumi.Input['PipelineEnvironmentArgs']]):
|
|
320
322
|
pulumi.set(self, "environment", value)
|
|
321
323
|
|
|
322
|
-
@property
|
|
324
|
+
@_builtins.property
|
|
323
325
|
@pulumi.getter(name="eventLog")
|
|
324
326
|
def event_log(self) -> Optional[pulumi.Input['PipelineEventLogArgs']]:
|
|
325
327
|
"""
|
|
326
|
-
an optional block specifying a table where
|
|
328
|
+
an optional block specifying a table where LDP Event Log will be stored. Consists of the following fields:
|
|
327
329
|
"""
|
|
328
330
|
return pulumi.get(self, "event_log")
|
|
329
331
|
|
|
@@ -331,16 +333,16 @@ class PipelineArgs:
|
|
|
331
333
|
def event_log(self, value: Optional[pulumi.Input['PipelineEventLogArgs']]):
|
|
332
334
|
pulumi.set(self, "event_log", value)
|
|
333
335
|
|
|
334
|
-
@property
|
|
336
|
+
@_builtins.property
|
|
335
337
|
@pulumi.getter(name="expectedLastModified")
|
|
336
|
-
def expected_last_modified(self) -> Optional[pulumi.Input[
|
|
338
|
+
def expected_last_modified(self) -> Optional[pulumi.Input[_builtins.int]]:
|
|
337
339
|
return pulumi.get(self, "expected_last_modified")
|
|
338
340
|
|
|
339
341
|
@expected_last_modified.setter
|
|
340
|
-
def expected_last_modified(self, value: Optional[pulumi.Input[
|
|
342
|
+
def expected_last_modified(self, value: Optional[pulumi.Input[_builtins.int]]):
|
|
341
343
|
pulumi.set(self, "expected_last_modified", value)
|
|
342
344
|
|
|
343
|
-
@property
|
|
345
|
+
@_builtins.property
|
|
344
346
|
@pulumi.getter
|
|
345
347
|
def filters(self) -> Optional[pulumi.Input['PipelineFiltersArgs']]:
|
|
346
348
|
"""
|
|
@@ -352,7 +354,7 @@ class PipelineArgs:
|
|
|
352
354
|
def filters(self, value: Optional[pulumi.Input['PipelineFiltersArgs']]):
|
|
353
355
|
pulumi.set(self, "filters", value)
|
|
354
356
|
|
|
355
|
-
@property
|
|
357
|
+
@_builtins.property
|
|
356
358
|
@pulumi.getter(name="gatewayDefinition")
|
|
357
359
|
def gateway_definition(self) -> Optional[pulumi.Input['PipelineGatewayDefinitionArgs']]:
|
|
358
360
|
"""
|
|
@@ -364,16 +366,16 @@ class PipelineArgs:
|
|
|
364
366
|
def gateway_definition(self, value: Optional[pulumi.Input['PipelineGatewayDefinitionArgs']]):
|
|
365
367
|
pulumi.set(self, "gateway_definition", value)
|
|
366
368
|
|
|
367
|
-
@property
|
|
369
|
+
@_builtins.property
|
|
368
370
|
@pulumi.getter
|
|
369
|
-
def health(self) -> Optional[pulumi.Input[
|
|
371
|
+
def health(self) -> Optional[pulumi.Input[_builtins.str]]:
|
|
370
372
|
return pulumi.get(self, "health")
|
|
371
373
|
|
|
372
374
|
@health.setter
|
|
373
|
-
def health(self, value: Optional[pulumi.Input[
|
|
375
|
+
def health(self, value: Optional[pulumi.Input[_builtins.str]]):
|
|
374
376
|
pulumi.set(self, "health", value)
|
|
375
377
|
|
|
376
|
-
@property
|
|
378
|
+
@_builtins.property
|
|
377
379
|
@pulumi.getter(name="ingestionDefinition")
|
|
378
380
|
def ingestion_definition(self) -> Optional[pulumi.Input['PipelineIngestionDefinitionArgs']]:
|
|
379
381
|
return pulumi.get(self, "ingestion_definition")
|
|
@@ -382,16 +384,16 @@ class PipelineArgs:
|
|
|
382
384
|
def ingestion_definition(self, value: Optional[pulumi.Input['PipelineIngestionDefinitionArgs']]):
|
|
383
385
|
pulumi.set(self, "ingestion_definition", value)
|
|
384
386
|
|
|
385
|
-
@property
|
|
387
|
+
@_builtins.property
|
|
386
388
|
@pulumi.getter(name="lastModified")
|
|
387
|
-
def last_modified(self) -> Optional[pulumi.Input[
|
|
389
|
+
def last_modified(self) -> Optional[pulumi.Input[_builtins.int]]:
|
|
388
390
|
return pulumi.get(self, "last_modified")
|
|
389
391
|
|
|
390
392
|
@last_modified.setter
|
|
391
|
-
def last_modified(self, value: Optional[pulumi.Input[
|
|
393
|
+
def last_modified(self, value: Optional[pulumi.Input[_builtins.int]]):
|
|
392
394
|
pulumi.set(self, "last_modified", value)
|
|
393
395
|
|
|
394
|
-
@property
|
|
396
|
+
@_builtins.property
|
|
395
397
|
@pulumi.getter(name="latestUpdates")
|
|
396
398
|
def latest_updates(self) -> Optional[pulumi.Input[Sequence[pulumi.Input['PipelineLatestUpdateArgs']]]]:
|
|
397
399
|
return pulumi.get(self, "latest_updates")
|
|
@@ -400,7 +402,7 @@ class PipelineArgs:
|
|
|
400
402
|
def latest_updates(self, value: Optional[pulumi.Input[Sequence[pulumi.Input['PipelineLatestUpdateArgs']]]]):
|
|
401
403
|
pulumi.set(self, "latest_updates", value)
|
|
402
404
|
|
|
403
|
-
@property
|
|
405
|
+
@_builtins.property
|
|
404
406
|
@pulumi.getter
|
|
405
407
|
def libraries(self) -> Optional[pulumi.Input[Sequence[pulumi.Input['PipelineLibraryArgs']]]]:
|
|
406
408
|
"""
|
|
@@ -412,19 +414,19 @@ class PipelineArgs:
|
|
|
412
414
|
def libraries(self, value: Optional[pulumi.Input[Sequence[pulumi.Input['PipelineLibraryArgs']]]]):
|
|
413
415
|
pulumi.set(self, "libraries", value)
|
|
414
416
|
|
|
415
|
-
@property
|
|
417
|
+
@_builtins.property
|
|
416
418
|
@pulumi.getter
|
|
417
|
-
def name(self) -> Optional[pulumi.Input[
|
|
419
|
+
def name(self) -> Optional[pulumi.Input[_builtins.str]]:
|
|
418
420
|
"""
|
|
419
421
|
A user-friendly name for this pipeline. The name can be used to identify pipeline jobs in the UI.
|
|
420
422
|
"""
|
|
421
423
|
return pulumi.get(self, "name")
|
|
422
424
|
|
|
423
425
|
@name.setter
|
|
424
|
-
def name(self, value: Optional[pulumi.Input[
|
|
426
|
+
def name(self, value: Optional[pulumi.Input[_builtins.str]]):
|
|
425
427
|
pulumi.set(self, "name", value)
|
|
426
428
|
|
|
427
|
-
@property
|
|
429
|
+
@_builtins.property
|
|
428
430
|
@pulumi.getter
|
|
429
431
|
def notifications(self) -> Optional[pulumi.Input[Sequence[pulumi.Input['PipelineNotificationArgs']]]]:
|
|
430
432
|
return pulumi.get(self, "notifications")
|
|
@@ -433,19 +435,19 @@ class PipelineArgs:
|
|
|
433
435
|
def notifications(self, value: Optional[pulumi.Input[Sequence[pulumi.Input['PipelineNotificationArgs']]]]):
|
|
434
436
|
pulumi.set(self, "notifications", value)
|
|
435
437
|
|
|
436
|
-
@property
|
|
438
|
+
@_builtins.property
|
|
437
439
|
@pulumi.getter
|
|
438
|
-
def photon(self) -> Optional[pulumi.Input[
|
|
440
|
+
def photon(self) -> Optional[pulumi.Input[_builtins.bool]]:
|
|
439
441
|
"""
|
|
440
442
|
A flag indicating whether to use Photon engine. The default value is `false`.
|
|
441
443
|
"""
|
|
442
444
|
return pulumi.get(self, "photon")
|
|
443
445
|
|
|
444
446
|
@photon.setter
|
|
445
|
-
def photon(self, value: Optional[pulumi.Input[
|
|
447
|
+
def photon(self, value: Optional[pulumi.Input[_builtins.bool]]):
|
|
446
448
|
pulumi.set(self, "photon", value)
|
|
447
449
|
|
|
448
|
-
@property
|
|
450
|
+
@_builtins.property
|
|
449
451
|
@pulumi.getter(name="restartWindow")
|
|
450
452
|
def restart_window(self) -> Optional[pulumi.Input['PipelineRestartWindowArgs']]:
|
|
451
453
|
return pulumi.get(self, "restart_window")
|
|
@@ -454,19 +456,19 @@ class PipelineArgs:
|
|
|
454
456
|
def restart_window(self, value: Optional[pulumi.Input['PipelineRestartWindowArgs']]):
|
|
455
457
|
pulumi.set(self, "restart_window", value)
|
|
456
458
|
|
|
457
|
-
@property
|
|
459
|
+
@_builtins.property
|
|
458
460
|
@pulumi.getter(name="rootPath")
|
|
459
|
-
def root_path(self) -> Optional[pulumi.Input[
|
|
461
|
+
def root_path(self) -> Optional[pulumi.Input[_builtins.str]]:
|
|
460
462
|
"""
|
|
461
463
|
An optional string specifying the root path for this pipeline. This is used as the root directory when editing the pipeline in the Databricks user interface and it is added to `sys.path` when executing Python sources during pipeline execution.
|
|
462
464
|
"""
|
|
463
465
|
return pulumi.get(self, "root_path")
|
|
464
466
|
|
|
465
467
|
@root_path.setter
|
|
466
|
-
def root_path(self, value: Optional[pulumi.Input[
|
|
468
|
+
def root_path(self, value: Optional[pulumi.Input[_builtins.str]]):
|
|
467
469
|
pulumi.set(self, "root_path", value)
|
|
468
470
|
|
|
469
|
-
@property
|
|
471
|
+
@_builtins.property
|
|
470
472
|
@pulumi.getter(name="runAs")
|
|
471
473
|
def run_as(self) -> Optional[pulumi.Input['PipelineRunAsArgs']]:
|
|
472
474
|
return pulumi.get(self, "run_as")
|
|
@@ -475,85 +477,85 @@ class PipelineArgs:
|
|
|
475
477
|
def run_as(self, value: Optional[pulumi.Input['PipelineRunAsArgs']]):
|
|
476
478
|
pulumi.set(self, "run_as", value)
|
|
477
479
|
|
|
478
|
-
@property
|
|
480
|
+
@_builtins.property
|
|
479
481
|
@pulumi.getter(name="runAsUserName")
|
|
480
|
-
def run_as_user_name(self) -> Optional[pulumi.Input[
|
|
482
|
+
def run_as_user_name(self) -> Optional[pulumi.Input[_builtins.str]]:
|
|
481
483
|
return pulumi.get(self, "run_as_user_name")
|
|
482
484
|
|
|
483
485
|
@run_as_user_name.setter
|
|
484
|
-
def run_as_user_name(self, value: Optional[pulumi.Input[
|
|
486
|
+
def run_as_user_name(self, value: Optional[pulumi.Input[_builtins.str]]):
|
|
485
487
|
pulumi.set(self, "run_as_user_name", value)
|
|
486
488
|
|
|
487
|
-
@property
|
|
489
|
+
@_builtins.property
|
|
488
490
|
@pulumi.getter
|
|
489
|
-
def schema(self) -> Optional[pulumi.Input[
|
|
491
|
+
def schema(self) -> Optional[pulumi.Input[_builtins.str]]:
|
|
490
492
|
"""
|
|
491
493
|
The default schema (database) where tables are read from or published to. The presence of this attribute implies that the pipeline is in direct publishing mode.
|
|
492
494
|
"""
|
|
493
495
|
return pulumi.get(self, "schema")
|
|
494
496
|
|
|
495
497
|
@schema.setter
|
|
496
|
-
def schema(self, value: Optional[pulumi.Input[
|
|
498
|
+
def schema(self, value: Optional[pulumi.Input[_builtins.str]]):
|
|
497
499
|
pulumi.set(self, "schema", value)
|
|
498
500
|
|
|
499
|
-
@property
|
|
501
|
+
@_builtins.property
|
|
500
502
|
@pulumi.getter
|
|
501
|
-
def serverless(self) -> Optional[pulumi.Input[
|
|
503
|
+
def serverless(self) -> Optional[pulumi.Input[_builtins.bool]]:
|
|
502
504
|
"""
|
|
503
|
-
An optional flag indicating if serverless compute should be used for this
|
|
505
|
+
An optional flag indicating if serverless compute should be used for this Lakeflow Declarative Pipeline. Requires `catalog` to be set, as it could be used only with Unity Catalog.
|
|
504
506
|
"""
|
|
505
507
|
return pulumi.get(self, "serverless")
|
|
506
508
|
|
|
507
509
|
@serverless.setter
|
|
508
|
-
def serverless(self, value: Optional[pulumi.Input[
|
|
510
|
+
def serverless(self, value: Optional[pulumi.Input[_builtins.bool]]):
|
|
509
511
|
pulumi.set(self, "serverless", value)
|
|
510
512
|
|
|
511
|
-
@property
|
|
513
|
+
@_builtins.property
|
|
512
514
|
@pulumi.getter
|
|
513
|
-
def state(self) -> Optional[pulumi.Input[
|
|
515
|
+
def state(self) -> Optional[pulumi.Input[_builtins.str]]:
|
|
514
516
|
return pulumi.get(self, "state")
|
|
515
517
|
|
|
516
518
|
@state.setter
|
|
517
|
-
def state(self, value: Optional[pulumi.Input[
|
|
519
|
+
def state(self, value: Optional[pulumi.Input[_builtins.str]]):
|
|
518
520
|
pulumi.set(self, "state", value)
|
|
519
521
|
|
|
520
|
-
@property
|
|
522
|
+
@_builtins.property
|
|
521
523
|
@pulumi.getter
|
|
522
|
-
def storage(self) -> Optional[pulumi.Input[
|
|
524
|
+
def storage(self) -> Optional[pulumi.Input[_builtins.str]]:
|
|
523
525
|
"""
|
|
524
|
-
A location on
|
|
526
|
+
A location on cloud storage where output data and metadata required for pipeline execution are stored. By default, tables are stored in a subdirectory of this location. *Change of this parameter forces recreation of the pipeline.* (Conflicts with `catalog`).
|
|
525
527
|
"""
|
|
526
528
|
return pulumi.get(self, "storage")
|
|
527
529
|
|
|
528
530
|
@storage.setter
|
|
529
|
-
def storage(self, value: Optional[pulumi.Input[
|
|
531
|
+
def storage(self, value: Optional[pulumi.Input[_builtins.str]]):
|
|
530
532
|
pulumi.set(self, "storage", value)
|
|
531
533
|
|
|
532
|
-
@property
|
|
534
|
+
@_builtins.property
|
|
533
535
|
@pulumi.getter
|
|
534
|
-
def tags(self) -> Optional[pulumi.Input[Mapping[str, pulumi.Input[
|
|
536
|
+
def tags(self) -> Optional[pulumi.Input[Mapping[str, pulumi.Input[_builtins.str]]]]:
|
|
535
537
|
"""
|
|
536
538
|
A map of tags associated with the pipeline. These are forwarded to the cluster as cluster tags, and are therefore subject to the same limitations. A maximum of 25 tags can be added to the pipeline.
|
|
537
539
|
"""
|
|
538
540
|
return pulumi.get(self, "tags")
|
|
539
541
|
|
|
540
542
|
@tags.setter
|
|
541
|
-
def tags(self, value: Optional[pulumi.Input[Mapping[str, pulumi.Input[
|
|
543
|
+
def tags(self, value: Optional[pulumi.Input[Mapping[str, pulumi.Input[_builtins.str]]]]):
|
|
542
544
|
pulumi.set(self, "tags", value)
|
|
543
545
|
|
|
544
|
-
@property
|
|
546
|
+
@_builtins.property
|
|
545
547
|
@pulumi.getter
|
|
546
|
-
def target(self) -> Optional[pulumi.Input[
|
|
548
|
+
def target(self) -> Optional[pulumi.Input[_builtins.str]]:
|
|
547
549
|
"""
|
|
548
550
|
The name of a database (in either the Hive metastore or in a UC catalog) for persisting pipeline output data. Configuring the target setting allows you to view and query the pipeline output data from the Databricks UI.
|
|
549
551
|
"""
|
|
550
552
|
return pulumi.get(self, "target")
|
|
551
553
|
|
|
552
554
|
@target.setter
|
|
553
|
-
def target(self, value: Optional[pulumi.Input[
|
|
555
|
+
def target(self, value: Optional[pulumi.Input[_builtins.str]]):
|
|
554
556
|
pulumi.set(self, "target", value)
|
|
555
557
|
|
|
556
|
-
@property
|
|
558
|
+
@_builtins.property
|
|
557
559
|
@pulumi.getter
|
|
558
560
|
def trigger(self) -> Optional[pulumi.Input['PipelineTriggerArgs']]:
|
|
559
561
|
return pulumi.get(self, "trigger")
|
|
@@ -562,85 +564,95 @@ class PipelineArgs:
|
|
|
562
564
|
def trigger(self, value: Optional[pulumi.Input['PipelineTriggerArgs']]):
|
|
563
565
|
pulumi.set(self, "trigger", value)
|
|
564
566
|
|
|
565
|
-
@property
|
|
567
|
+
@_builtins.property
|
|
566
568
|
@pulumi.getter
|
|
567
|
-
def url(self) -> Optional[pulumi.Input[
|
|
569
|
+
def url(self) -> Optional[pulumi.Input[_builtins.str]]:
|
|
568
570
|
"""
|
|
569
|
-
URL of the
|
|
571
|
+
URL of the Lakeflow Declarative Pipeline on the given workspace.
|
|
570
572
|
"""
|
|
571
573
|
return pulumi.get(self, "url")
|
|
572
574
|
|
|
573
575
|
@url.setter
|
|
574
|
-
def url(self, value: Optional[pulumi.Input[
|
|
576
|
+
def url(self, value: Optional[pulumi.Input[_builtins.str]]):
|
|
575
577
|
pulumi.set(self, "url", value)
|
|
576
578
|
|
|
579
|
+
@_builtins.property
|
|
580
|
+
@pulumi.getter(name="usagePolicyId")
|
|
581
|
+
def usage_policy_id(self) -> Optional[pulumi.Input[_builtins.str]]:
|
|
582
|
+
return pulumi.get(self, "usage_policy_id")
|
|
583
|
+
|
|
584
|
+
@usage_policy_id.setter
|
|
585
|
+
def usage_policy_id(self, value: Optional[pulumi.Input[_builtins.str]]):
|
|
586
|
+
pulumi.set(self, "usage_policy_id", value)
|
|
587
|
+
|
|
577
588
|
|
|
578
589
|
@pulumi.input_type
|
|
579
590
|
class _PipelineState:
|
|
580
591
|
def __init__(__self__, *,
|
|
581
|
-
allow_duplicate_names: Optional[pulumi.Input[
|
|
582
|
-
budget_policy_id: Optional[pulumi.Input[
|
|
583
|
-
catalog: Optional[pulumi.Input[
|
|
584
|
-
cause: Optional[pulumi.Input[
|
|
585
|
-
channel: Optional[pulumi.Input[
|
|
586
|
-
cluster_id: Optional[pulumi.Input[
|
|
592
|
+
allow_duplicate_names: Optional[pulumi.Input[_builtins.bool]] = None,
|
|
593
|
+
budget_policy_id: Optional[pulumi.Input[_builtins.str]] = None,
|
|
594
|
+
catalog: Optional[pulumi.Input[_builtins.str]] = None,
|
|
595
|
+
cause: Optional[pulumi.Input[_builtins.str]] = None,
|
|
596
|
+
channel: Optional[pulumi.Input[_builtins.str]] = None,
|
|
597
|
+
cluster_id: Optional[pulumi.Input[_builtins.str]] = None,
|
|
587
598
|
clusters: Optional[pulumi.Input[Sequence[pulumi.Input['PipelineClusterArgs']]]] = None,
|
|
588
|
-
configuration: Optional[pulumi.Input[Mapping[str, pulumi.Input[
|
|
589
|
-
continuous: Optional[pulumi.Input[
|
|
590
|
-
creator_user_name: Optional[pulumi.Input[
|
|
599
|
+
configuration: Optional[pulumi.Input[Mapping[str, pulumi.Input[_builtins.str]]]] = None,
|
|
600
|
+
continuous: Optional[pulumi.Input[_builtins.bool]] = None,
|
|
601
|
+
creator_user_name: Optional[pulumi.Input[_builtins.str]] = None,
|
|
591
602
|
deployment: Optional[pulumi.Input['PipelineDeploymentArgs']] = None,
|
|
592
|
-
development: Optional[pulumi.Input[
|
|
593
|
-
edition: Optional[pulumi.Input[
|
|
603
|
+
development: Optional[pulumi.Input[_builtins.bool]] = None,
|
|
604
|
+
edition: Optional[pulumi.Input[_builtins.str]] = None,
|
|
594
605
|
environment: Optional[pulumi.Input['PipelineEnvironmentArgs']] = None,
|
|
595
606
|
event_log: Optional[pulumi.Input['PipelineEventLogArgs']] = None,
|
|
596
|
-
expected_last_modified: Optional[pulumi.Input[
|
|
607
|
+
expected_last_modified: Optional[pulumi.Input[_builtins.int]] = None,
|
|
597
608
|
filters: Optional[pulumi.Input['PipelineFiltersArgs']] = None,
|
|
598
609
|
gateway_definition: Optional[pulumi.Input['PipelineGatewayDefinitionArgs']] = None,
|
|
599
|
-
health: Optional[pulumi.Input[
|
|
610
|
+
health: Optional[pulumi.Input[_builtins.str]] = None,
|
|
600
611
|
ingestion_definition: Optional[pulumi.Input['PipelineIngestionDefinitionArgs']] = None,
|
|
601
|
-
last_modified: Optional[pulumi.Input[
|
|
612
|
+
last_modified: Optional[pulumi.Input[_builtins.int]] = None,
|
|
602
613
|
latest_updates: Optional[pulumi.Input[Sequence[pulumi.Input['PipelineLatestUpdateArgs']]]] = None,
|
|
603
614
|
libraries: Optional[pulumi.Input[Sequence[pulumi.Input['PipelineLibraryArgs']]]] = None,
|
|
604
|
-
name: Optional[pulumi.Input[
|
|
615
|
+
name: Optional[pulumi.Input[_builtins.str]] = None,
|
|
605
616
|
notifications: Optional[pulumi.Input[Sequence[pulumi.Input['PipelineNotificationArgs']]]] = None,
|
|
606
|
-
photon: Optional[pulumi.Input[
|
|
617
|
+
photon: Optional[pulumi.Input[_builtins.bool]] = None,
|
|
607
618
|
restart_window: Optional[pulumi.Input['PipelineRestartWindowArgs']] = None,
|
|
608
|
-
root_path: Optional[pulumi.Input[
|
|
619
|
+
root_path: Optional[pulumi.Input[_builtins.str]] = None,
|
|
609
620
|
run_as: Optional[pulumi.Input['PipelineRunAsArgs']] = None,
|
|
610
|
-
run_as_user_name: Optional[pulumi.Input[
|
|
611
|
-
schema: Optional[pulumi.Input[
|
|
612
|
-
serverless: Optional[pulumi.Input[
|
|
613
|
-
state: Optional[pulumi.Input[
|
|
614
|
-
storage: Optional[pulumi.Input[
|
|
615
|
-
tags: Optional[pulumi.Input[Mapping[str, pulumi.Input[
|
|
616
|
-
target: Optional[pulumi.Input[
|
|
621
|
+
run_as_user_name: Optional[pulumi.Input[_builtins.str]] = None,
|
|
622
|
+
schema: Optional[pulumi.Input[_builtins.str]] = None,
|
|
623
|
+
serverless: Optional[pulumi.Input[_builtins.bool]] = None,
|
|
624
|
+
state: Optional[pulumi.Input[_builtins.str]] = None,
|
|
625
|
+
storage: Optional[pulumi.Input[_builtins.str]] = None,
|
|
626
|
+
tags: Optional[pulumi.Input[Mapping[str, pulumi.Input[_builtins.str]]]] = None,
|
|
627
|
+
target: Optional[pulumi.Input[_builtins.str]] = None,
|
|
617
628
|
trigger: Optional[pulumi.Input['PipelineTriggerArgs']] = None,
|
|
618
|
-
url: Optional[pulumi.Input[
|
|
629
|
+
url: Optional[pulumi.Input[_builtins.str]] = None,
|
|
630
|
+
usage_policy_id: Optional[pulumi.Input[_builtins.str]] = None):
|
|
619
631
|
"""
|
|
620
632
|
Input properties used for looking up and filtering Pipeline resources.
|
|
621
|
-
:param pulumi.Input[
|
|
622
|
-
:param pulumi.Input[
|
|
623
|
-
:param pulumi.Input[
|
|
624
|
-
:param pulumi.Input[
|
|
625
|
-
:param pulumi.Input[Sequence[pulumi.Input['PipelineClusterArgs']]] clusters: blocks - Clusters to run the pipeline. If none is specified, pipelines will automatically select a default cluster configuration for the pipeline. *Please note that
|
|
626
|
-
:param pulumi.Input[Mapping[str, pulumi.Input[
|
|
627
|
-
:param pulumi.Input[
|
|
633
|
+
:param pulumi.Input[_builtins.bool] allow_duplicate_names: Optional boolean flag. If false, deployment will fail if name conflicts with that of another pipeline. default is `false`.
|
|
634
|
+
:param pulumi.Input[_builtins.str] budget_policy_id: optional string specifying ID of the budget policy for this Lakeflow Declarative Pipeline.
|
|
635
|
+
:param pulumi.Input[_builtins.str] catalog: The name of catalog in Unity Catalog. *Change of this parameter forces recreation of the pipeline.* (Conflicts with `storage`).
|
|
636
|
+
:param pulumi.Input[_builtins.str] channel: optional name of the release channel for Spark version used by Lakeflow Declarative Pipeline. Supported values are: `CURRENT` (default) and `PREVIEW`.
|
|
637
|
+
:param pulumi.Input[Sequence[pulumi.Input['PipelineClusterArgs']]] clusters: blocks - Clusters to run the pipeline. If none is specified, pipelines will automatically select a default cluster configuration for the pipeline. *Please note that Lakeflow Declarative Pipeline clusters are supporting only subset of attributes as described in [documentation](https://docs.databricks.com/api/workspace/pipelines/create#clusters).* Also, note that `autoscale` block is extended with the `mode` parameter that controls the autoscaling algorithm (possible values are `ENHANCED` for new, enhanced autoscaling algorithm, or `LEGACY` for old algorithm).
|
|
638
|
+
:param pulumi.Input[Mapping[str, pulumi.Input[_builtins.str]]] configuration: An optional list of values to apply to the entire pipeline. Elements must be formatted as key:value pairs.
|
|
639
|
+
:param pulumi.Input[_builtins.bool] continuous: A flag indicating whether to run the pipeline continuously. The default value is `false`.
|
|
628
640
|
:param pulumi.Input['PipelineDeploymentArgs'] deployment: Deployment type of this pipeline. Supports following attributes:
|
|
629
|
-
:param pulumi.Input[
|
|
630
|
-
:param pulumi.Input[
|
|
631
|
-
:param pulumi.Input['PipelineEventLogArgs'] event_log: an optional block specifying a table where
|
|
641
|
+
:param pulumi.Input[_builtins.bool] development: A flag indicating whether to run the pipeline in development mode. The default value is `false`.
|
|
642
|
+
:param pulumi.Input[_builtins.str] edition: optional name of the [product edition](https://docs.databricks.com/aws/en/dlt/configure-pipeline#choose-a-product-edition). Supported values are: `CORE`, `PRO`, `ADVANCED` (default). Not required when `serverless` is set to `true`.
|
|
643
|
+
:param pulumi.Input['PipelineEventLogArgs'] event_log: an optional block specifying a table where LDP Event Log will be stored. Consists of the following fields:
|
|
632
644
|
:param pulumi.Input['PipelineFiltersArgs'] filters: Filters on which Pipeline packages to include in the deployed graph. This block consists of following attributes:
|
|
633
645
|
:param pulumi.Input['PipelineGatewayDefinitionArgs'] gateway_definition: The definition of a gateway pipeline to support CDC. Consists of following attributes:
|
|
634
646
|
:param pulumi.Input[Sequence[pulumi.Input['PipelineLibraryArgs']]] libraries: blocks - Specifies pipeline code.
|
|
635
|
-
:param pulumi.Input[
|
|
636
|
-
:param pulumi.Input[
|
|
637
|
-
:param pulumi.Input[
|
|
638
|
-
:param pulumi.Input[
|
|
639
|
-
:param pulumi.Input[
|
|
640
|
-
:param pulumi.Input[
|
|
641
|
-
:param pulumi.Input[Mapping[str, pulumi.Input[
|
|
642
|
-
:param pulumi.Input[
|
|
643
|
-
:param pulumi.Input[
|
|
647
|
+
:param pulumi.Input[_builtins.str] name: A user-friendly name for this pipeline. The name can be used to identify pipeline jobs in the UI.
|
|
648
|
+
:param pulumi.Input[_builtins.bool] photon: A flag indicating whether to use Photon engine. The default value is `false`.
|
|
649
|
+
:param pulumi.Input[_builtins.str] root_path: An optional string specifying the root path for this pipeline. This is used as the root directory when editing the pipeline in the Databricks user interface and it is added to `sys.path` when executing Python sources during pipeline execution.
|
|
650
|
+
:param pulumi.Input[_builtins.str] schema: The default schema (database) where tables are read from or published to. The presence of this attribute implies that the pipeline is in direct publishing mode.
|
|
651
|
+
:param pulumi.Input[_builtins.bool] serverless: An optional flag indicating if serverless compute should be used for this Lakeflow Declarative Pipeline. Requires `catalog` to be set, as it could be used only with Unity Catalog.
|
|
652
|
+
:param pulumi.Input[_builtins.str] storage: A location on cloud storage where output data and metadata required for pipeline execution are stored. By default, tables are stored in a subdirectory of this location. *Change of this parameter forces recreation of the pipeline.* (Conflicts with `catalog`).
|
|
653
|
+
:param pulumi.Input[Mapping[str, pulumi.Input[_builtins.str]]] tags: A map of tags associated with the pipeline. These are forwarded to the cluster as cluster tags, and are therefore subject to the same limitations. A maximum of 25 tags can be added to the pipeline.
|
|
654
|
+
:param pulumi.Input[_builtins.str] target: The name of a database (in either the Hive metastore or in a UC catalog) for persisting pipeline output data. Configuring the target setting allows you to view and query the pipeline output data from the Databricks UI.
|
|
655
|
+
:param pulumi.Input[_builtins.str] url: URL of the Lakeflow Declarative Pipeline on the given workspace.
|
|
644
656
|
"""
|
|
645
657
|
if allow_duplicate_names is not None:
|
|
646
658
|
pulumi.set(__self__, "allow_duplicate_names", allow_duplicate_names)
|
|
@@ -718,78 +730,80 @@ class _PipelineState:
|
|
|
718
730
|
pulumi.set(__self__, "trigger", trigger)
|
|
719
731
|
if url is not None:
|
|
720
732
|
pulumi.set(__self__, "url", url)
|
|
733
|
+
if usage_policy_id is not None:
|
|
734
|
+
pulumi.set(__self__, "usage_policy_id", usage_policy_id)
|
|
721
735
|
|
|
722
|
-
@property
|
|
736
|
+
@_builtins.property
|
|
723
737
|
@pulumi.getter(name="allowDuplicateNames")
|
|
724
|
-
def allow_duplicate_names(self) -> Optional[pulumi.Input[
|
|
738
|
+
def allow_duplicate_names(self) -> Optional[pulumi.Input[_builtins.bool]]:
|
|
725
739
|
"""
|
|
726
740
|
Optional boolean flag. If false, deployment will fail if name conflicts with that of another pipeline. default is `false`.
|
|
727
741
|
"""
|
|
728
742
|
return pulumi.get(self, "allow_duplicate_names")
|
|
729
743
|
|
|
730
744
|
@allow_duplicate_names.setter
|
|
731
|
-
def allow_duplicate_names(self, value: Optional[pulumi.Input[
|
|
745
|
+
def allow_duplicate_names(self, value: Optional[pulumi.Input[_builtins.bool]]):
|
|
732
746
|
pulumi.set(self, "allow_duplicate_names", value)
|
|
733
747
|
|
|
734
|
-
@property
|
|
748
|
+
@_builtins.property
|
|
735
749
|
@pulumi.getter(name="budgetPolicyId")
|
|
736
|
-
def budget_policy_id(self) -> Optional[pulumi.Input[
|
|
750
|
+
def budget_policy_id(self) -> Optional[pulumi.Input[_builtins.str]]:
|
|
737
751
|
"""
|
|
738
|
-
optional string specifying ID of the budget policy for this
|
|
752
|
+
optional string specifying ID of the budget policy for this Lakeflow Declarative Pipeline.
|
|
739
753
|
"""
|
|
740
754
|
return pulumi.get(self, "budget_policy_id")
|
|
741
755
|
|
|
742
756
|
@budget_policy_id.setter
|
|
743
|
-
def budget_policy_id(self, value: Optional[pulumi.Input[
|
|
757
|
+
def budget_policy_id(self, value: Optional[pulumi.Input[_builtins.str]]):
|
|
744
758
|
pulumi.set(self, "budget_policy_id", value)
|
|
745
759
|
|
|
746
|
-
@property
|
|
760
|
+
@_builtins.property
|
|
747
761
|
@pulumi.getter
|
|
748
|
-
def catalog(self) -> Optional[pulumi.Input[
|
|
762
|
+
def catalog(self) -> Optional[pulumi.Input[_builtins.str]]:
|
|
749
763
|
"""
|
|
750
764
|
The name of catalog in Unity Catalog. *Change of this parameter forces recreation of the pipeline.* (Conflicts with `storage`).
|
|
751
765
|
"""
|
|
752
766
|
return pulumi.get(self, "catalog")
|
|
753
767
|
|
|
754
768
|
@catalog.setter
|
|
755
|
-
def catalog(self, value: Optional[pulumi.Input[
|
|
769
|
+
def catalog(self, value: Optional[pulumi.Input[_builtins.str]]):
|
|
756
770
|
pulumi.set(self, "catalog", value)
|
|
757
771
|
|
|
758
|
-
@property
|
|
772
|
+
@_builtins.property
|
|
759
773
|
@pulumi.getter
|
|
760
|
-
def cause(self) -> Optional[pulumi.Input[
|
|
774
|
+
def cause(self) -> Optional[pulumi.Input[_builtins.str]]:
|
|
761
775
|
return pulumi.get(self, "cause")
|
|
762
776
|
|
|
763
777
|
@cause.setter
|
|
764
|
-
def cause(self, value: Optional[pulumi.Input[
|
|
778
|
+
def cause(self, value: Optional[pulumi.Input[_builtins.str]]):
|
|
765
779
|
pulumi.set(self, "cause", value)
|
|
766
780
|
|
|
767
|
-
@property
|
|
781
|
+
@_builtins.property
|
|
768
782
|
@pulumi.getter
|
|
769
|
-
def channel(self) -> Optional[pulumi.Input[
|
|
783
|
+
def channel(self) -> Optional[pulumi.Input[_builtins.str]]:
|
|
770
784
|
"""
|
|
771
|
-
optional name of the release channel for Spark version used by
|
|
785
|
+
optional name of the release channel for Spark version used by Lakeflow Declarative Pipeline. Supported values are: `CURRENT` (default) and `PREVIEW`.
|
|
772
786
|
"""
|
|
773
787
|
return pulumi.get(self, "channel")
|
|
774
788
|
|
|
775
789
|
@channel.setter
|
|
776
|
-
def channel(self, value: Optional[pulumi.Input[
|
|
790
|
+
def channel(self, value: Optional[pulumi.Input[_builtins.str]]):
|
|
777
791
|
pulumi.set(self, "channel", value)
|
|
778
792
|
|
|
779
|
-
@property
|
|
793
|
+
@_builtins.property
|
|
780
794
|
@pulumi.getter(name="clusterId")
|
|
781
|
-
def cluster_id(self) -> Optional[pulumi.Input[
|
|
795
|
+
def cluster_id(self) -> Optional[pulumi.Input[_builtins.str]]:
|
|
782
796
|
return pulumi.get(self, "cluster_id")
|
|
783
797
|
|
|
784
798
|
@cluster_id.setter
|
|
785
|
-
def cluster_id(self, value: Optional[pulumi.Input[
|
|
799
|
+
def cluster_id(self, value: Optional[pulumi.Input[_builtins.str]]):
|
|
786
800
|
pulumi.set(self, "cluster_id", value)
|
|
787
801
|
|
|
788
|
-
@property
|
|
802
|
+
@_builtins.property
|
|
789
803
|
@pulumi.getter
|
|
790
804
|
def clusters(self) -> Optional[pulumi.Input[Sequence[pulumi.Input['PipelineClusterArgs']]]]:
|
|
791
805
|
"""
|
|
792
|
-
blocks - Clusters to run the pipeline. If none is specified, pipelines will automatically select a default cluster configuration for the pipeline. *Please note that
|
|
806
|
+
blocks - Clusters to run the pipeline. If none is specified, pipelines will automatically select a default cluster configuration for the pipeline. *Please note that Lakeflow Declarative Pipeline clusters are supporting only subset of attributes as described in [documentation](https://docs.databricks.com/api/workspace/pipelines/create#clusters).* Also, note that `autoscale` block is extended with the `mode` parameter that controls the autoscaling algorithm (possible values are `ENHANCED` for new, enhanced autoscaling algorithm, or `LEGACY` for old algorithm).
|
|
793
807
|
"""
|
|
794
808
|
return pulumi.get(self, "clusters")
|
|
795
809
|
|
|
@@ -797,40 +811,40 @@ class _PipelineState:
|
|
|
797
811
|
def clusters(self, value: Optional[pulumi.Input[Sequence[pulumi.Input['PipelineClusterArgs']]]]):
|
|
798
812
|
pulumi.set(self, "clusters", value)
|
|
799
813
|
|
|
800
|
-
@property
|
|
814
|
+
@_builtins.property
|
|
801
815
|
@pulumi.getter
|
|
802
|
-
def configuration(self) -> Optional[pulumi.Input[Mapping[str, pulumi.Input[
|
|
816
|
+
def configuration(self) -> Optional[pulumi.Input[Mapping[str, pulumi.Input[_builtins.str]]]]:
|
|
803
817
|
"""
|
|
804
818
|
An optional list of values to apply to the entire pipeline. Elements must be formatted as key:value pairs.
|
|
805
819
|
"""
|
|
806
820
|
return pulumi.get(self, "configuration")
|
|
807
821
|
|
|
808
822
|
@configuration.setter
|
|
809
|
-
def configuration(self, value: Optional[pulumi.Input[Mapping[str, pulumi.Input[
|
|
823
|
+
def configuration(self, value: Optional[pulumi.Input[Mapping[str, pulumi.Input[_builtins.str]]]]):
|
|
810
824
|
pulumi.set(self, "configuration", value)
|
|
811
825
|
|
|
812
|
-
@property
|
|
826
|
+
@_builtins.property
|
|
813
827
|
@pulumi.getter
|
|
814
|
-
def continuous(self) -> Optional[pulumi.Input[
|
|
828
|
+
def continuous(self) -> Optional[pulumi.Input[_builtins.bool]]:
|
|
815
829
|
"""
|
|
816
830
|
A flag indicating whether to run the pipeline continuously. The default value is `false`.
|
|
817
831
|
"""
|
|
818
832
|
return pulumi.get(self, "continuous")
|
|
819
833
|
|
|
820
834
|
@continuous.setter
|
|
821
|
-
def continuous(self, value: Optional[pulumi.Input[
|
|
835
|
+
def continuous(self, value: Optional[pulumi.Input[_builtins.bool]]):
|
|
822
836
|
pulumi.set(self, "continuous", value)
|
|
823
837
|
|
|
824
|
-
@property
|
|
838
|
+
@_builtins.property
|
|
825
839
|
@pulumi.getter(name="creatorUserName")
|
|
826
|
-
def creator_user_name(self) -> Optional[pulumi.Input[
|
|
840
|
+
def creator_user_name(self) -> Optional[pulumi.Input[_builtins.str]]:
|
|
827
841
|
return pulumi.get(self, "creator_user_name")
|
|
828
842
|
|
|
829
843
|
@creator_user_name.setter
|
|
830
|
-
def creator_user_name(self, value: Optional[pulumi.Input[
|
|
844
|
+
def creator_user_name(self, value: Optional[pulumi.Input[_builtins.str]]):
|
|
831
845
|
pulumi.set(self, "creator_user_name", value)
|
|
832
846
|
|
|
833
|
-
@property
|
|
847
|
+
@_builtins.property
|
|
834
848
|
@pulumi.getter
|
|
835
849
|
def deployment(self) -> Optional[pulumi.Input['PipelineDeploymentArgs']]:
|
|
836
850
|
"""
|
|
@@ -842,31 +856,31 @@ class _PipelineState:
|
|
|
842
856
|
def deployment(self, value: Optional[pulumi.Input['PipelineDeploymentArgs']]):
|
|
843
857
|
pulumi.set(self, "deployment", value)
|
|
844
858
|
|
|
845
|
-
@property
|
|
859
|
+
@_builtins.property
|
|
846
860
|
@pulumi.getter
|
|
847
|
-
def development(self) -> Optional[pulumi.Input[
|
|
861
|
+
def development(self) -> Optional[pulumi.Input[_builtins.bool]]:
|
|
848
862
|
"""
|
|
849
863
|
A flag indicating whether to run the pipeline in development mode. The default value is `false`.
|
|
850
864
|
"""
|
|
851
865
|
return pulumi.get(self, "development")
|
|
852
866
|
|
|
853
867
|
@development.setter
|
|
854
|
-
def development(self, value: Optional[pulumi.Input[
|
|
868
|
+
def development(self, value: Optional[pulumi.Input[_builtins.bool]]):
|
|
855
869
|
pulumi.set(self, "development", value)
|
|
856
870
|
|
|
857
|
-
@property
|
|
871
|
+
@_builtins.property
|
|
858
872
|
@pulumi.getter
|
|
859
|
-
def edition(self) -> Optional[pulumi.Input[
|
|
873
|
+
def edition(self) -> Optional[pulumi.Input[_builtins.str]]:
|
|
860
874
|
"""
|
|
861
875
|
optional name of the [product edition](https://docs.databricks.com/aws/en/dlt/configure-pipeline#choose-a-product-edition). Supported values are: `CORE`, `PRO`, `ADVANCED` (default). Not required when `serverless` is set to `true`.
|
|
862
876
|
"""
|
|
863
877
|
return pulumi.get(self, "edition")
|
|
864
878
|
|
|
865
879
|
@edition.setter
|
|
866
|
-
def edition(self, value: Optional[pulumi.Input[
|
|
880
|
+
def edition(self, value: Optional[pulumi.Input[_builtins.str]]):
|
|
867
881
|
pulumi.set(self, "edition", value)
|
|
868
882
|
|
|
869
|
-
@property
|
|
883
|
+
@_builtins.property
|
|
870
884
|
@pulumi.getter
|
|
871
885
|
def environment(self) -> Optional[pulumi.Input['PipelineEnvironmentArgs']]:
|
|
872
886
|
return pulumi.get(self, "environment")
|
|
@@ -875,11 +889,11 @@ class _PipelineState:
|
|
|
875
889
|
def environment(self, value: Optional[pulumi.Input['PipelineEnvironmentArgs']]):
|
|
876
890
|
pulumi.set(self, "environment", value)
|
|
877
891
|
|
|
878
|
-
@property
|
|
892
|
+
@_builtins.property
|
|
879
893
|
@pulumi.getter(name="eventLog")
|
|
880
894
|
def event_log(self) -> Optional[pulumi.Input['PipelineEventLogArgs']]:
|
|
881
895
|
"""
|
|
882
|
-
an optional block specifying a table where
|
|
896
|
+
an optional block specifying a table where LDP Event Log will be stored. Consists of the following fields:
|
|
883
897
|
"""
|
|
884
898
|
return pulumi.get(self, "event_log")
|
|
885
899
|
|
|
@@ -887,16 +901,16 @@ class _PipelineState:
|
|
|
887
901
|
def event_log(self, value: Optional[pulumi.Input['PipelineEventLogArgs']]):
|
|
888
902
|
pulumi.set(self, "event_log", value)
|
|
889
903
|
|
|
890
|
-
@property
|
|
904
|
+
@_builtins.property
|
|
891
905
|
@pulumi.getter(name="expectedLastModified")
|
|
892
|
-
def expected_last_modified(self) -> Optional[pulumi.Input[
|
|
906
|
+
def expected_last_modified(self) -> Optional[pulumi.Input[_builtins.int]]:
|
|
893
907
|
return pulumi.get(self, "expected_last_modified")
|
|
894
908
|
|
|
895
909
|
@expected_last_modified.setter
|
|
896
|
-
def expected_last_modified(self, value: Optional[pulumi.Input[
|
|
910
|
+
def expected_last_modified(self, value: Optional[pulumi.Input[_builtins.int]]):
|
|
897
911
|
pulumi.set(self, "expected_last_modified", value)
|
|
898
912
|
|
|
899
|
-
@property
|
|
913
|
+
@_builtins.property
|
|
900
914
|
@pulumi.getter
|
|
901
915
|
def filters(self) -> Optional[pulumi.Input['PipelineFiltersArgs']]:
|
|
902
916
|
"""
|
|
@@ -908,7 +922,7 @@ class _PipelineState:
|
|
|
908
922
|
def filters(self, value: Optional[pulumi.Input['PipelineFiltersArgs']]):
|
|
909
923
|
pulumi.set(self, "filters", value)
|
|
910
924
|
|
|
911
|
-
@property
|
|
925
|
+
@_builtins.property
|
|
912
926
|
@pulumi.getter(name="gatewayDefinition")
|
|
913
927
|
def gateway_definition(self) -> Optional[pulumi.Input['PipelineGatewayDefinitionArgs']]:
|
|
914
928
|
"""
|
|
@@ -920,16 +934,16 @@ class _PipelineState:
|
|
|
920
934
|
def gateway_definition(self, value: Optional[pulumi.Input['PipelineGatewayDefinitionArgs']]):
|
|
921
935
|
pulumi.set(self, "gateway_definition", value)
|
|
922
936
|
|
|
923
|
-
@property
|
|
937
|
+
@_builtins.property
|
|
924
938
|
@pulumi.getter
|
|
925
|
-
def health(self) -> Optional[pulumi.Input[
|
|
939
|
+
def health(self) -> Optional[pulumi.Input[_builtins.str]]:
|
|
926
940
|
return pulumi.get(self, "health")
|
|
927
941
|
|
|
928
942
|
@health.setter
|
|
929
|
-
def health(self, value: Optional[pulumi.Input[
|
|
943
|
+
def health(self, value: Optional[pulumi.Input[_builtins.str]]):
|
|
930
944
|
pulumi.set(self, "health", value)
|
|
931
945
|
|
|
932
|
-
@property
|
|
946
|
+
@_builtins.property
|
|
933
947
|
@pulumi.getter(name="ingestionDefinition")
|
|
934
948
|
def ingestion_definition(self) -> Optional[pulumi.Input['PipelineIngestionDefinitionArgs']]:
|
|
935
949
|
return pulumi.get(self, "ingestion_definition")
|
|
@@ -938,16 +952,16 @@ class _PipelineState:
|
|
|
938
952
|
def ingestion_definition(self, value: Optional[pulumi.Input['PipelineIngestionDefinitionArgs']]):
|
|
939
953
|
pulumi.set(self, "ingestion_definition", value)
|
|
940
954
|
|
|
941
|
-
@property
|
|
955
|
+
@_builtins.property
|
|
942
956
|
@pulumi.getter(name="lastModified")
|
|
943
|
-
def last_modified(self) -> Optional[pulumi.Input[
|
|
957
|
+
def last_modified(self) -> Optional[pulumi.Input[_builtins.int]]:
|
|
944
958
|
return pulumi.get(self, "last_modified")
|
|
945
959
|
|
|
946
960
|
@last_modified.setter
|
|
947
|
-
def last_modified(self, value: Optional[pulumi.Input[
|
|
961
|
+
def last_modified(self, value: Optional[pulumi.Input[_builtins.int]]):
|
|
948
962
|
pulumi.set(self, "last_modified", value)
|
|
949
963
|
|
|
950
|
-
@property
|
|
964
|
+
@_builtins.property
|
|
951
965
|
@pulumi.getter(name="latestUpdates")
|
|
952
966
|
def latest_updates(self) -> Optional[pulumi.Input[Sequence[pulumi.Input['PipelineLatestUpdateArgs']]]]:
|
|
953
967
|
return pulumi.get(self, "latest_updates")
|
|
@@ -956,7 +970,7 @@ class _PipelineState:
|
|
|
956
970
|
def latest_updates(self, value: Optional[pulumi.Input[Sequence[pulumi.Input['PipelineLatestUpdateArgs']]]]):
|
|
957
971
|
pulumi.set(self, "latest_updates", value)
|
|
958
972
|
|
|
959
|
-
@property
|
|
973
|
+
@_builtins.property
|
|
960
974
|
@pulumi.getter
|
|
961
975
|
def libraries(self) -> Optional[pulumi.Input[Sequence[pulumi.Input['PipelineLibraryArgs']]]]:
|
|
962
976
|
"""
|
|
@@ -968,19 +982,19 @@ class _PipelineState:
|
|
|
968
982
|
def libraries(self, value: Optional[pulumi.Input[Sequence[pulumi.Input['PipelineLibraryArgs']]]]):
|
|
969
983
|
pulumi.set(self, "libraries", value)
|
|
970
984
|
|
|
971
|
-
@property
|
|
985
|
+
@_builtins.property
|
|
972
986
|
@pulumi.getter
|
|
973
|
-
def name(self) -> Optional[pulumi.Input[
|
|
987
|
+
def name(self) -> Optional[pulumi.Input[_builtins.str]]:
|
|
974
988
|
"""
|
|
975
989
|
A user-friendly name for this pipeline. The name can be used to identify pipeline jobs in the UI.
|
|
976
990
|
"""
|
|
977
991
|
return pulumi.get(self, "name")
|
|
978
992
|
|
|
979
993
|
@name.setter
|
|
980
|
-
def name(self, value: Optional[pulumi.Input[
|
|
994
|
+
def name(self, value: Optional[pulumi.Input[_builtins.str]]):
|
|
981
995
|
pulumi.set(self, "name", value)
|
|
982
996
|
|
|
983
|
-
@property
|
|
997
|
+
@_builtins.property
|
|
984
998
|
@pulumi.getter
|
|
985
999
|
def notifications(self) -> Optional[pulumi.Input[Sequence[pulumi.Input['PipelineNotificationArgs']]]]:
|
|
986
1000
|
return pulumi.get(self, "notifications")
|
|
@@ -989,19 +1003,19 @@ class _PipelineState:
|
|
|
989
1003
|
def notifications(self, value: Optional[pulumi.Input[Sequence[pulumi.Input['PipelineNotificationArgs']]]]):
|
|
990
1004
|
pulumi.set(self, "notifications", value)
|
|
991
1005
|
|
|
992
|
-
@property
|
|
1006
|
+
@_builtins.property
|
|
993
1007
|
@pulumi.getter
|
|
994
|
-
def photon(self) -> Optional[pulumi.Input[
|
|
1008
|
+
def photon(self) -> Optional[pulumi.Input[_builtins.bool]]:
|
|
995
1009
|
"""
|
|
996
1010
|
A flag indicating whether to use Photon engine. The default value is `false`.
|
|
997
1011
|
"""
|
|
998
1012
|
return pulumi.get(self, "photon")
|
|
999
1013
|
|
|
1000
1014
|
@photon.setter
|
|
1001
|
-
def photon(self, value: Optional[pulumi.Input[
|
|
1015
|
+
def photon(self, value: Optional[pulumi.Input[_builtins.bool]]):
|
|
1002
1016
|
pulumi.set(self, "photon", value)
|
|
1003
1017
|
|
|
1004
|
-
@property
|
|
1018
|
+
@_builtins.property
|
|
1005
1019
|
@pulumi.getter(name="restartWindow")
|
|
1006
1020
|
def restart_window(self) -> Optional[pulumi.Input['PipelineRestartWindowArgs']]:
|
|
1007
1021
|
return pulumi.get(self, "restart_window")
|
|
@@ -1010,19 +1024,19 @@ class _PipelineState:
|
|
|
1010
1024
|
def restart_window(self, value: Optional[pulumi.Input['PipelineRestartWindowArgs']]):
|
|
1011
1025
|
pulumi.set(self, "restart_window", value)
|
|
1012
1026
|
|
|
1013
|
-
@property
|
|
1027
|
+
@_builtins.property
|
|
1014
1028
|
@pulumi.getter(name="rootPath")
|
|
1015
|
-
def root_path(self) -> Optional[pulumi.Input[
|
|
1029
|
+
def root_path(self) -> Optional[pulumi.Input[_builtins.str]]:
|
|
1016
1030
|
"""
|
|
1017
1031
|
An optional string specifying the root path for this pipeline. This is used as the root directory when editing the pipeline in the Databricks user interface and it is added to `sys.path` when executing Python sources during pipeline execution.
|
|
1018
1032
|
"""
|
|
1019
1033
|
return pulumi.get(self, "root_path")
|
|
1020
1034
|
|
|
1021
1035
|
@root_path.setter
|
|
1022
|
-
def root_path(self, value: Optional[pulumi.Input[
|
|
1036
|
+
def root_path(self, value: Optional[pulumi.Input[_builtins.str]]):
|
|
1023
1037
|
pulumi.set(self, "root_path", value)
|
|
1024
1038
|
|
|
1025
|
-
@property
|
|
1039
|
+
@_builtins.property
|
|
1026
1040
|
@pulumi.getter(name="runAs")
|
|
1027
1041
|
def run_as(self) -> Optional[pulumi.Input['PipelineRunAsArgs']]:
|
|
1028
1042
|
return pulumi.get(self, "run_as")
|
|
@@ -1031,85 +1045,85 @@ class _PipelineState:
|
|
|
1031
1045
|
def run_as(self, value: Optional[pulumi.Input['PipelineRunAsArgs']]):
|
|
1032
1046
|
pulumi.set(self, "run_as", value)
|
|
1033
1047
|
|
|
1034
|
-
@property
|
|
1048
|
+
@_builtins.property
|
|
1035
1049
|
@pulumi.getter(name="runAsUserName")
|
|
1036
|
-
def run_as_user_name(self) -> Optional[pulumi.Input[
|
|
1050
|
+
def run_as_user_name(self) -> Optional[pulumi.Input[_builtins.str]]:
|
|
1037
1051
|
return pulumi.get(self, "run_as_user_name")
|
|
1038
1052
|
|
|
1039
1053
|
@run_as_user_name.setter
|
|
1040
|
-
def run_as_user_name(self, value: Optional[pulumi.Input[
|
|
1054
|
+
def run_as_user_name(self, value: Optional[pulumi.Input[_builtins.str]]):
|
|
1041
1055
|
pulumi.set(self, "run_as_user_name", value)
|
|
1042
1056
|
|
|
1043
|
-
@property
|
|
1057
|
+
@_builtins.property
|
|
1044
1058
|
@pulumi.getter
|
|
1045
|
-
def schema(self) -> Optional[pulumi.Input[
|
|
1059
|
+
def schema(self) -> Optional[pulumi.Input[_builtins.str]]:
|
|
1046
1060
|
"""
|
|
1047
1061
|
The default schema (database) where tables are read from or published to. The presence of this attribute implies that the pipeline is in direct publishing mode.
|
|
1048
1062
|
"""
|
|
1049
1063
|
return pulumi.get(self, "schema")
|
|
1050
1064
|
|
|
1051
1065
|
@schema.setter
|
|
1052
|
-
def schema(self, value: Optional[pulumi.Input[
|
|
1066
|
+
def schema(self, value: Optional[pulumi.Input[_builtins.str]]):
|
|
1053
1067
|
pulumi.set(self, "schema", value)
|
|
1054
1068
|
|
|
1055
|
-
@property
|
|
1069
|
+
@_builtins.property
|
|
1056
1070
|
@pulumi.getter
|
|
1057
|
-
def serverless(self) -> Optional[pulumi.Input[
|
|
1071
|
+
def serverless(self) -> Optional[pulumi.Input[_builtins.bool]]:
|
|
1058
1072
|
"""
|
|
1059
|
-
An optional flag indicating if serverless compute should be used for this
|
|
1073
|
+
An optional flag indicating if serverless compute should be used for this Lakeflow Declarative Pipeline. Requires `catalog` to be set, as it could be used only with Unity Catalog.
|
|
1060
1074
|
"""
|
|
1061
1075
|
return pulumi.get(self, "serverless")
|
|
1062
1076
|
|
|
1063
1077
|
@serverless.setter
|
|
1064
|
-
def serverless(self, value: Optional[pulumi.Input[
|
|
1078
|
+
def serverless(self, value: Optional[pulumi.Input[_builtins.bool]]):
|
|
1065
1079
|
pulumi.set(self, "serverless", value)
|
|
1066
1080
|
|
|
1067
|
-
@property
|
|
1081
|
+
@_builtins.property
|
|
1068
1082
|
@pulumi.getter
|
|
1069
|
-
def state(self) -> Optional[pulumi.Input[
|
|
1083
|
+
def state(self) -> Optional[pulumi.Input[_builtins.str]]:
|
|
1070
1084
|
return pulumi.get(self, "state")
|
|
1071
1085
|
|
|
1072
1086
|
@state.setter
|
|
1073
|
-
def state(self, value: Optional[pulumi.Input[
|
|
1087
|
+
def state(self, value: Optional[pulumi.Input[_builtins.str]]):
|
|
1074
1088
|
pulumi.set(self, "state", value)
|
|
1075
1089
|
|
|
1076
|
-
@property
|
|
1090
|
+
@_builtins.property
|
|
1077
1091
|
@pulumi.getter
|
|
1078
|
-
def storage(self) -> Optional[pulumi.Input[
|
|
1092
|
+
def storage(self) -> Optional[pulumi.Input[_builtins.str]]:
|
|
1079
1093
|
"""
|
|
1080
|
-
A location on
|
|
1094
|
+
A location on cloud storage where output data and metadata required for pipeline execution are stored. By default, tables are stored in a subdirectory of this location. *Change of this parameter forces recreation of the pipeline.* (Conflicts with `catalog`).
|
|
1081
1095
|
"""
|
|
1082
1096
|
return pulumi.get(self, "storage")
|
|
1083
1097
|
|
|
1084
1098
|
@storage.setter
|
|
1085
|
-
def storage(self, value: Optional[pulumi.Input[
|
|
1099
|
+
def storage(self, value: Optional[pulumi.Input[_builtins.str]]):
|
|
1086
1100
|
pulumi.set(self, "storage", value)
|
|
1087
1101
|
|
|
1088
|
-
@property
|
|
1102
|
+
@_builtins.property
|
|
1089
1103
|
@pulumi.getter
|
|
1090
|
-
def tags(self) -> Optional[pulumi.Input[Mapping[str, pulumi.Input[
|
|
1104
|
+
def tags(self) -> Optional[pulumi.Input[Mapping[str, pulumi.Input[_builtins.str]]]]:
|
|
1091
1105
|
"""
|
|
1092
1106
|
A map of tags associated with the pipeline. These are forwarded to the cluster as cluster tags, and are therefore subject to the same limitations. A maximum of 25 tags can be added to the pipeline.
|
|
1093
1107
|
"""
|
|
1094
1108
|
return pulumi.get(self, "tags")
|
|
1095
1109
|
|
|
1096
1110
|
@tags.setter
|
|
1097
|
-
def tags(self, value: Optional[pulumi.Input[Mapping[str, pulumi.Input[
|
|
1111
|
+
def tags(self, value: Optional[pulumi.Input[Mapping[str, pulumi.Input[_builtins.str]]]]):
|
|
1098
1112
|
pulumi.set(self, "tags", value)
|
|
1099
1113
|
|
|
1100
|
-
@property
|
|
1114
|
+
@_builtins.property
|
|
1101
1115
|
@pulumi.getter
|
|
1102
|
-
def target(self) -> Optional[pulumi.Input[
|
|
1116
|
+
def target(self) -> Optional[pulumi.Input[_builtins.str]]:
|
|
1103
1117
|
"""
|
|
1104
1118
|
The name of a database (in either the Hive metastore or in a UC catalog) for persisting pipeline output data. Configuring the target setting allows you to view and query the pipeline output data from the Databricks UI.
|
|
1105
1119
|
"""
|
|
1106
1120
|
return pulumi.get(self, "target")
|
|
1107
1121
|
|
|
1108
1122
|
@target.setter
|
|
1109
|
-
def target(self, value: Optional[pulumi.Input[
|
|
1123
|
+
def target(self, value: Optional[pulumi.Input[_builtins.str]]):
|
|
1110
1124
|
pulumi.set(self, "target", value)
|
|
1111
1125
|
|
|
1112
|
-
@property
|
|
1126
|
+
@_builtins.property
|
|
1113
1127
|
@pulumi.getter
|
|
1114
1128
|
def trigger(self) -> Optional[pulumi.Input['PipelineTriggerArgs']]:
|
|
1115
1129
|
return pulumi.get(self, "trigger")
|
|
@@ -1118,18 +1132,27 @@ class _PipelineState:
|
|
|
1118
1132
|
def trigger(self, value: Optional[pulumi.Input['PipelineTriggerArgs']]):
|
|
1119
1133
|
pulumi.set(self, "trigger", value)
|
|
1120
1134
|
|
|
1121
|
-
@property
|
|
1135
|
+
@_builtins.property
|
|
1122
1136
|
@pulumi.getter
|
|
1123
|
-
def url(self) -> Optional[pulumi.Input[
|
|
1137
|
+
def url(self) -> Optional[pulumi.Input[_builtins.str]]:
|
|
1124
1138
|
"""
|
|
1125
|
-
URL of the
|
|
1139
|
+
URL of the Lakeflow Declarative Pipeline on the given workspace.
|
|
1126
1140
|
"""
|
|
1127
1141
|
return pulumi.get(self, "url")
|
|
1128
1142
|
|
|
1129
1143
|
@url.setter
|
|
1130
|
-
def url(self, value: Optional[pulumi.Input[
|
|
1144
|
+
def url(self, value: Optional[pulumi.Input[_builtins.str]]):
|
|
1131
1145
|
pulumi.set(self, "url", value)
|
|
1132
1146
|
|
|
1147
|
+
@_builtins.property
|
|
1148
|
+
@pulumi.getter(name="usagePolicyId")
|
|
1149
|
+
def usage_policy_id(self) -> Optional[pulumi.Input[_builtins.str]]:
|
|
1150
|
+
return pulumi.get(self, "usage_policy_id")
|
|
1151
|
+
|
|
1152
|
+
@usage_policy_id.setter
|
|
1153
|
+
def usage_policy_id(self, value: Optional[pulumi.Input[_builtins.str]]):
|
|
1154
|
+
pulumi.set(self, "usage_policy_id", value)
|
|
1155
|
+
|
|
1133
1156
|
|
|
1134
1157
|
@pulumi.type_token("databricks:index/pipeline:Pipeline")
|
|
1135
1158
|
class Pipeline(pulumi.CustomResource):
|
|
@@ -1137,47 +1160,48 @@ class Pipeline(pulumi.CustomResource):
|
|
|
1137
1160
|
def __init__(__self__,
|
|
1138
1161
|
resource_name: str,
|
|
1139
1162
|
opts: Optional[pulumi.ResourceOptions] = None,
|
|
1140
|
-
allow_duplicate_names: Optional[pulumi.Input[
|
|
1141
|
-
budget_policy_id: Optional[pulumi.Input[
|
|
1142
|
-
catalog: Optional[pulumi.Input[
|
|
1143
|
-
cause: Optional[pulumi.Input[
|
|
1144
|
-
channel: Optional[pulumi.Input[
|
|
1145
|
-
cluster_id: Optional[pulumi.Input[
|
|
1163
|
+
allow_duplicate_names: Optional[pulumi.Input[_builtins.bool]] = None,
|
|
1164
|
+
budget_policy_id: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1165
|
+
catalog: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1166
|
+
cause: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1167
|
+
channel: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1168
|
+
cluster_id: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1146
1169
|
clusters: Optional[pulumi.Input[Sequence[pulumi.Input[Union['PipelineClusterArgs', 'PipelineClusterArgsDict']]]]] = None,
|
|
1147
|
-
configuration: Optional[pulumi.Input[Mapping[str, pulumi.Input[
|
|
1148
|
-
continuous: Optional[pulumi.Input[
|
|
1149
|
-
creator_user_name: Optional[pulumi.Input[
|
|
1170
|
+
configuration: Optional[pulumi.Input[Mapping[str, pulumi.Input[_builtins.str]]]] = None,
|
|
1171
|
+
continuous: Optional[pulumi.Input[_builtins.bool]] = None,
|
|
1172
|
+
creator_user_name: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1150
1173
|
deployment: Optional[pulumi.Input[Union['PipelineDeploymentArgs', 'PipelineDeploymentArgsDict']]] = None,
|
|
1151
|
-
development: Optional[pulumi.Input[
|
|
1152
|
-
edition: Optional[pulumi.Input[
|
|
1174
|
+
development: Optional[pulumi.Input[_builtins.bool]] = None,
|
|
1175
|
+
edition: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1153
1176
|
environment: Optional[pulumi.Input[Union['PipelineEnvironmentArgs', 'PipelineEnvironmentArgsDict']]] = None,
|
|
1154
1177
|
event_log: Optional[pulumi.Input[Union['PipelineEventLogArgs', 'PipelineEventLogArgsDict']]] = None,
|
|
1155
|
-
expected_last_modified: Optional[pulumi.Input[
|
|
1178
|
+
expected_last_modified: Optional[pulumi.Input[_builtins.int]] = None,
|
|
1156
1179
|
filters: Optional[pulumi.Input[Union['PipelineFiltersArgs', 'PipelineFiltersArgsDict']]] = None,
|
|
1157
1180
|
gateway_definition: Optional[pulumi.Input[Union['PipelineGatewayDefinitionArgs', 'PipelineGatewayDefinitionArgsDict']]] = None,
|
|
1158
|
-
health: Optional[pulumi.Input[
|
|
1181
|
+
health: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1159
1182
|
ingestion_definition: Optional[pulumi.Input[Union['PipelineIngestionDefinitionArgs', 'PipelineIngestionDefinitionArgsDict']]] = None,
|
|
1160
|
-
last_modified: Optional[pulumi.Input[
|
|
1183
|
+
last_modified: Optional[pulumi.Input[_builtins.int]] = None,
|
|
1161
1184
|
latest_updates: Optional[pulumi.Input[Sequence[pulumi.Input[Union['PipelineLatestUpdateArgs', 'PipelineLatestUpdateArgsDict']]]]] = None,
|
|
1162
1185
|
libraries: Optional[pulumi.Input[Sequence[pulumi.Input[Union['PipelineLibraryArgs', 'PipelineLibraryArgsDict']]]]] = None,
|
|
1163
|
-
name: Optional[pulumi.Input[
|
|
1186
|
+
name: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1164
1187
|
notifications: Optional[pulumi.Input[Sequence[pulumi.Input[Union['PipelineNotificationArgs', 'PipelineNotificationArgsDict']]]]] = None,
|
|
1165
|
-
photon: Optional[pulumi.Input[
|
|
1188
|
+
photon: Optional[pulumi.Input[_builtins.bool]] = None,
|
|
1166
1189
|
restart_window: Optional[pulumi.Input[Union['PipelineRestartWindowArgs', 'PipelineRestartWindowArgsDict']]] = None,
|
|
1167
|
-
root_path: Optional[pulumi.Input[
|
|
1190
|
+
root_path: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1168
1191
|
run_as: Optional[pulumi.Input[Union['PipelineRunAsArgs', 'PipelineRunAsArgsDict']]] = None,
|
|
1169
|
-
run_as_user_name: Optional[pulumi.Input[
|
|
1170
|
-
schema: Optional[pulumi.Input[
|
|
1171
|
-
serverless: Optional[pulumi.Input[
|
|
1172
|
-
state: Optional[pulumi.Input[
|
|
1173
|
-
storage: Optional[pulumi.Input[
|
|
1174
|
-
tags: Optional[pulumi.Input[Mapping[str, pulumi.Input[
|
|
1175
|
-
target: Optional[pulumi.Input[
|
|
1192
|
+
run_as_user_name: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1193
|
+
schema: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1194
|
+
serverless: Optional[pulumi.Input[_builtins.bool]] = None,
|
|
1195
|
+
state: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1196
|
+
storage: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1197
|
+
tags: Optional[pulumi.Input[Mapping[str, pulumi.Input[_builtins.str]]]] = None,
|
|
1198
|
+
target: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1176
1199
|
trigger: Optional[pulumi.Input[Union['PipelineTriggerArgs', 'PipelineTriggerArgsDict']]] = None,
|
|
1177
|
-
url: Optional[pulumi.Input[
|
|
1200
|
+
url: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1201
|
+
usage_policy_id: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1178
1202
|
__props__=None):
|
|
1179
1203
|
"""
|
|
1180
|
-
Use `Pipeline` to deploy [
|
|
1204
|
+
Use `Pipeline` to deploy [Lakeflow Declarative Pipelines](https://docs.databricks.com/aws/en/dlt).
|
|
1181
1205
|
|
|
1182
1206
|
> This resource can only be used with a workspace-level provider!
|
|
1183
1207
|
|
|
@@ -1187,11 +1211,12 @@ class Pipeline(pulumi.CustomResource):
|
|
|
1187
1211
|
import pulumi
|
|
1188
1212
|
import pulumi_databricks as databricks
|
|
1189
1213
|
|
|
1190
|
-
|
|
1191
|
-
|
|
1214
|
+
ldp_demo = databricks.Notebook("ldp_demo")
|
|
1215
|
+
ldp_demo_repo = databricks.Repo("ldp_demo")
|
|
1192
1216
|
this = databricks.Pipeline("this",
|
|
1193
1217
|
name="Pipeline Name",
|
|
1194
|
-
|
|
1218
|
+
catalog="main",
|
|
1219
|
+
schema="ldp_demo",
|
|
1195
1220
|
configuration={
|
|
1196
1221
|
"key1": "value1",
|
|
1197
1222
|
"key2": "value2",
|
|
@@ -1215,17 +1240,17 @@ class Pipeline(pulumi.CustomResource):
|
|
|
1215
1240
|
libraries=[
|
|
1216
1241
|
{
|
|
1217
1242
|
"notebook": {
|
|
1218
|
-
"path":
|
|
1243
|
+
"path": ldp_demo.id,
|
|
1219
1244
|
},
|
|
1220
1245
|
},
|
|
1221
1246
|
{
|
|
1222
1247
|
"file": {
|
|
1223
|
-
"path":
|
|
1248
|
+
"path": ldp_demo_repo.path.apply(lambda path: f"{path}/pipeline.sql"),
|
|
1224
1249
|
},
|
|
1225
1250
|
},
|
|
1226
1251
|
{
|
|
1227
1252
|
"glob": {
|
|
1228
|
-
"include":
|
|
1253
|
+
"include": ldp_demo_repo.path.apply(lambda path: f"{path}/subfolder/**"),
|
|
1229
1254
|
},
|
|
1230
1255
|
},
|
|
1231
1256
|
],
|
|
@@ -1249,7 +1274,7 @@ class Pipeline(pulumi.CustomResource):
|
|
|
1249
1274
|
The following resources are often used in the same context:
|
|
1250
1275
|
|
|
1251
1276
|
* End to end workspace management guide.
|
|
1252
|
-
* get_pipelines to retrieve [
|
|
1277
|
+
* get_pipelines to retrieve [Lakeflow Declarative Pipelines](https://docs.databricks.com/aws/en/dlt) data.
|
|
1253
1278
|
* Cluster to create [Databricks Clusters](https://docs.databricks.com/clusters/index.html).
|
|
1254
1279
|
* Job to manage [Databricks Jobs](https://docs.databricks.com/jobs.html) to run non-interactive code in a databricks_cluster.
|
|
1255
1280
|
* Notebook to manage [Databricks Notebooks](https://docs.databricks.com/notebooks/index.html).
|
|
@@ -1278,29 +1303,29 @@ class Pipeline(pulumi.CustomResource):
|
|
|
1278
1303
|
|
|
1279
1304
|
:param str resource_name: The name of the resource.
|
|
1280
1305
|
:param pulumi.ResourceOptions opts: Options for the resource.
|
|
1281
|
-
:param pulumi.Input[
|
|
1282
|
-
:param pulumi.Input[
|
|
1283
|
-
:param pulumi.Input[
|
|
1284
|
-
:param pulumi.Input[
|
|
1285
|
-
:param pulumi.Input[Sequence[pulumi.Input[Union['PipelineClusterArgs', 'PipelineClusterArgsDict']]]] clusters: blocks - Clusters to run the pipeline. If none is specified, pipelines will automatically select a default cluster configuration for the pipeline. *Please note that
|
|
1286
|
-
:param pulumi.Input[Mapping[str, pulumi.Input[
|
|
1287
|
-
:param pulumi.Input[
|
|
1306
|
+
:param pulumi.Input[_builtins.bool] allow_duplicate_names: Optional boolean flag. If false, deployment will fail if name conflicts with that of another pipeline. default is `false`.
|
|
1307
|
+
:param pulumi.Input[_builtins.str] budget_policy_id: optional string specifying ID of the budget policy for this Lakeflow Declarative Pipeline.
|
|
1308
|
+
:param pulumi.Input[_builtins.str] catalog: The name of catalog in Unity Catalog. *Change of this parameter forces recreation of the pipeline.* (Conflicts with `storage`).
|
|
1309
|
+
:param pulumi.Input[_builtins.str] channel: optional name of the release channel for Spark version used by Lakeflow Declarative Pipeline. Supported values are: `CURRENT` (default) and `PREVIEW`.
|
|
1310
|
+
:param pulumi.Input[Sequence[pulumi.Input[Union['PipelineClusterArgs', 'PipelineClusterArgsDict']]]] clusters: blocks - Clusters to run the pipeline. If none is specified, pipelines will automatically select a default cluster configuration for the pipeline. *Please note that Lakeflow Declarative Pipeline clusters are supporting only subset of attributes as described in [documentation](https://docs.databricks.com/api/workspace/pipelines/create#clusters).* Also, note that `autoscale` block is extended with the `mode` parameter that controls the autoscaling algorithm (possible values are `ENHANCED` for new, enhanced autoscaling algorithm, or `LEGACY` for old algorithm).
|
|
1311
|
+
:param pulumi.Input[Mapping[str, pulumi.Input[_builtins.str]]] configuration: An optional list of values to apply to the entire pipeline. Elements must be formatted as key:value pairs.
|
|
1312
|
+
:param pulumi.Input[_builtins.bool] continuous: A flag indicating whether to run the pipeline continuously. The default value is `false`.
|
|
1288
1313
|
:param pulumi.Input[Union['PipelineDeploymentArgs', 'PipelineDeploymentArgsDict']] deployment: Deployment type of this pipeline. Supports following attributes:
|
|
1289
|
-
:param pulumi.Input[
|
|
1290
|
-
:param pulumi.Input[
|
|
1291
|
-
:param pulumi.Input[Union['PipelineEventLogArgs', 'PipelineEventLogArgsDict']] event_log: an optional block specifying a table where
|
|
1314
|
+
:param pulumi.Input[_builtins.bool] development: A flag indicating whether to run the pipeline in development mode. The default value is `false`.
|
|
1315
|
+
:param pulumi.Input[_builtins.str] edition: optional name of the [product edition](https://docs.databricks.com/aws/en/dlt/configure-pipeline#choose-a-product-edition). Supported values are: `CORE`, `PRO`, `ADVANCED` (default). Not required when `serverless` is set to `true`.
|
|
1316
|
+
:param pulumi.Input[Union['PipelineEventLogArgs', 'PipelineEventLogArgsDict']] event_log: an optional block specifying a table where LDP Event Log will be stored. Consists of the following fields:
|
|
1292
1317
|
:param pulumi.Input[Union['PipelineFiltersArgs', 'PipelineFiltersArgsDict']] filters: Filters on which Pipeline packages to include in the deployed graph. This block consists of following attributes:
|
|
1293
1318
|
:param pulumi.Input[Union['PipelineGatewayDefinitionArgs', 'PipelineGatewayDefinitionArgsDict']] gateway_definition: The definition of a gateway pipeline to support CDC. Consists of following attributes:
|
|
1294
1319
|
:param pulumi.Input[Sequence[pulumi.Input[Union['PipelineLibraryArgs', 'PipelineLibraryArgsDict']]]] libraries: blocks - Specifies pipeline code.
|
|
1295
|
-
:param pulumi.Input[
|
|
1296
|
-
:param pulumi.Input[
|
|
1297
|
-
:param pulumi.Input[
|
|
1298
|
-
:param pulumi.Input[
|
|
1299
|
-
:param pulumi.Input[
|
|
1300
|
-
:param pulumi.Input[
|
|
1301
|
-
:param pulumi.Input[Mapping[str, pulumi.Input[
|
|
1302
|
-
:param pulumi.Input[
|
|
1303
|
-
:param pulumi.Input[
|
|
1320
|
+
:param pulumi.Input[_builtins.str] name: A user-friendly name for this pipeline. The name can be used to identify pipeline jobs in the UI.
|
|
1321
|
+
:param pulumi.Input[_builtins.bool] photon: A flag indicating whether to use Photon engine. The default value is `false`.
|
|
1322
|
+
:param pulumi.Input[_builtins.str] root_path: An optional string specifying the root path for this pipeline. This is used as the root directory when editing the pipeline in the Databricks user interface and it is added to `sys.path` when executing Python sources during pipeline execution.
|
|
1323
|
+
:param pulumi.Input[_builtins.str] schema: The default schema (database) where tables are read from or published to. The presence of this attribute implies that the pipeline is in direct publishing mode.
|
|
1324
|
+
:param pulumi.Input[_builtins.bool] serverless: An optional flag indicating if serverless compute should be used for this Lakeflow Declarative Pipeline. Requires `catalog` to be set, as it could be used only with Unity Catalog.
|
|
1325
|
+
:param pulumi.Input[_builtins.str] storage: A location on cloud storage where output data and metadata required for pipeline execution are stored. By default, tables are stored in a subdirectory of this location. *Change of this parameter forces recreation of the pipeline.* (Conflicts with `catalog`).
|
|
1326
|
+
:param pulumi.Input[Mapping[str, pulumi.Input[_builtins.str]]] tags: A map of tags associated with the pipeline. These are forwarded to the cluster as cluster tags, and are therefore subject to the same limitations. A maximum of 25 tags can be added to the pipeline.
|
|
1327
|
+
:param pulumi.Input[_builtins.str] target: The name of a database (in either the Hive metastore or in a UC catalog) for persisting pipeline output data. Configuring the target setting allows you to view and query the pipeline output data from the Databricks UI.
|
|
1328
|
+
:param pulumi.Input[_builtins.str] url: URL of the Lakeflow Declarative Pipeline on the given workspace.
|
|
1304
1329
|
"""
|
|
1305
1330
|
...
|
|
1306
1331
|
@overload
|
|
@@ -1309,7 +1334,7 @@ class Pipeline(pulumi.CustomResource):
|
|
|
1309
1334
|
args: Optional[PipelineArgs] = None,
|
|
1310
1335
|
opts: Optional[pulumi.ResourceOptions] = None):
|
|
1311
1336
|
"""
|
|
1312
|
-
Use `Pipeline` to deploy [
|
|
1337
|
+
Use `Pipeline` to deploy [Lakeflow Declarative Pipelines](https://docs.databricks.com/aws/en/dlt).
|
|
1313
1338
|
|
|
1314
1339
|
> This resource can only be used with a workspace-level provider!
|
|
1315
1340
|
|
|
@@ -1319,11 +1344,12 @@ class Pipeline(pulumi.CustomResource):
|
|
|
1319
1344
|
import pulumi
|
|
1320
1345
|
import pulumi_databricks as databricks
|
|
1321
1346
|
|
|
1322
|
-
|
|
1323
|
-
|
|
1347
|
+
ldp_demo = databricks.Notebook("ldp_demo")
|
|
1348
|
+
ldp_demo_repo = databricks.Repo("ldp_demo")
|
|
1324
1349
|
this = databricks.Pipeline("this",
|
|
1325
1350
|
name="Pipeline Name",
|
|
1326
|
-
|
|
1351
|
+
catalog="main",
|
|
1352
|
+
schema="ldp_demo",
|
|
1327
1353
|
configuration={
|
|
1328
1354
|
"key1": "value1",
|
|
1329
1355
|
"key2": "value2",
|
|
@@ -1347,17 +1373,17 @@ class Pipeline(pulumi.CustomResource):
|
|
|
1347
1373
|
libraries=[
|
|
1348
1374
|
{
|
|
1349
1375
|
"notebook": {
|
|
1350
|
-
"path":
|
|
1376
|
+
"path": ldp_demo.id,
|
|
1351
1377
|
},
|
|
1352
1378
|
},
|
|
1353
1379
|
{
|
|
1354
1380
|
"file": {
|
|
1355
|
-
"path":
|
|
1381
|
+
"path": ldp_demo_repo.path.apply(lambda path: f"{path}/pipeline.sql"),
|
|
1356
1382
|
},
|
|
1357
1383
|
},
|
|
1358
1384
|
{
|
|
1359
1385
|
"glob": {
|
|
1360
|
-
"include":
|
|
1386
|
+
"include": ldp_demo_repo.path.apply(lambda path: f"{path}/subfolder/**"),
|
|
1361
1387
|
},
|
|
1362
1388
|
},
|
|
1363
1389
|
],
|
|
@@ -1381,7 +1407,7 @@ class Pipeline(pulumi.CustomResource):
|
|
|
1381
1407
|
The following resources are often used in the same context:
|
|
1382
1408
|
|
|
1383
1409
|
* End to end workspace management guide.
|
|
1384
|
-
* get_pipelines to retrieve [
|
|
1410
|
+
* get_pipelines to retrieve [Lakeflow Declarative Pipelines](https://docs.databricks.com/aws/en/dlt) data.
|
|
1385
1411
|
* Cluster to create [Databricks Clusters](https://docs.databricks.com/clusters/index.html).
|
|
1386
1412
|
* Job to manage [Databricks Jobs](https://docs.databricks.com/jobs.html) to run non-interactive code in a databricks_cluster.
|
|
1387
1413
|
* Notebook to manage [Databricks Notebooks](https://docs.databricks.com/notebooks/index.html).
|
|
@@ -1423,44 +1449,45 @@ class Pipeline(pulumi.CustomResource):
|
|
|
1423
1449
|
def _internal_init(__self__,
|
|
1424
1450
|
resource_name: str,
|
|
1425
1451
|
opts: Optional[pulumi.ResourceOptions] = None,
|
|
1426
|
-
allow_duplicate_names: Optional[pulumi.Input[
|
|
1427
|
-
budget_policy_id: Optional[pulumi.Input[
|
|
1428
|
-
catalog: Optional[pulumi.Input[
|
|
1429
|
-
cause: Optional[pulumi.Input[
|
|
1430
|
-
channel: Optional[pulumi.Input[
|
|
1431
|
-
cluster_id: Optional[pulumi.Input[
|
|
1452
|
+
allow_duplicate_names: Optional[pulumi.Input[_builtins.bool]] = None,
|
|
1453
|
+
budget_policy_id: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1454
|
+
catalog: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1455
|
+
cause: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1456
|
+
channel: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1457
|
+
cluster_id: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1432
1458
|
clusters: Optional[pulumi.Input[Sequence[pulumi.Input[Union['PipelineClusterArgs', 'PipelineClusterArgsDict']]]]] = None,
|
|
1433
|
-
configuration: Optional[pulumi.Input[Mapping[str, pulumi.Input[
|
|
1434
|
-
continuous: Optional[pulumi.Input[
|
|
1435
|
-
creator_user_name: Optional[pulumi.Input[
|
|
1459
|
+
configuration: Optional[pulumi.Input[Mapping[str, pulumi.Input[_builtins.str]]]] = None,
|
|
1460
|
+
continuous: Optional[pulumi.Input[_builtins.bool]] = None,
|
|
1461
|
+
creator_user_name: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1436
1462
|
deployment: Optional[pulumi.Input[Union['PipelineDeploymentArgs', 'PipelineDeploymentArgsDict']]] = None,
|
|
1437
|
-
development: Optional[pulumi.Input[
|
|
1438
|
-
edition: Optional[pulumi.Input[
|
|
1463
|
+
development: Optional[pulumi.Input[_builtins.bool]] = None,
|
|
1464
|
+
edition: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1439
1465
|
environment: Optional[pulumi.Input[Union['PipelineEnvironmentArgs', 'PipelineEnvironmentArgsDict']]] = None,
|
|
1440
1466
|
event_log: Optional[pulumi.Input[Union['PipelineEventLogArgs', 'PipelineEventLogArgsDict']]] = None,
|
|
1441
|
-
expected_last_modified: Optional[pulumi.Input[
|
|
1467
|
+
expected_last_modified: Optional[pulumi.Input[_builtins.int]] = None,
|
|
1442
1468
|
filters: Optional[pulumi.Input[Union['PipelineFiltersArgs', 'PipelineFiltersArgsDict']]] = None,
|
|
1443
1469
|
gateway_definition: Optional[pulumi.Input[Union['PipelineGatewayDefinitionArgs', 'PipelineGatewayDefinitionArgsDict']]] = None,
|
|
1444
|
-
health: Optional[pulumi.Input[
|
|
1470
|
+
health: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1445
1471
|
ingestion_definition: Optional[pulumi.Input[Union['PipelineIngestionDefinitionArgs', 'PipelineIngestionDefinitionArgsDict']]] = None,
|
|
1446
|
-
last_modified: Optional[pulumi.Input[
|
|
1472
|
+
last_modified: Optional[pulumi.Input[_builtins.int]] = None,
|
|
1447
1473
|
latest_updates: Optional[pulumi.Input[Sequence[pulumi.Input[Union['PipelineLatestUpdateArgs', 'PipelineLatestUpdateArgsDict']]]]] = None,
|
|
1448
1474
|
libraries: Optional[pulumi.Input[Sequence[pulumi.Input[Union['PipelineLibraryArgs', 'PipelineLibraryArgsDict']]]]] = None,
|
|
1449
|
-
name: Optional[pulumi.Input[
|
|
1475
|
+
name: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1450
1476
|
notifications: Optional[pulumi.Input[Sequence[pulumi.Input[Union['PipelineNotificationArgs', 'PipelineNotificationArgsDict']]]]] = None,
|
|
1451
|
-
photon: Optional[pulumi.Input[
|
|
1477
|
+
photon: Optional[pulumi.Input[_builtins.bool]] = None,
|
|
1452
1478
|
restart_window: Optional[pulumi.Input[Union['PipelineRestartWindowArgs', 'PipelineRestartWindowArgsDict']]] = None,
|
|
1453
|
-
root_path: Optional[pulumi.Input[
|
|
1479
|
+
root_path: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1454
1480
|
run_as: Optional[pulumi.Input[Union['PipelineRunAsArgs', 'PipelineRunAsArgsDict']]] = None,
|
|
1455
|
-
run_as_user_name: Optional[pulumi.Input[
|
|
1456
|
-
schema: Optional[pulumi.Input[
|
|
1457
|
-
serverless: Optional[pulumi.Input[
|
|
1458
|
-
state: Optional[pulumi.Input[
|
|
1459
|
-
storage: Optional[pulumi.Input[
|
|
1460
|
-
tags: Optional[pulumi.Input[Mapping[str, pulumi.Input[
|
|
1461
|
-
target: Optional[pulumi.Input[
|
|
1481
|
+
run_as_user_name: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1482
|
+
schema: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1483
|
+
serverless: Optional[pulumi.Input[_builtins.bool]] = None,
|
|
1484
|
+
state: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1485
|
+
storage: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1486
|
+
tags: Optional[pulumi.Input[Mapping[str, pulumi.Input[_builtins.str]]]] = None,
|
|
1487
|
+
target: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1462
1488
|
trigger: Optional[pulumi.Input[Union['PipelineTriggerArgs', 'PipelineTriggerArgsDict']]] = None,
|
|
1463
|
-
url: Optional[pulumi.Input[
|
|
1489
|
+
url: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1490
|
+
usage_policy_id: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1464
1491
|
__props__=None):
|
|
1465
1492
|
opts = pulumi.ResourceOptions.merge(_utilities.get_resource_opts_defaults(), opts)
|
|
1466
1493
|
if not isinstance(opts, pulumi.ResourceOptions):
|
|
@@ -1508,6 +1535,7 @@ class Pipeline(pulumi.CustomResource):
|
|
|
1508
1535
|
__props__.__dict__["target"] = target
|
|
1509
1536
|
__props__.__dict__["trigger"] = trigger
|
|
1510
1537
|
__props__.__dict__["url"] = url
|
|
1538
|
+
__props__.__dict__["usage_policy_id"] = usage_policy_id
|
|
1511
1539
|
super(Pipeline, __self__).__init__(
|
|
1512
1540
|
'databricks:index/pipeline:Pipeline',
|
|
1513
1541
|
resource_name,
|
|
@@ -1518,44 +1546,45 @@ class Pipeline(pulumi.CustomResource):
|
|
|
1518
1546
|
def get(resource_name: str,
|
|
1519
1547
|
id: pulumi.Input[str],
|
|
1520
1548
|
opts: Optional[pulumi.ResourceOptions] = None,
|
|
1521
|
-
allow_duplicate_names: Optional[pulumi.Input[
|
|
1522
|
-
budget_policy_id: Optional[pulumi.Input[
|
|
1523
|
-
catalog: Optional[pulumi.Input[
|
|
1524
|
-
cause: Optional[pulumi.Input[
|
|
1525
|
-
channel: Optional[pulumi.Input[
|
|
1526
|
-
cluster_id: Optional[pulumi.Input[
|
|
1549
|
+
allow_duplicate_names: Optional[pulumi.Input[_builtins.bool]] = None,
|
|
1550
|
+
budget_policy_id: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1551
|
+
catalog: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1552
|
+
cause: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1553
|
+
channel: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1554
|
+
cluster_id: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1527
1555
|
clusters: Optional[pulumi.Input[Sequence[pulumi.Input[Union['PipelineClusterArgs', 'PipelineClusterArgsDict']]]]] = None,
|
|
1528
|
-
configuration: Optional[pulumi.Input[Mapping[str, pulumi.Input[
|
|
1529
|
-
continuous: Optional[pulumi.Input[
|
|
1530
|
-
creator_user_name: Optional[pulumi.Input[
|
|
1556
|
+
configuration: Optional[pulumi.Input[Mapping[str, pulumi.Input[_builtins.str]]]] = None,
|
|
1557
|
+
continuous: Optional[pulumi.Input[_builtins.bool]] = None,
|
|
1558
|
+
creator_user_name: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1531
1559
|
deployment: Optional[pulumi.Input[Union['PipelineDeploymentArgs', 'PipelineDeploymentArgsDict']]] = None,
|
|
1532
|
-
development: Optional[pulumi.Input[
|
|
1533
|
-
edition: Optional[pulumi.Input[
|
|
1560
|
+
development: Optional[pulumi.Input[_builtins.bool]] = None,
|
|
1561
|
+
edition: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1534
1562
|
environment: Optional[pulumi.Input[Union['PipelineEnvironmentArgs', 'PipelineEnvironmentArgsDict']]] = None,
|
|
1535
1563
|
event_log: Optional[pulumi.Input[Union['PipelineEventLogArgs', 'PipelineEventLogArgsDict']]] = None,
|
|
1536
|
-
expected_last_modified: Optional[pulumi.Input[
|
|
1564
|
+
expected_last_modified: Optional[pulumi.Input[_builtins.int]] = None,
|
|
1537
1565
|
filters: Optional[pulumi.Input[Union['PipelineFiltersArgs', 'PipelineFiltersArgsDict']]] = None,
|
|
1538
1566
|
gateway_definition: Optional[pulumi.Input[Union['PipelineGatewayDefinitionArgs', 'PipelineGatewayDefinitionArgsDict']]] = None,
|
|
1539
|
-
health: Optional[pulumi.Input[
|
|
1567
|
+
health: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1540
1568
|
ingestion_definition: Optional[pulumi.Input[Union['PipelineIngestionDefinitionArgs', 'PipelineIngestionDefinitionArgsDict']]] = None,
|
|
1541
|
-
last_modified: Optional[pulumi.Input[
|
|
1569
|
+
last_modified: Optional[pulumi.Input[_builtins.int]] = None,
|
|
1542
1570
|
latest_updates: Optional[pulumi.Input[Sequence[pulumi.Input[Union['PipelineLatestUpdateArgs', 'PipelineLatestUpdateArgsDict']]]]] = None,
|
|
1543
1571
|
libraries: Optional[pulumi.Input[Sequence[pulumi.Input[Union['PipelineLibraryArgs', 'PipelineLibraryArgsDict']]]]] = None,
|
|
1544
|
-
name: Optional[pulumi.Input[
|
|
1572
|
+
name: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1545
1573
|
notifications: Optional[pulumi.Input[Sequence[pulumi.Input[Union['PipelineNotificationArgs', 'PipelineNotificationArgsDict']]]]] = None,
|
|
1546
|
-
photon: Optional[pulumi.Input[
|
|
1574
|
+
photon: Optional[pulumi.Input[_builtins.bool]] = None,
|
|
1547
1575
|
restart_window: Optional[pulumi.Input[Union['PipelineRestartWindowArgs', 'PipelineRestartWindowArgsDict']]] = None,
|
|
1548
|
-
root_path: Optional[pulumi.Input[
|
|
1576
|
+
root_path: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1549
1577
|
run_as: Optional[pulumi.Input[Union['PipelineRunAsArgs', 'PipelineRunAsArgsDict']]] = None,
|
|
1550
|
-
run_as_user_name: Optional[pulumi.Input[
|
|
1551
|
-
schema: Optional[pulumi.Input[
|
|
1552
|
-
serverless: Optional[pulumi.Input[
|
|
1553
|
-
state: Optional[pulumi.Input[
|
|
1554
|
-
storage: Optional[pulumi.Input[
|
|
1555
|
-
tags: Optional[pulumi.Input[Mapping[str, pulumi.Input[
|
|
1556
|
-
target: Optional[pulumi.Input[
|
|
1578
|
+
run_as_user_name: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1579
|
+
schema: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1580
|
+
serverless: Optional[pulumi.Input[_builtins.bool]] = None,
|
|
1581
|
+
state: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1582
|
+
storage: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1583
|
+
tags: Optional[pulumi.Input[Mapping[str, pulumi.Input[_builtins.str]]]] = None,
|
|
1584
|
+
target: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1557
1585
|
trigger: Optional[pulumi.Input[Union['PipelineTriggerArgs', 'PipelineTriggerArgsDict']]] = None,
|
|
1558
|
-
url: Optional[pulumi.Input[
|
|
1586
|
+
url: Optional[pulumi.Input[_builtins.str]] = None,
|
|
1587
|
+
usage_policy_id: Optional[pulumi.Input[_builtins.str]] = None) -> 'Pipeline':
|
|
1559
1588
|
"""
|
|
1560
1589
|
Get an existing Pipeline resource's state with the given name, id, and optional extra
|
|
1561
1590
|
properties used to qualify the lookup.
|
|
@@ -1563,29 +1592,29 @@ class Pipeline(pulumi.CustomResource):
|
|
|
1563
1592
|
:param str resource_name: The unique name of the resulting resource.
|
|
1564
1593
|
:param pulumi.Input[str] id: The unique provider ID of the resource to lookup.
|
|
1565
1594
|
:param pulumi.ResourceOptions opts: Options for the resource.
|
|
1566
|
-
:param pulumi.Input[
|
|
1567
|
-
:param pulumi.Input[
|
|
1568
|
-
:param pulumi.Input[
|
|
1569
|
-
:param pulumi.Input[
|
|
1570
|
-
:param pulumi.Input[Sequence[pulumi.Input[Union['PipelineClusterArgs', 'PipelineClusterArgsDict']]]] clusters: blocks - Clusters to run the pipeline. If none is specified, pipelines will automatically select a default cluster configuration for the pipeline. *Please note that
|
|
1571
|
-
:param pulumi.Input[Mapping[str, pulumi.Input[
|
|
1572
|
-
:param pulumi.Input[
|
|
1595
|
+
:param pulumi.Input[_builtins.bool] allow_duplicate_names: Optional boolean flag. If false, deployment will fail if name conflicts with that of another pipeline. default is `false`.
|
|
1596
|
+
:param pulumi.Input[_builtins.str] budget_policy_id: optional string specifying ID of the budget policy for this Lakeflow Declarative Pipeline.
|
|
1597
|
+
:param pulumi.Input[_builtins.str] catalog: The name of catalog in Unity Catalog. *Change of this parameter forces recreation of the pipeline.* (Conflicts with `storage`).
|
|
1598
|
+
:param pulumi.Input[_builtins.str] channel: optional name of the release channel for Spark version used by Lakeflow Declarative Pipeline. Supported values are: `CURRENT` (default) and `PREVIEW`.
|
|
1599
|
+
:param pulumi.Input[Sequence[pulumi.Input[Union['PipelineClusterArgs', 'PipelineClusterArgsDict']]]] clusters: blocks - Clusters to run the pipeline. If none is specified, pipelines will automatically select a default cluster configuration for the pipeline. *Please note that Lakeflow Declarative Pipeline clusters are supporting only subset of attributes as described in [documentation](https://docs.databricks.com/api/workspace/pipelines/create#clusters).* Also, note that `autoscale` block is extended with the `mode` parameter that controls the autoscaling algorithm (possible values are `ENHANCED` for new, enhanced autoscaling algorithm, or `LEGACY` for old algorithm).
|
|
1600
|
+
:param pulumi.Input[Mapping[str, pulumi.Input[_builtins.str]]] configuration: An optional list of values to apply to the entire pipeline. Elements must be formatted as key:value pairs.
|
|
1601
|
+
:param pulumi.Input[_builtins.bool] continuous: A flag indicating whether to run the pipeline continuously. The default value is `false`.
|
|
1573
1602
|
:param pulumi.Input[Union['PipelineDeploymentArgs', 'PipelineDeploymentArgsDict']] deployment: Deployment type of this pipeline. Supports following attributes:
|
|
1574
|
-
:param pulumi.Input[
|
|
1575
|
-
:param pulumi.Input[
|
|
1576
|
-
:param pulumi.Input[Union['PipelineEventLogArgs', 'PipelineEventLogArgsDict']] event_log: an optional block specifying a table where
|
|
1603
|
+
:param pulumi.Input[_builtins.bool] development: A flag indicating whether to run the pipeline in development mode. The default value is `false`.
|
|
1604
|
+
:param pulumi.Input[_builtins.str] edition: optional name of the [product edition](https://docs.databricks.com/aws/en/dlt/configure-pipeline#choose-a-product-edition). Supported values are: `CORE`, `PRO`, `ADVANCED` (default). Not required when `serverless` is set to `true`.
|
|
1605
|
+
:param pulumi.Input[Union['PipelineEventLogArgs', 'PipelineEventLogArgsDict']] event_log: an optional block specifying a table where LDP Event Log will be stored. Consists of the following fields:
|
|
1577
1606
|
:param pulumi.Input[Union['PipelineFiltersArgs', 'PipelineFiltersArgsDict']] filters: Filters on which Pipeline packages to include in the deployed graph. This block consists of following attributes:
|
|
1578
1607
|
:param pulumi.Input[Union['PipelineGatewayDefinitionArgs', 'PipelineGatewayDefinitionArgsDict']] gateway_definition: The definition of a gateway pipeline to support CDC. Consists of following attributes:
|
|
1579
1608
|
:param pulumi.Input[Sequence[pulumi.Input[Union['PipelineLibraryArgs', 'PipelineLibraryArgsDict']]]] libraries: blocks - Specifies pipeline code.
|
|
1580
|
-
:param pulumi.Input[
|
|
1581
|
-
:param pulumi.Input[
|
|
1582
|
-
:param pulumi.Input[
|
|
1583
|
-
:param pulumi.Input[
|
|
1584
|
-
:param pulumi.Input[
|
|
1585
|
-
:param pulumi.Input[
|
|
1586
|
-
:param pulumi.Input[Mapping[str, pulumi.Input[
|
|
1587
|
-
:param pulumi.Input[
|
|
1588
|
-
:param pulumi.Input[
|
|
1609
|
+
:param pulumi.Input[_builtins.str] name: A user-friendly name for this pipeline. The name can be used to identify pipeline jobs in the UI.
|
|
1610
|
+
:param pulumi.Input[_builtins.bool] photon: A flag indicating whether to use Photon engine. The default value is `false`.
|
|
1611
|
+
:param pulumi.Input[_builtins.str] root_path: An optional string specifying the root path for this pipeline. This is used as the root directory when editing the pipeline in the Databricks user interface and it is added to `sys.path` when executing Python sources during pipeline execution.
|
|
1612
|
+
:param pulumi.Input[_builtins.str] schema: The default schema (database) where tables are read from or published to. The presence of this attribute implies that the pipeline is in direct publishing mode.
|
|
1613
|
+
:param pulumi.Input[_builtins.bool] serverless: An optional flag indicating if serverless compute should be used for this Lakeflow Declarative Pipeline. Requires `catalog` to be set, as it could be used only with Unity Catalog.
|
|
1614
|
+
:param pulumi.Input[_builtins.str] storage: A location on cloud storage where output data and metadata required for pipeline execution are stored. By default, tables are stored in a subdirectory of this location. *Change of this parameter forces recreation of the pipeline.* (Conflicts with `catalog`).
|
|
1615
|
+
:param pulumi.Input[Mapping[str, pulumi.Input[_builtins.str]]] tags: A map of tags associated with the pipeline. These are forwarded to the cluster as cluster tags, and are therefore subject to the same limitations. A maximum of 25 tags can be added to the pipeline.
|
|
1616
|
+
:param pulumi.Input[_builtins.str] target: The name of a database (in either the Hive metastore or in a UC catalog) for persisting pipeline output data. Configuring the target setting allows you to view and query the pipeline output data from the Databricks UI.
|
|
1617
|
+
:param pulumi.Input[_builtins.str] url: URL of the Lakeflow Declarative Pipeline on the given workspace.
|
|
1589
1618
|
"""
|
|
1590
1619
|
opts = pulumi.ResourceOptions.merge(opts, pulumi.ResourceOptions(id=id))
|
|
1591
1620
|
|
|
@@ -1629,80 +1658,81 @@ class Pipeline(pulumi.CustomResource):
|
|
|
1629
1658
|
__props__.__dict__["target"] = target
|
|
1630
1659
|
__props__.__dict__["trigger"] = trigger
|
|
1631
1660
|
__props__.__dict__["url"] = url
|
|
1661
|
+
__props__.__dict__["usage_policy_id"] = usage_policy_id
|
|
1632
1662
|
return Pipeline(resource_name, opts=opts, __props__=__props__)
|
|
1633
1663
|
|
|
1634
|
-
@property
|
|
1664
|
+
@_builtins.property
|
|
1635
1665
|
@pulumi.getter(name="allowDuplicateNames")
|
|
1636
|
-
def allow_duplicate_names(self) -> pulumi.Output[Optional[
|
|
1666
|
+
def allow_duplicate_names(self) -> pulumi.Output[Optional[_builtins.bool]]:
|
|
1637
1667
|
"""
|
|
1638
1668
|
Optional boolean flag. If false, deployment will fail if name conflicts with that of another pipeline. default is `false`.
|
|
1639
1669
|
"""
|
|
1640
1670
|
return pulumi.get(self, "allow_duplicate_names")
|
|
1641
1671
|
|
|
1642
|
-
@property
|
|
1672
|
+
@_builtins.property
|
|
1643
1673
|
@pulumi.getter(name="budgetPolicyId")
|
|
1644
|
-
def budget_policy_id(self) -> pulumi.Output[Optional[
|
|
1674
|
+
def budget_policy_id(self) -> pulumi.Output[Optional[_builtins.str]]:
|
|
1645
1675
|
"""
|
|
1646
|
-
optional string specifying ID of the budget policy for this
|
|
1676
|
+
optional string specifying ID of the budget policy for this Lakeflow Declarative Pipeline.
|
|
1647
1677
|
"""
|
|
1648
1678
|
return pulumi.get(self, "budget_policy_id")
|
|
1649
1679
|
|
|
1650
|
-
@property
|
|
1680
|
+
@_builtins.property
|
|
1651
1681
|
@pulumi.getter
|
|
1652
|
-
def catalog(self) -> pulumi.Output[Optional[
|
|
1682
|
+
def catalog(self) -> pulumi.Output[Optional[_builtins.str]]:
|
|
1653
1683
|
"""
|
|
1654
1684
|
The name of catalog in Unity Catalog. *Change of this parameter forces recreation of the pipeline.* (Conflicts with `storage`).
|
|
1655
1685
|
"""
|
|
1656
1686
|
return pulumi.get(self, "catalog")
|
|
1657
1687
|
|
|
1658
|
-
@property
|
|
1688
|
+
@_builtins.property
|
|
1659
1689
|
@pulumi.getter
|
|
1660
|
-
def cause(self) -> pulumi.Output[
|
|
1690
|
+
def cause(self) -> pulumi.Output[_builtins.str]:
|
|
1661
1691
|
return pulumi.get(self, "cause")
|
|
1662
1692
|
|
|
1663
|
-
@property
|
|
1693
|
+
@_builtins.property
|
|
1664
1694
|
@pulumi.getter
|
|
1665
|
-
def channel(self) -> pulumi.Output[Optional[
|
|
1695
|
+
def channel(self) -> pulumi.Output[Optional[_builtins.str]]:
|
|
1666
1696
|
"""
|
|
1667
|
-
optional name of the release channel for Spark version used by
|
|
1697
|
+
optional name of the release channel for Spark version used by Lakeflow Declarative Pipeline. Supported values are: `CURRENT` (default) and `PREVIEW`.
|
|
1668
1698
|
"""
|
|
1669
1699
|
return pulumi.get(self, "channel")
|
|
1670
1700
|
|
|
1671
|
-
@property
|
|
1701
|
+
@_builtins.property
|
|
1672
1702
|
@pulumi.getter(name="clusterId")
|
|
1673
|
-
def cluster_id(self) -> pulumi.Output[
|
|
1703
|
+
def cluster_id(self) -> pulumi.Output[_builtins.str]:
|
|
1674
1704
|
return pulumi.get(self, "cluster_id")
|
|
1675
1705
|
|
|
1676
|
-
@property
|
|
1706
|
+
@_builtins.property
|
|
1677
1707
|
@pulumi.getter
|
|
1678
1708
|
def clusters(self) -> pulumi.Output[Optional[Sequence['outputs.PipelineCluster']]]:
|
|
1679
1709
|
"""
|
|
1680
|
-
blocks - Clusters to run the pipeline. If none is specified, pipelines will automatically select a default cluster configuration for the pipeline. *Please note that
|
|
1710
|
+
blocks - Clusters to run the pipeline. If none is specified, pipelines will automatically select a default cluster configuration for the pipeline. *Please note that Lakeflow Declarative Pipeline clusters are supporting only subset of attributes as described in [documentation](https://docs.databricks.com/api/workspace/pipelines/create#clusters).* Also, note that `autoscale` block is extended with the `mode` parameter that controls the autoscaling algorithm (possible values are `ENHANCED` for new, enhanced autoscaling algorithm, or `LEGACY` for old algorithm).
|
|
1681
1711
|
"""
|
|
1682
1712
|
return pulumi.get(self, "clusters")
|
|
1683
1713
|
|
|
1684
|
-
@property
|
|
1714
|
+
@_builtins.property
|
|
1685
1715
|
@pulumi.getter
|
|
1686
|
-
def configuration(self) -> pulumi.Output[Optional[Mapping[str,
|
|
1716
|
+
def configuration(self) -> pulumi.Output[Optional[Mapping[str, _builtins.str]]]:
|
|
1687
1717
|
"""
|
|
1688
1718
|
An optional list of values to apply to the entire pipeline. Elements must be formatted as key:value pairs.
|
|
1689
1719
|
"""
|
|
1690
1720
|
return pulumi.get(self, "configuration")
|
|
1691
1721
|
|
|
1692
|
-
@property
|
|
1722
|
+
@_builtins.property
|
|
1693
1723
|
@pulumi.getter
|
|
1694
|
-
def continuous(self) -> pulumi.Output[Optional[
|
|
1724
|
+
def continuous(self) -> pulumi.Output[Optional[_builtins.bool]]:
|
|
1695
1725
|
"""
|
|
1696
1726
|
A flag indicating whether to run the pipeline continuously. The default value is `false`.
|
|
1697
1727
|
"""
|
|
1698
1728
|
return pulumi.get(self, "continuous")
|
|
1699
1729
|
|
|
1700
|
-
@property
|
|
1730
|
+
@_builtins.property
|
|
1701
1731
|
@pulumi.getter(name="creatorUserName")
|
|
1702
|
-
def creator_user_name(self) -> pulumi.Output[
|
|
1732
|
+
def creator_user_name(self) -> pulumi.Output[_builtins.str]:
|
|
1703
1733
|
return pulumi.get(self, "creator_user_name")
|
|
1704
1734
|
|
|
1705
|
-
@property
|
|
1735
|
+
@_builtins.property
|
|
1706
1736
|
@pulumi.getter
|
|
1707
1737
|
def deployment(self) -> pulumi.Output[Optional['outputs.PipelineDeployment']]:
|
|
1708
1738
|
"""
|
|
@@ -1710,41 +1740,41 @@ class Pipeline(pulumi.CustomResource):
|
|
|
1710
1740
|
"""
|
|
1711
1741
|
return pulumi.get(self, "deployment")
|
|
1712
1742
|
|
|
1713
|
-
@property
|
|
1743
|
+
@_builtins.property
|
|
1714
1744
|
@pulumi.getter
|
|
1715
|
-
def development(self) -> pulumi.Output[Optional[
|
|
1745
|
+
def development(self) -> pulumi.Output[Optional[_builtins.bool]]:
|
|
1716
1746
|
"""
|
|
1717
1747
|
A flag indicating whether to run the pipeline in development mode. The default value is `false`.
|
|
1718
1748
|
"""
|
|
1719
1749
|
return pulumi.get(self, "development")
|
|
1720
1750
|
|
|
1721
|
-
@property
|
|
1751
|
+
@_builtins.property
|
|
1722
1752
|
@pulumi.getter
|
|
1723
|
-
def edition(self) -> pulumi.Output[Optional[
|
|
1753
|
+
def edition(self) -> pulumi.Output[Optional[_builtins.str]]:
|
|
1724
1754
|
"""
|
|
1725
1755
|
optional name of the [product edition](https://docs.databricks.com/aws/en/dlt/configure-pipeline#choose-a-product-edition). Supported values are: `CORE`, `PRO`, `ADVANCED` (default). Not required when `serverless` is set to `true`.
|
|
1726
1756
|
"""
|
|
1727
1757
|
return pulumi.get(self, "edition")
|
|
1728
1758
|
|
|
1729
|
-
@property
|
|
1759
|
+
@_builtins.property
|
|
1730
1760
|
@pulumi.getter
|
|
1731
1761
|
def environment(self) -> pulumi.Output[Optional['outputs.PipelineEnvironment']]:
|
|
1732
1762
|
return pulumi.get(self, "environment")
|
|
1733
1763
|
|
|
1734
|
-
@property
|
|
1764
|
+
@_builtins.property
|
|
1735
1765
|
@pulumi.getter(name="eventLog")
|
|
1736
1766
|
def event_log(self) -> pulumi.Output[Optional['outputs.PipelineEventLog']]:
|
|
1737
1767
|
"""
|
|
1738
|
-
an optional block specifying a table where
|
|
1768
|
+
an optional block specifying a table where LDP Event Log will be stored. Consists of the following fields:
|
|
1739
1769
|
"""
|
|
1740
1770
|
return pulumi.get(self, "event_log")
|
|
1741
1771
|
|
|
1742
|
-
@property
|
|
1772
|
+
@_builtins.property
|
|
1743
1773
|
@pulumi.getter(name="expectedLastModified")
|
|
1744
|
-
def expected_last_modified(self) -> pulumi.Output[Optional[
|
|
1774
|
+
def expected_last_modified(self) -> pulumi.Output[Optional[_builtins.int]]:
|
|
1745
1775
|
return pulumi.get(self, "expected_last_modified")
|
|
1746
1776
|
|
|
1747
|
-
@property
|
|
1777
|
+
@_builtins.property
|
|
1748
1778
|
@pulumi.getter
|
|
1749
1779
|
def filters(self) -> pulumi.Output[Optional['outputs.PipelineFilters']]:
|
|
1750
1780
|
"""
|
|
@@ -1752,7 +1782,7 @@ class Pipeline(pulumi.CustomResource):
|
|
|
1752
1782
|
"""
|
|
1753
1783
|
return pulumi.get(self, "filters")
|
|
1754
1784
|
|
|
1755
|
-
@property
|
|
1785
|
+
@_builtins.property
|
|
1756
1786
|
@pulumi.getter(name="gatewayDefinition")
|
|
1757
1787
|
def gateway_definition(self) -> pulumi.Output[Optional['outputs.PipelineGatewayDefinition']]:
|
|
1758
1788
|
"""
|
|
@@ -1760,27 +1790,27 @@ class Pipeline(pulumi.CustomResource):
|
|
|
1760
1790
|
"""
|
|
1761
1791
|
return pulumi.get(self, "gateway_definition")
|
|
1762
1792
|
|
|
1763
|
-
@property
|
|
1793
|
+
@_builtins.property
|
|
1764
1794
|
@pulumi.getter
|
|
1765
|
-
def health(self) -> pulumi.Output[
|
|
1795
|
+
def health(self) -> pulumi.Output[_builtins.str]:
|
|
1766
1796
|
return pulumi.get(self, "health")
|
|
1767
1797
|
|
|
1768
|
-
@property
|
|
1798
|
+
@_builtins.property
|
|
1769
1799
|
@pulumi.getter(name="ingestionDefinition")
|
|
1770
1800
|
def ingestion_definition(self) -> pulumi.Output[Optional['outputs.PipelineIngestionDefinition']]:
|
|
1771
1801
|
return pulumi.get(self, "ingestion_definition")
|
|
1772
1802
|
|
|
1773
|
-
@property
|
|
1803
|
+
@_builtins.property
|
|
1774
1804
|
@pulumi.getter(name="lastModified")
|
|
1775
|
-
def last_modified(self) -> pulumi.Output[
|
|
1805
|
+
def last_modified(self) -> pulumi.Output[_builtins.int]:
|
|
1776
1806
|
return pulumi.get(self, "last_modified")
|
|
1777
1807
|
|
|
1778
|
-
@property
|
|
1808
|
+
@_builtins.property
|
|
1779
1809
|
@pulumi.getter(name="latestUpdates")
|
|
1780
1810
|
def latest_updates(self) -> pulumi.Output[Sequence['outputs.PipelineLatestUpdate']]:
|
|
1781
1811
|
return pulumi.get(self, "latest_updates")
|
|
1782
1812
|
|
|
1783
|
-
@property
|
|
1813
|
+
@_builtins.property
|
|
1784
1814
|
@pulumi.getter
|
|
1785
1815
|
def libraries(self) -> pulumi.Output[Optional[Sequence['outputs.PipelineLibrary']]]:
|
|
1786
1816
|
"""
|
|
@@ -1788,105 +1818,110 @@ class Pipeline(pulumi.CustomResource):
|
|
|
1788
1818
|
"""
|
|
1789
1819
|
return pulumi.get(self, "libraries")
|
|
1790
1820
|
|
|
1791
|
-
@property
|
|
1821
|
+
@_builtins.property
|
|
1792
1822
|
@pulumi.getter
|
|
1793
|
-
def name(self) -> pulumi.Output[
|
|
1823
|
+
def name(self) -> pulumi.Output[_builtins.str]:
|
|
1794
1824
|
"""
|
|
1795
1825
|
A user-friendly name for this pipeline. The name can be used to identify pipeline jobs in the UI.
|
|
1796
1826
|
"""
|
|
1797
1827
|
return pulumi.get(self, "name")
|
|
1798
1828
|
|
|
1799
|
-
@property
|
|
1829
|
+
@_builtins.property
|
|
1800
1830
|
@pulumi.getter
|
|
1801
1831
|
def notifications(self) -> pulumi.Output[Optional[Sequence['outputs.PipelineNotification']]]:
|
|
1802
1832
|
return pulumi.get(self, "notifications")
|
|
1803
1833
|
|
|
1804
|
-
@property
|
|
1834
|
+
@_builtins.property
|
|
1805
1835
|
@pulumi.getter
|
|
1806
|
-
def photon(self) -> pulumi.Output[Optional[
|
|
1836
|
+
def photon(self) -> pulumi.Output[Optional[_builtins.bool]]:
|
|
1807
1837
|
"""
|
|
1808
1838
|
A flag indicating whether to use Photon engine. The default value is `false`.
|
|
1809
1839
|
"""
|
|
1810
1840
|
return pulumi.get(self, "photon")
|
|
1811
1841
|
|
|
1812
|
-
@property
|
|
1842
|
+
@_builtins.property
|
|
1813
1843
|
@pulumi.getter(name="restartWindow")
|
|
1814
1844
|
def restart_window(self) -> pulumi.Output[Optional['outputs.PipelineRestartWindow']]:
|
|
1815
1845
|
return pulumi.get(self, "restart_window")
|
|
1816
1846
|
|
|
1817
|
-
@property
|
|
1847
|
+
@_builtins.property
|
|
1818
1848
|
@pulumi.getter(name="rootPath")
|
|
1819
|
-
def root_path(self) -> pulumi.Output[Optional[
|
|
1849
|
+
def root_path(self) -> pulumi.Output[Optional[_builtins.str]]:
|
|
1820
1850
|
"""
|
|
1821
1851
|
An optional string specifying the root path for this pipeline. This is used as the root directory when editing the pipeline in the Databricks user interface and it is added to `sys.path` when executing Python sources during pipeline execution.
|
|
1822
1852
|
"""
|
|
1823
1853
|
return pulumi.get(self, "root_path")
|
|
1824
1854
|
|
|
1825
|
-
@property
|
|
1855
|
+
@_builtins.property
|
|
1826
1856
|
@pulumi.getter(name="runAs")
|
|
1827
|
-
def run_as(self) -> pulumi.Output['outputs.PipelineRunAs']:
|
|
1857
|
+
def run_as(self) -> pulumi.Output[Optional['outputs.PipelineRunAs']]:
|
|
1828
1858
|
return pulumi.get(self, "run_as")
|
|
1829
1859
|
|
|
1830
|
-
@property
|
|
1860
|
+
@_builtins.property
|
|
1831
1861
|
@pulumi.getter(name="runAsUserName")
|
|
1832
|
-
def run_as_user_name(self) -> pulumi.Output[
|
|
1862
|
+
def run_as_user_name(self) -> pulumi.Output[_builtins.str]:
|
|
1833
1863
|
return pulumi.get(self, "run_as_user_name")
|
|
1834
1864
|
|
|
1835
|
-
@property
|
|
1865
|
+
@_builtins.property
|
|
1836
1866
|
@pulumi.getter
|
|
1837
|
-
def schema(self) -> pulumi.Output[Optional[
|
|
1867
|
+
def schema(self) -> pulumi.Output[Optional[_builtins.str]]:
|
|
1838
1868
|
"""
|
|
1839
1869
|
The default schema (database) where tables are read from or published to. The presence of this attribute implies that the pipeline is in direct publishing mode.
|
|
1840
1870
|
"""
|
|
1841
1871
|
return pulumi.get(self, "schema")
|
|
1842
1872
|
|
|
1843
|
-
@property
|
|
1873
|
+
@_builtins.property
|
|
1844
1874
|
@pulumi.getter
|
|
1845
|
-
def serverless(self) -> pulumi.Output[Optional[
|
|
1875
|
+
def serverless(self) -> pulumi.Output[Optional[_builtins.bool]]:
|
|
1846
1876
|
"""
|
|
1847
|
-
An optional flag indicating if serverless compute should be used for this
|
|
1877
|
+
An optional flag indicating if serverless compute should be used for this Lakeflow Declarative Pipeline. Requires `catalog` to be set, as it could be used only with Unity Catalog.
|
|
1848
1878
|
"""
|
|
1849
1879
|
return pulumi.get(self, "serverless")
|
|
1850
1880
|
|
|
1851
|
-
@property
|
|
1881
|
+
@_builtins.property
|
|
1852
1882
|
@pulumi.getter
|
|
1853
|
-
def state(self) -> pulumi.Output[
|
|
1883
|
+
def state(self) -> pulumi.Output[_builtins.str]:
|
|
1854
1884
|
return pulumi.get(self, "state")
|
|
1855
1885
|
|
|
1856
|
-
@property
|
|
1886
|
+
@_builtins.property
|
|
1857
1887
|
@pulumi.getter
|
|
1858
|
-
def storage(self) -> pulumi.Output[Optional[
|
|
1888
|
+
def storage(self) -> pulumi.Output[Optional[_builtins.str]]:
|
|
1859
1889
|
"""
|
|
1860
|
-
A location on
|
|
1890
|
+
A location on cloud storage where output data and metadata required for pipeline execution are stored. By default, tables are stored in a subdirectory of this location. *Change of this parameter forces recreation of the pipeline.* (Conflicts with `catalog`).
|
|
1861
1891
|
"""
|
|
1862
1892
|
return pulumi.get(self, "storage")
|
|
1863
1893
|
|
|
1864
|
-
@property
|
|
1894
|
+
@_builtins.property
|
|
1865
1895
|
@pulumi.getter
|
|
1866
|
-
def tags(self) -> pulumi.Output[Optional[Mapping[str,
|
|
1896
|
+
def tags(self) -> pulumi.Output[Optional[Mapping[str, _builtins.str]]]:
|
|
1867
1897
|
"""
|
|
1868
1898
|
A map of tags associated with the pipeline. These are forwarded to the cluster as cluster tags, and are therefore subject to the same limitations. A maximum of 25 tags can be added to the pipeline.
|
|
1869
1899
|
"""
|
|
1870
1900
|
return pulumi.get(self, "tags")
|
|
1871
1901
|
|
|
1872
|
-
@property
|
|
1902
|
+
@_builtins.property
|
|
1873
1903
|
@pulumi.getter
|
|
1874
|
-
def target(self) -> pulumi.Output[Optional[
|
|
1904
|
+
def target(self) -> pulumi.Output[Optional[_builtins.str]]:
|
|
1875
1905
|
"""
|
|
1876
1906
|
The name of a database (in either the Hive metastore or in a UC catalog) for persisting pipeline output data. Configuring the target setting allows you to view and query the pipeline output data from the Databricks UI.
|
|
1877
1907
|
"""
|
|
1878
1908
|
return pulumi.get(self, "target")
|
|
1879
1909
|
|
|
1880
|
-
@property
|
|
1910
|
+
@_builtins.property
|
|
1881
1911
|
@pulumi.getter
|
|
1882
1912
|
def trigger(self) -> pulumi.Output[Optional['outputs.PipelineTrigger']]:
|
|
1883
1913
|
return pulumi.get(self, "trigger")
|
|
1884
1914
|
|
|
1885
|
-
@property
|
|
1915
|
+
@_builtins.property
|
|
1886
1916
|
@pulumi.getter
|
|
1887
|
-
def url(self) -> pulumi.Output[
|
|
1917
|
+
def url(self) -> pulumi.Output[_builtins.str]:
|
|
1888
1918
|
"""
|
|
1889
|
-
URL of the
|
|
1919
|
+
URL of the Lakeflow Declarative Pipeline on the given workspace.
|
|
1890
1920
|
"""
|
|
1891
1921
|
return pulumi.get(self, "url")
|
|
1892
1922
|
|
|
1923
|
+
@_builtins.property
|
|
1924
|
+
@pulumi.getter(name="usagePolicyId")
|
|
1925
|
+
def usage_policy_id(self) -> pulumi.Output[Optional[_builtins.str]]:
|
|
1926
|
+
return pulumi.get(self, "usage_policy_id")
|
|
1927
|
+
|