prefect-client 2.14.12__py3-none-any.whl → 2.14.13__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/client/orchestration.py +27 -0
- prefect/client/schemas/objects.py +12 -1
- prefect/engine.py +66 -0
- prefect/events/actions.py +13 -2
- prefect/events/schemas.py +1 -1
- prefect/flows.py +2 -0
- prefect/runner/runner.py +2 -2
- prefect/runner/storage.py +39 -24
- prefect/server/api/collections_data/views/aggregate-worker-metadata.json +1535 -0
- prefect/tasks.py +28 -0
- prefect/utilities/processutils.py +7 -1
- {prefect_client-2.14.12.dist-info → prefect_client-2.14.13.dist-info}/METADATA +1 -1
- {prefect_client-2.14.12.dist-info → prefect_client-2.14.13.dist-info}/RECORD +16 -15
- {prefect_client-2.14.12.dist-info → prefect_client-2.14.13.dist-info}/LICENSE +0 -0
- {prefect_client-2.14.12.dist-info → prefect_client-2.14.13.dist-info}/WHEEL +0 -0
- {prefect_client-2.14.12.dist-info → prefect_client-2.14.13.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,1535 @@
|
|
1
|
+
{
|
2
|
+
"prefect": {
|
3
|
+
"prefect-agent": {
|
4
|
+
"type": "prefect-agent",
|
5
|
+
"documentation_url": "https://docs.prefect.io/latest/concepts/work-pools/#agent-overview",
|
6
|
+
"display_name": "Prefect Agent",
|
7
|
+
"logo_url": "https://cdn.sanity.io/images/3ugk85nk/production/c771bb53894c877e169c8db158c5598558b8f175-24x24.svg",
|
8
|
+
"install_command": "pip install prefect",
|
9
|
+
"default_base_job_configuration": {},
|
10
|
+
"description": "Execute flow runs on heterogeneous infrastructure using infrastructure blocks."
|
11
|
+
},
|
12
|
+
"process": {
|
13
|
+
"default_base_job_configuration": {
|
14
|
+
"job_configuration": {
|
15
|
+
"command": "{{ command }}",
|
16
|
+
"env": "{{ env }}",
|
17
|
+
"labels": "{{ labels }}",
|
18
|
+
"name": "{{ name }}",
|
19
|
+
"stream_output": "{{ stream_output }}",
|
20
|
+
"working_dir": "{{ working_dir }}"
|
21
|
+
},
|
22
|
+
"variables": {
|
23
|
+
"type": "object",
|
24
|
+
"properties": {
|
25
|
+
"name": {
|
26
|
+
"title": "Name",
|
27
|
+
"description": "Name given to infrastructure created by a worker.",
|
28
|
+
"type": "string"
|
29
|
+
},
|
30
|
+
"env": {
|
31
|
+
"title": "Environment Variables",
|
32
|
+
"description": "Environment variables to set when starting a flow run.",
|
33
|
+
"type": "object",
|
34
|
+
"additionalProperties": {
|
35
|
+
"type": "string"
|
36
|
+
}
|
37
|
+
},
|
38
|
+
"labels": {
|
39
|
+
"title": "Labels",
|
40
|
+
"description": "Labels applied to infrastructure created by a worker.",
|
41
|
+
"type": "object",
|
42
|
+
"additionalProperties": {
|
43
|
+
"type": "string"
|
44
|
+
}
|
45
|
+
},
|
46
|
+
"command": {
|
47
|
+
"title": "Command",
|
48
|
+
"description": "The command to use when starting a flow run. In most cases, this should be left blank and the command will be automatically generated by the worker.",
|
49
|
+
"type": "string"
|
50
|
+
},
|
51
|
+
"stream_output": {
|
52
|
+
"title": "Stream Output",
|
53
|
+
"description": "If enabled, workers will stream output from flow run processes to local standard output.",
|
54
|
+
"default": true,
|
55
|
+
"type": "boolean"
|
56
|
+
},
|
57
|
+
"working_dir": {
|
58
|
+
"title": "Working Directory",
|
59
|
+
"description": "If provided, workers will open flow run processes within the specified path as the working directory. Otherwise, a temporary directory will be created.",
|
60
|
+
"type": "string",
|
61
|
+
"format": "path"
|
62
|
+
}
|
63
|
+
}
|
64
|
+
}
|
65
|
+
},
|
66
|
+
"description": "Execute flow runs as subprocesses on a worker. Works well for local execution when first getting started.",
|
67
|
+
"display_name": "Local Subprocess",
|
68
|
+
"documentation_url": "https://docs.prefect.io/latest/api-ref/prefect/workers/process/",
|
69
|
+
"install_command": "pip install prefect",
|
70
|
+
"is_beta": false,
|
71
|
+
"logo_url": "https://cdn.sanity.io/images/3ugk85nk/production/356e6766a91baf20e1d08bbe16e8b5aaef4d8643-48x48.png",
|
72
|
+
"type": "process"
|
73
|
+
}
|
74
|
+
},
|
75
|
+
"prefect-aws": {
|
76
|
+
"ecs": {
|
77
|
+
"default_base_job_configuration": {
|
78
|
+
"job_configuration": {
|
79
|
+
"command": "{{ command }}",
|
80
|
+
"env": "{{ env }}",
|
81
|
+
"labels": "{{ labels }}",
|
82
|
+
"name": "{{ name }}",
|
83
|
+
"aws_credentials": "{{ aws_credentials }}",
|
84
|
+
"task_definition": {
|
85
|
+
"containerDefinitions": [
|
86
|
+
{
|
87
|
+
"image": "{{ image }}",
|
88
|
+
"name": "{{ container_name }}"
|
89
|
+
}
|
90
|
+
],
|
91
|
+
"cpu": "{{ cpu }}",
|
92
|
+
"family": "{{ family }}",
|
93
|
+
"memory": "{{ memory }}",
|
94
|
+
"executionRoleArn": "{{ execution_role_arn }}"
|
95
|
+
},
|
96
|
+
"task_run_request": {
|
97
|
+
"launchType": "{{ launch_type }}",
|
98
|
+
"cluster": "{{ cluster }}",
|
99
|
+
"overrides": {
|
100
|
+
"containerOverrides": [
|
101
|
+
{
|
102
|
+
"name": "{{ container_name }}",
|
103
|
+
"command": "{{ command }}",
|
104
|
+
"environment": "{{ env }}",
|
105
|
+
"cpu": "{{ cpu }}",
|
106
|
+
"memory": "{{ memory }}"
|
107
|
+
}
|
108
|
+
],
|
109
|
+
"cpu": "{{ cpu }}",
|
110
|
+
"memory": "{{ memory }}",
|
111
|
+
"taskRoleArn": "{{ task_role_arn }}"
|
112
|
+
},
|
113
|
+
"tags": "{{ labels }}",
|
114
|
+
"taskDefinition": "{{ task_definition_arn }}"
|
115
|
+
},
|
116
|
+
"configure_cloudwatch_logs": "{{ configure_cloudwatch_logs }}",
|
117
|
+
"cloudwatch_logs_options": "{{ cloudwatch_logs_options }}",
|
118
|
+
"network_configuration": "{{ network_configuration }}",
|
119
|
+
"stream_output": "{{ stream_output }}",
|
120
|
+
"task_start_timeout_seconds": "{{ task_start_timeout_seconds }}",
|
121
|
+
"task_watch_poll_interval": "{{ task_watch_poll_interval }}",
|
122
|
+
"auto_deregister_task_definition": "{{ auto_deregister_task_definition }}",
|
123
|
+
"vpc_id": "{{ vpc_id }}",
|
124
|
+
"container_name": "{{ container_name }}",
|
125
|
+
"cluster": "{{ cluster }}"
|
126
|
+
},
|
127
|
+
"variables": {
|
128
|
+
"description": "Variables for templating an ECS job.",
|
129
|
+
"type": "object",
|
130
|
+
"properties": {
|
131
|
+
"name": {
|
132
|
+
"title": "Name",
|
133
|
+
"description": "Name given to infrastructure created by a worker.",
|
134
|
+
"type": "string"
|
135
|
+
},
|
136
|
+
"env": {
|
137
|
+
"title": "Environment Variables",
|
138
|
+
"description": "Environment variables to provide to the task run. These variables are set on the Prefect container at task runtime. These will not be set on the task definition.",
|
139
|
+
"type": "object",
|
140
|
+
"additionalProperties": {
|
141
|
+
"type": "string"
|
142
|
+
}
|
143
|
+
},
|
144
|
+
"labels": {
|
145
|
+
"title": "Labels",
|
146
|
+
"description": "Labels applied to infrastructure created by a worker.",
|
147
|
+
"type": "object",
|
148
|
+
"additionalProperties": {
|
149
|
+
"type": "string"
|
150
|
+
}
|
151
|
+
},
|
152
|
+
"command": {
|
153
|
+
"title": "Command",
|
154
|
+
"description": "The command to use when starting a flow run. In most cases, this should be left blank and the command will be automatically generated by the worker.",
|
155
|
+
"type": "string"
|
156
|
+
},
|
157
|
+
"task_definition_arn": {
|
158
|
+
"title": "Task Definition Arn",
|
159
|
+
"description": "An identifier for an existing task definition to use. If set, options that require changes to the task definition will be ignored. All contents of the task definition in the job configuration will be ignored.",
|
160
|
+
"type": "string"
|
161
|
+
},
|
162
|
+
"aws_credentials": {
|
163
|
+
"title": "AWS Credentials",
|
164
|
+
"description": "The AWS credentials to use to connect to ECS. If not provided, credentials will be inferred from the local environment following AWS's boto client's rules.",
|
165
|
+
"allOf": [
|
166
|
+
{
|
167
|
+
"$ref": "#/definitions/AwsCredentials"
|
168
|
+
}
|
169
|
+
]
|
170
|
+
},
|
171
|
+
"cluster": {
|
172
|
+
"title": "Cluster",
|
173
|
+
"description": "The ECS cluster to run the task in. An ARN or name may be provided. If not provided, the default cluster will be used.",
|
174
|
+
"type": "string"
|
175
|
+
},
|
176
|
+
"family": {
|
177
|
+
"title": "Family",
|
178
|
+
"description": "A family for the task definition. If not provided, it will be inferred from the task definition. If the task definition does not have a family, the name will be generated. When flow and deployment metadata is available, the generated name will include their names. Values for this field will be slugified to match AWS character requirements.",
|
179
|
+
"type": "string"
|
180
|
+
},
|
181
|
+
"launch_type": {
|
182
|
+
"title": "Launch Type",
|
183
|
+
"description": "The type of ECS task run infrastructure that should be used. Note that 'FARGATE_SPOT' is not a formal ECS launch type, but we will configure the proper capacity provider strategy if set here.",
|
184
|
+
"default": "FARGATE",
|
185
|
+
"enum": ["FARGATE", "EC2", "EXTERNAL", "FARGATE_SPOT"],
|
186
|
+
"type": "string"
|
187
|
+
},
|
188
|
+
"image": {
|
189
|
+
"title": "Image",
|
190
|
+
"description": "The image to use for the Prefect container in the task. If this value is not null, it will override the value in the task definition. This value defaults to a Prefect base image matching your local versions.",
|
191
|
+
"type": "string"
|
192
|
+
},
|
193
|
+
"cpu": {
|
194
|
+
"title": "CPU",
|
195
|
+
"description": "The amount of CPU to provide to the ECS task. Valid amounts are specified in the AWS documentation. If not provided, a default value of 1024 will be used unless present on the task definition.",
|
196
|
+
"type": "integer"
|
197
|
+
},
|
198
|
+
"memory": {
|
199
|
+
"title": "Memory",
|
200
|
+
"description": "The amount of memory to provide to the ECS task. Valid amounts are specified in the AWS documentation. If not provided, a default value of 2048 will be used unless present on the task definition.",
|
201
|
+
"type": "integer"
|
202
|
+
},
|
203
|
+
"container_name": {
|
204
|
+
"title": "Container Name",
|
205
|
+
"description": "The name of the container flow run orchestration will occur in. If not specified, a default value of prefect will be used and if that is not found in the task definition the first container will be used.",
|
206
|
+
"type": "string"
|
207
|
+
},
|
208
|
+
"task_role_arn": {
|
209
|
+
"title": "Task Role ARN",
|
210
|
+
"description": "A role to attach to the task run. This controls the permissions of the task while it is running.",
|
211
|
+
"type": "string"
|
212
|
+
},
|
213
|
+
"execution_role_arn": {
|
214
|
+
"title": "Execution Role ARN",
|
215
|
+
"description": "An execution role to use for the task. This controls the permissions of the task when it is launching. If this value is not null, it will override the value in the task definition. An execution role must be provided to capture logs from the container.",
|
216
|
+
"type": "string"
|
217
|
+
},
|
218
|
+
"vpc_id": {
|
219
|
+
"title": "VPC ID",
|
220
|
+
"description": "The AWS VPC to link the task run to. This is only applicable when using the 'awsvpc' network mode for your task. FARGATE tasks require this network mode, but for EC2 tasks the default network mode is 'bridge'. If using the 'awsvpc' network mode and this field is null, your default VPC will be used. If no default VPC can be found, the task run will fail.",
|
221
|
+
"type": "string"
|
222
|
+
},
|
223
|
+
"configure_cloudwatch_logs": {
|
224
|
+
"title": "Configure Cloudwatch Logs",
|
225
|
+
"description": "If enabled, the Prefect container will be configured to send its output to the AWS CloudWatch logs service. This functionality requires an execution role with logs:CreateLogStream, logs:CreateLogGroup, and logs:PutLogEvents permissions. The default for this field is `False` unless `stream_output` is set.",
|
226
|
+
"type": "boolean"
|
227
|
+
},
|
228
|
+
"cloudwatch_logs_options": {
|
229
|
+
"title": "Cloudwatch Logs Options",
|
230
|
+
"description": "When `configure_cloudwatch_logs` is enabled, this setting may be used to pass additional options to the CloudWatch logs configuration or override the default options. See the [AWS documentation](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/using_awslogs.html#create_awslogs_logdriver_options) for available options. ",
|
231
|
+
"type": "object",
|
232
|
+
"additionalProperties": {
|
233
|
+
"type": "string"
|
234
|
+
}
|
235
|
+
},
|
236
|
+
"network_configuration": {
|
237
|
+
"title": "Network Configuration",
|
238
|
+
"description": "When `network_configuration` is supplied it will override ECS Worker'sawsvpcConfiguration that defined in the ECS task executing your workload. See the [AWS documentation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-ecs-service-awsvpcconfiguration.html) for available options.",
|
239
|
+
"type": "object"
|
240
|
+
},
|
241
|
+
"stream_output": {
|
242
|
+
"title": "Stream Output",
|
243
|
+
"description": "If enabled, logs will be streamed from the Prefect container to the local console. Unless you have configured AWS CloudWatch logs manually on your task definition, this requires the same prerequisites outlined in `configure_cloudwatch_logs`.",
|
244
|
+
"type": "boolean"
|
245
|
+
},
|
246
|
+
"task_start_timeout_seconds": {
|
247
|
+
"title": "Task Start Timeout Seconds",
|
248
|
+
"description": "The amount of time to watch for the start of the ECS task before marking it as failed. The task must enter a RUNNING state to be considered started.",
|
249
|
+
"default": 300,
|
250
|
+
"type": "integer"
|
251
|
+
},
|
252
|
+
"task_watch_poll_interval": {
|
253
|
+
"title": "Task Watch Poll Interval",
|
254
|
+
"description": "The amount of time to wait between AWS API calls while monitoring the state of an ECS task.",
|
255
|
+
"default": 5.0,
|
256
|
+
"type": "number"
|
257
|
+
},
|
258
|
+
"auto_deregister_task_definition": {
|
259
|
+
"title": "Auto Deregister Task Definition",
|
260
|
+
"description": "If enabled, any task definitions that are created by this block will be deregistered. Existing task definitions linked by ARN will never be deregistered. Deregistering a task definition does not remove it from your AWS account, instead it will be marked as INACTIVE.",
|
261
|
+
"default": false,
|
262
|
+
"type": "boolean"
|
263
|
+
}
|
264
|
+
},
|
265
|
+
"definitions": {
|
266
|
+
"AwsClientParameters": {
|
267
|
+
"title": "AwsClientParameters",
|
268
|
+
"description": "Model used to manage extra parameters that you can pass when you initialize\nthe Client. If you want to find more information, see\n[boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html)\nfor more info about the possible client configurations.\n\nAttributes:\n api_version: The API version to use. By default, botocore will\n use the latest API version when creating a client. You only need\n to specify this parameter if you want to use a previous API version\n of the client.\n use_ssl: Whether or not to use SSL. By default, SSL is used.\n Note that not all services support non-ssl connections.\n verify: Whether or not to verify SSL certificates. By default\n SSL certificates are verified. If False, SSL will still be used\n (unless use_ssl is False), but SSL certificates\n will not be verified. Passing a file path to this is deprecated.\n verify_cert_path: A filename of the CA cert bundle to\n use. You can specify this argument if you want to use a\n different CA cert bundle than the one used by botocore.\n endpoint_url: The complete URL to use for the constructed\n client. Normally, botocore will automatically construct the\n appropriate URL to use when communicating with a service. You\n can specify a complete URL (including the \"http/https\" scheme)\n to override this behavior. If this value is provided,\n then ``use_ssl`` is ignored.\n config: Advanced configuration for Botocore clients. See\n [botocore docs](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html)\n for more details.",
|
269
|
+
"type": "object",
|
270
|
+
"properties": {
|
271
|
+
"api_version": {
|
272
|
+
"title": "API Version",
|
273
|
+
"description": "The API version to use.",
|
274
|
+
"type": "string"
|
275
|
+
},
|
276
|
+
"use_ssl": {
|
277
|
+
"title": "Use SSL",
|
278
|
+
"description": "Whether or not to use SSL.",
|
279
|
+
"default": true,
|
280
|
+
"type": "boolean"
|
281
|
+
},
|
282
|
+
"verify": {
|
283
|
+
"title": "Verify",
|
284
|
+
"description": "Whether or not to verify SSL certificates.",
|
285
|
+
"default": true,
|
286
|
+
"anyOf": [
|
287
|
+
{
|
288
|
+
"type": "boolean"
|
289
|
+
},
|
290
|
+
{
|
291
|
+
"type": "string",
|
292
|
+
"format": "file-path"
|
293
|
+
}
|
294
|
+
]
|
295
|
+
},
|
296
|
+
"verify_cert_path": {
|
297
|
+
"title": "Certificate Authority Bundle File Path",
|
298
|
+
"description": "Path to the CA cert bundle to use.",
|
299
|
+
"format": "file-path",
|
300
|
+
"type": "string"
|
301
|
+
},
|
302
|
+
"endpoint_url": {
|
303
|
+
"title": "Endpoint URL",
|
304
|
+
"description": "The complete URL to use for the constructed client.",
|
305
|
+
"type": "string"
|
306
|
+
},
|
307
|
+
"config": {
|
308
|
+
"title": "Botocore Config",
|
309
|
+
"description": "Advanced configuration for Botocore clients.",
|
310
|
+
"type": "object"
|
311
|
+
}
|
312
|
+
}
|
313
|
+
},
|
314
|
+
"AwsCredentials": {
|
315
|
+
"title": "AwsCredentials",
|
316
|
+
"description": "Block used to manage authentication with AWS. AWS authentication is\nhandled via the `boto3` module. Refer to the\n[boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html)\nfor more info about the possible credential configurations.",
|
317
|
+
"type": "object",
|
318
|
+
"properties": {
|
319
|
+
"aws_access_key_id": {
|
320
|
+
"title": "AWS Access Key ID",
|
321
|
+
"description": "A specific AWS access key ID.",
|
322
|
+
"type": "string"
|
323
|
+
},
|
324
|
+
"aws_secret_access_key": {
|
325
|
+
"title": "AWS Access Key Secret",
|
326
|
+
"description": "A specific AWS secret access key.",
|
327
|
+
"type": "string",
|
328
|
+
"writeOnly": true,
|
329
|
+
"format": "password"
|
330
|
+
},
|
331
|
+
"aws_session_token": {
|
332
|
+
"title": "AWS Session Token",
|
333
|
+
"description": "The session key for your AWS account. This is only needed when you are using temporary credentials.",
|
334
|
+
"type": "string"
|
335
|
+
},
|
336
|
+
"profile_name": {
|
337
|
+
"title": "Profile Name",
|
338
|
+
"description": "The profile to use when creating your session.",
|
339
|
+
"type": "string"
|
340
|
+
},
|
341
|
+
"region_name": {
|
342
|
+
"title": "Region Name",
|
343
|
+
"description": "The AWS Region where you want to create new connections.",
|
344
|
+
"type": "string"
|
345
|
+
},
|
346
|
+
"aws_client_parameters": {
|
347
|
+
"title": "AWS Client Parameters",
|
348
|
+
"description": "Extra parameters to initialize the Client.",
|
349
|
+
"allOf": [
|
350
|
+
{
|
351
|
+
"$ref": "#/definitions/AwsClientParameters"
|
352
|
+
}
|
353
|
+
]
|
354
|
+
}
|
355
|
+
},
|
356
|
+
"block_type_slug": "aws-credentials",
|
357
|
+
"secret_fields": ["aws_secret_access_key"],
|
358
|
+
"block_schema_references": {}
|
359
|
+
}
|
360
|
+
}
|
361
|
+
}
|
362
|
+
},
|
363
|
+
"description": "Execute flow runs within containers on AWS ECS. Works with EC2 and Fargate clusters. Requires an AWS account.",
|
364
|
+
"display_name": "AWS Elastic Container Service",
|
365
|
+
"documentation_url": "https://prefecthq.github.io/prefect-aws/ecs_worker/",
|
366
|
+
"install_command": "pip install prefect-aws",
|
367
|
+
"is_beta": false,
|
368
|
+
"logo_url": "https://cdn.sanity.io/images/3ugk85nk/production/d74b16fe84ce626345adf235a47008fea2869a60-225x225.png",
|
369
|
+
"type": "ecs"
|
370
|
+
}
|
371
|
+
},
|
372
|
+
"prefect-azure": {
|
373
|
+
"azure-container-instance": {
|
374
|
+
"default_base_job_configuration": {
|
375
|
+
"job_configuration": {
|
376
|
+
"command": "{{ command }}",
|
377
|
+
"env": "{{ env }}",
|
378
|
+
"labels": "{{ labels }}",
|
379
|
+
"name": "{{ name }}",
|
380
|
+
"image": "{{ image }}",
|
381
|
+
"resource_group_name": "{{ resource_group_name }}",
|
382
|
+
"subscription_id": "{{ subscription_id }}",
|
383
|
+
"identities": "{{ identities }}",
|
384
|
+
"entrypoint": "{{ entrypoint }}",
|
385
|
+
"image_registry": "{{ image_registry }}",
|
386
|
+
"cpu": "{{ cpu }}",
|
387
|
+
"gpu_count": "{{ gpu_count }}",
|
388
|
+
"gpu_sku": "{{ gpu_sku }}",
|
389
|
+
"memory": "{{ memory }}",
|
390
|
+
"subnet_ids": "{{ subnet_ids }}",
|
391
|
+
"dns_servers": "{{ dns_servers }}",
|
392
|
+
"stream_output": "{{ stream_output }}",
|
393
|
+
"aci_credentials": "{{ aci_credentials }}",
|
394
|
+
"task_start_timeout_seconds": "{{ task_start_timeout_seconds }}",
|
395
|
+
"task_watch_poll_interval": "{{ task_watch_poll_interval }}",
|
396
|
+
"arm_template": {
|
397
|
+
"$schema": "https://schema.management.azure.com/schemas/2019-08-01/deploymentTemplate.json#",
|
398
|
+
"contentVersion": "1.0.0.0",
|
399
|
+
"parameters": {
|
400
|
+
"location": {
|
401
|
+
"type": "string",
|
402
|
+
"defaultValue": "[resourceGroup().location]",
|
403
|
+
"metadata": {
|
404
|
+
"description": "Location for all resources."
|
405
|
+
}
|
406
|
+
},
|
407
|
+
"container_group_name": {
|
408
|
+
"type": "string",
|
409
|
+
"defaultValue": "[uniqueString(resourceGroup().id)]",
|
410
|
+
"metadata": {
|
411
|
+
"description": "The name of the container group to create."
|
412
|
+
}
|
413
|
+
},
|
414
|
+
"container_name": {
|
415
|
+
"type": "string",
|
416
|
+
"defaultValue": "[uniqueString(resourceGroup().id)]",
|
417
|
+
"metadata": {
|
418
|
+
"description": "The name of the container to create."
|
419
|
+
}
|
420
|
+
}
|
421
|
+
},
|
422
|
+
"resources": [
|
423
|
+
{
|
424
|
+
"type": "Microsoft.ContainerInstance/containerGroups",
|
425
|
+
"apiVersion": "2022-09-01",
|
426
|
+
"name": "[parameters('container_group_name')]",
|
427
|
+
"location": "[parameters('location')]",
|
428
|
+
"properties": {
|
429
|
+
"containers": [
|
430
|
+
{
|
431
|
+
"name": "[parameters('container_name')]",
|
432
|
+
"properties": {
|
433
|
+
"image": "{{ image }}",
|
434
|
+
"command": "{{ command }}",
|
435
|
+
"resources": {
|
436
|
+
"requests": {
|
437
|
+
"cpu": "{{ cpu }}",
|
438
|
+
"memoryInGB": "{{ memory }}"
|
439
|
+
}
|
440
|
+
},
|
441
|
+
"environmentVariables": []
|
442
|
+
}
|
443
|
+
}
|
444
|
+
],
|
445
|
+
"osType": "Linux",
|
446
|
+
"restartPolicy": "Never"
|
447
|
+
}
|
448
|
+
}
|
449
|
+
]
|
450
|
+
}
|
451
|
+
},
|
452
|
+
"variables": {
|
453
|
+
"description": "Variables for an Azure Container Instance flow run.",
|
454
|
+
"type": "object",
|
455
|
+
"properties": {
|
456
|
+
"name": {
|
457
|
+
"title": "Name",
|
458
|
+
"description": "Name given to infrastructure created by a worker.",
|
459
|
+
"type": "string"
|
460
|
+
},
|
461
|
+
"env": {
|
462
|
+
"title": "Environment Variables",
|
463
|
+
"description": "Environment variables to set when starting a flow run.",
|
464
|
+
"type": "object",
|
465
|
+
"additionalProperties": {
|
466
|
+
"type": "string"
|
467
|
+
}
|
468
|
+
},
|
469
|
+
"labels": {
|
470
|
+
"title": "Labels",
|
471
|
+
"description": "Labels applied to infrastructure created by a worker.",
|
472
|
+
"type": "object",
|
473
|
+
"additionalProperties": {
|
474
|
+
"type": "string"
|
475
|
+
}
|
476
|
+
},
|
477
|
+
"command": {
|
478
|
+
"title": "Command",
|
479
|
+
"description": "The command to use when starting a flow run. In most cases, this should be left blank and the command will be automatically generated by the worker.",
|
480
|
+
"type": "string"
|
481
|
+
},
|
482
|
+
"image": {
|
483
|
+
"title": "Image",
|
484
|
+
"description": "The image to use for the Prefect container in the task. This value defaults to a Prefect base image matching your local versions.",
|
485
|
+
"type": "string"
|
486
|
+
},
|
487
|
+
"resource_group_name": {
|
488
|
+
"title": "Azure Resource Group Name",
|
489
|
+
"description": "The name of the Azure Resource Group in which to run Prefect ACI tasks.",
|
490
|
+
"type": "string"
|
491
|
+
},
|
492
|
+
"subscription_id": {
|
493
|
+
"title": "Azure Subscription ID",
|
494
|
+
"description": "The ID of the Azure subscription to create containers under.",
|
495
|
+
"type": "string",
|
496
|
+
"writeOnly": true,
|
497
|
+
"format": "password"
|
498
|
+
},
|
499
|
+
"identities": {
|
500
|
+
"title": "Identities",
|
501
|
+
"description": "A list of user-assigned identities to associate with the container group. The identities should be an ARM resource IDs in the form: '/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}'.",
|
502
|
+
"type": "array",
|
503
|
+
"items": {
|
504
|
+
"type": "string"
|
505
|
+
}
|
506
|
+
},
|
507
|
+
"entrypoint": {
|
508
|
+
"title": "Entrypoint",
|
509
|
+
"description": "The entrypoint of the container you wish you run. This value defaults to the entrypoint used by Prefect images and should only be changed when using a custom image that is not based on an official Prefect image. Any commands set on deployments will be passed to the entrypoint as parameters.",
|
510
|
+
"default": "/opt/prefect/entrypoint.sh",
|
511
|
+
"type": "string"
|
512
|
+
},
|
513
|
+
"image_registry": {
|
514
|
+
"title": "Image Registry (Optional)",
|
515
|
+
"description": "To use any private container registry with a username and password, choose DockerRegistry. To use a private Azure Container Registry with a managed identity, choose ACRManagedIdentity.",
|
516
|
+
"anyOf": [
|
517
|
+
{
|
518
|
+
"$ref": "#/definitions/DockerRegistry"
|
519
|
+
},
|
520
|
+
{
|
521
|
+
"$ref": "#/definitions/ACRManagedIdentity"
|
522
|
+
}
|
523
|
+
]
|
524
|
+
},
|
525
|
+
"cpu": {
|
526
|
+
"title": "CPU",
|
527
|
+
"description": "The number of virtual CPUs to assign to the task container. If not provided, a default value of 1.0 will be used.",
|
528
|
+
"default": 1.0,
|
529
|
+
"type": "number"
|
530
|
+
},
|
531
|
+
"gpu_count": {
|
532
|
+
"title": "GPU Count",
|
533
|
+
"description": "The number of GPUs to assign to the task container. If not provided, no GPU will be used.",
|
534
|
+
"type": "integer"
|
535
|
+
},
|
536
|
+
"gpu_sku": {
|
537
|
+
"title": "GPU SKU",
|
538
|
+
"description": "The Azure GPU SKU to use. See the ACI documentation for a list of GPU SKUs available in each Azure region.",
|
539
|
+
"type": "string"
|
540
|
+
},
|
541
|
+
"memory": {
|
542
|
+
"title": "Memory",
|
543
|
+
"description": "The amount of memory in gigabytes to provide to the ACI task. Valid amounts are specified in the Azure documentation. If not provided, a default value of 1.0 will be used unless present on the task definition.",
|
544
|
+
"default": 1.0,
|
545
|
+
"type": "number"
|
546
|
+
},
|
547
|
+
"subnet_ids": {
|
548
|
+
"title": "Subnet IDs",
|
549
|
+
"description": "A list of subnet IDs to associate with the container group. ",
|
550
|
+
"type": "array",
|
551
|
+
"items": {
|
552
|
+
"type": "string"
|
553
|
+
}
|
554
|
+
},
|
555
|
+
"dns_servers": {
|
556
|
+
"title": "DNS Servers",
|
557
|
+
"description": "A list of DNS servers to associate with the container group.",
|
558
|
+
"type": "array",
|
559
|
+
"items": {
|
560
|
+
"type": "string"
|
561
|
+
}
|
562
|
+
},
|
563
|
+
"aci_credentials": {
|
564
|
+
"title": "Aci Credentials",
|
565
|
+
"description": "The credentials to use to authenticate with Azure.",
|
566
|
+
"allOf": [
|
567
|
+
{
|
568
|
+
"$ref": "#/definitions/AzureContainerInstanceCredentials"
|
569
|
+
}
|
570
|
+
]
|
571
|
+
},
|
572
|
+
"stream_output": {
|
573
|
+
"title": "Stream Output",
|
574
|
+
"description": "If `True`, logs will be streamed from the Prefect container to the local console.",
|
575
|
+
"default": false,
|
576
|
+
"type": "boolean"
|
577
|
+
},
|
578
|
+
"task_start_timeout_seconds": {
|
579
|
+
"title": "Task Start Timeout Seconds",
|
580
|
+
"description": "The amount of time to watch for the start of the ACI container. before marking it as failed.",
|
581
|
+
"default": 240,
|
582
|
+
"type": "integer"
|
583
|
+
},
|
584
|
+
"task_watch_poll_interval": {
|
585
|
+
"title": "Task Watch Poll Interval",
|
586
|
+
"description": "The number of seconds to wait between Azure API calls while monitoring the state of an Azure Container Instances task.",
|
587
|
+
"default": 5.0,
|
588
|
+
"type": "number"
|
589
|
+
}
|
590
|
+
},
|
591
|
+
"required": ["resource_group_name", "subscription_id"],
|
592
|
+
"definitions": {
|
593
|
+
"DockerRegistry": {
|
594
|
+
"title": "DockerRegistry",
|
595
|
+
"description": "Connects to a Docker registry.\n\nRequires a Docker Engine to be connectable.",
|
596
|
+
"type": "object",
|
597
|
+
"properties": {
|
598
|
+
"username": {
|
599
|
+
"title": "Username",
|
600
|
+
"description": "The username to log into the registry with.",
|
601
|
+
"type": "string"
|
602
|
+
},
|
603
|
+
"password": {
|
604
|
+
"title": "Password",
|
605
|
+
"description": "The password to log into the registry with.",
|
606
|
+
"type": "string",
|
607
|
+
"writeOnly": true,
|
608
|
+
"format": "password"
|
609
|
+
},
|
610
|
+
"registry_url": {
|
611
|
+
"title": "Registry Url",
|
612
|
+
"description": "The URL to the registry. Generally, \"http\" or \"https\" can be omitted.",
|
613
|
+
"type": "string"
|
614
|
+
},
|
615
|
+
"reauth": {
|
616
|
+
"title": "Reauth",
|
617
|
+
"description": "Whether or not to reauthenticate on each interaction.",
|
618
|
+
"default": true,
|
619
|
+
"type": "boolean"
|
620
|
+
}
|
621
|
+
},
|
622
|
+
"required": ["username", "password", "registry_url"],
|
623
|
+
"block_type_slug": "docker-registry",
|
624
|
+
"secret_fields": ["password"],
|
625
|
+
"block_schema_references": {}
|
626
|
+
},
|
627
|
+
"ACRManagedIdentity": {
|
628
|
+
"title": "ACRManagedIdentity",
|
629
|
+
"description": "Use a Managed Identity to access Azure Container registry. Requires the\nuser-assigned managed identity be available to the ACI container group.",
|
630
|
+
"type": "object",
|
631
|
+
"properties": {
|
632
|
+
"registry_url": {
|
633
|
+
"title": "Registry URL",
|
634
|
+
"description": "The URL to the registry, such as myregistry.azurecr.io. Generally, 'http' or 'https' can be omitted.",
|
635
|
+
"type": "string"
|
636
|
+
},
|
637
|
+
"identity": {
|
638
|
+
"title": "Identity",
|
639
|
+
"description": "The user-assigned Azure managed identity for the private registry.",
|
640
|
+
"type": "string"
|
641
|
+
}
|
642
|
+
},
|
643
|
+
"required": ["registry_url", "identity"]
|
644
|
+
},
|
645
|
+
"AzureContainerInstanceCredentials": {
|
646
|
+
"title": "AzureContainerInstanceCredentials",
|
647
|
+
"description": "Block used to manage Azure Container Instances authentication. Stores Azure Service\nPrincipal authentication data.",
|
648
|
+
"type": "object",
|
649
|
+
"properties": {
|
650
|
+
"client_id": {
|
651
|
+
"title": "Client ID",
|
652
|
+
"description": "The service principal client ID. If none of client_id, tenant_id, and client_secret are provided, will use DefaultAzureCredential; else will need to provide all three to use ClientSecretCredential.",
|
653
|
+
"type": "string"
|
654
|
+
},
|
655
|
+
"tenant_id": {
|
656
|
+
"title": "Tenant ID",
|
657
|
+
"description": "The service principal tenant ID.If none of client_id, tenant_id, and client_secret are provided, will use DefaultAzureCredential; else will need to provide all three to use ClientSecretCredential.",
|
658
|
+
"type": "string"
|
659
|
+
},
|
660
|
+
"client_secret": {
|
661
|
+
"title": "Client Secret",
|
662
|
+
"description": "The service principal client secret.If none of client_id, tenant_id, and client_secret are provided, will use DefaultAzureCredential; else will need to provide all three to use ClientSecretCredential.",
|
663
|
+
"type": "string",
|
664
|
+
"writeOnly": true,
|
665
|
+
"format": "password"
|
666
|
+
},
|
667
|
+
"credential_kwargs": {
|
668
|
+
"title": "Additional Credential Keyword Arguments",
|
669
|
+
"description": "Additional keyword arguments to pass to `ClientSecretCredential` or `DefaultAzureCredential`.",
|
670
|
+
"type": "object"
|
671
|
+
}
|
672
|
+
},
|
673
|
+
"block_type_slug": "azure-container-instance-credentials",
|
674
|
+
"secret_fields": ["client_secret"],
|
675
|
+
"block_schema_references": {}
|
676
|
+
}
|
677
|
+
}
|
678
|
+
}
|
679
|
+
},
|
680
|
+
"description": "Execute flow runs within containers on Azure's Container Instances service. Requires an Azure account.",
|
681
|
+
"display_name": "Azure Container Instances",
|
682
|
+
"documentation_url": "https://prefecthq.github.io/prefect-azure/container_instance_worker/",
|
683
|
+
"install_command": "pip install prefect-azure",
|
684
|
+
"is_beta": false,
|
685
|
+
"logo_url": "https://cdn.sanity.io/images/3ugk85nk/production/54e3fa7e00197a4fbd1d82ed62494cb58d08c96a-250x250.png",
|
686
|
+
"type": "azure-container-instance"
|
687
|
+
}
|
688
|
+
},
|
689
|
+
"prefect-docker": {
|
690
|
+
"docker": {
|
691
|
+
"default_base_job_configuration": {
|
692
|
+
"job_configuration": {
|
693
|
+
"command": "{{ command }}",
|
694
|
+
"env": "{{ env }}",
|
695
|
+
"labels": "{{ labels }}",
|
696
|
+
"name": "{{ name }}",
|
697
|
+
"image": "{{ image }}",
|
698
|
+
"image_pull_policy": "{{ image_pull_policy }}",
|
699
|
+
"networks": "{{ networks }}",
|
700
|
+
"network_mode": "{{ network_mode }}",
|
701
|
+
"auto_remove": "{{ auto_remove }}",
|
702
|
+
"volumes": "{{ volumes }}",
|
703
|
+
"stream_output": "{{ stream_output }}",
|
704
|
+
"mem_limit": "{{ mem_limit }}",
|
705
|
+
"memswap_limit": "{{ memswap_limit }}",
|
706
|
+
"privileged": "{{ privileged }}"
|
707
|
+
},
|
708
|
+
"variables": {
|
709
|
+
"description": "Configuration class used by the Docker worker.\n\nAn instance of this class is passed to the Docker worker's `run` method\nfor each flow run. It contains all the information necessary to execute the\nflow run as a Docker container.\n\nAttributes:\n name: The name to give to created Docker containers.\n command: The command executed in created Docker containers to kick off\n flow run execution.\n env: The environment variables to set in created Docker containers.\n labels: The labels to set on created Docker containers.\n image: The image reference of a container image to use for created jobs.\n If not set, the latest Prefect image will be used.\n image_pull_policy: The image pull policy to use when pulling images.\n networks: Docker networks that created containers should be connected to.\n network_mode: The network mode for the created containers (e.g. host, bridge).\n If 'networks' is set, this cannot be set.\n auto_remove: If set, containers will be deleted on completion.\n volumes: Docker volumes that should be mounted in created containers.\n stream_output: If set, the output from created containers will be streamed\n to local standard output.\n mem_limit: Memory limit of created containers. Accepts a value\n with a unit identifier (e.g. 100000b, 1000k, 128m, 1g.) If a value is\n given without a unit, bytes are assumed.\n memswap_limit: Total memory (memory + swap), -1 to disable swap. Should only be\n set if `mem_limit` is also set. If `mem_limit` is set, this defaults to\n allowing the container to use as much swap as memory. For example, if\n `mem_limit` is 300m and `memswap_limit` is not set, containers can use\n 600m in total of memory and swap.\n privileged: Give extended privileges to created containers.",
|
710
|
+
"type": "object",
|
711
|
+
"properties": {
|
712
|
+
"command": {
|
713
|
+
"title": "Command",
|
714
|
+
"description": "The command to use when starting a flow run. In most cases, this should be left blank and the command will be automatically generated by the worker.",
|
715
|
+
"type": "string"
|
716
|
+
},
|
717
|
+
"env": {
|
718
|
+
"title": "Environment Variables",
|
719
|
+
"description": "Environment variables to set when starting a flow run.",
|
720
|
+
"type": "object",
|
721
|
+
"additionalProperties": {
|
722
|
+
"type": "string"
|
723
|
+
}
|
724
|
+
},
|
725
|
+
"labels": {
|
726
|
+
"title": "Labels",
|
727
|
+
"description": "Labels applied to infrastructure created by the worker using this job configuration.",
|
728
|
+
"type": "object",
|
729
|
+
"additionalProperties": {
|
730
|
+
"type": "string"
|
731
|
+
}
|
732
|
+
},
|
733
|
+
"name": {
|
734
|
+
"title": "Name",
|
735
|
+
"description": "Name given to infrastructure created by the worker using this job configuration.",
|
736
|
+
"type": "string"
|
737
|
+
},
|
738
|
+
"image": {
|
739
|
+
"title": "Image",
|
740
|
+
"description": "The image reference of a container image to use for created jobs. If not set, the latest Prefect image will be used.",
|
741
|
+
"example": "docker.io/prefecthq/prefect:2-latest",
|
742
|
+
"type": "string"
|
743
|
+
},
|
744
|
+
"image_pull_policy": {
|
745
|
+
"title": "Image Pull Policy",
|
746
|
+
"description": "The image pull policy to use when pulling images.",
|
747
|
+
"enum": ["IfNotPresent", "Always", "Never"],
|
748
|
+
"type": "string"
|
749
|
+
},
|
750
|
+
"networks": {
|
751
|
+
"title": "Networks",
|
752
|
+
"description": "Docker networks that created containers should be connected to.",
|
753
|
+
"type": "array",
|
754
|
+
"items": {
|
755
|
+
"type": "string"
|
756
|
+
}
|
757
|
+
},
|
758
|
+
"network_mode": {
|
759
|
+
"title": "Network Mode",
|
760
|
+
"description": "The network mode for the created containers (e.g. host, bridge). If 'networks' is set, this cannot be set.",
|
761
|
+
"type": "string"
|
762
|
+
},
|
763
|
+
"auto_remove": {
|
764
|
+
"title": "Auto Remove",
|
765
|
+
"description": "If set, containers will be deleted on completion.",
|
766
|
+
"default": false,
|
767
|
+
"type": "boolean"
|
768
|
+
},
|
769
|
+
"volumes": {
|
770
|
+
"title": "Volumes",
|
771
|
+
"description": "A list of volume to mount into created containers.",
|
772
|
+
"example": ["/my/local/path:/path/in/container"],
|
773
|
+
"type": "array",
|
774
|
+
"items": {
|
775
|
+
"type": "string"
|
776
|
+
}
|
777
|
+
},
|
778
|
+
"stream_output": {
|
779
|
+
"title": "Stream Output",
|
780
|
+
"description": "If set, the output from created containers will be streamed to local standard output.",
|
781
|
+
"default": true,
|
782
|
+
"type": "boolean"
|
783
|
+
},
|
784
|
+
"mem_limit": {
|
785
|
+
"title": "Memory Limit",
|
786
|
+
"description": "Memory limit of created containers. Accepts a value with a unit identifier (e.g. 100000b, 1000k, 128m, 1g.) If a value is given without a unit, bytes are assumed.",
|
787
|
+
"type": "string"
|
788
|
+
},
|
789
|
+
"memswap_limit": {
|
790
|
+
"title": "Memory Swap Limit",
|
791
|
+
"description": "Total memory (memory + swap), -1 to disable swap. Should only be set if `mem_limit` is also set. If `mem_limit` is set, this defaults toallowing the container to use as much swap as memory. For example, if `mem_limit` is 300m and `memswap_limit` is not set, containers can use 600m in total of memory and swap.",
|
792
|
+
"type": "string"
|
793
|
+
},
|
794
|
+
"privileged": {
|
795
|
+
"title": "Privileged",
|
796
|
+
"description": "Give extended privileges to created container.",
|
797
|
+
"default": false,
|
798
|
+
"type": "boolean"
|
799
|
+
}
|
800
|
+
}
|
801
|
+
}
|
802
|
+
},
|
803
|
+
"description": "Execute flow runs within Docker containers. Works well for managing flow execution environments via Docker images. Requires access to a running Docker daemon.",
|
804
|
+
"display_name": "Docker",
|
805
|
+
"documentation_url": "https://prefecthq.github.io/prefect-docker/worker/",
|
806
|
+
"install_command": "pip install prefect-docker",
|
807
|
+
"is_beta": false,
|
808
|
+
"logo_url": "https://cdn.sanity.io/images/3ugk85nk/production/14a315b79990200db7341e42553e23650b34bb96-250x250.png",
|
809
|
+
"type": "docker"
|
810
|
+
}
|
811
|
+
},
|
812
|
+
"prefect-gcp": {
|
813
|
+
"cloud-run": {
|
814
|
+
"default_base_job_configuration": {
|
815
|
+
"job_configuration": {
|
816
|
+
"command": "{{ command }}",
|
817
|
+
"env": "{{ env }}",
|
818
|
+
"labels": "{{ labels }}",
|
819
|
+
"name": "{{ name }}",
|
820
|
+
"region": "{{ region }}",
|
821
|
+
"credentials": "{{ credentials }}",
|
822
|
+
"job_body": {
|
823
|
+
"apiVersion": "run.googleapis.com/v1",
|
824
|
+
"kind": "Job",
|
825
|
+
"metadata": {
|
826
|
+
"name": "{{ name }}",
|
827
|
+
"annotations": {
|
828
|
+
"run.googleapis.com/launch-stage": "BETA",
|
829
|
+
"run.googleapis.com/vpc-access-connector": "{{ vpc_connector_name }}"
|
830
|
+
}
|
831
|
+
},
|
832
|
+
"spec": {
|
833
|
+
"template": {
|
834
|
+
"spec": {
|
835
|
+
"template": {
|
836
|
+
"spec": {
|
837
|
+
"containers": [
|
838
|
+
{
|
839
|
+
"image": "{{ image }}",
|
840
|
+
"command": "{{ command }}",
|
841
|
+
"resources": {
|
842
|
+
"limits": {
|
843
|
+
"cpu": "{{ cpu }}",
|
844
|
+
"memory": "{{ memory }}"
|
845
|
+
},
|
846
|
+
"requests": {
|
847
|
+
"cpu": "{{ cpu }}",
|
848
|
+
"memory": "{{ memory }}"
|
849
|
+
}
|
850
|
+
}
|
851
|
+
}
|
852
|
+
],
|
853
|
+
"timeoutSeconds": "{{ timeout }}",
|
854
|
+
"serviceAccountName": "{{ service_account_name }}"
|
855
|
+
}
|
856
|
+
}
|
857
|
+
}
|
858
|
+
}
|
859
|
+
}
|
860
|
+
},
|
861
|
+
"timeout": "{{ timeout }}",
|
862
|
+
"keep_job": "{{ keep_job }}"
|
863
|
+
},
|
864
|
+
"variables": {
|
865
|
+
"description": "Default variables for the Cloud Run worker.\n\nThe schema for this class is used to populate the `variables` section of the default\nbase job template.",
|
866
|
+
"type": "object",
|
867
|
+
"properties": {
|
868
|
+
"name": {
|
869
|
+
"title": "Name",
|
870
|
+
"description": "Name given to infrastructure created by a worker.",
|
871
|
+
"type": "string"
|
872
|
+
},
|
873
|
+
"env": {
|
874
|
+
"title": "Environment Variables",
|
875
|
+
"description": "Environment variables to set when starting a flow run.",
|
876
|
+
"type": "object",
|
877
|
+
"additionalProperties": {
|
878
|
+
"type": "string"
|
879
|
+
}
|
880
|
+
},
|
881
|
+
"labels": {
|
882
|
+
"title": "Labels",
|
883
|
+
"description": "Labels applied to infrastructure created by a worker.",
|
884
|
+
"type": "object",
|
885
|
+
"additionalProperties": {
|
886
|
+
"type": "string"
|
887
|
+
}
|
888
|
+
},
|
889
|
+
"command": {
|
890
|
+
"title": "Command",
|
891
|
+
"description": "The command to use when starting a flow run. In most cases, this should be left blank and the command will be automatically generated by the worker.",
|
892
|
+
"type": "string"
|
893
|
+
},
|
894
|
+
"region": {
|
895
|
+
"title": "Region",
|
896
|
+
"description": "The region where the Cloud Run Job resides.",
|
897
|
+
"default": "us-central1",
|
898
|
+
"example": "us-central1",
|
899
|
+
"type": "string"
|
900
|
+
},
|
901
|
+
"credentials": {
|
902
|
+
"title": "GCP Credentials",
|
903
|
+
"description": "The GCP Credentials used to initiate the Cloud Run Job. If not provided credentials will be inferred from the local environment.",
|
904
|
+
"allOf": [
|
905
|
+
{
|
906
|
+
"$ref": "#/definitions/GcpCredentials"
|
907
|
+
}
|
908
|
+
]
|
909
|
+
},
|
910
|
+
"image": {
|
911
|
+
"title": "Image Name",
|
912
|
+
"description": "The image to use for a new Cloud Run Job. If not set, the latest Prefect image will be used. See https://cloud.google.com/run/docs/deploying#images.",
|
913
|
+
"example": "docker.io/prefecthq/prefect:2-latest",
|
914
|
+
"type": "string"
|
915
|
+
},
|
916
|
+
"cpu": {
|
917
|
+
"title": "CPU",
|
918
|
+
"description": "The amount of compute allocated to the Cloud Run Job. (1000m = 1 CPU). See https://cloud.google.com/run/docs/configuring/cpu#setting-jobs.",
|
919
|
+
"pattern": "^(\\d*000)m$",
|
920
|
+
"example": "1000m",
|
921
|
+
"type": "string"
|
922
|
+
},
|
923
|
+
"memory": {
|
924
|
+
"title": "Memory",
|
925
|
+
"description": "The amount of memory allocated to the Cloud Run Job. Must be specified in units of 'G', 'Gi', 'M', or 'Mi'. See https://cloud.google.com/run/docs/configuring/memory-limits#setting.",
|
926
|
+
"pattern": "^\\d+(?:G|Gi|M|Mi)$",
|
927
|
+
"example": "512Mi",
|
928
|
+
"type": "string"
|
929
|
+
},
|
930
|
+
"vpc_connector_name": {
|
931
|
+
"title": "VPC Connector Name",
|
932
|
+
"description": "The name of the VPC connector to use for the Cloud Run Job.",
|
933
|
+
"type": "string"
|
934
|
+
},
|
935
|
+
"service_account_name": {
|
936
|
+
"title": "Service Account Name",
|
937
|
+
"description": "The name of the service account to use for the task execution of Cloud Run Job. By default Cloud Run jobs run as the default Compute Engine Service Account. ",
|
938
|
+
"example": "service-account@example.iam.gserviceaccount.com",
|
939
|
+
"type": "string"
|
940
|
+
},
|
941
|
+
"keep_job": {
|
942
|
+
"title": "Keep Job After Completion",
|
943
|
+
"description": "Keep the completed Cloud Run Job after it has run.",
|
944
|
+
"default": false,
|
945
|
+
"type": "boolean"
|
946
|
+
},
|
947
|
+
"timeout": {
|
948
|
+
"title": "Job Timeout",
|
949
|
+
"description": "The length of time that Prefect will wait for Cloud Run Job state changes.",
|
950
|
+
"default": 600,
|
951
|
+
"exclusiveMinimum": 0,
|
952
|
+
"maximum": 3600,
|
953
|
+
"type": "integer"
|
954
|
+
}
|
955
|
+
},
|
956
|
+
"definitions": {
|
957
|
+
"GcpCredentials": {
|
958
|
+
"title": "GcpCredentials",
|
959
|
+
"description": "Block used to manage authentication with GCP. Google authentication is\nhandled via the `google.oauth2` module or through the CLI.\nSpecify either one of service `account_file` or `service_account_info`; if both\nare not specified, the client will try to detect the credentials following Google's\n[Application Default Credentials](https://cloud.google.com/docs/authentication/application-default-credentials).\nSee Google's [Authentication documentation](https://cloud.google.com/docs/authentication#service-accounts)\nfor details on inference and recommended authentication patterns.",
|
960
|
+
"type": "object",
|
961
|
+
"properties": {
|
962
|
+
"service_account_file": {
|
963
|
+
"title": "Service Account File",
|
964
|
+
"description": "Path to the service account JSON keyfile.",
|
965
|
+
"type": "string",
|
966
|
+
"format": "path"
|
967
|
+
},
|
968
|
+
"service_account_info": {
|
969
|
+
"title": "Service Account Info",
|
970
|
+
"description": "The contents of the keyfile as a dict.",
|
971
|
+
"type": "object"
|
972
|
+
},
|
973
|
+
"project": {
|
974
|
+
"title": "Project",
|
975
|
+
"description": "The GCP project to use for the client.",
|
976
|
+
"type": "string"
|
977
|
+
}
|
978
|
+
},
|
979
|
+
"block_type_slug": "gcp-credentials",
|
980
|
+
"secret_fields": ["service_account_info.*"],
|
981
|
+
"block_schema_references": {}
|
982
|
+
}
|
983
|
+
}
|
984
|
+
}
|
985
|
+
},
|
986
|
+
"description": "Execute flow runs within containers on Google Cloud Run. Requires a Google Cloud Platform account.",
|
987
|
+
"display_name": "Google Cloud Run",
|
988
|
+
"documentation_url": "https://prefecthq.github.io/prefect-gcp/cloud_run_worker/",
|
989
|
+
"install_command": "pip install prefect-gcp",
|
990
|
+
"is_beta": false,
|
991
|
+
"logo_url": "https://cdn.sanity.io/images/3ugk85nk/production/10424e311932e31c477ac2b9ef3d53cefbaad708-250x250.png",
|
992
|
+
"type": "cloud-run"
|
993
|
+
},
|
994
|
+
"cloud-run-v2": {
|
995
|
+
"default_base_job_configuration": {
|
996
|
+
"job_configuration": {
|
997
|
+
"command": "{{ command }}",
|
998
|
+
"env": "{{ env }}",
|
999
|
+
"labels": "{{ labels }}",
|
1000
|
+
"name": "{{ name }}",
|
1001
|
+
"credentials": "{{ credentials }}",
|
1002
|
+
"job_body": {
|
1003
|
+
"client": "prefect",
|
1004
|
+
"launchStage": "{{ launch_stage }}",
|
1005
|
+
"template": {
|
1006
|
+
"template": {
|
1007
|
+
"maxRetries": "{{ max_retries }}",
|
1008
|
+
"timeout": "{{ timeout }}",
|
1009
|
+
"containers": [
|
1010
|
+
{
|
1011
|
+
"env": [],
|
1012
|
+
"image": "{{ image }}",
|
1013
|
+
"command": "{{ command }}",
|
1014
|
+
"args": "{{ args }}",
|
1015
|
+
"resources": {
|
1016
|
+
"limits": {
|
1017
|
+
"cpu": "{{ cpu }}",
|
1018
|
+
"memory": "{{ memory }}"
|
1019
|
+
}
|
1020
|
+
}
|
1021
|
+
}
|
1022
|
+
]
|
1023
|
+
}
|
1024
|
+
}
|
1025
|
+
},
|
1026
|
+
"keep_job": "{{ keep_job }}",
|
1027
|
+
"region": "{{ region }}",
|
1028
|
+
"timeout": "{{ timeout }}"
|
1029
|
+
},
|
1030
|
+
"variables": {
|
1031
|
+
"description": "Default variables for the Cloud Run worker V2.\n\nThe schema for this class is used to populate the `variables` section of the\ndefault base job template.",
|
1032
|
+
"type": "object",
|
1033
|
+
"properties": {
|
1034
|
+
"name": {
|
1035
|
+
"title": "Name",
|
1036
|
+
"description": "Name given to infrastructure created by a worker.",
|
1037
|
+
"type": "string"
|
1038
|
+
},
|
1039
|
+
"env": {
|
1040
|
+
"title": "Environment Variables",
|
1041
|
+
"description": "Environment variables to set when starting a flow run.",
|
1042
|
+
"type": "object",
|
1043
|
+
"additionalProperties": {
|
1044
|
+
"type": "string"
|
1045
|
+
}
|
1046
|
+
},
|
1047
|
+
"labels": {
|
1048
|
+
"title": "Labels",
|
1049
|
+
"description": "Labels applied to infrastructure created by a worker.",
|
1050
|
+
"type": "object",
|
1051
|
+
"additionalProperties": {
|
1052
|
+
"type": "string"
|
1053
|
+
}
|
1054
|
+
},
|
1055
|
+
"command": {
|
1056
|
+
"title": "Command",
|
1057
|
+
"description": "The command to use when starting a flow run. In most cases, this should be left blank and the command will be automatically generated by the worker.",
|
1058
|
+
"type": "string"
|
1059
|
+
},
|
1060
|
+
"credentials": {
|
1061
|
+
"title": "GCP Credentials",
|
1062
|
+
"description": "The GCP Credentials used to connect to Cloud Run. If not provided credentials will be inferred from the local environment.",
|
1063
|
+
"allOf": [
|
1064
|
+
{
|
1065
|
+
"$ref": "#/definitions/GcpCredentials"
|
1066
|
+
}
|
1067
|
+
]
|
1068
|
+
},
|
1069
|
+
"region": {
|
1070
|
+
"title": "Region",
|
1071
|
+
"description": "The region in which to run the Cloud Run job",
|
1072
|
+
"default": "us-central1",
|
1073
|
+
"type": "string"
|
1074
|
+
},
|
1075
|
+
"image": {
|
1076
|
+
"title": "Image Name",
|
1077
|
+
"description": "The image to use for the Cloud Run job. If not provided the default Prefect image will be used.",
|
1078
|
+
"default": "prefecthq/prefect:2-latest",
|
1079
|
+
"type": "string"
|
1080
|
+
},
|
1081
|
+
"args": {
|
1082
|
+
"title": "Args",
|
1083
|
+
"description": "The arguments to pass to the Cloud Run Job V2's entrypoint command.",
|
1084
|
+
"type": "array",
|
1085
|
+
"items": {
|
1086
|
+
"type": "string"
|
1087
|
+
}
|
1088
|
+
},
|
1089
|
+
"keep_job": {
|
1090
|
+
"title": "Keep Job After Completion",
|
1091
|
+
"description": "Keep the completed Cloud run job on Google Cloud Platform.",
|
1092
|
+
"default": false,
|
1093
|
+
"type": "boolean"
|
1094
|
+
},
|
1095
|
+
"launch_stage": {
|
1096
|
+
"title": "Launch Stage",
|
1097
|
+
"description": "The launch stage of the Cloud Run Job V2. See https://cloud.google.com/run/docs/about-features-categories for additional details.",
|
1098
|
+
"default": "BETA",
|
1099
|
+
"enum": [
|
1100
|
+
"ALPHA",
|
1101
|
+
"BETA",
|
1102
|
+
"GA",
|
1103
|
+
"DEPRECATED",
|
1104
|
+
"EARLY_ACCESS",
|
1105
|
+
"PRELAUNCH",
|
1106
|
+
"UNIMPLEMENTED",
|
1107
|
+
"LAUNCH_TAG_UNSPECIFIED"
|
1108
|
+
],
|
1109
|
+
"type": "string"
|
1110
|
+
},
|
1111
|
+
"max_retries": {
|
1112
|
+
"title": "Max Retries",
|
1113
|
+
"description": "The number of times to retry the Cloud Run job.",
|
1114
|
+
"default": 0,
|
1115
|
+
"type": "integer"
|
1116
|
+
},
|
1117
|
+
"cpu": {
|
1118
|
+
"title": "CPU",
|
1119
|
+
"description": "The CPU to allocate to the Cloud Run job.",
|
1120
|
+
"default": "1000m",
|
1121
|
+
"type": "string"
|
1122
|
+
},
|
1123
|
+
"memory": {
|
1124
|
+
"title": "Memory",
|
1125
|
+
"description": "The memory to allocate to the Cloud Run job along with the units, whichcould be: G, Gi, M, Mi.",
|
1126
|
+
"default": "512Mi",
|
1127
|
+
"example": "512Mi",
|
1128
|
+
"pattern": "^\\d+(?:G|Gi|M|Mi)$",
|
1129
|
+
"type": "string"
|
1130
|
+
},
|
1131
|
+
"timeout": {
|
1132
|
+
"title": "Job Timeout",
|
1133
|
+
"description": "The length of time that Prefect will wait for a Cloud Run Job to complete before raising an exception (maximum of 86400 seconds, 1 day).",
|
1134
|
+
"default": 600,
|
1135
|
+
"exclusiveMinimum": 0,
|
1136
|
+
"maximum": 86400,
|
1137
|
+
"type": "integer"
|
1138
|
+
},
|
1139
|
+
"vpc_connector_name": {
|
1140
|
+
"title": "VPC Connector Name",
|
1141
|
+
"description": "The name of the VPC connector to use for the Cloud Run job.",
|
1142
|
+
"type": "string"
|
1143
|
+
}
|
1144
|
+
},
|
1145
|
+
"definitions": {
|
1146
|
+
"GcpCredentials": {
|
1147
|
+
"title": "GcpCredentials",
|
1148
|
+
"description": "Block used to manage authentication with GCP. Google authentication is\nhandled via the `google.oauth2` module or through the CLI.\nSpecify either one of service `account_file` or `service_account_info`; if both\nare not specified, the client will try to detect the credentials following Google's\n[Application Default Credentials](https://cloud.google.com/docs/authentication/application-default-credentials).\nSee Google's [Authentication documentation](https://cloud.google.com/docs/authentication#service-accounts)\nfor details on inference and recommended authentication patterns.",
|
1149
|
+
"type": "object",
|
1150
|
+
"properties": {
|
1151
|
+
"service_account_file": {
|
1152
|
+
"title": "Service Account File",
|
1153
|
+
"description": "Path to the service account JSON keyfile.",
|
1154
|
+
"type": "string",
|
1155
|
+
"format": "path"
|
1156
|
+
},
|
1157
|
+
"service_account_info": {
|
1158
|
+
"title": "Service Account Info",
|
1159
|
+
"description": "The contents of the keyfile as a dict.",
|
1160
|
+
"type": "object"
|
1161
|
+
},
|
1162
|
+
"project": {
|
1163
|
+
"title": "Project",
|
1164
|
+
"description": "The GCP project to use for the client.",
|
1165
|
+
"type": "string"
|
1166
|
+
}
|
1167
|
+
},
|
1168
|
+
"block_type_slug": "gcp-credentials",
|
1169
|
+
"secret_fields": ["service_account_info.*"],
|
1170
|
+
"block_schema_references": {}
|
1171
|
+
}
|
1172
|
+
}
|
1173
|
+
}
|
1174
|
+
},
|
1175
|
+
"description": "Execute flow runs within containers on Google Cloud Run (V2 API). Requires a Google Cloud Platform account.",
|
1176
|
+
"display_name": "Google Cloud Run V2",
|
1177
|
+
"documentation_url": "https://prefecthq.github.io/prefect-gcp/worker_v2/",
|
1178
|
+
"install_command": "pip install prefect-gcp",
|
1179
|
+
"is_beta": false,
|
1180
|
+
"logo_url": "https://cdn.sanity.io/images/3ugk85nk/production/10424e311932e31c477ac2b9ef3d53cefbaad708-250x250.png",
|
1181
|
+
"type": "cloud-run-v2"
|
1182
|
+
},
|
1183
|
+
"vertex-ai": {
|
1184
|
+
"default_base_job_configuration": {
|
1185
|
+
"job_configuration": {
|
1186
|
+
"command": "{{ command }}",
|
1187
|
+
"env": "{{ env }}",
|
1188
|
+
"labels": "{{ labels }}",
|
1189
|
+
"name": "{{ name }}",
|
1190
|
+
"region": "{{ region }}",
|
1191
|
+
"credentials": "{{ credentials }}",
|
1192
|
+
"job_spec": {
|
1193
|
+
"service_account_name": "{{ service_account_name }}",
|
1194
|
+
"network": "{{ network }}",
|
1195
|
+
"reserved_ip_ranges": "{{ reserved_ip_ranges }}",
|
1196
|
+
"maximum_run_time_hours": "{{ maximum_run_time_hours }}",
|
1197
|
+
"worker_pool_specs": [
|
1198
|
+
{
|
1199
|
+
"replica_count": 1,
|
1200
|
+
"container_spec": {
|
1201
|
+
"image_uri": "{{ image }}",
|
1202
|
+
"command": "{{ command }}",
|
1203
|
+
"args": []
|
1204
|
+
},
|
1205
|
+
"machine_spec": {
|
1206
|
+
"machine_type": "{{ machine_type }}",
|
1207
|
+
"accelerator_type": "{{ accelerator_type }}",
|
1208
|
+
"accelerator_count": "{{ accelerator_count }}"
|
1209
|
+
},
|
1210
|
+
"disk_spec": {
|
1211
|
+
"boot_disk_type": "{{ boot_disk_type }}",
|
1212
|
+
"boot_disk_size_gb": "{{ boot_disk_size_gb }}"
|
1213
|
+
}
|
1214
|
+
}
|
1215
|
+
]
|
1216
|
+
},
|
1217
|
+
"job_watch_poll_interval": "{{ job_watch_poll_interval }}"
|
1218
|
+
},
|
1219
|
+
"variables": {
|
1220
|
+
"description": "Default variables for the Vertex AI worker.\n\nThe schema for this class is used to populate the `variables` section of the default\nbase job template.",
|
1221
|
+
"type": "object",
|
1222
|
+
"properties": {
|
1223
|
+
"name": {
|
1224
|
+
"title": "Name",
|
1225
|
+
"description": "Name given to infrastructure created by a worker.",
|
1226
|
+
"type": "string"
|
1227
|
+
},
|
1228
|
+
"env": {
|
1229
|
+
"title": "Environment Variables",
|
1230
|
+
"description": "Environment variables to set when starting a flow run.",
|
1231
|
+
"type": "object",
|
1232
|
+
"additionalProperties": {
|
1233
|
+
"type": "string"
|
1234
|
+
}
|
1235
|
+
},
|
1236
|
+
"labels": {
|
1237
|
+
"title": "Labels",
|
1238
|
+
"description": "Labels applied to infrastructure created by a worker.",
|
1239
|
+
"type": "object",
|
1240
|
+
"additionalProperties": {
|
1241
|
+
"type": "string"
|
1242
|
+
}
|
1243
|
+
},
|
1244
|
+
"command": {
|
1245
|
+
"title": "Command",
|
1246
|
+
"description": "The command to use when starting a flow run. In most cases, this should be left blank and the command will be automatically generated by the worker.",
|
1247
|
+
"type": "string"
|
1248
|
+
},
|
1249
|
+
"region": {
|
1250
|
+
"title": "Region",
|
1251
|
+
"description": "The region where the Vertex AI Job resides.",
|
1252
|
+
"example": "us-central1",
|
1253
|
+
"type": "string"
|
1254
|
+
},
|
1255
|
+
"image": {
|
1256
|
+
"title": "Image Name",
|
1257
|
+
"description": "The URI of a container image in the Container or Artifact Registry, used to run your Vertex AI Job. Note that Vertex AI will need accessto the project and region where the container image is stored. See https://cloud.google.com/vertex-ai/docs/training/create-custom-container",
|
1258
|
+
"example": "gcr.io/your-project/your-repo:latest",
|
1259
|
+
"type": "string"
|
1260
|
+
},
|
1261
|
+
"credentials": {
|
1262
|
+
"title": "GCP Credentials",
|
1263
|
+
"description": "The GCP Credentials used to initiate the Vertex AI Job. If not provided credentials will be inferred from the local environment.",
|
1264
|
+
"allOf": [
|
1265
|
+
{
|
1266
|
+
"$ref": "#/definitions/GcpCredentials"
|
1267
|
+
}
|
1268
|
+
]
|
1269
|
+
},
|
1270
|
+
"machine_type": {
|
1271
|
+
"title": "Machine Type",
|
1272
|
+
"description": "The machine type to use for the run, which controls the available CPU and memory. See https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec",
|
1273
|
+
"default": "n1-standard-4",
|
1274
|
+
"type": "string"
|
1275
|
+
},
|
1276
|
+
"accelerator_type": {
|
1277
|
+
"title": "Accelerator Type",
|
1278
|
+
"description": "The type of accelerator to attach to the machine. See https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec",
|
1279
|
+
"example": "NVIDIA_TESLA_K80",
|
1280
|
+
"type": "string"
|
1281
|
+
},
|
1282
|
+
"accelerator_count": {
|
1283
|
+
"title": "Accelerator Count",
|
1284
|
+
"description": "The number of accelerators to attach to the machine. See https://cloud.google.com/vertex-ai/docs/reference/rest/v1/MachineSpec",
|
1285
|
+
"example": 1,
|
1286
|
+
"type": "integer"
|
1287
|
+
},
|
1288
|
+
"boot_disk_type": {
|
1289
|
+
"title": "Boot Disk Type",
|
1290
|
+
"description": "The type of boot disk to attach to the machine.",
|
1291
|
+
"default": "pd-ssd",
|
1292
|
+
"type": "string"
|
1293
|
+
},
|
1294
|
+
"boot_disk_size_gb": {
|
1295
|
+
"title": "Boot Disk Size (GB)",
|
1296
|
+
"description": "The size of the boot disk to attach to the machine, in gigabytes.",
|
1297
|
+
"default": 100,
|
1298
|
+
"type": "integer"
|
1299
|
+
},
|
1300
|
+
"maximum_run_time_hours": {
|
1301
|
+
"title": "Maximum Run Time (Hours)",
|
1302
|
+
"description": "The maximum job running time, in hours",
|
1303
|
+
"default": 1,
|
1304
|
+
"type": "integer"
|
1305
|
+
},
|
1306
|
+
"network": {
|
1307
|
+
"title": "Network",
|
1308
|
+
"description": "The full name of the Compute Engine networkto which the Job should be peered. Private services access must already be configured for the network. If left unspecified, the job is not peered with any network. For example: projects/12345/global/networks/myVPC",
|
1309
|
+
"type": "string"
|
1310
|
+
},
|
1311
|
+
"reserved_ip_ranges": {
|
1312
|
+
"title": "Reserved IP Ranges",
|
1313
|
+
"description": "A list of names for the reserved ip ranges under the VPC network that can be used for this job. If set, we will deploy the job within the provided ip ranges. Otherwise, the job will be deployed to any ip ranges under the provided VPC network.",
|
1314
|
+
"type": "array",
|
1315
|
+
"items": {
|
1316
|
+
"type": "string"
|
1317
|
+
}
|
1318
|
+
},
|
1319
|
+
"service_account_name": {
|
1320
|
+
"title": "Service Account Name",
|
1321
|
+
"description": "Specifies the service account to use as the run-as account in Vertex AI. The worker submitting jobs must have act-as permission on this run-as account. If unspecified, the AI Platform Custom Code Service Agent for the CustomJob's project is used. Takes precedence over the service account found in GCP credentials, and required if a service account cannot be detected in GCP credentials.",
|
1322
|
+
"type": "string"
|
1323
|
+
},
|
1324
|
+
"job_watch_poll_interval": {
|
1325
|
+
"title": "Poll Interval (Seconds)",
|
1326
|
+
"description": "The amount of time to wait between GCP API calls while monitoring the state of a Vertex AI Job.",
|
1327
|
+
"default": 5.0,
|
1328
|
+
"type": "number"
|
1329
|
+
}
|
1330
|
+
},
|
1331
|
+
"required": ["region", "image"],
|
1332
|
+
"definitions": {
|
1333
|
+
"GcpCredentials": {
|
1334
|
+
"title": "GcpCredentials",
|
1335
|
+
"description": "Block used to manage authentication with GCP. Google authentication is\nhandled via the `google.oauth2` module or through the CLI.\nSpecify either one of service `account_file` or `service_account_info`; if both\nare not specified, the client will try to detect the credentials following Google's\n[Application Default Credentials](https://cloud.google.com/docs/authentication/application-default-credentials).\nSee Google's [Authentication documentation](https://cloud.google.com/docs/authentication#service-accounts)\nfor details on inference and recommended authentication patterns.",
|
1336
|
+
"type": "object",
|
1337
|
+
"properties": {
|
1338
|
+
"service_account_file": {
|
1339
|
+
"title": "Service Account File",
|
1340
|
+
"description": "Path to the service account JSON keyfile.",
|
1341
|
+
"type": "string",
|
1342
|
+
"format": "path"
|
1343
|
+
},
|
1344
|
+
"service_account_info": {
|
1345
|
+
"title": "Service Account Info",
|
1346
|
+
"description": "The contents of the keyfile as a dict.",
|
1347
|
+
"type": "object"
|
1348
|
+
},
|
1349
|
+
"project": {
|
1350
|
+
"title": "Project",
|
1351
|
+
"description": "The GCP project to use for the client.",
|
1352
|
+
"type": "string"
|
1353
|
+
}
|
1354
|
+
},
|
1355
|
+
"block_type_slug": "gcp-credentials",
|
1356
|
+
"secret_fields": ["service_account_info.*"],
|
1357
|
+
"block_schema_references": {}
|
1358
|
+
}
|
1359
|
+
}
|
1360
|
+
}
|
1361
|
+
},
|
1362
|
+
"description": "Execute flow runs within containers on Google Vertex AI. Requires a Google Cloud Platform account.",
|
1363
|
+
"display_name": "Google Vertex AI",
|
1364
|
+
"documentation_url": "https://prefecthq.github.io/prefect-gcp/vertex_worker/",
|
1365
|
+
"install_command": "pip install prefect-gcp",
|
1366
|
+
"is_beta": false,
|
1367
|
+
"logo_url": "https://cdn.sanity.io/images/3ugk85nk/production/10424e311932e31c477ac2b9ef3d53cefbaad708-250x250.png",
|
1368
|
+
"type": "vertex-ai"
|
1369
|
+
}
|
1370
|
+
},
|
1371
|
+
"prefect-kubernetes": {
|
1372
|
+
"kubernetes": {
|
1373
|
+
"default_base_job_configuration": {
|
1374
|
+
"job_configuration": {
|
1375
|
+
"command": "{{ command }}",
|
1376
|
+
"env": "{{ env }}",
|
1377
|
+
"labels": "{{ labels }}",
|
1378
|
+
"name": "{{ name }}",
|
1379
|
+
"namespace": "{{ namespace }}",
|
1380
|
+
"job_manifest": {
|
1381
|
+
"apiVersion": "batch/v1",
|
1382
|
+
"kind": "Job",
|
1383
|
+
"metadata": {
|
1384
|
+
"labels": "{{ labels }}",
|
1385
|
+
"namespace": "{{ namespace }}",
|
1386
|
+
"generateName": "{{ name }}-"
|
1387
|
+
},
|
1388
|
+
"spec": {
|
1389
|
+
"backoffLimit": 0,
|
1390
|
+
"ttlSecondsAfterFinished": "{{ finished_job_ttl }}",
|
1391
|
+
"template": {
|
1392
|
+
"spec": {
|
1393
|
+
"parallelism": 1,
|
1394
|
+
"completions": 1,
|
1395
|
+
"restartPolicy": "Never",
|
1396
|
+
"serviceAccountName": "{{ service_account_name }}",
|
1397
|
+
"containers": [
|
1398
|
+
{
|
1399
|
+
"name": "prefect-job",
|
1400
|
+
"env": "{{ env }}",
|
1401
|
+
"image": "{{ image }}",
|
1402
|
+
"imagePullPolicy": "{{ image_pull_policy }}",
|
1403
|
+
"args": "{{ command }}"
|
1404
|
+
}
|
1405
|
+
]
|
1406
|
+
}
|
1407
|
+
}
|
1408
|
+
}
|
1409
|
+
},
|
1410
|
+
"cluster_config": "{{ cluster_config }}",
|
1411
|
+
"job_watch_timeout_seconds": "{{ job_watch_timeout_seconds }}",
|
1412
|
+
"pod_watch_timeout_seconds": "{{ pod_watch_timeout_seconds }}",
|
1413
|
+
"stream_output": "{{ stream_output }}"
|
1414
|
+
},
|
1415
|
+
"variables": {
|
1416
|
+
"description": "Default variables for the Kubernetes worker.\n\nThe schema for this class is used to populate the `variables` section of the default\nbase job template.",
|
1417
|
+
"type": "object",
|
1418
|
+
"properties": {
|
1419
|
+
"name": {
|
1420
|
+
"title": "Name",
|
1421
|
+
"description": "Name given to infrastructure created by a worker.",
|
1422
|
+
"type": "string"
|
1423
|
+
},
|
1424
|
+
"env": {
|
1425
|
+
"title": "Environment Variables",
|
1426
|
+
"description": "Environment variables to set when starting a flow run.",
|
1427
|
+
"type": "object",
|
1428
|
+
"additionalProperties": {
|
1429
|
+
"type": "string"
|
1430
|
+
}
|
1431
|
+
},
|
1432
|
+
"labels": {
|
1433
|
+
"title": "Labels",
|
1434
|
+
"description": "Labels applied to infrastructure created by a worker.",
|
1435
|
+
"type": "object",
|
1436
|
+
"additionalProperties": {
|
1437
|
+
"type": "string"
|
1438
|
+
}
|
1439
|
+
},
|
1440
|
+
"command": {
|
1441
|
+
"title": "Command",
|
1442
|
+
"description": "The command to use when starting a flow run. In most cases, this should be left blank and the command will be automatically generated by the worker.",
|
1443
|
+
"type": "string"
|
1444
|
+
},
|
1445
|
+
"namespace": {
|
1446
|
+
"title": "Namespace",
|
1447
|
+
"description": "The Kubernetes namespace to create jobs within.",
|
1448
|
+
"default": "default",
|
1449
|
+
"type": "string"
|
1450
|
+
},
|
1451
|
+
"image": {
|
1452
|
+
"title": "Image",
|
1453
|
+
"description": "The image reference of a container image to use for created jobs. If not set, the latest Prefect image will be used.",
|
1454
|
+
"example": "docker.io/prefecthq/prefect:2-latest",
|
1455
|
+
"type": "string"
|
1456
|
+
},
|
1457
|
+
"service_account_name": {
|
1458
|
+
"title": "Service Account Name",
|
1459
|
+
"description": "The Kubernetes service account to use for job creation.",
|
1460
|
+
"type": "string"
|
1461
|
+
},
|
1462
|
+
"image_pull_policy": {
|
1463
|
+
"title": "Image Pull Policy",
|
1464
|
+
"description": "The Kubernetes image pull policy to use for job containers.",
|
1465
|
+
"default": "IfNotPresent",
|
1466
|
+
"enum": ["IfNotPresent", "Always", "Never"],
|
1467
|
+
"type": "string"
|
1468
|
+
},
|
1469
|
+
"finished_job_ttl": {
|
1470
|
+
"title": "Finished Job TTL",
|
1471
|
+
"description": "The number of seconds to retain jobs after completion. If set, finished jobs will be cleaned up by Kubernetes after the given delay. If not set, jobs will be retained indefinitely.",
|
1472
|
+
"type": "integer"
|
1473
|
+
},
|
1474
|
+
"job_watch_timeout_seconds": {
|
1475
|
+
"title": "Job Watch Timeout Seconds",
|
1476
|
+
"description": "Number of seconds to wait for each event emitted by a job before timing out. If not set, the worker will wait for each event indefinitely.",
|
1477
|
+
"type": "integer"
|
1478
|
+
},
|
1479
|
+
"pod_watch_timeout_seconds": {
|
1480
|
+
"title": "Pod Watch Timeout Seconds",
|
1481
|
+
"description": "Number of seconds to watch for pod creation before timing out.",
|
1482
|
+
"default": 60,
|
1483
|
+
"type": "integer"
|
1484
|
+
},
|
1485
|
+
"stream_output": {
|
1486
|
+
"title": "Stream Output",
|
1487
|
+
"description": "If set, output will be streamed from the job to local standard output.",
|
1488
|
+
"default": true,
|
1489
|
+
"type": "boolean"
|
1490
|
+
},
|
1491
|
+
"cluster_config": {
|
1492
|
+
"title": "Cluster Config",
|
1493
|
+
"description": "The Kubernetes cluster config to use for job creation.",
|
1494
|
+
"allOf": [
|
1495
|
+
{
|
1496
|
+
"$ref": "#/definitions/KubernetesClusterConfig"
|
1497
|
+
}
|
1498
|
+
]
|
1499
|
+
}
|
1500
|
+
},
|
1501
|
+
"definitions": {
|
1502
|
+
"KubernetesClusterConfig": {
|
1503
|
+
"title": "KubernetesClusterConfig",
|
1504
|
+
"description": "Stores configuration for interaction with Kubernetes clusters.\n\nSee `from_file` for creation.",
|
1505
|
+
"type": "object",
|
1506
|
+
"properties": {
|
1507
|
+
"config": {
|
1508
|
+
"title": "Config",
|
1509
|
+
"description": "The entire contents of a kubectl config file.",
|
1510
|
+
"type": "object"
|
1511
|
+
},
|
1512
|
+
"context_name": {
|
1513
|
+
"title": "Context Name",
|
1514
|
+
"description": "The name of the kubectl context to use.",
|
1515
|
+
"type": "string"
|
1516
|
+
}
|
1517
|
+
},
|
1518
|
+
"required": ["config", "context_name"],
|
1519
|
+
"block_type_slug": "kubernetes-cluster-config",
|
1520
|
+
"secret_fields": [],
|
1521
|
+
"block_schema_references": {}
|
1522
|
+
}
|
1523
|
+
}
|
1524
|
+
}
|
1525
|
+
},
|
1526
|
+
"description": "Execute flow runs within jobs scheduled on a Kubernetes cluster. Requires a Kubernetes cluster.",
|
1527
|
+
"display_name": "Kubernetes",
|
1528
|
+
"documentation_url": "https://prefecthq.github.io/prefect-kubernetes/worker/",
|
1529
|
+
"install_command": "pip install prefect-kubernetes",
|
1530
|
+
"is_beta": false,
|
1531
|
+
"logo_url": "https://cdn.sanity.io/images/3ugk85nk/production/2d0b896006ad463b49c28aaac14f31e00e32cfab-250x250.png",
|
1532
|
+
"type": "kubernetes"
|
1533
|
+
}
|
1534
|
+
}
|
1535
|
+
}
|