cloud-submit 0.5__tar.gz → 0.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cloud_submit-0.5 → cloud_submit-0.6}/PKG-INFO +1 -1
- {cloud_submit-0.5 → cloud_submit-0.6}/pyproject.toml +1 -1
- {cloud_submit-0.5 → cloud_submit-0.6}/src/cloud_submit/cli.py +51 -2
- {cloud_submit-0.5 → cloud_submit-0.6}/src/cloud_submit/controller.py +39 -0
- {cloud_submit-0.5 → cloud_submit-0.6}/src/cloud_submit/environment_handler.py +8 -0
- {cloud_submit-0.5 → cloud_submit-0.6}/src/cloud_submit/envs/aws/remote_environment_handler.py +168 -19
- {cloud_submit-0.5 → cloud_submit-0.6}/src/cloud_submit/envs/local/environment_handler.py +9 -0
- {cloud_submit-0.5 → cloud_submit-0.6}/src/cloud_submit/execution/config.py +6 -2
- {cloud_submit-0.5 → cloud_submit-0.6}/.gitignore +0 -0
- {cloud_submit-0.5 → cloud_submit-0.6}/LICENSE +0 -0
- {cloud_submit-0.5 → cloud_submit-0.6}/README.md +0 -0
- {cloud_submit-0.5 → cloud_submit-0.6}/docs/examples/aws/src/csub/build_config.py +0 -0
- {cloud_submit-0.5 → cloud_submit-0.6}/docs/examples/aws/src/fit.py +0 -0
- {cloud_submit-0.5 → cloud_submit-0.6}/docs/examples/aws/src/generate.py +0 -0
- {cloud_submit-0.5 → cloud_submit-0.6}/docs/examples/aws/src/parameters.py +0 -0
- {cloud_submit-0.5 → cloud_submit-0.6}/docs/examples/aws/userconfig/template.yaml +0 -0
- {cloud_submit-0.5 → cloud_submit-0.6}/docs/examples/basic/src/csub/build_config.py +0 -0
- {cloud_submit-0.5 → cloud_submit-0.6}/docs/examples/basic/src/fit.py +0 -0
- {cloud_submit-0.5 → cloud_submit-0.6}/docs/examples/basic/src/generate.py +0 -0
- {cloud_submit-0.5 → cloud_submit-0.6}/docs/examples/basic/src/parameters.py +0 -0
- {cloud_submit-0.5 → cloud_submit-0.6}/docs/examples/basic/userconfig/template.yaml +0 -0
- {cloud_submit-0.5 → cloud_submit-0.6}/docs/examples/multi-worker/src/csub/build_config.py +0 -0
- {cloud_submit-0.5 → cloud_submit-0.6}/docs/examples/multi-worker/src/fit.py +0 -0
- {cloud_submit-0.5 → cloud_submit-0.6}/docs/examples/multi-worker/src/generate.py +0 -0
- {cloud_submit-0.5 → cloud_submit-0.6}/docs/examples/multi-worker/src/parameters.py +0 -0
- {cloud_submit-0.5 → cloud_submit-0.6}/docs/examples/multi-worker/userconfig/template.yaml +0 -0
- {cloud_submit-0.5 → cloud_submit-0.6}/src/cloud_submit/__init__.py +0 -0
- {cloud_submit-0.5 → cloud_submit-0.6}/src/cloud_submit/config.py +0 -0
- {cloud_submit-0.5 → cloud_submit-0.6}/src/cloud_submit/envs/aws/__init__.py +0 -0
- {cloud_submit-0.5 → cloud_submit-0.6}/src/cloud_submit/envs/aws/local_environment_handler.py +0 -0
- {cloud_submit-0.5 → cloud_submit-0.6}/src/cloud_submit/envs/aws/local_execution_handler.py +0 -0
- {cloud_submit-0.5 → cloud_submit-0.6}/src/cloud_submit/envs/aws/remote_execution_handler.py +0 -0
- {cloud_submit-0.5 → cloud_submit-0.6}/src/cloud_submit/envs/aws/s3_tools.py +0 -0
- {cloud_submit-0.5 → cloud_submit-0.6}/src/cloud_submit/envs/local/execution_handler.py +0 -0
- {cloud_submit-0.5 → cloud_submit-0.6}/src/cloud_submit/execution/base_execution_handler.py +0 -0
- {cloud_submit-0.5 → cloud_submit-0.6}/src/cloud_submit/execution/execute.py +0 -0
- {cloud_submit-0.5 → cloud_submit-0.6}/src/cloud_submit/execution/utils.py +0 -0
- {cloud_submit-0.5 → cloud_submit-0.6}/src/cloud_submit/images.py +0 -0
- {cloud_submit-0.5 → cloud_submit-0.6}/src/cloud_submit/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: cloud-submit
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6
|
|
4
4
|
Summary: A tool for building processing pipelines in Python and executing them locally or in the cloud.
|
|
5
5
|
Project-URL: Homepage, https://github.com/mwiebusch78/cloud-submit
|
|
6
6
|
Project-URL: Bug Tracker, https://github.com/mwiebusch78/cloud-submit/issues
|
|
@@ -178,8 +178,7 @@ def list_pipelines(ctx, steps, pipelines):
|
|
|
178
178
|
Run the pipeline PIPELINE. Specify STEPS to select a subset of steps to run.
|
|
179
179
|
STEPS should be a comma-separated list of step names or step ranges of the form
|
|
180
180
|
start:end where start and end are step names. If steps is not specified all
|
|
181
|
-
steps are executed.
|
|
182
|
-
re-built (but you can use --no-rebuild to prevent that).
|
|
181
|
+
steps are executed.
|
|
183
182
|
"""
|
|
184
183
|
)
|
|
185
184
|
@click.option(
|
|
@@ -273,6 +272,56 @@ def run_pipeline(
|
|
|
273
272
|
abort(str(e))
|
|
274
273
|
|
|
275
274
|
|
|
275
|
+
@pipelines.command(
|
|
276
|
+
name='deploy',
|
|
277
|
+
help="""Deploy a pipeline.
|
|
278
|
+
|
|
279
|
+
Deploy the pipeline PIPELINE.
|
|
280
|
+
"""
|
|
281
|
+
)
|
|
282
|
+
@click.option(
|
|
283
|
+
'--env', '-e',
|
|
284
|
+
type=str,
|
|
285
|
+
default=None,
|
|
286
|
+
help='The name of the environment to use for the deployment.',
|
|
287
|
+
)
|
|
288
|
+
@click.option(
|
|
289
|
+
'--build-env',
|
|
290
|
+
type=str,
|
|
291
|
+
default=None,
|
|
292
|
+
help='The name of the environment to use for building images.',
|
|
293
|
+
)
|
|
294
|
+
@click.argument(
|
|
295
|
+
'pipeline',
|
|
296
|
+
type=str,
|
|
297
|
+
required=True,
|
|
298
|
+
)
|
|
299
|
+
@click.pass_context
|
|
300
|
+
def deploy_pipeline(
|
|
301
|
+
ctx,
|
|
302
|
+
env,
|
|
303
|
+
build_env,
|
|
304
|
+
pipeline,
|
|
305
|
+
):
|
|
306
|
+
init(ctx)
|
|
307
|
+
config = ctx.obj['config']
|
|
308
|
+
controller = ctx.obj['controller']
|
|
309
|
+
|
|
310
|
+
try:
|
|
311
|
+
pipeline = config.pipelines[pipeline]
|
|
312
|
+
except KeyError:
|
|
313
|
+
abort(f'Pipeline not found: {pipeline}')
|
|
314
|
+
|
|
315
|
+
try:
|
|
316
|
+
deployment_data = controller.deploy_pipeline(
|
|
317
|
+
pipeline.name,
|
|
318
|
+
env=env,
|
|
319
|
+
build_env=build_env,
|
|
320
|
+
)
|
|
321
|
+
except CloudSubmitError as e:
|
|
322
|
+
abort(str(e))
|
|
323
|
+
|
|
324
|
+
|
|
276
325
|
# images subgroup
|
|
277
326
|
|
|
278
327
|
|
|
@@ -444,6 +444,45 @@ class Controller:
|
|
|
444
444
|
'start_timestamp': now,
|
|
445
445
|
}
|
|
446
446
|
|
|
447
|
+
def deploy_pipeline(
|
|
448
|
+
self,
|
|
449
|
+
pipeline,
|
|
450
|
+
env=None,
|
|
451
|
+
build_env=None,
|
|
452
|
+
):
|
|
453
|
+
try:
|
|
454
|
+
pipeline = self._config.pipelines[pipeline]
|
|
455
|
+
except KeyError:
|
|
456
|
+
raise CloudSubmitError(f'Pipeline not found: {pipeline}')
|
|
457
|
+
|
|
458
|
+
if build_env is None:
|
|
459
|
+
build_env = env
|
|
460
|
+
|
|
461
|
+
env_handler = self._config.get_run_env(env)
|
|
462
|
+
now = dt.datetime.now(tz=dt.UTC)
|
|
463
|
+
build_id = env_handler.generate_build_id(now, None)
|
|
464
|
+
|
|
465
|
+
with self._config.in_project_root():
|
|
466
|
+
images = [step.image for step in pipeline.steps]
|
|
467
|
+
self.build_images(images, build_id=build_id, env=build_env)
|
|
468
|
+
refs = {}
|
|
469
|
+
for step in pipeline.steps:
|
|
470
|
+
image = self._config.images.get(step.image)
|
|
471
|
+
if image is None:
|
|
472
|
+
raise CloudSubmitError(
|
|
473
|
+
f'Cannot find declaration for image {step.image}.')
|
|
474
|
+
repo = env_handler.get_image_repo(image)
|
|
475
|
+
refs[step.name] = ':'.join([repo, build_id])
|
|
476
|
+
|
|
477
|
+
temp_path = 'temp/deploy'
|
|
478
|
+
ensure_path(temp_path, clear=True)
|
|
479
|
+
env_handler.deploy_pipeline(
|
|
480
|
+
pipeline=pipeline,
|
|
481
|
+
image_refs=refs,
|
|
482
|
+
temp_path=temp_path,
|
|
483
|
+
)
|
|
484
|
+
|
|
485
|
+
|
|
447
486
|
def print_logs(
|
|
448
487
|
self,
|
|
449
488
|
run_id,
|
{cloud_submit-0.5 → cloud_submit-0.6}/src/cloud_submit/envs/aws/remote_environment_handler.py
RENAMED
|
@@ -35,6 +35,8 @@ class RemoteAWSEnv(LocalAWSEnv):
|
|
|
35
35
|
aws_command='aws',
|
|
36
36
|
container_aws_command='aws',
|
|
37
37
|
log_group_prefix='/ecs/csub/',
|
|
38
|
+
scheduler_timezone=None,
|
|
39
|
+
scheduler_role_arn=None,
|
|
38
40
|
):
|
|
39
41
|
super().__init__(
|
|
40
42
|
name=name,
|
|
@@ -59,6 +61,8 @@ class RemoteAWSEnv(LocalAWSEnv):
|
|
|
59
61
|
self._ecs_task_role_arn = str(ecs_task_role_arn)
|
|
60
62
|
self._stepfunctions_role_arn = str(stepfunctions_role_arn)
|
|
61
63
|
self._log_group_prefix = str(log_group_prefix)
|
|
64
|
+
self._scheduler_timezone = scheduler_timezone
|
|
65
|
+
self._scheduler_role_arn = scheduler_role_arn
|
|
62
66
|
|
|
63
67
|
del self._envvars['CSUB_AWS_PROFILE']
|
|
64
68
|
del self._envvars['CSUB_AWS_REGION']
|
|
@@ -78,11 +82,12 @@ class RemoteAWSEnv(LocalAWSEnv):
|
|
|
78
82
|
os.path.join(path, 's3_tools.py'),
|
|
79
83
|
)
|
|
80
84
|
|
|
81
|
-
def build_image(self, path, image, build_id):
|
|
85
|
+
def build_image(self, path, image, build_id, docker_flags):
|
|
82
86
|
return self._build_image(
|
|
83
87
|
path=path,
|
|
84
88
|
image=image,
|
|
85
89
|
build_id=build_id,
|
|
90
|
+
docker_flags=docker_flags,
|
|
86
91
|
push=True,
|
|
87
92
|
)
|
|
88
93
|
|
|
@@ -90,8 +95,16 @@ class RemoteAWSEnv(LocalAWSEnv):
|
|
|
90
95
|
repo_name = self.get_image_repo(image)
|
|
91
96
|
return self._list_remote_image_tags(repo_name)
|
|
92
97
|
|
|
93
|
-
def _make_workflow_name(self, run_id):
|
|
94
|
-
|
|
98
|
+
def _make_workflow_name(self, pipeline_name, run_id):
|
|
99
|
+
if run_id is None:
|
|
100
|
+
parts = ['deployed', self._project, self._user, pipeline_name]
|
|
101
|
+
else:
|
|
102
|
+
parts = ['adhoc', self._project, self._user, run_id]
|
|
103
|
+
return '--'.join(parts)
|
|
104
|
+
|
|
105
|
+
def _make_schedule_name(self, pipeline_name):
|
|
106
|
+
parts = [self._project, self._user, pipeline_name]
|
|
107
|
+
return '--'.join(parts)
|
|
95
108
|
|
|
96
109
|
def _make_workflow_arn(self, workflow_name):
|
|
97
110
|
return ':'.join([
|
|
@@ -100,6 +113,13 @@ class RemoteAWSEnv(LocalAWSEnv):
|
|
|
100
113
|
'stateMachine', workflow_name,
|
|
101
114
|
])
|
|
102
115
|
|
|
116
|
+
def _make_schedule_arn(self, schedule_name):
|
|
117
|
+
return ':'.join([
|
|
118
|
+
'arn', 'aws', 'scheduler',
|
|
119
|
+
self._aws_region, self._aws_account_id,
|
|
120
|
+
f'schedule/default/{schedule_name}',
|
|
121
|
+
])
|
|
122
|
+
|
|
103
123
|
def _check_workflow_exists(self, workflow_name):
|
|
104
124
|
workflow_arn = self._make_workflow_arn(workflow_name)
|
|
105
125
|
result = run_command(
|
|
@@ -118,6 +138,24 @@ class RemoteAWSEnv(LocalAWSEnv):
|
|
|
118
138
|
arns = set(s['stateMachineArn'] for s in result['stateMachines'])
|
|
119
139
|
return workflow_arn in arns
|
|
120
140
|
|
|
141
|
+
def _check_schedule_exists(self, schedule_name):
|
|
142
|
+
schedule_arn = self._make_schedule_arn(schedule_name)
|
|
143
|
+
result = run_command(
|
|
144
|
+
[
|
|
145
|
+
self._aws_command,
|
|
146
|
+
'scheduler',
|
|
147
|
+
'list-schedules',
|
|
148
|
+
'--profile', self._aws_profile,
|
|
149
|
+
'--region', self._aws_region,
|
|
150
|
+
],
|
|
151
|
+
stdout=subprocess.PIPE,
|
|
152
|
+
text=True,
|
|
153
|
+
hide_stderr=True,
|
|
154
|
+
)
|
|
155
|
+
result = json.loads(result.stdout)
|
|
156
|
+
arns = set(s['Arn'] for s in result['Schedules'])
|
|
157
|
+
return schedule_arn in arns
|
|
158
|
+
|
|
121
159
|
def _group_steps(self, pipeline, image_refs):
|
|
122
160
|
groups = []
|
|
123
161
|
last_spec = None
|
|
@@ -156,11 +194,20 @@ class RemoteAWSEnv(LocalAWSEnv):
|
|
|
156
194
|
for name, value in self._secrets.items()
|
|
157
195
|
if value is not None
|
|
158
196
|
]
|
|
197
|
+
if timestamp is None or run_id is None:
|
|
198
|
+
timestamp = "{% $states.context.Execution.StartTime %}"
|
|
199
|
+
run_id = (
|
|
200
|
+
"{% $fromMillis($toMillis($states.context.Execution.StartTime), "
|
|
201
|
+
"'[Y0001][M01][D01]-[H01][m01][s01]') %}"
|
|
202
|
+
)
|
|
203
|
+
else:
|
|
204
|
+
timestamp = timestamp.isoformat()
|
|
205
|
+
|
|
159
206
|
return {
|
|
160
207
|
f"RegisterTask{task_index}": {
|
|
161
208
|
"Type": "Task",
|
|
162
209
|
"Resource": "arn:aws:states:::aws-sdk:ecs:registerTaskDefinition",
|
|
163
|
-
"
|
|
210
|
+
"Arguments": {
|
|
164
211
|
"Family": f"{workflow_name}--{task_index}",
|
|
165
212
|
"RequiresCompatibilities": ["EC2"],
|
|
166
213
|
"NetworkMode": "host",
|
|
@@ -183,7 +230,7 @@ class RemoteAWSEnv(LocalAWSEnv):
|
|
|
183
230
|
"Environment": [
|
|
184
231
|
{
|
|
185
232
|
"Name": "CSUB_TIMESTAMP",
|
|
186
|
-
"Value": timestamp
|
|
233
|
+
"Value": timestamp,
|
|
187
234
|
},
|
|
188
235
|
{
|
|
189
236
|
"Name": "CSUB_RUN_ID",
|
|
@@ -219,15 +266,16 @@ class RemoteAWSEnv(LocalAWSEnv):
|
|
|
219
266
|
}
|
|
220
267
|
]
|
|
221
268
|
},
|
|
222
|
-
"
|
|
269
|
+
"Output": "{% {'RegisteredTask': $states.result} %}",
|
|
223
270
|
"Next": f"RunTask{task_index}"
|
|
224
271
|
},
|
|
225
272
|
f"RunTask{task_index}": {
|
|
226
273
|
"Type": "Task",
|
|
227
274
|
"Resource": "arn:aws:states:::ecs:runTask.sync",
|
|
228
|
-
"
|
|
275
|
+
"Arguments": {
|
|
229
276
|
"Cluster": self._ecs_cluster_arn,
|
|
230
|
-
"TaskDefinition
|
|
277
|
+
"TaskDefinition":
|
|
278
|
+
"{% $states.input.RegisteredTask.TaskDefinition.TaskDefinitionArn %}",
|
|
231
279
|
"CapacityProviderStrategy": [
|
|
232
280
|
{
|
|
233
281
|
"CapacityProvider": self._ecs_capacity_provider,
|
|
@@ -251,11 +299,11 @@ class RemoteAWSEnv(LocalAWSEnv):
|
|
|
251
299
|
}
|
|
252
300
|
]
|
|
253
301
|
},
|
|
254
|
-
"
|
|
302
|
+
"Output": "{% $merge([$states.input, {'ExecutionResult': $states.result}]) %}",
|
|
255
303
|
"Catch": [
|
|
256
304
|
{
|
|
257
305
|
"ErrorEquals": ["States.ALL"],
|
|
258
|
-
"
|
|
306
|
+
"Output": "{% $merge([$states.input, {'ExecutionError': $states.errorOutput}]) %}",
|
|
259
307
|
"Next": f"DeregisterTask{task_index}OnFailure"
|
|
260
308
|
}
|
|
261
309
|
],
|
|
@@ -264,8 +312,8 @@ class RemoteAWSEnv(LocalAWSEnv):
|
|
|
264
312
|
f"DeregisterTask{task_index}OnSuccess": {
|
|
265
313
|
"Type": "Task",
|
|
266
314
|
"Resource": "arn:aws:states:::aws-sdk:ecs:deregisterTaskDefinition",
|
|
267
|
-
"
|
|
268
|
-
"TaskDefinition
|
|
315
|
+
"Arguments": {
|
|
316
|
+
"TaskDefinition": "{% $states.input.RegisteredTask.TaskDefinition.TaskDefinitionArn %}"
|
|
269
317
|
},
|
|
270
318
|
("End" if is_last else "Next"): (
|
|
271
319
|
True if is_last else f"RegisterTask{task_index + 1}"),
|
|
@@ -273,8 +321,8 @@ class RemoteAWSEnv(LocalAWSEnv):
|
|
|
273
321
|
f"DeregisterTask{task_index}OnFailure": {
|
|
274
322
|
"Type": "Task",
|
|
275
323
|
"Resource": "arn:aws:states:::aws-sdk:ecs:deregisterTaskDefinition",
|
|
276
|
-
"
|
|
277
|
-
"TaskDefinition
|
|
324
|
+
"Arguments": {
|
|
325
|
+
"TaskDefinition": "{% $states.input.RegisteredTask.TaskDefinition.TaskDefinitionArn %}"
|
|
278
326
|
},
|
|
279
327
|
"Next": "WorkflowFailed"
|
|
280
328
|
},
|
|
@@ -324,6 +372,7 @@ class RemoteAWSEnv(LocalAWSEnv):
|
|
|
324
372
|
}
|
|
325
373
|
|
|
326
374
|
return {
|
|
375
|
+
"QueryLanguage": "JSONata",
|
|
327
376
|
"StartAt": "RegisterTask0",
|
|
328
377
|
"States": states,
|
|
329
378
|
}
|
|
@@ -337,7 +386,7 @@ class RemoteAWSEnv(LocalAWSEnv):
|
|
|
337
386
|
run_id,
|
|
338
387
|
temp_path
|
|
339
388
|
):
|
|
340
|
-
workflow_name = self._make_workflow_name(run_id)
|
|
389
|
+
workflow_name = self._make_workflow_name(None, run_id)
|
|
341
390
|
workflow = self._build_workflow(
|
|
342
391
|
pipeline=pipeline,
|
|
343
392
|
image_refs=image_refs,
|
|
@@ -348,7 +397,7 @@ class RemoteAWSEnv(LocalAWSEnv):
|
|
|
348
397
|
|
|
349
398
|
workflow_path = os.path.join(temp_path, f'{workflow_name}.json')
|
|
350
399
|
with open(workflow_path, 'w') as stream:
|
|
351
|
-
json.dump(workflow, stream)
|
|
400
|
+
json.dump(workflow, stream, indent=4)
|
|
352
401
|
|
|
353
402
|
workflow_arn = self._make_workflow_arn(workflow_name)
|
|
354
403
|
if self._check_workflow_exists(workflow_name):
|
|
@@ -381,8 +430,9 @@ class RemoteAWSEnv(LocalAWSEnv):
|
|
|
381
430
|
)
|
|
382
431
|
|
|
383
432
|
print('Clearing output and temporary artifacts.')
|
|
384
|
-
|
|
385
|
-
|
|
433
|
+
for kind, artifacts in overwrite_artifacts.items():
|
|
434
|
+
self.remove_remote_artifacts(
|
|
435
|
+
kind, artifacts, [[run_id]]*len(artifacts))
|
|
386
436
|
|
|
387
437
|
result = run_command(
|
|
388
438
|
[
|
|
@@ -402,7 +452,7 @@ class RemoteAWSEnv(LocalAWSEnv):
|
|
|
402
452
|
|
|
403
453
|
def print_logs(self, run_id, start_timestamp, stream=False):
|
|
404
454
|
print('\nStreaming logs...')
|
|
405
|
-
workflow_name = self._make_workflow_name(run_id)
|
|
455
|
+
workflow_name = self._make_workflow_name(None, run_id)
|
|
406
456
|
command = [
|
|
407
457
|
self._aws_command,
|
|
408
458
|
'logs',
|
|
@@ -416,3 +466,102 @@ class RemoteAWSEnv(LocalAWSEnv):
|
|
|
416
466
|
command.append('--follow')
|
|
417
467
|
command.append(self._log_group_prefix + workflow_name)
|
|
418
468
|
run_command(command)
|
|
469
|
+
|
|
470
|
+
def deploy_pipeline(
|
|
471
|
+
self,
|
|
472
|
+
pipeline,
|
|
473
|
+
image_refs,
|
|
474
|
+
temp_path,
|
|
475
|
+
schedule=True,
|
|
476
|
+
):
|
|
477
|
+
workflow_name = self._make_workflow_name(pipeline.name, None)
|
|
478
|
+
workflow = self._build_workflow(
|
|
479
|
+
pipeline=pipeline,
|
|
480
|
+
image_refs=image_refs,
|
|
481
|
+
workflow_name=workflow_name,
|
|
482
|
+
timestamp=None,
|
|
483
|
+
run_id=None,
|
|
484
|
+
)
|
|
485
|
+
|
|
486
|
+
workflow_path = os.path.join(temp_path, f'{workflow_name}.json')
|
|
487
|
+
with open(workflow_path, 'w') as stream:
|
|
488
|
+
json.dump(workflow, stream, indent=4)
|
|
489
|
+
|
|
490
|
+
workflow_arn = self._make_workflow_arn(workflow_name)
|
|
491
|
+
if self._check_workflow_exists(workflow_name):
|
|
492
|
+
run_command(
|
|
493
|
+
[
|
|
494
|
+
self._aws_command,
|
|
495
|
+
'stepfunctions',
|
|
496
|
+
'update-state-machine',
|
|
497
|
+
'--profile', self._aws_profile,
|
|
498
|
+
'--region', self._aws_region,
|
|
499
|
+
'--state-machine-arn', workflow_arn,
|
|
500
|
+
'--role-arn', self._stepfunctions_role_arn,
|
|
501
|
+
'--definition', f'file://{workflow_path}',
|
|
502
|
+
],
|
|
503
|
+
stdout=subprocess.DEVNULL,
|
|
504
|
+
)
|
|
505
|
+
else:
|
|
506
|
+
run_command(
|
|
507
|
+
[
|
|
508
|
+
self._aws_command,
|
|
509
|
+
'stepfunctions',
|
|
510
|
+
'create-state-machine',
|
|
511
|
+
'--profile', self._aws_profile,
|
|
512
|
+
'--region', self._aws_region,
|
|
513
|
+
'--name', workflow_name,
|
|
514
|
+
'--role-arn', self._stepfunctions_role_arn,
|
|
515
|
+
'--definition', f'file://{workflow_path}',
|
|
516
|
+
],
|
|
517
|
+
stdout=subprocess.DEVNULL,
|
|
518
|
+
)
|
|
519
|
+
print(f'Created step function {workflow_arn}.')
|
|
520
|
+
|
|
521
|
+
if schedule and pipeline.schedule is not None:
|
|
522
|
+
if self._scheduler_timezone is None:
|
|
523
|
+
raise CloudSubmitError(
|
|
524
|
+
'You must specify the schedule timezone '
|
|
525
|
+
'to use automatic scheduling.'
|
|
526
|
+
)
|
|
527
|
+
if self._scheduler_role_arn is None:
|
|
528
|
+
raise CloudSubmitError(
|
|
529
|
+
'You must specify the scheduler role ARN '
|
|
530
|
+
'to use automatic scheduling.'
|
|
531
|
+
)
|
|
532
|
+
schedule_name = self._make_schedule_name(pipeline.name)
|
|
533
|
+
target = json.dumps({
|
|
534
|
+
'Arn': workflow_arn,
|
|
535
|
+
'RoleArn': self._scheduler_role_arn,
|
|
536
|
+
})
|
|
537
|
+
options = [
|
|
538
|
+
'--profile', self._aws_profile,
|
|
539
|
+
'--region', self._aws_region,
|
|
540
|
+
'--name', schedule_name,
|
|
541
|
+
'--schedule-expression', f'cron({pipeline.schedule})',
|
|
542
|
+
'--schedule-expression-timezone', self._scheduler_timezone,
|
|
543
|
+
'--flexible-time-window', '{"Mode": "OFF"}',
|
|
544
|
+
'--target', target,
|
|
545
|
+
]
|
|
546
|
+
if self._check_schedule_exists(schedule_name):
|
|
547
|
+
run_command(
|
|
548
|
+
[
|
|
549
|
+
self._aws_command,
|
|
550
|
+
'scheduler',
|
|
551
|
+
'update-schedule',
|
|
552
|
+
*options
|
|
553
|
+
],
|
|
554
|
+
stdout=subprocess.DEVNULL,
|
|
555
|
+
)
|
|
556
|
+
else:
|
|
557
|
+
run_command(
|
|
558
|
+
[
|
|
559
|
+
self._aws_command,
|
|
560
|
+
'scheduler',
|
|
561
|
+
'create-schedule',
|
|
562
|
+
*options,
|
|
563
|
+
],
|
|
564
|
+
stdout=subprocess.DEVNULL,
|
|
565
|
+
)
|
|
566
|
+
print(f'Created schedule {schedule_name}.')
|
|
567
|
+
|
|
@@ -170,3 +170,12 @@ class LocalEnv(EnvironmentHandler):
|
|
|
170
170
|
def print_logs(self, run_id, start_timestamp, stream=False):
|
|
171
171
|
raise CloudSubmitError(
|
|
172
172
|
'Log retrieval is not supported by this environment.')
|
|
173
|
+
|
|
174
|
+
def deploy_pipeline(
|
|
175
|
+
self,
|
|
176
|
+
pipeline,
|
|
177
|
+
image_refs,
|
|
178
|
+
temp_path
|
|
179
|
+
):
|
|
180
|
+
raise CloudSubmitError(
|
|
181
|
+
'Deployment is not supported by this environment.')
|
|
@@ -194,10 +194,12 @@ class Pipeline:
|
|
|
194
194
|
name,
|
|
195
195
|
steps,
|
|
196
196
|
default_submit_timestamp=None,
|
|
197
|
+
schedule=None,
|
|
197
198
|
):
|
|
198
199
|
self.name = name
|
|
199
200
|
self.steps = list(steps)
|
|
200
201
|
self.default_submit_timestamp = to_utc(default_submit_timestamp)
|
|
202
|
+
self.schedule = schedule
|
|
201
203
|
|
|
202
204
|
step_names = set()
|
|
203
205
|
for step in self.steps:
|
|
@@ -212,7 +214,8 @@ class Pipeline:
|
|
|
212
214
|
return {
|
|
213
215
|
'name': self.name,
|
|
214
216
|
'steps': [s.to_dict() for s in self.steps],
|
|
215
|
-
'default_submit_timestamp': self.default_submit_timestamp
|
|
217
|
+
'default_submit_timestamp': self.default_submit_timestamp,
|
|
218
|
+
'schedule': self.schedule,
|
|
216
219
|
}
|
|
217
220
|
|
|
218
221
|
@staticmethod
|
|
@@ -220,7 +223,8 @@ class Pipeline:
|
|
|
220
223
|
return Pipeline(
|
|
221
224
|
name=obj['name'],
|
|
222
225
|
steps=[Step.from_dict(s) for s in obj['steps']],
|
|
223
|
-
default_submit_timestamp=obj['default_submit_timestamp']
|
|
226
|
+
default_submit_timestamp=obj['default_submit_timestamp'],
|
|
227
|
+
schedule=obj['schedule'],
|
|
224
228
|
)
|
|
225
229
|
|
|
226
230
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{cloud_submit-0.5 → cloud_submit-0.6}/src/cloud_submit/envs/aws/local_environment_handler.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|