cloud-submit 0.5.1__tar.gz → 0.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cloud_submit-0.5.1 → cloud_submit-0.6}/PKG-INFO +1 -1
- {cloud_submit-0.5.1 → cloud_submit-0.6}/pyproject.toml +1 -1
- {cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/cli.py +51 -2
- {cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/controller.py +39 -0
- {cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/environment_handler.py +8 -0
- {cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/envs/aws/remote_environment_handler.py +163 -16
- {cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/envs/local/environment_handler.py +9 -0
- {cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/execution/config.py +6 -2
- {cloud_submit-0.5.1 → cloud_submit-0.6}/.gitignore +0 -0
- {cloud_submit-0.5.1 → cloud_submit-0.6}/LICENSE +0 -0
- {cloud_submit-0.5.1 → cloud_submit-0.6}/README.md +0 -0
- {cloud_submit-0.5.1 → cloud_submit-0.6}/docs/examples/aws/src/csub/build_config.py +0 -0
- {cloud_submit-0.5.1 → cloud_submit-0.6}/docs/examples/aws/src/fit.py +0 -0
- {cloud_submit-0.5.1 → cloud_submit-0.6}/docs/examples/aws/src/generate.py +0 -0
- {cloud_submit-0.5.1 → cloud_submit-0.6}/docs/examples/aws/src/parameters.py +0 -0
- {cloud_submit-0.5.1 → cloud_submit-0.6}/docs/examples/aws/userconfig/template.yaml +0 -0
- {cloud_submit-0.5.1 → cloud_submit-0.6}/docs/examples/basic/src/csub/build_config.py +0 -0
- {cloud_submit-0.5.1 → cloud_submit-0.6}/docs/examples/basic/src/fit.py +0 -0
- {cloud_submit-0.5.1 → cloud_submit-0.6}/docs/examples/basic/src/generate.py +0 -0
- {cloud_submit-0.5.1 → cloud_submit-0.6}/docs/examples/basic/src/parameters.py +0 -0
- {cloud_submit-0.5.1 → cloud_submit-0.6}/docs/examples/basic/userconfig/template.yaml +0 -0
- {cloud_submit-0.5.1 → cloud_submit-0.6}/docs/examples/multi-worker/src/csub/build_config.py +0 -0
- {cloud_submit-0.5.1 → cloud_submit-0.6}/docs/examples/multi-worker/src/fit.py +0 -0
- {cloud_submit-0.5.1 → cloud_submit-0.6}/docs/examples/multi-worker/src/generate.py +0 -0
- {cloud_submit-0.5.1 → cloud_submit-0.6}/docs/examples/multi-worker/src/parameters.py +0 -0
- {cloud_submit-0.5.1 → cloud_submit-0.6}/docs/examples/multi-worker/userconfig/template.yaml +0 -0
- {cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/__init__.py +0 -0
- {cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/config.py +0 -0
- {cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/envs/aws/__init__.py +0 -0
- {cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/envs/aws/local_environment_handler.py +0 -0
- {cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/envs/aws/local_execution_handler.py +0 -0
- {cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/envs/aws/remote_execution_handler.py +0 -0
- {cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/envs/aws/s3_tools.py +0 -0
- {cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/envs/local/execution_handler.py +0 -0
- {cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/execution/base_execution_handler.py +0 -0
- {cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/execution/execute.py +0 -0
- {cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/execution/utils.py +0 -0
- {cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/images.py +0 -0
- {cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: cloud-submit
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6
|
|
4
4
|
Summary: A tool for building processing pipelines in Python and executing them locally or in the cloud.
|
|
5
5
|
Project-URL: Homepage, https://github.com/mwiebusch78/cloud-submit
|
|
6
6
|
Project-URL: Bug Tracker, https://github.com/mwiebusch78/cloud-submit/issues
|
|
@@ -178,8 +178,7 @@ def list_pipelines(ctx, steps, pipelines):
|
|
|
178
178
|
Run the pipeline PIPELINE. Specify STEPS to select a subset of steps to run.
|
|
179
179
|
STEPS should be a comma-separated list of step names or step ranges of the form
|
|
180
180
|
start:end where start and end are step names. If steps is not specified all
|
|
181
|
-
steps are executed.
|
|
182
|
-
re-built (but you can use --no-rebuild to prevent that).
|
|
181
|
+
steps are executed.
|
|
183
182
|
"""
|
|
184
183
|
)
|
|
185
184
|
@click.option(
|
|
@@ -273,6 +272,56 @@ def run_pipeline(
|
|
|
273
272
|
abort(str(e))
|
|
274
273
|
|
|
275
274
|
|
|
275
|
+
@pipelines.command(
|
|
276
|
+
name='deploy',
|
|
277
|
+
help="""Deploy a pipeline.
|
|
278
|
+
|
|
279
|
+
Deploy the pipeline PIPELINE.
|
|
280
|
+
"""
|
|
281
|
+
)
|
|
282
|
+
@click.option(
|
|
283
|
+
'--env', '-e',
|
|
284
|
+
type=str,
|
|
285
|
+
default=None,
|
|
286
|
+
help='The name of the environment to use for the deployment.',
|
|
287
|
+
)
|
|
288
|
+
@click.option(
|
|
289
|
+
'--build-env',
|
|
290
|
+
type=str,
|
|
291
|
+
default=None,
|
|
292
|
+
help='The name of the environment to use for building images.',
|
|
293
|
+
)
|
|
294
|
+
@click.argument(
|
|
295
|
+
'pipeline',
|
|
296
|
+
type=str,
|
|
297
|
+
required=True,
|
|
298
|
+
)
|
|
299
|
+
@click.pass_context
|
|
300
|
+
def deploy_pipeline(
|
|
301
|
+
ctx,
|
|
302
|
+
env,
|
|
303
|
+
build_env,
|
|
304
|
+
pipeline,
|
|
305
|
+
):
|
|
306
|
+
init(ctx)
|
|
307
|
+
config = ctx.obj['config']
|
|
308
|
+
controller = ctx.obj['controller']
|
|
309
|
+
|
|
310
|
+
try:
|
|
311
|
+
pipeline = config.pipelines[pipeline]
|
|
312
|
+
except KeyError:
|
|
313
|
+
abort(f'Pipeline not found: {pipeline}')
|
|
314
|
+
|
|
315
|
+
try:
|
|
316
|
+
deployment_data = controller.deploy_pipeline(
|
|
317
|
+
pipeline.name,
|
|
318
|
+
env=env,
|
|
319
|
+
build_env=build_env,
|
|
320
|
+
)
|
|
321
|
+
except CloudSubmitError as e:
|
|
322
|
+
abort(str(e))
|
|
323
|
+
|
|
324
|
+
|
|
276
325
|
# images subgroup
|
|
277
326
|
|
|
278
327
|
|
|
@@ -444,6 +444,45 @@ class Controller:
|
|
|
444
444
|
'start_timestamp': now,
|
|
445
445
|
}
|
|
446
446
|
|
|
447
|
+
def deploy_pipeline(
|
|
448
|
+
self,
|
|
449
|
+
pipeline,
|
|
450
|
+
env=None,
|
|
451
|
+
build_env=None,
|
|
452
|
+
):
|
|
453
|
+
try:
|
|
454
|
+
pipeline = self._config.pipelines[pipeline]
|
|
455
|
+
except KeyError:
|
|
456
|
+
raise CloudSubmitError(f'Pipeline not found: {pipeline}')
|
|
457
|
+
|
|
458
|
+
if build_env is None:
|
|
459
|
+
build_env = env
|
|
460
|
+
|
|
461
|
+
env_handler = self._config.get_run_env(env)
|
|
462
|
+
now = dt.datetime.now(tz=dt.UTC)
|
|
463
|
+
build_id = env_handler.generate_build_id(now, None)
|
|
464
|
+
|
|
465
|
+
with self._config.in_project_root():
|
|
466
|
+
images = [step.image for step in pipeline.steps]
|
|
467
|
+
self.build_images(images, build_id=build_id, env=build_env)
|
|
468
|
+
refs = {}
|
|
469
|
+
for step in pipeline.steps:
|
|
470
|
+
image = self._config.images.get(step.image)
|
|
471
|
+
if image is None:
|
|
472
|
+
raise CloudSubmitError(
|
|
473
|
+
f'Cannot find declaration for image {step.image}.')
|
|
474
|
+
repo = env_handler.get_image_repo(image)
|
|
475
|
+
refs[step.name] = ':'.join([repo, build_id])
|
|
476
|
+
|
|
477
|
+
temp_path = 'temp/deploy'
|
|
478
|
+
ensure_path(temp_path, clear=True)
|
|
479
|
+
env_handler.deploy_pipeline(
|
|
480
|
+
pipeline=pipeline,
|
|
481
|
+
image_refs=refs,
|
|
482
|
+
temp_path=temp_path,
|
|
483
|
+
)
|
|
484
|
+
|
|
485
|
+
|
|
447
486
|
def print_logs(
|
|
448
487
|
self,
|
|
449
488
|
run_id,
|
{cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/envs/aws/remote_environment_handler.py
RENAMED
|
@@ -35,6 +35,8 @@ class RemoteAWSEnv(LocalAWSEnv):
|
|
|
35
35
|
aws_command='aws',
|
|
36
36
|
container_aws_command='aws',
|
|
37
37
|
log_group_prefix='/ecs/csub/',
|
|
38
|
+
scheduler_timezone=None,
|
|
39
|
+
scheduler_role_arn=None,
|
|
38
40
|
):
|
|
39
41
|
super().__init__(
|
|
40
42
|
name=name,
|
|
@@ -59,6 +61,8 @@ class RemoteAWSEnv(LocalAWSEnv):
|
|
|
59
61
|
self._ecs_task_role_arn = str(ecs_task_role_arn)
|
|
60
62
|
self._stepfunctions_role_arn = str(stepfunctions_role_arn)
|
|
61
63
|
self._log_group_prefix = str(log_group_prefix)
|
|
64
|
+
self._scheduler_timezone = scheduler_timezone
|
|
65
|
+
self._scheduler_role_arn = scheduler_role_arn
|
|
62
66
|
|
|
63
67
|
del self._envvars['CSUB_AWS_PROFILE']
|
|
64
68
|
del self._envvars['CSUB_AWS_REGION']
|
|
@@ -91,8 +95,16 @@ class RemoteAWSEnv(LocalAWSEnv):
|
|
|
91
95
|
repo_name = self.get_image_repo(image)
|
|
92
96
|
return self._list_remote_image_tags(repo_name)
|
|
93
97
|
|
|
94
|
-
def _make_workflow_name(self, run_id):
|
|
95
|
-
|
|
98
|
+
def _make_workflow_name(self, pipeline_name, run_id):
|
|
99
|
+
if run_id is None:
|
|
100
|
+
parts = ['deployed', self._project, self._user, pipeline_name]
|
|
101
|
+
else:
|
|
102
|
+
parts = ['adhoc', self._project, self._user, run_id]
|
|
103
|
+
return '--'.join(parts)
|
|
104
|
+
|
|
105
|
+
def _make_schedule_name(self, pipeline_name):
|
|
106
|
+
parts = [self._project, self._user, pipeline_name]
|
|
107
|
+
return '--'.join(parts)
|
|
96
108
|
|
|
97
109
|
def _make_workflow_arn(self, workflow_name):
|
|
98
110
|
return ':'.join([
|
|
@@ -101,6 +113,13 @@ class RemoteAWSEnv(LocalAWSEnv):
|
|
|
101
113
|
'stateMachine', workflow_name,
|
|
102
114
|
])
|
|
103
115
|
|
|
116
|
+
def _make_schedule_arn(self, schedule_name):
|
|
117
|
+
return ':'.join([
|
|
118
|
+
'arn', 'aws', 'scheduler',
|
|
119
|
+
self._aws_region, self._aws_account_id,
|
|
120
|
+
f'schedule/default/{schedule_name}',
|
|
121
|
+
])
|
|
122
|
+
|
|
104
123
|
def _check_workflow_exists(self, workflow_name):
|
|
105
124
|
workflow_arn = self._make_workflow_arn(workflow_name)
|
|
106
125
|
result = run_command(
|
|
@@ -119,6 +138,24 @@ class RemoteAWSEnv(LocalAWSEnv):
|
|
|
119
138
|
arns = set(s['stateMachineArn'] for s in result['stateMachines'])
|
|
120
139
|
return workflow_arn in arns
|
|
121
140
|
|
|
141
|
+
def _check_schedule_exists(self, schedule_name):
|
|
142
|
+
schedule_arn = self._make_schedule_arn(schedule_name)
|
|
143
|
+
result = run_command(
|
|
144
|
+
[
|
|
145
|
+
self._aws_command,
|
|
146
|
+
'scheduler',
|
|
147
|
+
'list-schedules',
|
|
148
|
+
'--profile', self._aws_profile,
|
|
149
|
+
'--region', self._aws_region,
|
|
150
|
+
],
|
|
151
|
+
stdout=subprocess.PIPE,
|
|
152
|
+
text=True,
|
|
153
|
+
hide_stderr=True,
|
|
154
|
+
)
|
|
155
|
+
result = json.loads(result.stdout)
|
|
156
|
+
arns = set(s['Arn'] for s in result['Schedules'])
|
|
157
|
+
return schedule_arn in arns
|
|
158
|
+
|
|
122
159
|
def _group_steps(self, pipeline, image_refs):
|
|
123
160
|
groups = []
|
|
124
161
|
last_spec = None
|
|
@@ -157,11 +194,20 @@ class RemoteAWSEnv(LocalAWSEnv):
|
|
|
157
194
|
for name, value in self._secrets.items()
|
|
158
195
|
if value is not None
|
|
159
196
|
]
|
|
197
|
+
if timestamp is None or run_id is None:
|
|
198
|
+
timestamp = "{% $states.context.Execution.StartTime %}"
|
|
199
|
+
run_id = (
|
|
200
|
+
"{% $fromMillis($toMillis($states.context.Execution.StartTime), "
|
|
201
|
+
"'[Y0001][M01][D01]-[H01][m01][s01]') %}"
|
|
202
|
+
)
|
|
203
|
+
else:
|
|
204
|
+
timestamp = timestamp.isoformat()
|
|
205
|
+
|
|
160
206
|
return {
|
|
161
207
|
f"RegisterTask{task_index}": {
|
|
162
208
|
"Type": "Task",
|
|
163
209
|
"Resource": "arn:aws:states:::aws-sdk:ecs:registerTaskDefinition",
|
|
164
|
-
"
|
|
210
|
+
"Arguments": {
|
|
165
211
|
"Family": f"{workflow_name}--{task_index}",
|
|
166
212
|
"RequiresCompatibilities": ["EC2"],
|
|
167
213
|
"NetworkMode": "host",
|
|
@@ -184,7 +230,7 @@ class RemoteAWSEnv(LocalAWSEnv):
|
|
|
184
230
|
"Environment": [
|
|
185
231
|
{
|
|
186
232
|
"Name": "CSUB_TIMESTAMP",
|
|
187
|
-
"Value": timestamp
|
|
233
|
+
"Value": timestamp,
|
|
188
234
|
},
|
|
189
235
|
{
|
|
190
236
|
"Name": "CSUB_RUN_ID",
|
|
@@ -220,15 +266,16 @@ class RemoteAWSEnv(LocalAWSEnv):
|
|
|
220
266
|
}
|
|
221
267
|
]
|
|
222
268
|
},
|
|
223
|
-
"
|
|
269
|
+
"Output": "{% {'RegisteredTask': $states.result} %}",
|
|
224
270
|
"Next": f"RunTask{task_index}"
|
|
225
271
|
},
|
|
226
272
|
f"RunTask{task_index}": {
|
|
227
273
|
"Type": "Task",
|
|
228
274
|
"Resource": "arn:aws:states:::ecs:runTask.sync",
|
|
229
|
-
"
|
|
275
|
+
"Arguments": {
|
|
230
276
|
"Cluster": self._ecs_cluster_arn,
|
|
231
|
-
"TaskDefinition
|
|
277
|
+
"TaskDefinition":
|
|
278
|
+
"{% $states.input.RegisteredTask.TaskDefinition.TaskDefinitionArn %}",
|
|
232
279
|
"CapacityProviderStrategy": [
|
|
233
280
|
{
|
|
234
281
|
"CapacityProvider": self._ecs_capacity_provider,
|
|
@@ -252,11 +299,11 @@ class RemoteAWSEnv(LocalAWSEnv):
|
|
|
252
299
|
}
|
|
253
300
|
]
|
|
254
301
|
},
|
|
255
|
-
"
|
|
302
|
+
"Output": "{% $merge([$states.input, {'ExecutionResult': $states.result}]) %}",
|
|
256
303
|
"Catch": [
|
|
257
304
|
{
|
|
258
305
|
"ErrorEquals": ["States.ALL"],
|
|
259
|
-
"
|
|
306
|
+
"Output": "{% $merge([$states.input, {'ExecutionError': $states.errorOutput}]) %}",
|
|
260
307
|
"Next": f"DeregisterTask{task_index}OnFailure"
|
|
261
308
|
}
|
|
262
309
|
],
|
|
@@ -265,8 +312,8 @@ class RemoteAWSEnv(LocalAWSEnv):
|
|
|
265
312
|
f"DeregisterTask{task_index}OnSuccess": {
|
|
266
313
|
"Type": "Task",
|
|
267
314
|
"Resource": "arn:aws:states:::aws-sdk:ecs:deregisterTaskDefinition",
|
|
268
|
-
"
|
|
269
|
-
"TaskDefinition
|
|
315
|
+
"Arguments": {
|
|
316
|
+
"TaskDefinition": "{% $states.input.RegisteredTask.TaskDefinition.TaskDefinitionArn %}"
|
|
270
317
|
},
|
|
271
318
|
("End" if is_last else "Next"): (
|
|
272
319
|
True if is_last else f"RegisterTask{task_index + 1}"),
|
|
@@ -274,8 +321,8 @@ class RemoteAWSEnv(LocalAWSEnv):
|
|
|
274
321
|
f"DeregisterTask{task_index}OnFailure": {
|
|
275
322
|
"Type": "Task",
|
|
276
323
|
"Resource": "arn:aws:states:::aws-sdk:ecs:deregisterTaskDefinition",
|
|
277
|
-
"
|
|
278
|
-
"TaskDefinition
|
|
324
|
+
"Arguments": {
|
|
325
|
+
"TaskDefinition": "{% $states.input.RegisteredTask.TaskDefinition.TaskDefinitionArn %}"
|
|
279
326
|
},
|
|
280
327
|
"Next": "WorkflowFailed"
|
|
281
328
|
},
|
|
@@ -325,6 +372,7 @@ class RemoteAWSEnv(LocalAWSEnv):
|
|
|
325
372
|
}
|
|
326
373
|
|
|
327
374
|
return {
|
|
375
|
+
"QueryLanguage": "JSONata",
|
|
328
376
|
"StartAt": "RegisterTask0",
|
|
329
377
|
"States": states,
|
|
330
378
|
}
|
|
@@ -338,7 +386,7 @@ class RemoteAWSEnv(LocalAWSEnv):
|
|
|
338
386
|
run_id,
|
|
339
387
|
temp_path
|
|
340
388
|
):
|
|
341
|
-
workflow_name = self._make_workflow_name(run_id)
|
|
389
|
+
workflow_name = self._make_workflow_name(None, run_id)
|
|
342
390
|
workflow = self._build_workflow(
|
|
343
391
|
pipeline=pipeline,
|
|
344
392
|
image_refs=image_refs,
|
|
@@ -349,7 +397,7 @@ class RemoteAWSEnv(LocalAWSEnv):
|
|
|
349
397
|
|
|
350
398
|
workflow_path = os.path.join(temp_path, f'{workflow_name}.json')
|
|
351
399
|
with open(workflow_path, 'w') as stream:
|
|
352
|
-
json.dump(workflow, stream)
|
|
400
|
+
json.dump(workflow, stream, indent=4)
|
|
353
401
|
|
|
354
402
|
workflow_arn = self._make_workflow_arn(workflow_name)
|
|
355
403
|
if self._check_workflow_exists(workflow_name):
|
|
@@ -404,7 +452,7 @@ class RemoteAWSEnv(LocalAWSEnv):
|
|
|
404
452
|
|
|
405
453
|
def print_logs(self, run_id, start_timestamp, stream=False):
|
|
406
454
|
print('\nStreaming logs...')
|
|
407
|
-
workflow_name = self._make_workflow_name(run_id)
|
|
455
|
+
workflow_name = self._make_workflow_name(None, run_id)
|
|
408
456
|
command = [
|
|
409
457
|
self._aws_command,
|
|
410
458
|
'logs',
|
|
@@ -418,3 +466,102 @@ class RemoteAWSEnv(LocalAWSEnv):
|
|
|
418
466
|
command.append('--follow')
|
|
419
467
|
command.append(self._log_group_prefix + workflow_name)
|
|
420
468
|
run_command(command)
|
|
469
|
+
|
|
470
|
+
def deploy_pipeline(
|
|
471
|
+
self,
|
|
472
|
+
pipeline,
|
|
473
|
+
image_refs,
|
|
474
|
+
temp_path,
|
|
475
|
+
schedule=True,
|
|
476
|
+
):
|
|
477
|
+
workflow_name = self._make_workflow_name(pipeline.name, None)
|
|
478
|
+
workflow = self._build_workflow(
|
|
479
|
+
pipeline=pipeline,
|
|
480
|
+
image_refs=image_refs,
|
|
481
|
+
workflow_name=workflow_name,
|
|
482
|
+
timestamp=None,
|
|
483
|
+
run_id=None,
|
|
484
|
+
)
|
|
485
|
+
|
|
486
|
+
workflow_path = os.path.join(temp_path, f'{workflow_name}.json')
|
|
487
|
+
with open(workflow_path, 'w') as stream:
|
|
488
|
+
json.dump(workflow, stream, indent=4)
|
|
489
|
+
|
|
490
|
+
workflow_arn = self._make_workflow_arn(workflow_name)
|
|
491
|
+
if self._check_workflow_exists(workflow_name):
|
|
492
|
+
run_command(
|
|
493
|
+
[
|
|
494
|
+
self._aws_command,
|
|
495
|
+
'stepfunctions',
|
|
496
|
+
'update-state-machine',
|
|
497
|
+
'--profile', self._aws_profile,
|
|
498
|
+
'--region', self._aws_region,
|
|
499
|
+
'--state-machine-arn', workflow_arn,
|
|
500
|
+
'--role-arn', self._stepfunctions_role_arn,
|
|
501
|
+
'--definition', f'file://{workflow_path}',
|
|
502
|
+
],
|
|
503
|
+
stdout=subprocess.DEVNULL,
|
|
504
|
+
)
|
|
505
|
+
else:
|
|
506
|
+
run_command(
|
|
507
|
+
[
|
|
508
|
+
self._aws_command,
|
|
509
|
+
'stepfunctions',
|
|
510
|
+
'create-state-machine',
|
|
511
|
+
'--profile', self._aws_profile,
|
|
512
|
+
'--region', self._aws_region,
|
|
513
|
+
'--name', workflow_name,
|
|
514
|
+
'--role-arn', self._stepfunctions_role_arn,
|
|
515
|
+
'--definition', f'file://{workflow_path}',
|
|
516
|
+
],
|
|
517
|
+
stdout=subprocess.DEVNULL,
|
|
518
|
+
)
|
|
519
|
+
print(f'Created step function {workflow_arn}.')
|
|
520
|
+
|
|
521
|
+
if schedule and pipeline.schedule is not None:
|
|
522
|
+
if self._scheduler_timezone is None:
|
|
523
|
+
raise CloudSubmitError(
|
|
524
|
+
'You must specify the schedule timezone '
|
|
525
|
+
'to use automatic scheduling.'
|
|
526
|
+
)
|
|
527
|
+
if self._scheduler_role_arn is None:
|
|
528
|
+
raise CloudSubmitError(
|
|
529
|
+
'You must specify the scheduler role ARN '
|
|
530
|
+
'to use automatic scheduling.'
|
|
531
|
+
)
|
|
532
|
+
schedule_name = self._make_schedule_name(pipeline.name)
|
|
533
|
+
target = json.dumps({
|
|
534
|
+
'Arn': workflow_arn,
|
|
535
|
+
'RoleArn': self._scheduler_role_arn,
|
|
536
|
+
})
|
|
537
|
+
options = [
|
|
538
|
+
'--profile', self._aws_profile,
|
|
539
|
+
'--region', self._aws_region,
|
|
540
|
+
'--name', schedule_name,
|
|
541
|
+
'--schedule-expression', f'cron({pipeline.schedule})',
|
|
542
|
+
'--schedule-expression-timezone', self._scheduler_timezone,
|
|
543
|
+
'--flexible-time-window', '{"Mode": "OFF"}',
|
|
544
|
+
'--target', target,
|
|
545
|
+
]
|
|
546
|
+
if self._check_schedule_exists(schedule_name):
|
|
547
|
+
run_command(
|
|
548
|
+
[
|
|
549
|
+
self._aws_command,
|
|
550
|
+
'scheduler',
|
|
551
|
+
'update-schedule',
|
|
552
|
+
*options
|
|
553
|
+
],
|
|
554
|
+
stdout=subprocess.DEVNULL,
|
|
555
|
+
)
|
|
556
|
+
else:
|
|
557
|
+
run_command(
|
|
558
|
+
[
|
|
559
|
+
self._aws_command,
|
|
560
|
+
'scheduler',
|
|
561
|
+
'create-schedule',
|
|
562
|
+
*options,
|
|
563
|
+
],
|
|
564
|
+
stdout=subprocess.DEVNULL,
|
|
565
|
+
)
|
|
566
|
+
print(f'Created schedule {schedule_name}.')
|
|
567
|
+
|
|
@@ -170,3 +170,12 @@ class LocalEnv(EnvironmentHandler):
|
|
|
170
170
|
def print_logs(self, run_id, start_timestamp, stream=False):
|
|
171
171
|
raise CloudSubmitError(
|
|
172
172
|
'Log retrieval is not supported by this environment.')
|
|
173
|
+
|
|
174
|
+
def deploy_pipeline(
|
|
175
|
+
self,
|
|
176
|
+
pipeline,
|
|
177
|
+
image_refs,
|
|
178
|
+
temp_path
|
|
179
|
+
):
|
|
180
|
+
raise CloudSubmitError(
|
|
181
|
+
'Deployment is not supported by this environment.')
|
|
@@ -194,10 +194,12 @@ class Pipeline:
|
|
|
194
194
|
name,
|
|
195
195
|
steps,
|
|
196
196
|
default_submit_timestamp=None,
|
|
197
|
+
schedule=None,
|
|
197
198
|
):
|
|
198
199
|
self.name = name
|
|
199
200
|
self.steps = list(steps)
|
|
200
201
|
self.default_submit_timestamp = to_utc(default_submit_timestamp)
|
|
202
|
+
self.schedule = schedule
|
|
201
203
|
|
|
202
204
|
step_names = set()
|
|
203
205
|
for step in self.steps:
|
|
@@ -212,7 +214,8 @@ class Pipeline:
|
|
|
212
214
|
return {
|
|
213
215
|
'name': self.name,
|
|
214
216
|
'steps': [s.to_dict() for s in self.steps],
|
|
215
|
-
'default_submit_timestamp': self.default_submit_timestamp
|
|
217
|
+
'default_submit_timestamp': self.default_submit_timestamp,
|
|
218
|
+
'schedule': self.schedule,
|
|
216
219
|
}
|
|
217
220
|
|
|
218
221
|
@staticmethod
|
|
@@ -220,7 +223,8 @@ class Pipeline:
|
|
|
220
223
|
return Pipeline(
|
|
221
224
|
name=obj['name'],
|
|
222
225
|
steps=[Step.from_dict(s) for s in obj['steps']],
|
|
223
|
-
default_submit_timestamp=obj['default_submit_timestamp']
|
|
226
|
+
default_submit_timestamp=obj['default_submit_timestamp'],
|
|
227
|
+
schedule=obj['schedule'],
|
|
224
228
|
)
|
|
225
229
|
|
|
226
230
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/envs/aws/local_environment_handler.py
RENAMED
|
File without changes
|
{cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/envs/aws/local_execution_handler.py
RENAMED
|
File without changes
|
{cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/envs/aws/remote_execution_handler.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{cloud_submit-0.5.1 → cloud_submit-0.6}/src/cloud_submit/execution/base_execution_handler.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|