cloud-submit 0.3.1__tar.gz → 0.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cloud_submit-0.3.1 → cloud_submit-0.5}/PKG-INFO +1 -1
- {cloud_submit-0.3.1 → cloud_submit-0.5}/pyproject.toml +1 -1
- {cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/cli.py +24 -4
- {cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/controller.py +80 -42
- {cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/environment_handler.py +31 -21
- {cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/envs/aws/local_environment_handler.py +42 -11
- {cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/envs/local/environment_handler.py +9 -8
- {cloud_submit-0.3.1 → cloud_submit-0.5}/.gitignore +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.5}/LICENSE +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.5}/README.md +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.5}/docs/examples/aws/src/csub/build_config.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.5}/docs/examples/aws/src/fit.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.5}/docs/examples/aws/src/generate.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.5}/docs/examples/aws/src/parameters.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.5}/docs/examples/aws/userconfig/template.yaml +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.5}/docs/examples/basic/src/csub/build_config.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.5}/docs/examples/basic/src/fit.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.5}/docs/examples/basic/src/generate.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.5}/docs/examples/basic/src/parameters.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.5}/docs/examples/basic/userconfig/template.yaml +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.5}/docs/examples/multi-worker/src/csub/build_config.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.5}/docs/examples/multi-worker/src/fit.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.5}/docs/examples/multi-worker/src/generate.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.5}/docs/examples/multi-worker/src/parameters.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.5}/docs/examples/multi-worker/userconfig/template.yaml +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/__init__.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/config.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/envs/aws/__init__.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/envs/aws/local_execution_handler.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/envs/aws/remote_environment_handler.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/envs/aws/remote_execution_handler.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/envs/aws/s3_tools.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/envs/local/execution_handler.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/execution/base_execution_handler.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/execution/config.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/execution/execute.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/execution/utils.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/images.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: cloud-submit
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5
|
|
4
4
|
Summary: A tool for building processing pipelines in Python and executing them locally or in the cloud.
|
|
5
5
|
Project-URL: Homepage, https://github.com/mwiebusch78/cloud-submit
|
|
6
6
|
Project-URL: Bug Tracker, https://github.com/mwiebusch78/cloud-submit/issues
|
|
@@ -82,7 +82,7 @@ def get_steps(pipeline, steps_arg):
|
|
|
82
82
|
tokens = steps_arg.split(',')
|
|
83
83
|
all_steps = [step.name for step in pipeline.steps]
|
|
84
84
|
indices = {step: i for i, step in enumerate(all_steps)}
|
|
85
|
-
|
|
85
|
+
|
|
86
86
|
selected = set()
|
|
87
87
|
for token in tokens:
|
|
88
88
|
token = token.split(':')
|
|
@@ -99,7 +99,7 @@ def get_steps(pipeline, steps_arg):
|
|
|
99
99
|
end = indices[token[1]]
|
|
100
100
|
except KeyError:
|
|
101
101
|
raise CloudSubmitError(f'Invalid step name: {token[1]}')
|
|
102
|
-
for step in all_steps[
|
|
102
|
+
for step in all_steps[begin:end+1]:
|
|
103
103
|
selected.add(step)
|
|
104
104
|
else:
|
|
105
105
|
raise CloudSubmitError(f'Invalid step range: {":".join(token)}')
|
|
@@ -306,9 +306,19 @@ def images():
|
|
|
306
306
|
is_flag=True,
|
|
307
307
|
help='Ignore IMAGES arguments and build all base images.',
|
|
308
308
|
)
|
|
309
|
+
@click.option(
|
|
310
|
+
'--no-cache',
|
|
311
|
+
is_flag=True,
|
|
312
|
+
help='Do not use docker cache when building images.',
|
|
313
|
+
)
|
|
314
|
+
@click.option(
|
|
315
|
+
'--pull',
|
|
316
|
+
is_flag=True,
|
|
317
|
+
help='Instruct docker to always attempt to pull all referenced images.',
|
|
318
|
+
)
|
|
309
319
|
@click.argument('images', type=str, nargs=-1)
|
|
310
320
|
@click.pass_context
|
|
311
|
-
def build_images(ctx, env, build_id, build_all, images):
|
|
321
|
+
def build_images(ctx, env, build_id, build_all, no_cache, pull, images):
|
|
312
322
|
init(ctx)
|
|
313
323
|
config = ctx.obj['config']
|
|
314
324
|
controller = ctx.obj['controller']
|
|
@@ -321,8 +331,18 @@ def build_images(ctx, env, build_id, build_all, images):
|
|
|
321
331
|
name for name, image in config.images.items()
|
|
322
332
|
if isinstance(image, BaseImage)
|
|
323
333
|
]
|
|
334
|
+
docker_flags = []
|
|
335
|
+
if no_cache:
|
|
336
|
+
docker_flags.append('--no-cache')
|
|
337
|
+
if pull:
|
|
338
|
+
docker_flags.append('--pull')
|
|
324
339
|
try:
|
|
325
|
-
controller.build_images(
|
|
340
|
+
controller.build_images(
|
|
341
|
+
images,
|
|
342
|
+
build_id=build_id,
|
|
343
|
+
env=env,
|
|
344
|
+
docker_flags=docker_flags,
|
|
345
|
+
)
|
|
326
346
|
except CloudSubmitError as e:
|
|
327
347
|
abort(str(e))
|
|
328
348
|
|
|
@@ -61,7 +61,7 @@ class Controller:
|
|
|
61
61
|
except FileNotFoundError:
|
|
62
62
|
pass
|
|
63
63
|
|
|
64
|
-
def _build_image(self, image, build_id, env, rebuild=False):
|
|
64
|
+
def _build_image(self, image, build_id, env, docker_flags, rebuild=False):
|
|
65
65
|
is_execution_image = isinstance(image, ExecutionImage)
|
|
66
66
|
|
|
67
67
|
if not rebuild and not is_execution_image:
|
|
@@ -96,7 +96,7 @@ class Controller:
|
|
|
96
96
|
image.setup_builddir(path, parent_ref)
|
|
97
97
|
|
|
98
98
|
print(f'Building image {image.name}.')
|
|
99
|
-
ref = env.build_image(path, image, build_id)
|
|
99
|
+
ref = env.build_image(path, image, build_id, docker_flags)
|
|
100
100
|
if ref is None:
|
|
101
101
|
raise CloudSubmitError(
|
|
102
102
|
'build_image method of environment handler did not '
|
|
@@ -105,10 +105,12 @@ class Controller:
|
|
|
105
105
|
if not is_execution_image:
|
|
106
106
|
self._save_image_ref(image, ref)
|
|
107
107
|
|
|
108
|
-
def build_images(self, images, build_id=None, env=None):
|
|
108
|
+
def build_images(self, images, build_id=None, env=None, docker_flags=None):
|
|
109
109
|
now = dt.datetime.now(tz=dt.UTC)
|
|
110
110
|
env_handler = self._config.get_build_env(env)
|
|
111
111
|
build_id = env_handler.generate_build_id(now, build_id)
|
|
112
|
+
if docker_flags is None:
|
|
113
|
+
docker_flags = []
|
|
112
114
|
|
|
113
115
|
images = set(images)
|
|
114
116
|
for image in images:
|
|
@@ -127,7 +129,7 @@ class Controller:
|
|
|
127
129
|
continue
|
|
128
130
|
rebuild = image_name in images
|
|
129
131
|
self._build_image(
|
|
130
|
-
image, build_id, env_handler, rebuild=rebuild)
|
|
132
|
+
image, build_id, env_handler, docker_flags, rebuild=rebuild)
|
|
131
133
|
|
|
132
134
|
def list_image_refs(self, images=None, ids=None, env=None, remote=False):
|
|
133
135
|
env_handler = self._config.get_build_env(env)
|
|
@@ -210,6 +212,22 @@ class Controller:
|
|
|
210
212
|
with self._config.in_project_root():
|
|
211
213
|
self._clear_image_ref(image)
|
|
212
214
|
|
|
215
|
+
def _split_kinds(self, artifacts):
|
|
216
|
+
kinds = {}
|
|
217
|
+
keys = []
|
|
218
|
+
for artifact in artifacts:
|
|
219
|
+
kind = artifact.kind
|
|
220
|
+
kinds[kind] = kinds.get(kind, [])
|
|
221
|
+
kinds[kind].append(artifact)
|
|
222
|
+
keys.append((kind, len(kinds[kind])-1))
|
|
223
|
+
return kinds, keys
|
|
224
|
+
|
|
225
|
+
def _merge_kinds(self, kinds, keys):
|
|
226
|
+
result = []
|
|
227
|
+
for kind, pos in keys:
|
|
228
|
+
result.append(kinds[kind][pos])
|
|
229
|
+
return result
|
|
230
|
+
|
|
213
231
|
def list_artifacts(
|
|
214
232
|
self,
|
|
215
233
|
artifact_names=None,
|
|
@@ -226,15 +244,19 @@ class Controller:
|
|
|
226
244
|
except KeyError:
|
|
227
245
|
raise CloudSubmitError(f'Unknown artifact name: {name}.')
|
|
228
246
|
artifacts.append(artifact.copy())
|
|
247
|
+
kinds, keys = self._split_kinds(artifacts)
|
|
229
248
|
|
|
230
249
|
env_handler = self._config.get_run_env(env)
|
|
231
250
|
with self._config.in_project_root():
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
251
|
+
runs = {}
|
|
252
|
+
for kind, artifact_list in kinds.items():
|
|
253
|
+
if remote:
|
|
254
|
+
runs[kind] = env_handler.list_remote_artifacts(
|
|
255
|
+
kind, artifact_list, run_ids=run_ids)
|
|
256
|
+
else:
|
|
257
|
+
runs[kind] = env_handler.list_local_artifacts(
|
|
258
|
+
kind, artifact_list, run_ids=run_ids)
|
|
259
|
+
runs = self._merge_kinds(runs, keys)
|
|
238
260
|
|
|
239
261
|
return [a.name for a in artifacts], runs
|
|
240
262
|
|
|
@@ -250,11 +272,15 @@ class Controller:
|
|
|
250
272
|
env_handler = self._config.get_run_env(env)
|
|
251
273
|
artifacts = [
|
|
252
274
|
self._config.artifacts[name].copy() for name in artifact_names]
|
|
275
|
+
kinds, keys = self._split_kinds(artifacts)
|
|
253
276
|
with self._config.in_project_root():
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
277
|
+
for kind, artifact_list in kinds.items():
|
|
278
|
+
if remote:
|
|
279
|
+
env_handler.remove_remote_artifacts(
|
|
280
|
+
kind, artifact_list, run_ids)
|
|
281
|
+
else:
|
|
282
|
+
env_handler.remove_local_artifacts(
|
|
283
|
+
kind, artifact_list, run_ids)
|
|
258
284
|
|
|
259
285
|
def pull_artifacts(
|
|
260
286
|
self,
|
|
@@ -265,9 +291,11 @@ class Controller:
|
|
|
265
291
|
env_handler = self._config.get_run_env(env)
|
|
266
292
|
artifacts = [
|
|
267
293
|
self._config.artifacts[name].copy() for name in artifact_names]
|
|
294
|
+
kinds, keys = self._split_kinds(artifacts)
|
|
268
295
|
with self._config.in_project_root():
|
|
269
|
-
|
|
270
|
-
|
|
296
|
+
for kind, artifact_list in kinds.items():
|
|
297
|
+
env_handler.remove_local_artifacts(kind, artifact_list, run_ids)
|
|
298
|
+
env_handler.pull_artifacts(kind, artifact_list, run_ids)
|
|
271
299
|
|
|
272
300
|
def push_artifacts(
|
|
273
301
|
self,
|
|
@@ -278,9 +306,12 @@ class Controller:
|
|
|
278
306
|
env_handler = self._config.get_run_env(env)
|
|
279
307
|
artifacts = [
|
|
280
308
|
self._config.artifacts[name].copy() for name in artifact_names]
|
|
309
|
+
kinds, keys = self._split_kinds(artifacts)
|
|
281
310
|
with self._config.in_project_root():
|
|
282
|
-
|
|
283
|
-
|
|
311
|
+
for kind, artifact_list in kinds.items():
|
|
312
|
+
env_handler.remove_remote_artifacts(
|
|
313
|
+
kind, artifact_list, run_ids)
|
|
314
|
+
env_handler.push_artifacts(kind, artifact_list, run_ids)
|
|
284
315
|
|
|
285
316
|
def copy_artifacts(
|
|
286
317
|
self,
|
|
@@ -298,17 +329,19 @@ class Controller:
|
|
|
298
329
|
self._config.artifacts[name].copy() for name in artifact_names
|
|
299
330
|
if self._config.artifacts[name].scope == 'run'
|
|
300
331
|
]
|
|
332
|
+
kinds, keys = self._split_kinds(artifacts)
|
|
301
333
|
with self._config.in_project_root():
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
334
|
+
for kind, artifact_list in kinds.items():
|
|
335
|
+
if remote:
|
|
336
|
+
env_handler.remove_remote_artifacts(
|
|
337
|
+
kind, artifact_list, [[to_run_id]]*len(artifact_list))
|
|
338
|
+
env_handler.copy_remote_artifacts(
|
|
339
|
+
kind, artifact_list, from_run_id, to_run_id)
|
|
340
|
+
else:
|
|
341
|
+
env_handler.remove_local_artifacts(
|
|
342
|
+
kind, artifact_list, [[to_run_id]]*len(artifact_list))
|
|
343
|
+
env_handler.copy_local_artifacts(
|
|
344
|
+
kind, artifact_list, from_run_id, to_run_id)
|
|
312
345
|
|
|
313
346
|
def move_artifacts(
|
|
314
347
|
self,
|
|
@@ -326,17 +359,19 @@ class Controller:
|
|
|
326
359
|
self._config.artifacts[name].copy() for name in artifact_names
|
|
327
360
|
if self._config.artifacts[name].scope == 'run'
|
|
328
361
|
]
|
|
362
|
+
kinds, keys = self._split_kinds(artifacts)
|
|
329
363
|
with self._config.in_project_root():
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
364
|
+
for kind, artifact_list in kinds.items():
|
|
365
|
+
if remote:
|
|
366
|
+
env_handler.remove_remote_artifacts(
|
|
367
|
+
kind, artifact_list, [[to_run_id]]*len(artifact_list))
|
|
368
|
+
env_handler.move_remote_artifacts(
|
|
369
|
+
kind, artifact_list, from_run_id, to_run_id)
|
|
370
|
+
else:
|
|
371
|
+
env_handler.remove_local_artifacts(
|
|
372
|
+
kind, artifact_list, [[to_run_id]]*len(artifact_list))
|
|
373
|
+
env_handler.move_local_artifacts(
|
|
374
|
+
kind, artifact_list, from_run_id, to_run_id)
|
|
340
375
|
|
|
341
376
|
def run_pipeline(
|
|
342
377
|
self,
|
|
@@ -384,11 +419,14 @@ class Controller:
|
|
|
384
419
|
refs[step.name] = ':'.join([repo, run_id])
|
|
385
420
|
|
|
386
421
|
for loc in step.outputs.values():
|
|
387
|
-
|
|
388
|
-
|
|
422
|
+
artifact = self._config.artifacts[loc.artifact_name].copy()
|
|
423
|
+
if artifact.scope == 'run':
|
|
424
|
+
overwrite_artifacts.append(artifact)
|
|
389
425
|
for loc in step.temporaries.values():
|
|
390
|
-
|
|
391
|
-
|
|
426
|
+
artifact = self._config.artifacts[loc.artifact_name].copy()
|
|
427
|
+
overwrite_artifacts.append(artifact)
|
|
428
|
+
|
|
429
|
+
overwrite_artifacts, _ = self._split_kinds(overwrite_artifacts)
|
|
392
430
|
|
|
393
431
|
temp_path = 'temp/run'
|
|
394
432
|
ensure_path(temp_path, clear=True)
|
|
@@ -235,13 +235,14 @@ class EnvironmentHandler:
|
|
|
235
235
|
def remove_remote_image_refs(self, refs):
|
|
236
236
|
raise NotImplementedError
|
|
237
237
|
|
|
238
|
-
def build_image(self, path, image, build_id):
|
|
238
|
+
def build_image(self, path, image, build_id, docker_flags):
|
|
239
239
|
repo = self.get_image_repo(image)
|
|
240
240
|
ref = ':'.join([repo, build_id])
|
|
241
241
|
command = [
|
|
242
242
|
self._docker_command,
|
|
243
243
|
'buildx',
|
|
244
244
|
'build',
|
|
245
|
+
*docker_flags,
|
|
245
246
|
*(image.args),
|
|
246
247
|
'-t', ref,
|
|
247
248
|
path,
|
|
@@ -252,9 +253,8 @@ class EnvironmentHandler:
|
|
|
252
253
|
def get_local_artifact_ref(self, artifact, run_id=None):
|
|
253
254
|
if artifact.kind != 'file':
|
|
254
255
|
raise CloudSubmitError(
|
|
255
|
-
f'Cannot get path for local artifact {artifact.name}
|
|
256
|
-
|
|
257
|
-
f'is of kind {repr(artifact.kind)}.'
|
|
256
|
+
f'Cannot get path for local artifact {artifact.name} '
|
|
257
|
+
f'of kind {repr(artifact.kind)}.'
|
|
258
258
|
)
|
|
259
259
|
if artifact.scope == 'project':
|
|
260
260
|
return os.path.join('artifacts', 'shared', artifact.name)
|
|
@@ -276,18 +276,18 @@ class EnvironmentHandler:
|
|
|
276
276
|
def get_remote_artifact_ref(self, artifact, run_id=None):
|
|
277
277
|
raise NotImplementedError
|
|
278
278
|
|
|
279
|
-
def list_local_artifacts(self, artifacts, run_ids=None):
|
|
279
|
+
def list_local_artifacts(self, kind, artifacts, run_ids=None):
|
|
280
|
+
if kind != 'file':
|
|
281
|
+
raise CloudSubmitError(
|
|
282
|
+
'Cannot list run IDs for local artifacts of kind '
|
|
283
|
+
f'{repr(kind)}. '
|
|
284
|
+
)
|
|
285
|
+
|
|
280
286
|
if run_ids is not None:
|
|
281
287
|
run_ids = set(run_ids)
|
|
282
288
|
|
|
283
289
|
results = []
|
|
284
290
|
for artifact in artifacts:
|
|
285
|
-
if artifact.kind != 'file':
|
|
286
|
-
raise CloudSubmitError(
|
|
287
|
-
f'Cannot list run IDs for local artifact {artifact.name}. '
|
|
288
|
-
"Only artifacts of kind 'file' are supported and this one "
|
|
289
|
-
f'is of kind {repr(artifact.kind)}.'
|
|
290
|
-
)
|
|
291
291
|
if artifact.scope == 'run':
|
|
292
292
|
pattern = os.path.join(
|
|
293
293
|
'artifacts', 'users', self._user,
|
|
@@ -316,27 +316,33 @@ class EnvironmentHandler:
|
|
|
316
316
|
results.append(ids)
|
|
317
317
|
return results
|
|
318
318
|
|
|
319
|
-
def list_remote_artifacts(self, artifacts, run_ids=None):
|
|
319
|
+
def list_remote_artifacts(self, kind, artifacts, run_ids=None):
|
|
320
320
|
raise NotImplementedError
|
|
321
321
|
|
|
322
|
-
def push_artifacts(self, artifacts, run_ids):
|
|
322
|
+
def push_artifacts(self, kind, artifacts, run_ids):
|
|
323
323
|
raise NotImplementedError
|
|
324
324
|
|
|
325
|
-
def pull_artifacts(self, artifacts, run_ids):
|
|
325
|
+
def pull_artifacts(self, kind, artifacts, run_ids):
|
|
326
326
|
raise NotImplementedError
|
|
327
327
|
|
|
328
|
-
def remove_local_artifacts(self, artifacts, run_ids):
|
|
328
|
+
def remove_local_artifacts(self, kind, artifacts, run_ids):
|
|
329
|
+
if kind != 'file':
|
|
330
|
+
raise CloudSubmitError(
|
|
331
|
+
f'Cannot remove local artifacts of kind {repr(kind)}. '
|
|
332
|
+
)
|
|
329
333
|
for artifact, runs in zip(artifacts, run_ids):
|
|
330
|
-
if artifact.kind != 'file':
|
|
331
|
-
continue
|
|
332
334
|
for run_id in runs:
|
|
333
335
|
path = self.get_local_artifact_ref(artifact, run_id)
|
|
334
336
|
clear_path(path)
|
|
335
337
|
|
|
336
|
-
def remove_remote_artifacts(self, artifacts, run_ids):
|
|
338
|
+
def remove_remote_artifacts(self, kind, artifacts, run_ids):
|
|
337
339
|
raise NotImplementedError
|
|
338
340
|
|
|
339
|
-
def copy_local_artifacts(self, artifacts, from_run_id, to_run_id):
|
|
341
|
+
def copy_local_artifacts(self, kind, artifacts, from_run_id, to_run_id):
|
|
342
|
+
if kind != 'file':
|
|
343
|
+
raise CloudSubmitError(
|
|
344
|
+
f'Cannot copy local artifacts of kind {repr(kind)}. '
|
|
345
|
+
)
|
|
340
346
|
for artifact in artifacts:
|
|
341
347
|
src_path = self.get_local_artifact_ref(artifact, from_run_id)
|
|
342
348
|
dst_path = self.get_local_artifact_ref(artifact, to_run_id)
|
|
@@ -350,10 +356,14 @@ class EnvironmentHandler:
|
|
|
350
356
|
else:
|
|
351
357
|
raise
|
|
352
358
|
|
|
353
|
-
def copy_remote_artifacts(self, artifacts, from_run_id, to_run_id):
|
|
359
|
+
def copy_remote_artifacts(self, kind, artifacts, from_run_id, to_run_id):
|
|
354
360
|
raise NotImplementedError
|
|
355
361
|
|
|
356
|
-
def move_local_artifacts(self, artifacts, from_run_id, to_run_id):
|
|
362
|
+
def move_local_artifacts(self, kind, artifacts, from_run_id, to_run_id):
|
|
363
|
+
if kind != 'file':
|
|
364
|
+
raise CloudSubmitError(
|
|
365
|
+
f'Cannot move local artifacts of kind {repr(kind)}. '
|
|
366
|
+
)
|
|
357
367
|
for artifact in artifacts:
|
|
358
368
|
src_path = self.get_local_artifact_ref(artifact, from_run_id)
|
|
359
369
|
dst_path = self.get_local_artifact_ref(artifact, to_run_id)
|
{cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/envs/aws/local_environment_handler.py
RENAMED
|
@@ -129,9 +129,9 @@ class LocalAWSEnv(LocalEnv):
|
|
|
129
129
|
os.path.join(path, 's3_tools.py'),
|
|
130
130
|
)
|
|
131
131
|
|
|
132
|
-
def _build_image(self, path, image, build_id, push):
|
|
132
|
+
def _build_image(self, path, image, build_id, docker_flags, push):
|
|
133
133
|
if not push:
|
|
134
|
-
ref = super().build_image(path, image, build_id)
|
|
134
|
+
ref = super().build_image(path, image, build_id, docker_flags)
|
|
135
135
|
else:
|
|
136
136
|
self._docker_login()
|
|
137
137
|
repo = self.get_image_repo(image)
|
|
@@ -140,6 +140,7 @@ class LocalAWSEnv(LocalEnv):
|
|
|
140
140
|
self._docker_command,
|
|
141
141
|
'buildx',
|
|
142
142
|
'build',
|
|
143
|
+
*docker_flags,
|
|
143
144
|
*(image.args),
|
|
144
145
|
]
|
|
145
146
|
if self._docker_platforms is not None:
|
|
@@ -148,11 +149,12 @@ class LocalAWSEnv(LocalEnv):
|
|
|
148
149
|
run_command(command)
|
|
149
150
|
return ref
|
|
150
151
|
|
|
151
|
-
def build_image(self, path, image, build_id):
|
|
152
|
+
def build_image(self, path, image, build_id, docker_flags):
|
|
152
153
|
return self._build_image(
|
|
153
154
|
path=path,
|
|
154
155
|
image=image,
|
|
155
156
|
build_id=build_id,
|
|
157
|
+
docker_flags=docker_flags,
|
|
156
158
|
push=isinstance(image, BaseImage),
|
|
157
159
|
)
|
|
158
160
|
|
|
@@ -242,7 +244,13 @@ class LocalAWSEnv(LocalEnv):
|
|
|
242
244
|
lines = lines.split('\n')
|
|
243
245
|
return [l.split()[-1] for l in lines]
|
|
244
246
|
|
|
245
|
-
def list_remote_artifacts(self, artifacts, run_ids=None):
|
|
247
|
+
def list_remote_artifacts(self, kind, artifacts, run_ids=None):
|
|
248
|
+
if kind != 'file':
|
|
249
|
+
raise CloudSubmitError(
|
|
250
|
+
'Cannot list run IDs for remote artifacts of kind '
|
|
251
|
+
f'{repr(kind)}. '
|
|
252
|
+
)
|
|
253
|
+
|
|
246
254
|
if run_ids is not None:
|
|
247
255
|
run_ids = set(run_ids)
|
|
248
256
|
|
|
@@ -250,8 +258,6 @@ class LocalAWSEnv(LocalEnv):
|
|
|
250
258
|
user_artifacts = set()
|
|
251
259
|
run_artifacts = set()
|
|
252
260
|
for a in artifacts:
|
|
253
|
-
if a.kind != 'file':
|
|
254
|
-
continue
|
|
255
261
|
if a.scope == 'project':
|
|
256
262
|
project_artifacts.add(a.name)
|
|
257
263
|
elif a.scope == 'user':
|
|
@@ -301,7 +307,12 @@ class LocalAWSEnv(LocalEnv):
|
|
|
301
307
|
results = [sorted(results[a.name]) for a in artifacts]
|
|
302
308
|
return results
|
|
303
309
|
|
|
304
|
-
def remove_remote_artifacts(self, artifacts, run_ids):
|
|
310
|
+
def remove_remote_artifacts(self, kind, artifacts, run_ids):
|
|
311
|
+
if kind != 'file':
|
|
312
|
+
raise CloudSubmitError(
|
|
313
|
+
f'Cannot remove remote artifacts of kind {repr(kind)}.'
|
|
314
|
+
)
|
|
315
|
+
|
|
305
316
|
path = '/'.join(
|
|
306
317
|
['s3:/', self._s3_bucket, self._s3_prefix, self._project])
|
|
307
318
|
command = [
|
|
@@ -352,7 +363,12 @@ class LocalAWSEnv(LocalEnv):
|
|
|
352
363
|
|
|
353
364
|
run_command(command)
|
|
354
365
|
|
|
355
|
-
def push_artifacts(self, artifacts, run_ids):
|
|
366
|
+
def push_artifacts(self, kind, artifacts, run_ids):
|
|
367
|
+
if kind != 'file':
|
|
368
|
+
raise CloudSubmitError(
|
|
369
|
+
f'Cannot push artifacts of kind {repr(kind)}.'
|
|
370
|
+
)
|
|
371
|
+
|
|
356
372
|
remote_path = '/'.join(
|
|
357
373
|
['s3:/', self._s3_bucket, self._s3_prefix, self._project])
|
|
358
374
|
local_path = 'artifacts'
|
|
@@ -402,7 +418,12 @@ class LocalAWSEnv(LocalEnv):
|
|
|
402
418
|
|
|
403
419
|
run_command(command)
|
|
404
420
|
|
|
405
|
-
def pull_artifacts(self, artifacts, run_ids):
|
|
421
|
+
def pull_artifacts(self, kind, artifacts, run_ids):
|
|
422
|
+
if kind != 'file':
|
|
423
|
+
raise CloudSubmitError(
|
|
424
|
+
f'Cannot pull artifacts of kind {repr(kind)}.'
|
|
425
|
+
)
|
|
426
|
+
|
|
406
427
|
remote_path = '/'.join(
|
|
407
428
|
['s3:/', self._s3_bucket, self._s3_prefix, self._project])
|
|
408
429
|
local_path = 'artifacts'
|
|
@@ -452,7 +473,12 @@ class LocalAWSEnv(LocalEnv):
|
|
|
452
473
|
|
|
453
474
|
run_command(command)
|
|
454
475
|
|
|
455
|
-
def copy_remote_artifacts(self, artifacts, from_run_id, to_run_id):
|
|
476
|
+
def copy_remote_artifacts(self, kind, artifacts, from_run_id, to_run_id):
|
|
477
|
+
if kind != 'file':
|
|
478
|
+
raise CloudSubmitError(
|
|
479
|
+
f'Cannot copy remote artifacts of kind {repr(kind)}.'
|
|
480
|
+
)
|
|
481
|
+
|
|
456
482
|
base_path = '/'.join([
|
|
457
483
|
's3:/', self._s3_bucket, self._s3_prefix, self._project,
|
|
458
484
|
'users', self._user, 'runs',
|
|
@@ -478,7 +504,12 @@ class LocalAWSEnv(LocalEnv):
|
|
|
478
504
|
])
|
|
479
505
|
run_command(command)
|
|
480
506
|
|
|
481
|
-
def move_remote_artifacts(self, artifacts, from_run_id, to_run_id):
|
|
507
|
+
def move_remote_artifacts(self, kind, artifacts, from_run_id, to_run_id):
|
|
508
|
+
if kind != 'file':
|
|
509
|
+
raise CloudSubmitError(
|
|
510
|
+
f'Cannot move remote artifacts of kind {repr(kind)}.'
|
|
511
|
+
)
|
|
512
|
+
|
|
482
513
|
base_path = '/'.join([
|
|
483
514
|
's3:/', self._s3_bucket, self._s3_prefix, self._project,
|
|
484
515
|
'users', self._user, 'runs',
|
|
@@ -79,16 +79,16 @@ class LocalEnv(EnvironmentHandler):
|
|
|
79
79
|
def get_remote_artifact_ref(self, artifact, run_id=None):
|
|
80
80
|
return ''
|
|
81
81
|
|
|
82
|
-
def list_remote_artifacts(self, artifacts, run_ids=None):
|
|
82
|
+
def list_remote_artifacts(self, kind, artifacts, run_ids=None):
|
|
83
83
|
return [[] for a in artifacts]
|
|
84
84
|
|
|
85
|
-
def remove_remote_artifacts(self, artifacts, run_ids):
|
|
85
|
+
def remove_remote_artifacts(self, kind, artifacts, run_ids):
|
|
86
86
|
pass
|
|
87
87
|
|
|
88
|
-
def copy_remote_artifacts(self, artifacts, from_run_id, to_run_id):
|
|
88
|
+
def copy_remote_artifacts(self, kind, artifacts, from_run_id, to_run_id):
|
|
89
89
|
pass
|
|
90
90
|
|
|
91
|
-
def move_remote_artifacts(self, artifacts, from_run_id, to_run_id):
|
|
91
|
+
def move_remote_artifacts(self, kind, artifacts, from_run_id, to_run_id):
|
|
92
92
|
pass
|
|
93
93
|
|
|
94
94
|
def run_pipeline(
|
|
@@ -109,10 +109,11 @@ class LocalEnv(EnvironmentHandler):
|
|
|
109
109
|
ensure_path(artifacts_user_path)
|
|
110
110
|
ensure_path(artifacts_run_path)
|
|
111
111
|
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
112
|
+
for kind, artifacts in overwrite_artifacts.items():
|
|
113
|
+
self.remove_local_artifacts(
|
|
114
|
+
kind, artifacts, [[run_id]]*len(artifacts))
|
|
115
|
+
self.remove_remote_artifacts(
|
|
116
|
+
kind, artifacts, [[run_id]]*len(artifacts))
|
|
116
117
|
|
|
117
118
|
for step in pipeline.steps:
|
|
118
119
|
if step.name not in image_refs:
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/envs/aws/local_execution_handler.py
RENAMED
|
File without changes
|
{cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/envs/aws/remote_environment_handler.py
RENAMED
|
File without changes
|
{cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/envs/aws/remote_execution_handler.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/execution/base_execution_handler.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|