cloud-submit 0.4__tar.gz → 0.5.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cloud_submit-0.4 → cloud_submit-0.5.1}/PKG-INFO +1 -1
- {cloud_submit-0.4 → cloud_submit-0.5.1}/pyproject.toml +1 -1
- {cloud_submit-0.4 → cloud_submit-0.5.1}/src/cloud_submit/controller.py +74 -38
- {cloud_submit-0.4 → cloud_submit-0.5.1}/src/cloud_submit/environment_handler.py +29 -20
- {cloud_submit-0.4 → cloud_submit-0.5.1}/src/cloud_submit/envs/aws/local_environment_handler.py +37 -8
- {cloud_submit-0.4 → cloud_submit-0.5.1}/src/cloud_submit/envs/aws/remote_environment_handler.py +5 -3
- {cloud_submit-0.4 → cloud_submit-0.5.1}/src/cloud_submit/envs/local/environment_handler.py +9 -8
- {cloud_submit-0.4 → cloud_submit-0.5.1}/.gitignore +0 -0
- {cloud_submit-0.4 → cloud_submit-0.5.1}/LICENSE +0 -0
- {cloud_submit-0.4 → cloud_submit-0.5.1}/README.md +0 -0
- {cloud_submit-0.4 → cloud_submit-0.5.1}/docs/examples/aws/src/csub/build_config.py +0 -0
- {cloud_submit-0.4 → cloud_submit-0.5.1}/docs/examples/aws/src/fit.py +0 -0
- {cloud_submit-0.4 → cloud_submit-0.5.1}/docs/examples/aws/src/generate.py +0 -0
- {cloud_submit-0.4 → cloud_submit-0.5.1}/docs/examples/aws/src/parameters.py +0 -0
- {cloud_submit-0.4 → cloud_submit-0.5.1}/docs/examples/aws/userconfig/template.yaml +0 -0
- {cloud_submit-0.4 → cloud_submit-0.5.1}/docs/examples/basic/src/csub/build_config.py +0 -0
- {cloud_submit-0.4 → cloud_submit-0.5.1}/docs/examples/basic/src/fit.py +0 -0
- {cloud_submit-0.4 → cloud_submit-0.5.1}/docs/examples/basic/src/generate.py +0 -0
- {cloud_submit-0.4 → cloud_submit-0.5.1}/docs/examples/basic/src/parameters.py +0 -0
- {cloud_submit-0.4 → cloud_submit-0.5.1}/docs/examples/basic/userconfig/template.yaml +0 -0
- {cloud_submit-0.4 → cloud_submit-0.5.1}/docs/examples/multi-worker/src/csub/build_config.py +0 -0
- {cloud_submit-0.4 → cloud_submit-0.5.1}/docs/examples/multi-worker/src/fit.py +0 -0
- {cloud_submit-0.4 → cloud_submit-0.5.1}/docs/examples/multi-worker/src/generate.py +0 -0
- {cloud_submit-0.4 → cloud_submit-0.5.1}/docs/examples/multi-worker/src/parameters.py +0 -0
- {cloud_submit-0.4 → cloud_submit-0.5.1}/docs/examples/multi-worker/userconfig/template.yaml +0 -0
- {cloud_submit-0.4 → cloud_submit-0.5.1}/src/cloud_submit/__init__.py +0 -0
- {cloud_submit-0.4 → cloud_submit-0.5.1}/src/cloud_submit/cli.py +0 -0
- {cloud_submit-0.4 → cloud_submit-0.5.1}/src/cloud_submit/config.py +0 -0
- {cloud_submit-0.4 → cloud_submit-0.5.1}/src/cloud_submit/envs/aws/__init__.py +0 -0
- {cloud_submit-0.4 → cloud_submit-0.5.1}/src/cloud_submit/envs/aws/local_execution_handler.py +0 -0
- {cloud_submit-0.4 → cloud_submit-0.5.1}/src/cloud_submit/envs/aws/remote_execution_handler.py +0 -0
- {cloud_submit-0.4 → cloud_submit-0.5.1}/src/cloud_submit/envs/aws/s3_tools.py +0 -0
- {cloud_submit-0.4 → cloud_submit-0.5.1}/src/cloud_submit/envs/local/execution_handler.py +0 -0
- {cloud_submit-0.4 → cloud_submit-0.5.1}/src/cloud_submit/execution/base_execution_handler.py +0 -0
- {cloud_submit-0.4 → cloud_submit-0.5.1}/src/cloud_submit/execution/config.py +0 -0
- {cloud_submit-0.4 → cloud_submit-0.5.1}/src/cloud_submit/execution/execute.py +0 -0
- {cloud_submit-0.4 → cloud_submit-0.5.1}/src/cloud_submit/execution/utils.py +0 -0
- {cloud_submit-0.4 → cloud_submit-0.5.1}/src/cloud_submit/images.py +0 -0
- {cloud_submit-0.4 → cloud_submit-0.5.1}/src/cloud_submit/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: cloud-submit
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.1
|
|
4
4
|
Summary: A tool for building processing pipelines in Python and executing them locally or in the cloud.
|
|
5
5
|
Project-URL: Homepage, https://github.com/mwiebusch78/cloud-submit
|
|
6
6
|
Project-URL: Bug Tracker, https://github.com/mwiebusch78/cloud-submit/issues
|
|
@@ -212,6 +212,22 @@ class Controller:
|
|
|
212
212
|
with self._config.in_project_root():
|
|
213
213
|
self._clear_image_ref(image)
|
|
214
214
|
|
|
215
|
+
def _split_kinds(self, artifacts):
|
|
216
|
+
kinds = {}
|
|
217
|
+
keys = []
|
|
218
|
+
for artifact in artifacts:
|
|
219
|
+
kind = artifact.kind
|
|
220
|
+
kinds[kind] = kinds.get(kind, [])
|
|
221
|
+
kinds[kind].append(artifact)
|
|
222
|
+
keys.append((kind, len(kinds[kind])-1))
|
|
223
|
+
return kinds, keys
|
|
224
|
+
|
|
225
|
+
def _merge_kinds(self, kinds, keys):
|
|
226
|
+
result = []
|
|
227
|
+
for kind, pos in keys:
|
|
228
|
+
result.append(kinds[kind][pos])
|
|
229
|
+
return result
|
|
230
|
+
|
|
215
231
|
def list_artifacts(
|
|
216
232
|
self,
|
|
217
233
|
artifact_names=None,
|
|
@@ -228,15 +244,19 @@ class Controller:
|
|
|
228
244
|
except KeyError:
|
|
229
245
|
raise CloudSubmitError(f'Unknown artifact name: {name}.')
|
|
230
246
|
artifacts.append(artifact.copy())
|
|
247
|
+
kinds, keys = self._split_kinds(artifacts)
|
|
231
248
|
|
|
232
249
|
env_handler = self._config.get_run_env(env)
|
|
233
250
|
with self._config.in_project_root():
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
251
|
+
runs = {}
|
|
252
|
+
for kind, artifact_list in kinds.items():
|
|
253
|
+
if remote:
|
|
254
|
+
runs[kind] = env_handler.list_remote_artifacts(
|
|
255
|
+
kind, artifact_list, run_ids=run_ids)
|
|
256
|
+
else:
|
|
257
|
+
runs[kind] = env_handler.list_local_artifacts(
|
|
258
|
+
kind, artifact_list, run_ids=run_ids)
|
|
259
|
+
runs = self._merge_kinds(runs, keys)
|
|
240
260
|
|
|
241
261
|
return [a.name for a in artifacts], runs
|
|
242
262
|
|
|
@@ -252,11 +272,15 @@ class Controller:
|
|
|
252
272
|
env_handler = self._config.get_run_env(env)
|
|
253
273
|
artifacts = [
|
|
254
274
|
self._config.artifacts[name].copy() for name in artifact_names]
|
|
275
|
+
kinds, keys = self._split_kinds(artifacts)
|
|
255
276
|
with self._config.in_project_root():
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
277
|
+
for kind, artifact_list in kinds.items():
|
|
278
|
+
if remote:
|
|
279
|
+
env_handler.remove_remote_artifacts(
|
|
280
|
+
kind, artifact_list, run_ids)
|
|
281
|
+
else:
|
|
282
|
+
env_handler.remove_local_artifacts(
|
|
283
|
+
kind, artifact_list, run_ids)
|
|
260
284
|
|
|
261
285
|
def pull_artifacts(
|
|
262
286
|
self,
|
|
@@ -267,9 +291,11 @@ class Controller:
|
|
|
267
291
|
env_handler = self._config.get_run_env(env)
|
|
268
292
|
artifacts = [
|
|
269
293
|
self._config.artifacts[name].copy() for name in artifact_names]
|
|
294
|
+
kinds, keys = self._split_kinds(artifacts)
|
|
270
295
|
with self._config.in_project_root():
|
|
271
|
-
|
|
272
|
-
|
|
296
|
+
for kind, artifact_list in kinds.items():
|
|
297
|
+
env_handler.remove_local_artifacts(kind, artifact_list, run_ids)
|
|
298
|
+
env_handler.pull_artifacts(kind, artifact_list, run_ids)
|
|
273
299
|
|
|
274
300
|
def push_artifacts(
|
|
275
301
|
self,
|
|
@@ -280,9 +306,12 @@ class Controller:
|
|
|
280
306
|
env_handler = self._config.get_run_env(env)
|
|
281
307
|
artifacts = [
|
|
282
308
|
self._config.artifacts[name].copy() for name in artifact_names]
|
|
309
|
+
kinds, keys = self._split_kinds(artifacts)
|
|
283
310
|
with self._config.in_project_root():
|
|
284
|
-
|
|
285
|
-
|
|
311
|
+
for kind, artifact_list in kinds.items():
|
|
312
|
+
env_handler.remove_remote_artifacts(
|
|
313
|
+
kind, artifact_list, run_ids)
|
|
314
|
+
env_handler.push_artifacts(kind, artifact_list, run_ids)
|
|
286
315
|
|
|
287
316
|
def copy_artifacts(
|
|
288
317
|
self,
|
|
@@ -300,17 +329,19 @@ class Controller:
|
|
|
300
329
|
self._config.artifacts[name].copy() for name in artifact_names
|
|
301
330
|
if self._config.artifacts[name].scope == 'run'
|
|
302
331
|
]
|
|
332
|
+
kinds, keys = self._split_kinds(artifacts)
|
|
303
333
|
with self._config.in_project_root():
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
334
|
+
for kind, artifact_list in kinds.items():
|
|
335
|
+
if remote:
|
|
336
|
+
env_handler.remove_remote_artifacts(
|
|
337
|
+
kind, artifact_list, [[to_run_id]]*len(artifact_list))
|
|
338
|
+
env_handler.copy_remote_artifacts(
|
|
339
|
+
kind, artifact_list, from_run_id, to_run_id)
|
|
340
|
+
else:
|
|
341
|
+
env_handler.remove_local_artifacts(
|
|
342
|
+
kind, artifact_list, [[to_run_id]]*len(artifact_list))
|
|
343
|
+
env_handler.copy_local_artifacts(
|
|
344
|
+
kind, artifact_list, from_run_id, to_run_id)
|
|
314
345
|
|
|
315
346
|
def move_artifacts(
|
|
316
347
|
self,
|
|
@@ -328,17 +359,19 @@ class Controller:
|
|
|
328
359
|
self._config.artifacts[name].copy() for name in artifact_names
|
|
329
360
|
if self._config.artifacts[name].scope == 'run'
|
|
330
361
|
]
|
|
362
|
+
kinds, keys = self._split_kinds(artifacts)
|
|
331
363
|
with self._config.in_project_root():
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
364
|
+
for kind, artifact_list in kinds.items():
|
|
365
|
+
if remote:
|
|
366
|
+
env_handler.remove_remote_artifacts(
|
|
367
|
+
kind, artifact_list, [[to_run_id]]*len(artifact_list))
|
|
368
|
+
env_handler.move_remote_artifacts(
|
|
369
|
+
kind, artifact_list, from_run_id, to_run_id)
|
|
370
|
+
else:
|
|
371
|
+
env_handler.remove_local_artifacts(
|
|
372
|
+
kind, artifact_list, [[to_run_id]]*len(artifact_list))
|
|
373
|
+
env_handler.move_local_artifacts(
|
|
374
|
+
kind, artifact_list, from_run_id, to_run_id)
|
|
342
375
|
|
|
343
376
|
def run_pipeline(
|
|
344
377
|
self,
|
|
@@ -386,11 +419,14 @@ class Controller:
|
|
|
386
419
|
refs[step.name] = ':'.join([repo, run_id])
|
|
387
420
|
|
|
388
421
|
for loc in step.outputs.values():
|
|
389
|
-
|
|
390
|
-
|
|
422
|
+
artifact = self._config.artifacts[loc.artifact_name].copy()
|
|
423
|
+
if artifact.scope == 'run':
|
|
424
|
+
overwrite_artifacts.append(artifact)
|
|
391
425
|
for loc in step.temporaries.values():
|
|
392
|
-
|
|
393
|
-
|
|
426
|
+
artifact = self._config.artifacts[loc.artifact_name].copy()
|
|
427
|
+
overwrite_artifacts.append(artifact)
|
|
428
|
+
|
|
429
|
+
overwrite_artifacts, _ = self._split_kinds(overwrite_artifacts)
|
|
394
430
|
|
|
395
431
|
temp_path = 'temp/run'
|
|
396
432
|
ensure_path(temp_path, clear=True)
|
|
@@ -253,9 +253,8 @@ class EnvironmentHandler:
|
|
|
253
253
|
def get_local_artifact_ref(self, artifact, run_id=None):
|
|
254
254
|
if artifact.kind != 'file':
|
|
255
255
|
raise CloudSubmitError(
|
|
256
|
-
f'Cannot get path for local artifact {artifact.name}
|
|
257
|
-
|
|
258
|
-
f'is of kind {repr(artifact.kind)}.'
|
|
256
|
+
f'Cannot get path for local artifact {artifact.name} '
|
|
257
|
+
f'of kind {repr(artifact.kind)}.'
|
|
259
258
|
)
|
|
260
259
|
if artifact.scope == 'project':
|
|
261
260
|
return os.path.join('artifacts', 'shared', artifact.name)
|
|
@@ -277,18 +276,18 @@ class EnvironmentHandler:
|
|
|
277
276
|
def get_remote_artifact_ref(self, artifact, run_id=None):
|
|
278
277
|
raise NotImplementedError
|
|
279
278
|
|
|
280
|
-
def list_local_artifacts(self, artifacts, run_ids=None):
|
|
279
|
+
def list_local_artifacts(self, kind, artifacts, run_ids=None):
|
|
280
|
+
if kind != 'file':
|
|
281
|
+
raise CloudSubmitError(
|
|
282
|
+
'Cannot list run IDs for local artifacts of kind '
|
|
283
|
+
f'{repr(kind)}. '
|
|
284
|
+
)
|
|
285
|
+
|
|
281
286
|
if run_ids is not None:
|
|
282
287
|
run_ids = set(run_ids)
|
|
283
288
|
|
|
284
289
|
results = []
|
|
285
290
|
for artifact in artifacts:
|
|
286
|
-
if artifact.kind != 'file':
|
|
287
|
-
raise CloudSubmitError(
|
|
288
|
-
f'Cannot list run IDs for local artifact {artifact.name}. '
|
|
289
|
-
"Only artifacts of kind 'file' are supported and this one "
|
|
290
|
-
f'is of kind {repr(artifact.kind)}.'
|
|
291
|
-
)
|
|
292
291
|
if artifact.scope == 'run':
|
|
293
292
|
pattern = os.path.join(
|
|
294
293
|
'artifacts', 'users', self._user,
|
|
@@ -317,27 +316,33 @@ class EnvironmentHandler:
|
|
|
317
316
|
results.append(ids)
|
|
318
317
|
return results
|
|
319
318
|
|
|
320
|
-
def list_remote_artifacts(self, artifacts, run_ids=None):
|
|
319
|
+
def list_remote_artifacts(self, kind, artifacts, run_ids=None):
|
|
321
320
|
raise NotImplementedError
|
|
322
321
|
|
|
323
|
-
def push_artifacts(self, artifacts, run_ids):
|
|
322
|
+
def push_artifacts(self, kind, artifacts, run_ids):
|
|
324
323
|
raise NotImplementedError
|
|
325
324
|
|
|
326
|
-
def pull_artifacts(self, artifacts, run_ids):
|
|
325
|
+
def pull_artifacts(self, kind, artifacts, run_ids):
|
|
327
326
|
raise NotImplementedError
|
|
328
327
|
|
|
329
|
-
def remove_local_artifacts(self, artifacts, run_ids):
|
|
328
|
+
def remove_local_artifacts(self, kind, artifacts, run_ids):
|
|
329
|
+
if kind != 'file':
|
|
330
|
+
raise CloudSubmitError(
|
|
331
|
+
f'Cannot remove local artifacts of kind {repr(kind)}. '
|
|
332
|
+
)
|
|
330
333
|
for artifact, runs in zip(artifacts, run_ids):
|
|
331
|
-
if artifact.kind != 'file':
|
|
332
|
-
continue
|
|
333
334
|
for run_id in runs:
|
|
334
335
|
path = self.get_local_artifact_ref(artifact, run_id)
|
|
335
336
|
clear_path(path)
|
|
336
337
|
|
|
337
|
-
def remove_remote_artifacts(self, artifacts, run_ids):
|
|
338
|
+
def remove_remote_artifacts(self, kind, artifacts, run_ids):
|
|
338
339
|
raise NotImplementedError
|
|
339
340
|
|
|
340
|
-
def copy_local_artifacts(self, artifacts, from_run_id, to_run_id):
|
|
341
|
+
def copy_local_artifacts(self, kind, artifacts, from_run_id, to_run_id):
|
|
342
|
+
if kind != 'file':
|
|
343
|
+
raise CloudSubmitError(
|
|
344
|
+
f'Cannot copy local artifacts of kind {repr(kind)}. '
|
|
345
|
+
)
|
|
341
346
|
for artifact in artifacts:
|
|
342
347
|
src_path = self.get_local_artifact_ref(artifact, from_run_id)
|
|
343
348
|
dst_path = self.get_local_artifact_ref(artifact, to_run_id)
|
|
@@ -351,10 +356,14 @@ class EnvironmentHandler:
|
|
|
351
356
|
else:
|
|
352
357
|
raise
|
|
353
358
|
|
|
354
|
-
def copy_remote_artifacts(self, artifacts, from_run_id, to_run_id):
|
|
359
|
+
def copy_remote_artifacts(self, kind, artifacts, from_run_id, to_run_id):
|
|
355
360
|
raise NotImplementedError
|
|
356
361
|
|
|
357
|
-
def move_local_artifacts(self, artifacts, from_run_id, to_run_id):
|
|
362
|
+
def move_local_artifacts(self, kind, artifacts, from_run_id, to_run_id):
|
|
363
|
+
if kind != 'file':
|
|
364
|
+
raise CloudSubmitError(
|
|
365
|
+
f'Cannot move local artifacts of kind {repr(kind)}. '
|
|
366
|
+
)
|
|
358
367
|
for artifact in artifacts:
|
|
359
368
|
src_path = self.get_local_artifact_ref(artifact, from_run_id)
|
|
360
369
|
dst_path = self.get_local_artifact_ref(artifact, to_run_id)
|
{cloud_submit-0.4 → cloud_submit-0.5.1}/src/cloud_submit/envs/aws/local_environment_handler.py
RENAMED
|
@@ -244,7 +244,13 @@ class LocalAWSEnv(LocalEnv):
|
|
|
244
244
|
lines = lines.split('\n')
|
|
245
245
|
return [l.split()[-1] for l in lines]
|
|
246
246
|
|
|
247
|
-
def list_remote_artifacts(self, artifacts, run_ids=None):
|
|
247
|
+
def list_remote_artifacts(self, kind, artifacts, run_ids=None):
|
|
248
|
+
if kind != 'file':
|
|
249
|
+
raise CloudSubmitError(
|
|
250
|
+
'Cannot list run IDs for remote artifacts of kind '
|
|
251
|
+
f'{repr(kind)}. '
|
|
252
|
+
)
|
|
253
|
+
|
|
248
254
|
if run_ids is not None:
|
|
249
255
|
run_ids = set(run_ids)
|
|
250
256
|
|
|
@@ -252,8 +258,6 @@ class LocalAWSEnv(LocalEnv):
|
|
|
252
258
|
user_artifacts = set()
|
|
253
259
|
run_artifacts = set()
|
|
254
260
|
for a in artifacts:
|
|
255
|
-
if a.kind != 'file':
|
|
256
|
-
continue
|
|
257
261
|
if a.scope == 'project':
|
|
258
262
|
project_artifacts.add(a.name)
|
|
259
263
|
elif a.scope == 'user':
|
|
@@ -303,7 +307,12 @@ class LocalAWSEnv(LocalEnv):
|
|
|
303
307
|
results = [sorted(results[a.name]) for a in artifacts]
|
|
304
308
|
return results
|
|
305
309
|
|
|
306
|
-
def remove_remote_artifacts(self, artifacts, run_ids):
|
|
310
|
+
def remove_remote_artifacts(self, kind, artifacts, run_ids):
|
|
311
|
+
if kind != 'file':
|
|
312
|
+
raise CloudSubmitError(
|
|
313
|
+
f'Cannot remove remote artifacts of kind {repr(kind)}.'
|
|
314
|
+
)
|
|
315
|
+
|
|
307
316
|
path = '/'.join(
|
|
308
317
|
['s3:/', self._s3_bucket, self._s3_prefix, self._project])
|
|
309
318
|
command = [
|
|
@@ -354,7 +363,12 @@ class LocalAWSEnv(LocalEnv):
|
|
|
354
363
|
|
|
355
364
|
run_command(command)
|
|
356
365
|
|
|
357
|
-
def push_artifacts(self, artifacts, run_ids):
|
|
366
|
+
def push_artifacts(self, kind, artifacts, run_ids):
|
|
367
|
+
if kind != 'file':
|
|
368
|
+
raise CloudSubmitError(
|
|
369
|
+
f'Cannot push artifacts of kind {repr(kind)}.'
|
|
370
|
+
)
|
|
371
|
+
|
|
358
372
|
remote_path = '/'.join(
|
|
359
373
|
['s3:/', self._s3_bucket, self._s3_prefix, self._project])
|
|
360
374
|
local_path = 'artifacts'
|
|
@@ -404,7 +418,12 @@ class LocalAWSEnv(LocalEnv):
|
|
|
404
418
|
|
|
405
419
|
run_command(command)
|
|
406
420
|
|
|
407
|
-
def pull_artifacts(self, artifacts, run_ids):
|
|
421
|
+
def pull_artifacts(self, kind, artifacts, run_ids):
|
|
422
|
+
if kind != 'file':
|
|
423
|
+
raise CloudSubmitError(
|
|
424
|
+
f'Cannot pull artifacts of kind {repr(kind)}.'
|
|
425
|
+
)
|
|
426
|
+
|
|
408
427
|
remote_path = '/'.join(
|
|
409
428
|
['s3:/', self._s3_bucket, self._s3_prefix, self._project])
|
|
410
429
|
local_path = 'artifacts'
|
|
@@ -454,7 +473,12 @@ class LocalAWSEnv(LocalEnv):
|
|
|
454
473
|
|
|
455
474
|
run_command(command)
|
|
456
475
|
|
|
457
|
-
def copy_remote_artifacts(self, artifacts, from_run_id, to_run_id):
|
|
476
|
+
def copy_remote_artifacts(self, kind, artifacts, from_run_id, to_run_id):
|
|
477
|
+
if kind != 'file':
|
|
478
|
+
raise CloudSubmitError(
|
|
479
|
+
f'Cannot copy remote artifacts of kind {repr(kind)}.'
|
|
480
|
+
)
|
|
481
|
+
|
|
458
482
|
base_path = '/'.join([
|
|
459
483
|
's3:/', self._s3_bucket, self._s3_prefix, self._project,
|
|
460
484
|
'users', self._user, 'runs',
|
|
@@ -480,7 +504,12 @@ class LocalAWSEnv(LocalEnv):
|
|
|
480
504
|
])
|
|
481
505
|
run_command(command)
|
|
482
506
|
|
|
483
|
-
def move_remote_artifacts(self, artifacts, from_run_id, to_run_id):
|
|
507
|
+
def move_remote_artifacts(self, kind, artifacts, from_run_id, to_run_id):
|
|
508
|
+
if kind != 'file':
|
|
509
|
+
raise CloudSubmitError(
|
|
510
|
+
f'Cannot move remote artifacts of kind {repr(kind)}.'
|
|
511
|
+
)
|
|
512
|
+
|
|
484
513
|
base_path = '/'.join([
|
|
485
514
|
's3:/', self._s3_bucket, self._s3_prefix, self._project,
|
|
486
515
|
'users', self._user, 'runs',
|
{cloud_submit-0.4 → cloud_submit-0.5.1}/src/cloud_submit/envs/aws/remote_environment_handler.py
RENAMED
|
@@ -78,11 +78,12 @@ class RemoteAWSEnv(LocalAWSEnv):
|
|
|
78
78
|
os.path.join(path, 's3_tools.py'),
|
|
79
79
|
)
|
|
80
80
|
|
|
81
|
-
def build_image(self, path, image, build_id):
|
|
81
|
+
def build_image(self, path, image, build_id, docker_flags):
|
|
82
82
|
return self._build_image(
|
|
83
83
|
path=path,
|
|
84
84
|
image=image,
|
|
85
85
|
build_id=build_id,
|
|
86
|
+
docker_flags=docker_flags,
|
|
86
87
|
push=True,
|
|
87
88
|
)
|
|
88
89
|
|
|
@@ -381,8 +382,9 @@ class RemoteAWSEnv(LocalAWSEnv):
|
|
|
381
382
|
)
|
|
382
383
|
|
|
383
384
|
print('Clearing output and temporary artifacts.')
|
|
384
|
-
|
|
385
|
-
|
|
385
|
+
for kind, artifacts in overwrite_artifacts.items():
|
|
386
|
+
self.remove_remote_artifacts(
|
|
387
|
+
kind, artifacts, [[run_id]]*len(artifacts))
|
|
386
388
|
|
|
387
389
|
result = run_command(
|
|
388
390
|
[
|
|
@@ -79,16 +79,16 @@ class LocalEnv(EnvironmentHandler):
|
|
|
79
79
|
def get_remote_artifact_ref(self, artifact, run_id=None):
|
|
80
80
|
return ''
|
|
81
81
|
|
|
82
|
-
def list_remote_artifacts(self, artifacts, run_ids=None):
|
|
82
|
+
def list_remote_artifacts(self, kind, artifacts, run_ids=None):
|
|
83
83
|
return [[] for a in artifacts]
|
|
84
84
|
|
|
85
|
-
def remove_remote_artifacts(self, artifacts, run_ids):
|
|
85
|
+
def remove_remote_artifacts(self, kind, artifacts, run_ids):
|
|
86
86
|
pass
|
|
87
87
|
|
|
88
|
-
def copy_remote_artifacts(self, artifacts, from_run_id, to_run_id):
|
|
88
|
+
def copy_remote_artifacts(self, kind, artifacts, from_run_id, to_run_id):
|
|
89
89
|
pass
|
|
90
90
|
|
|
91
|
-
def move_remote_artifacts(self, artifacts, from_run_id, to_run_id):
|
|
91
|
+
def move_remote_artifacts(self, kind, artifacts, from_run_id, to_run_id):
|
|
92
92
|
pass
|
|
93
93
|
|
|
94
94
|
def run_pipeline(
|
|
@@ -109,10 +109,11 @@ class LocalEnv(EnvironmentHandler):
|
|
|
109
109
|
ensure_path(artifacts_user_path)
|
|
110
110
|
ensure_path(artifacts_run_path)
|
|
111
111
|
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
112
|
+
for kind, artifacts in overwrite_artifacts.items():
|
|
113
|
+
self.remove_local_artifacts(
|
|
114
|
+
kind, artifacts, [[run_id]]*len(artifacts))
|
|
115
|
+
self.remove_remote_artifacts(
|
|
116
|
+
kind, artifacts, [[run_id]]*len(artifacts))
|
|
116
117
|
|
|
117
118
|
for step in pipeline.steps:
|
|
118
119
|
if step.name not in image_refs:
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{cloud_submit-0.4 → cloud_submit-0.5.1}/src/cloud_submit/envs/aws/local_execution_handler.py
RENAMED
|
File without changes
|
{cloud_submit-0.4 → cloud_submit-0.5.1}/src/cloud_submit/envs/aws/remote_execution_handler.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{cloud_submit-0.4 → cloud_submit-0.5.1}/src/cloud_submit/execution/base_execution_handler.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|