cloud-submit 0.3.1__tar.gz → 0.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. {cloud_submit-0.3.1 → cloud_submit-0.5}/PKG-INFO +1 -1
  2. {cloud_submit-0.3.1 → cloud_submit-0.5}/pyproject.toml +1 -1
  3. {cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/cli.py +24 -4
  4. {cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/controller.py +80 -42
  5. {cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/environment_handler.py +31 -21
  6. {cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/envs/aws/local_environment_handler.py +42 -11
  7. {cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/envs/local/environment_handler.py +9 -8
  8. {cloud_submit-0.3.1 → cloud_submit-0.5}/.gitignore +0 -0
  9. {cloud_submit-0.3.1 → cloud_submit-0.5}/LICENSE +0 -0
  10. {cloud_submit-0.3.1 → cloud_submit-0.5}/README.md +0 -0
  11. {cloud_submit-0.3.1 → cloud_submit-0.5}/docs/examples/aws/src/csub/build_config.py +0 -0
  12. {cloud_submit-0.3.1 → cloud_submit-0.5}/docs/examples/aws/src/fit.py +0 -0
  13. {cloud_submit-0.3.1 → cloud_submit-0.5}/docs/examples/aws/src/generate.py +0 -0
  14. {cloud_submit-0.3.1 → cloud_submit-0.5}/docs/examples/aws/src/parameters.py +0 -0
  15. {cloud_submit-0.3.1 → cloud_submit-0.5}/docs/examples/aws/userconfig/template.yaml +0 -0
  16. {cloud_submit-0.3.1 → cloud_submit-0.5}/docs/examples/basic/src/csub/build_config.py +0 -0
  17. {cloud_submit-0.3.1 → cloud_submit-0.5}/docs/examples/basic/src/fit.py +0 -0
  18. {cloud_submit-0.3.1 → cloud_submit-0.5}/docs/examples/basic/src/generate.py +0 -0
  19. {cloud_submit-0.3.1 → cloud_submit-0.5}/docs/examples/basic/src/parameters.py +0 -0
  20. {cloud_submit-0.3.1 → cloud_submit-0.5}/docs/examples/basic/userconfig/template.yaml +0 -0
  21. {cloud_submit-0.3.1 → cloud_submit-0.5}/docs/examples/multi-worker/src/csub/build_config.py +0 -0
  22. {cloud_submit-0.3.1 → cloud_submit-0.5}/docs/examples/multi-worker/src/fit.py +0 -0
  23. {cloud_submit-0.3.1 → cloud_submit-0.5}/docs/examples/multi-worker/src/generate.py +0 -0
  24. {cloud_submit-0.3.1 → cloud_submit-0.5}/docs/examples/multi-worker/src/parameters.py +0 -0
  25. {cloud_submit-0.3.1 → cloud_submit-0.5}/docs/examples/multi-worker/userconfig/template.yaml +0 -0
  26. {cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/__init__.py +0 -0
  27. {cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/config.py +0 -0
  28. {cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/envs/aws/__init__.py +0 -0
  29. {cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/envs/aws/local_execution_handler.py +0 -0
  30. {cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/envs/aws/remote_environment_handler.py +0 -0
  31. {cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/envs/aws/remote_execution_handler.py +0 -0
  32. {cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/envs/aws/s3_tools.py +0 -0
  33. {cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/envs/local/execution_handler.py +0 -0
  34. {cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/execution/base_execution_handler.py +0 -0
  35. {cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/execution/config.py +0 -0
  36. {cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/execution/execute.py +0 -0
  37. {cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/execution/utils.py +0 -0
  38. {cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/images.py +0 -0
  39. {cloud_submit-0.3.1 → cloud_submit-0.5}/src/cloud_submit/utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: cloud-submit
3
- Version: 0.3.1
3
+ Version: 0.5
4
4
  Summary: A tool for building processing pipelines in Python and executing them locally or in the cloud.
5
5
  Project-URL: Homepage, https://github.com/mwiebusch78/cloud-submit
6
6
  Project-URL: Bug Tracker, https://github.com/mwiebusch78/cloud-submit/issues
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "cloud-submit"
7
- version = "0.3.1"
7
+ version = "0.5"
8
8
  authors = [
9
9
  { name="Martin Wiebusch" },
10
10
  ]
@@ -82,7 +82,7 @@ def get_steps(pipeline, steps_arg):
82
82
  tokens = steps_arg.split(',')
83
83
  all_steps = [step.name for step in pipeline.steps]
84
84
  indices = {step: i for i, step in enumerate(all_steps)}
85
-
85
+
86
86
  selected = set()
87
87
  for token in tokens:
88
88
  token = token.split(':')
@@ -99,7 +99,7 @@ def get_steps(pipeline, steps_arg):
99
99
  end = indices[token[1]]
100
100
  except KeyError:
101
101
  raise CloudSubmitError(f'Invalid step name: {token[1]}')
102
- for step in all_steps[beg:end+1]:
102
+ for step in all_steps[begin:end+1]:
103
103
  selected.add(step)
104
104
  else:
105
105
  raise CloudSubmitError(f'Invalid step range: {":".join(token)}')
@@ -306,9 +306,19 @@ def images():
306
306
  is_flag=True,
307
307
  help='Ignore IMAGES arguments and build all base images.',
308
308
  )
309
+ @click.option(
310
+ '--no-cache',
311
+ is_flag=True,
312
+ help='Do not use docker cache when building images.',
313
+ )
314
+ @click.option(
315
+ '--pull',
316
+ is_flag=True,
317
+ help='Instruct docker to always attempt to pull all referenced images.',
318
+ )
309
319
  @click.argument('images', type=str, nargs=-1)
310
320
  @click.pass_context
311
- def build_images(ctx, env, build_id, build_all, images):
321
+ def build_images(ctx, env, build_id, build_all, no_cache, pull, images):
312
322
  init(ctx)
313
323
  config = ctx.obj['config']
314
324
  controller = ctx.obj['controller']
@@ -321,8 +331,18 @@ def build_images(ctx, env, build_id, build_all, images):
321
331
  name for name, image in config.images.items()
322
332
  if isinstance(image, BaseImage)
323
333
  ]
334
+ docker_flags = []
335
+ if no_cache:
336
+ docker_flags.append('--no-cache')
337
+ if pull:
338
+ docker_flags.append('--pull')
324
339
  try:
325
- controller.build_images(images, build_id=build_id, env=env)
340
+ controller.build_images(
341
+ images,
342
+ build_id=build_id,
343
+ env=env,
344
+ docker_flags=docker_flags,
345
+ )
326
346
  except CloudSubmitError as e:
327
347
  abort(str(e))
328
348
 
@@ -61,7 +61,7 @@ class Controller:
61
61
  except FileNotFoundError:
62
62
  pass
63
63
 
64
- def _build_image(self, image, build_id, env, rebuild=False):
64
+ def _build_image(self, image, build_id, env, docker_flags, rebuild=False):
65
65
  is_execution_image = isinstance(image, ExecutionImage)
66
66
 
67
67
  if not rebuild and not is_execution_image:
@@ -96,7 +96,7 @@ class Controller:
96
96
  image.setup_builddir(path, parent_ref)
97
97
 
98
98
  print(f'Building image {image.name}.')
99
- ref = env.build_image(path, image, build_id)
99
+ ref = env.build_image(path, image, build_id, docker_flags)
100
100
  if ref is None:
101
101
  raise CloudSubmitError(
102
102
  'build_image method of environment handler did not '
@@ -105,10 +105,12 @@ class Controller:
105
105
  if not is_execution_image:
106
106
  self._save_image_ref(image, ref)
107
107
 
108
- def build_images(self, images, build_id=None, env=None):
108
+ def build_images(self, images, build_id=None, env=None, docker_flags=None):
109
109
  now = dt.datetime.now(tz=dt.UTC)
110
110
  env_handler = self._config.get_build_env(env)
111
111
  build_id = env_handler.generate_build_id(now, build_id)
112
+ if docker_flags is None:
113
+ docker_flags = []
112
114
 
113
115
  images = set(images)
114
116
  for image in images:
@@ -127,7 +129,7 @@ class Controller:
127
129
  continue
128
130
  rebuild = image_name in images
129
131
  self._build_image(
130
- image, build_id, env_handler, rebuild=rebuild)
132
+ image, build_id, env_handler, docker_flags, rebuild=rebuild)
131
133
 
132
134
  def list_image_refs(self, images=None, ids=None, env=None, remote=False):
133
135
  env_handler = self._config.get_build_env(env)
@@ -210,6 +212,22 @@ class Controller:
210
212
  with self._config.in_project_root():
211
213
  self._clear_image_ref(image)
212
214
 
215
+ def _split_kinds(self, artifacts):
216
+ kinds = {}
217
+ keys = []
218
+ for artifact in artifacts:
219
+ kind = artifact.kind
220
+ kinds[kind] = kinds.get(kind, [])
221
+ kinds[kind].append(artifact)
222
+ keys.append((kind, len(kinds[kind])-1))
223
+ return kinds, keys
224
+
225
+ def _merge_kinds(self, kinds, keys):
226
+ result = []
227
+ for kind, pos in keys:
228
+ result.append(kinds[kind][pos])
229
+ return result
230
+
213
231
  def list_artifacts(
214
232
  self,
215
233
  artifact_names=None,
@@ -226,15 +244,19 @@ class Controller:
226
244
  except KeyError:
227
245
  raise CloudSubmitError(f'Unknown artifact name: {name}.')
228
246
  artifacts.append(artifact.copy())
247
+ kinds, keys = self._split_kinds(artifacts)
229
248
 
230
249
  env_handler = self._config.get_run_env(env)
231
250
  with self._config.in_project_root():
232
- if remote:
233
- runs = env_handler.list_remote_artifacts(
234
- artifacts, run_ids=run_ids)
235
- else:
236
- runs = env_handler.list_local_artifacts(
237
- artifacts, run_ids=run_ids)
251
+ runs = {}
252
+ for kind, artifact_list in kinds.items():
253
+ if remote:
254
+ runs[kind] = env_handler.list_remote_artifacts(
255
+ kind, artifact_list, run_ids=run_ids)
256
+ else:
257
+ runs[kind] = env_handler.list_local_artifacts(
258
+ kind, artifact_list, run_ids=run_ids)
259
+ runs = self._merge_kinds(runs, keys)
238
260
 
239
261
  return [a.name for a in artifacts], runs
240
262
 
@@ -250,11 +272,15 @@ class Controller:
250
272
  env_handler = self._config.get_run_env(env)
251
273
  artifacts = [
252
274
  self._config.artifacts[name].copy() for name in artifact_names]
275
+ kinds, keys = self._split_kinds(artifacts)
253
276
  with self._config.in_project_root():
254
- if remote:
255
- env_handler.remove_remote_artifacts(artifacts, run_ids)
256
- else:
257
- env_handler.remove_local_artifacts(artifacts, run_ids)
277
+ for kind, artifact_list in kinds.items():
278
+ if remote:
279
+ env_handler.remove_remote_artifacts(
280
+ kind, artifact_list, run_ids)
281
+ else:
282
+ env_handler.remove_local_artifacts(
283
+ kind, artifact_list, run_ids)
258
284
 
259
285
  def pull_artifacts(
260
286
  self,
@@ -265,9 +291,11 @@ class Controller:
265
291
  env_handler = self._config.get_run_env(env)
266
292
  artifacts = [
267
293
  self._config.artifacts[name].copy() for name in artifact_names]
294
+ kinds, keys = self._split_kinds(artifacts)
268
295
  with self._config.in_project_root():
269
- env_handler.remove_local_artifacts(artifacts, run_ids)
270
- env_handler.pull_artifacts(artifacts, run_ids)
296
+ for kind, artifact_list in kinds.items():
297
+ env_handler.remove_local_artifacts(kind, artifact_list, run_ids)
298
+ env_handler.pull_artifacts(kind, artifact_list, run_ids)
271
299
 
272
300
  def push_artifacts(
273
301
  self,
@@ -278,9 +306,12 @@ class Controller:
278
306
  env_handler = self._config.get_run_env(env)
279
307
  artifacts = [
280
308
  self._config.artifacts[name].copy() for name in artifact_names]
309
+ kinds, keys = self._split_kinds(artifacts)
281
310
  with self._config.in_project_root():
282
- env_handler.remove_remote_artifacts(artifacts, run_ids)
283
- env_handler.push_artifacts(artifacts, run_ids)
311
+ for kind, artifact_list in kinds.items():
312
+ env_handler.remove_remote_artifacts(
313
+ kind, artifact_list, run_ids)
314
+ env_handler.push_artifacts(kind, artifact_list, run_ids)
284
315
 
285
316
  def copy_artifacts(
286
317
  self,
@@ -298,17 +329,19 @@ class Controller:
298
329
  self._config.artifacts[name].copy() for name in artifact_names
299
330
  if self._config.artifacts[name].scope == 'run'
300
331
  ]
332
+ kinds, keys = self._split_kinds(artifacts)
301
333
  with self._config.in_project_root():
302
- if remote:
303
- env_handler.remove_remote_artifacts(
304
- artifacts, [[to_run_id]]*len(artifacts))
305
- env_handler.copy_remote_artifacts(
306
- artifacts, from_run_id, to_run_id)
307
- else:
308
- env_handler.remove_local_artifacts(
309
- artifacts, [[to_run_id]]*len(artifacts))
310
- env_handler.copy_local_artifacts(
311
- artifacts, from_run_id, to_run_id)
334
+ for kind, artifact_list in kinds.items():
335
+ if remote:
336
+ env_handler.remove_remote_artifacts(
337
+ kind, artifact_list, [[to_run_id]]*len(artifact_list))
338
+ env_handler.copy_remote_artifacts(
339
+ kind, artifact_list, from_run_id, to_run_id)
340
+ else:
341
+ env_handler.remove_local_artifacts(
342
+ kind, artifact_list, [[to_run_id]]*len(artifact_list))
343
+ env_handler.copy_local_artifacts(
344
+ kind, artifact_list, from_run_id, to_run_id)
312
345
 
313
346
  def move_artifacts(
314
347
  self,
@@ -326,17 +359,19 @@ class Controller:
326
359
  self._config.artifacts[name].copy() for name in artifact_names
327
360
  if self._config.artifacts[name].scope == 'run'
328
361
  ]
362
+ kinds, keys = self._split_kinds(artifacts)
329
363
  with self._config.in_project_root():
330
- if remote:
331
- env_handler.remove_remote_artifacts(
332
- artifacts, [[to_run_id]]*len(artifacts))
333
- env_handler.move_remote_artifacts(
334
- artifacts, from_run_id, to_run_id)
335
- else:
336
- env_handler.remove_local_artifacts(
337
- artifacts, [[to_run_id]]*len(artifacts))
338
- env_handler.move_local_artifacts(
339
- artifacts, from_run_id, to_run_id)
364
+ for kind, artifact_list in kinds.items():
365
+ if remote:
366
+ env_handler.remove_remote_artifacts(
367
+ kind, artifact_list, [[to_run_id]]*len(artifact_list))
368
+ env_handler.move_remote_artifacts(
369
+ kind, artifact_list, from_run_id, to_run_id)
370
+ else:
371
+ env_handler.remove_local_artifacts(
372
+ kind, artifact_list, [[to_run_id]]*len(artifact_list))
373
+ env_handler.move_local_artifacts(
374
+ kind, artifact_list, from_run_id, to_run_id)
340
375
 
341
376
  def run_pipeline(
342
377
  self,
@@ -384,11 +419,14 @@ class Controller:
384
419
  refs[step.name] = ':'.join([repo, run_id])
385
420
 
386
421
  for loc in step.outputs.values():
387
- overwrite_artifacts.append(
388
- self._config.artifacts[loc.artifact_name].copy())
422
+ artifact = self._config.artifacts[loc.artifact_name].copy()
423
+ if artifact.scope == 'run':
424
+ overwrite_artifacts.append(artifact)
389
425
  for loc in step.temporaries.values():
390
- overwrite_artifacts.append(
391
- self._config.artifacts[loc.artifact_name].copy())
426
+ artifact = self._config.artifacts[loc.artifact_name].copy()
427
+ overwrite_artifacts.append(artifact)
428
+
429
+ overwrite_artifacts, _ = self._split_kinds(overwrite_artifacts)
392
430
 
393
431
  temp_path = 'temp/run'
394
432
  ensure_path(temp_path, clear=True)
@@ -235,13 +235,14 @@ class EnvironmentHandler:
235
235
  def remove_remote_image_refs(self, refs):
236
236
  raise NotImplementedError
237
237
 
238
- def build_image(self, path, image, build_id):
238
+ def build_image(self, path, image, build_id, docker_flags):
239
239
  repo = self.get_image_repo(image)
240
240
  ref = ':'.join([repo, build_id])
241
241
  command = [
242
242
  self._docker_command,
243
243
  'buildx',
244
244
  'build',
245
+ *docker_flags,
245
246
  *(image.args),
246
247
  '-t', ref,
247
248
  path,
@@ -252,9 +253,8 @@ class EnvironmentHandler:
252
253
  def get_local_artifact_ref(self, artifact, run_id=None):
253
254
  if artifact.kind != 'file':
254
255
  raise CloudSubmitError(
255
- f'Cannot get path for local artifact {artifact.name}. '
256
- "Only artifacts of kind 'file' are supported and this one "
257
- f'is of kind {repr(artifact.kind)}.'
256
+ f'Cannot get path for local artifact {artifact.name} '
257
+ f'of kind {repr(artifact.kind)}.'
258
258
  )
259
259
  if artifact.scope == 'project':
260
260
  return os.path.join('artifacts', 'shared', artifact.name)
@@ -276,18 +276,18 @@ class EnvironmentHandler:
276
276
  def get_remote_artifact_ref(self, artifact, run_id=None):
277
277
  raise NotImplementedError
278
278
 
279
- def list_local_artifacts(self, artifacts, run_ids=None):
279
+ def list_local_artifacts(self, kind, artifacts, run_ids=None):
280
+ if kind != 'file':
281
+ raise CloudSubmitError(
282
+ 'Cannot list run IDs for local artifacts of kind '
283
+ f'{repr(kind)}. '
284
+ )
285
+
280
286
  if run_ids is not None:
281
287
  run_ids = set(run_ids)
282
288
 
283
289
  results = []
284
290
  for artifact in artifacts:
285
- if artifact.kind != 'file':
286
- raise CloudSubmitError(
287
- f'Cannot list run IDs for local artifact {artifact.name}. '
288
- "Only artifacts of kind 'file' are supported and this one "
289
- f'is of kind {repr(artifact.kind)}.'
290
- )
291
291
  if artifact.scope == 'run':
292
292
  pattern = os.path.join(
293
293
  'artifacts', 'users', self._user,
@@ -316,27 +316,33 @@ class EnvironmentHandler:
316
316
  results.append(ids)
317
317
  return results
318
318
 
319
- def list_remote_artifacts(self, artifacts, run_ids=None):
319
+ def list_remote_artifacts(self, kind, artifacts, run_ids=None):
320
320
  raise NotImplementedError
321
321
 
322
- def push_artifacts(self, artifacts, run_ids):
322
+ def push_artifacts(self, kind, artifacts, run_ids):
323
323
  raise NotImplementedError
324
324
 
325
- def pull_artifacts(self, artifacts, run_ids):
325
+ def pull_artifacts(self, kind, artifacts, run_ids):
326
326
  raise NotImplementedError
327
327
 
328
- def remove_local_artifacts(self, artifacts, run_ids):
328
+ def remove_local_artifacts(self, kind, artifacts, run_ids):
329
+ if kind != 'file':
330
+ raise CloudSubmitError(
331
+ f'Cannot remove local artifacts of kind {repr(kind)}. '
332
+ )
329
333
  for artifact, runs in zip(artifacts, run_ids):
330
- if artifact.kind != 'file':
331
- continue
332
334
  for run_id in runs:
333
335
  path = self.get_local_artifact_ref(artifact, run_id)
334
336
  clear_path(path)
335
337
 
336
- def remove_remote_artifacts(self, artifacts, run_ids):
338
+ def remove_remote_artifacts(self, kind, artifacts, run_ids):
337
339
  raise NotImplementedError
338
340
 
339
- def copy_local_artifacts(self, artifacts, from_run_id, to_run_id):
341
+ def copy_local_artifacts(self, kind, artifacts, from_run_id, to_run_id):
342
+ if kind != 'file':
343
+ raise CloudSubmitError(
344
+ f'Cannot copy local artifacts of kind {repr(kind)}. '
345
+ )
340
346
  for artifact in artifacts:
341
347
  src_path = self.get_local_artifact_ref(artifact, from_run_id)
342
348
  dst_path = self.get_local_artifact_ref(artifact, to_run_id)
@@ -350,10 +356,14 @@ class EnvironmentHandler:
350
356
  else:
351
357
  raise
352
358
 
353
- def copy_remote_artifacts(self, artifacts, from_run_id, to_run_id):
359
+ def copy_remote_artifacts(self, kind, artifacts, from_run_id, to_run_id):
354
360
  raise NotImplementedError
355
361
 
356
- def move_local_artifacts(self, artifacts, from_run_id, to_run_id):
362
+ def move_local_artifacts(self, kind, artifacts, from_run_id, to_run_id):
363
+ if kind != 'file':
364
+ raise CloudSubmitError(
365
+ f'Cannot move local artifacts of kind {repr(kind)}. '
366
+ )
357
367
  for artifact in artifacts:
358
368
  src_path = self.get_local_artifact_ref(artifact, from_run_id)
359
369
  dst_path = self.get_local_artifact_ref(artifact, to_run_id)
@@ -129,9 +129,9 @@ class LocalAWSEnv(LocalEnv):
129
129
  os.path.join(path, 's3_tools.py'),
130
130
  )
131
131
 
132
- def _build_image(self, path, image, build_id, push):
132
+ def _build_image(self, path, image, build_id, docker_flags, push):
133
133
  if not push:
134
- ref = super().build_image(path, image, build_id)
134
+ ref = super().build_image(path, image, build_id, docker_flags)
135
135
  else:
136
136
  self._docker_login()
137
137
  repo = self.get_image_repo(image)
@@ -140,6 +140,7 @@ class LocalAWSEnv(LocalEnv):
140
140
  self._docker_command,
141
141
  'buildx',
142
142
  'build',
143
+ *docker_flags,
143
144
  *(image.args),
144
145
  ]
145
146
  if self._docker_platforms is not None:
@@ -148,11 +149,12 @@ class LocalAWSEnv(LocalEnv):
148
149
  run_command(command)
149
150
  return ref
150
151
 
151
- def build_image(self, path, image, build_id):
152
+ def build_image(self, path, image, build_id, docker_flags):
152
153
  return self._build_image(
153
154
  path=path,
154
155
  image=image,
155
156
  build_id=build_id,
157
+ docker_flags=docker_flags,
156
158
  push=isinstance(image, BaseImage),
157
159
  )
158
160
 
@@ -242,7 +244,13 @@ class LocalAWSEnv(LocalEnv):
242
244
  lines = lines.split('\n')
243
245
  return [l.split()[-1] for l in lines]
244
246
 
245
- def list_remote_artifacts(self, artifacts, run_ids=None):
247
+ def list_remote_artifacts(self, kind, artifacts, run_ids=None):
248
+ if kind != 'file':
249
+ raise CloudSubmitError(
250
+ 'Cannot list run IDs for remote artifacts of kind '
251
+ f'{repr(kind)}. '
252
+ )
253
+
246
254
  if run_ids is not None:
247
255
  run_ids = set(run_ids)
248
256
 
@@ -250,8 +258,6 @@ class LocalAWSEnv(LocalEnv):
250
258
  user_artifacts = set()
251
259
  run_artifacts = set()
252
260
  for a in artifacts:
253
- if a.kind != 'file':
254
- continue
255
261
  if a.scope == 'project':
256
262
  project_artifacts.add(a.name)
257
263
  elif a.scope == 'user':
@@ -301,7 +307,12 @@ class LocalAWSEnv(LocalEnv):
301
307
  results = [sorted(results[a.name]) for a in artifacts]
302
308
  return results
303
309
 
304
- def remove_remote_artifacts(self, artifacts, run_ids):
310
+ def remove_remote_artifacts(self, kind, artifacts, run_ids):
311
+ if kind != 'file':
312
+ raise CloudSubmitError(
313
+ f'Cannot remove remote artifacts of kind {repr(kind)}.'
314
+ )
315
+
305
316
  path = '/'.join(
306
317
  ['s3:/', self._s3_bucket, self._s3_prefix, self._project])
307
318
  command = [
@@ -352,7 +363,12 @@ class LocalAWSEnv(LocalEnv):
352
363
 
353
364
  run_command(command)
354
365
 
355
- def push_artifacts(self, artifacts, run_ids):
366
+ def push_artifacts(self, kind, artifacts, run_ids):
367
+ if kind != 'file':
368
+ raise CloudSubmitError(
369
+ f'Cannot push artifacts of kind {repr(kind)}.'
370
+ )
371
+
356
372
  remote_path = '/'.join(
357
373
  ['s3:/', self._s3_bucket, self._s3_prefix, self._project])
358
374
  local_path = 'artifacts'
@@ -402,7 +418,12 @@ class LocalAWSEnv(LocalEnv):
402
418
 
403
419
  run_command(command)
404
420
 
405
- def pull_artifacts(self, artifacts, run_ids):
421
+ def pull_artifacts(self, kind, artifacts, run_ids):
422
+ if kind != 'file':
423
+ raise CloudSubmitError(
424
+ f'Cannot pull artifacts of kind {repr(kind)}.'
425
+ )
426
+
406
427
  remote_path = '/'.join(
407
428
  ['s3:/', self._s3_bucket, self._s3_prefix, self._project])
408
429
  local_path = 'artifacts'
@@ -452,7 +473,12 @@ class LocalAWSEnv(LocalEnv):
452
473
 
453
474
  run_command(command)
454
475
 
455
- def copy_remote_artifacts(self, artifacts, from_run_id, to_run_id):
476
+ def copy_remote_artifacts(self, kind, artifacts, from_run_id, to_run_id):
477
+ if kind != 'file':
478
+ raise CloudSubmitError(
479
+ f'Cannot copy remote artifacts of kind {repr(kind)}.'
480
+ )
481
+
456
482
  base_path = '/'.join([
457
483
  's3:/', self._s3_bucket, self._s3_prefix, self._project,
458
484
  'users', self._user, 'runs',
@@ -478,7 +504,12 @@ class LocalAWSEnv(LocalEnv):
478
504
  ])
479
505
  run_command(command)
480
506
 
481
- def move_remote_artifacts(self, artifacts, from_run_id, to_run_id):
507
+ def move_remote_artifacts(self, kind, artifacts, from_run_id, to_run_id):
508
+ if kind != 'file':
509
+ raise CloudSubmitError(
510
+ f'Cannot move remote artifacts of kind {repr(kind)}.'
511
+ )
512
+
482
513
  base_path = '/'.join([
483
514
  's3:/', self._s3_bucket, self._s3_prefix, self._project,
484
515
  'users', self._user, 'runs',
@@ -79,16 +79,16 @@ class LocalEnv(EnvironmentHandler):
79
79
  def get_remote_artifact_ref(self, artifact, run_id=None):
80
80
  return ''
81
81
 
82
- def list_remote_artifacts(self, artifacts, run_ids=None):
82
+ def list_remote_artifacts(self, kind, artifacts, run_ids=None):
83
83
  return [[] for a in artifacts]
84
84
 
85
- def remove_remote_artifacts(self, artifacts, run_ids):
85
+ def remove_remote_artifacts(self, kind, artifacts, run_ids):
86
86
  pass
87
87
 
88
- def copy_remote_artifacts(self, artifacts, from_run_id, to_run_id):
88
+ def copy_remote_artifacts(self, kind, artifacts, from_run_id, to_run_id):
89
89
  pass
90
90
 
91
- def move_remote_artifacts(self, artifacts, from_run_id, to_run_id):
91
+ def move_remote_artifacts(self, kind, artifacts, from_run_id, to_run_id):
92
92
  pass
93
93
 
94
94
  def run_pipeline(
@@ -109,10 +109,11 @@ class LocalEnv(EnvironmentHandler):
109
109
  ensure_path(artifacts_user_path)
110
110
  ensure_path(artifacts_run_path)
111
111
 
112
- self.remove_local_artifacts(
113
- overwrite_artifacts, [[run_id]]*len(overwrite_artifacts))
114
- self.remove_remote_artifacts(
115
- overwrite_artifacts, [[run_id]]*len(overwrite_artifacts))
112
+ for kind, artifacts in overwrite_artifacts.items():
113
+ self.remove_local_artifacts(
114
+ kind, artifacts, [[run_id]]*len(artifacts))
115
+ self.remove_remote_artifacts(
116
+ kind, artifacts, [[run_id]]*len(artifacts))
116
117
 
117
118
  for step in pipeline.steps:
118
119
  if step.name not in image_refs:
File without changes
File without changes
File without changes