cloud-submit 0.4__tar.gz → 0.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. {cloud_submit-0.4 → cloud_submit-0.5}/PKG-INFO +1 -1
  2. {cloud_submit-0.4 → cloud_submit-0.5}/pyproject.toml +1 -1
  3. {cloud_submit-0.4 → cloud_submit-0.5}/src/cloud_submit/controller.py +74 -38
  4. {cloud_submit-0.4 → cloud_submit-0.5}/src/cloud_submit/environment_handler.py +29 -20
  5. {cloud_submit-0.4 → cloud_submit-0.5}/src/cloud_submit/envs/aws/local_environment_handler.py +37 -8
  6. {cloud_submit-0.4 → cloud_submit-0.5}/src/cloud_submit/envs/local/environment_handler.py +9 -8
  7. {cloud_submit-0.4 → cloud_submit-0.5}/.gitignore +0 -0
  8. {cloud_submit-0.4 → cloud_submit-0.5}/LICENSE +0 -0
  9. {cloud_submit-0.4 → cloud_submit-0.5}/README.md +0 -0
  10. {cloud_submit-0.4 → cloud_submit-0.5}/docs/examples/aws/src/csub/build_config.py +0 -0
  11. {cloud_submit-0.4 → cloud_submit-0.5}/docs/examples/aws/src/fit.py +0 -0
  12. {cloud_submit-0.4 → cloud_submit-0.5}/docs/examples/aws/src/generate.py +0 -0
  13. {cloud_submit-0.4 → cloud_submit-0.5}/docs/examples/aws/src/parameters.py +0 -0
  14. {cloud_submit-0.4 → cloud_submit-0.5}/docs/examples/aws/userconfig/template.yaml +0 -0
  15. {cloud_submit-0.4 → cloud_submit-0.5}/docs/examples/basic/src/csub/build_config.py +0 -0
  16. {cloud_submit-0.4 → cloud_submit-0.5}/docs/examples/basic/src/fit.py +0 -0
  17. {cloud_submit-0.4 → cloud_submit-0.5}/docs/examples/basic/src/generate.py +0 -0
  18. {cloud_submit-0.4 → cloud_submit-0.5}/docs/examples/basic/src/parameters.py +0 -0
  19. {cloud_submit-0.4 → cloud_submit-0.5}/docs/examples/basic/userconfig/template.yaml +0 -0
  20. {cloud_submit-0.4 → cloud_submit-0.5}/docs/examples/multi-worker/src/csub/build_config.py +0 -0
  21. {cloud_submit-0.4 → cloud_submit-0.5}/docs/examples/multi-worker/src/fit.py +0 -0
  22. {cloud_submit-0.4 → cloud_submit-0.5}/docs/examples/multi-worker/src/generate.py +0 -0
  23. {cloud_submit-0.4 → cloud_submit-0.5}/docs/examples/multi-worker/src/parameters.py +0 -0
  24. {cloud_submit-0.4 → cloud_submit-0.5}/docs/examples/multi-worker/userconfig/template.yaml +0 -0
  25. {cloud_submit-0.4 → cloud_submit-0.5}/src/cloud_submit/__init__.py +0 -0
  26. {cloud_submit-0.4 → cloud_submit-0.5}/src/cloud_submit/cli.py +0 -0
  27. {cloud_submit-0.4 → cloud_submit-0.5}/src/cloud_submit/config.py +0 -0
  28. {cloud_submit-0.4 → cloud_submit-0.5}/src/cloud_submit/envs/aws/__init__.py +0 -0
  29. {cloud_submit-0.4 → cloud_submit-0.5}/src/cloud_submit/envs/aws/local_execution_handler.py +0 -0
  30. {cloud_submit-0.4 → cloud_submit-0.5}/src/cloud_submit/envs/aws/remote_environment_handler.py +0 -0
  31. {cloud_submit-0.4 → cloud_submit-0.5}/src/cloud_submit/envs/aws/remote_execution_handler.py +0 -0
  32. {cloud_submit-0.4 → cloud_submit-0.5}/src/cloud_submit/envs/aws/s3_tools.py +0 -0
  33. {cloud_submit-0.4 → cloud_submit-0.5}/src/cloud_submit/envs/local/execution_handler.py +0 -0
  34. {cloud_submit-0.4 → cloud_submit-0.5}/src/cloud_submit/execution/base_execution_handler.py +0 -0
  35. {cloud_submit-0.4 → cloud_submit-0.5}/src/cloud_submit/execution/config.py +0 -0
  36. {cloud_submit-0.4 → cloud_submit-0.5}/src/cloud_submit/execution/execute.py +0 -0
  37. {cloud_submit-0.4 → cloud_submit-0.5}/src/cloud_submit/execution/utils.py +0 -0
  38. {cloud_submit-0.4 → cloud_submit-0.5}/src/cloud_submit/images.py +0 -0
  39. {cloud_submit-0.4 → cloud_submit-0.5}/src/cloud_submit/utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: cloud-submit
3
- Version: 0.4
3
+ Version: 0.5
4
4
  Summary: A tool for building processing pipelines in Python and executing them locally or in the cloud.
5
5
  Project-URL: Homepage, https://github.com/mwiebusch78/cloud-submit
6
6
  Project-URL: Bug Tracker, https://github.com/mwiebusch78/cloud-submit/issues
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "cloud-submit"
7
- version = "0.4"
7
+ version = "0.5"
8
8
  authors = [
9
9
  { name="Martin Wiebusch" },
10
10
  ]
@@ -212,6 +212,22 @@ class Controller:
212
212
  with self._config.in_project_root():
213
213
  self._clear_image_ref(image)
214
214
 
215
+ def _split_kinds(self, artifacts):
216
+ kinds = {}
217
+ keys = []
218
+ for artifact in artifacts:
219
+ kind = artifact.kind
220
+ kinds[kind] = kinds.get(kind, [])
221
+ kinds[kind].append(artifact)
222
+ keys.append((kind, len(kinds[kind])-1))
223
+ return kinds, keys
224
+
225
+ def _merge_kinds(self, kinds, keys):
226
+ result = []
227
+ for kind, pos in keys:
228
+ result.append(kinds[kind][pos])
229
+ return result
230
+
215
231
  def list_artifacts(
216
232
  self,
217
233
  artifact_names=None,
@@ -228,15 +244,19 @@ class Controller:
228
244
  except KeyError:
229
245
  raise CloudSubmitError(f'Unknown artifact name: {name}.')
230
246
  artifacts.append(artifact.copy())
247
+ kinds, keys = self._split_kinds(artifacts)
231
248
 
232
249
  env_handler = self._config.get_run_env(env)
233
250
  with self._config.in_project_root():
234
- if remote:
235
- runs = env_handler.list_remote_artifacts(
236
- artifacts, run_ids=run_ids)
237
- else:
238
- runs = env_handler.list_local_artifacts(
239
- artifacts, run_ids=run_ids)
251
+ runs = {}
252
+ for kind, artifact_list in kinds.items():
253
+ if remote:
254
+ runs[kind] = env_handler.list_remote_artifacts(
255
+ kind, artifact_list, run_ids=run_ids)
256
+ else:
257
+ runs[kind] = env_handler.list_local_artifacts(
258
+ kind, artifact_list, run_ids=run_ids)
259
+ runs = self._merge_kinds(runs, keys)
240
260
 
241
261
  return [a.name for a in artifacts], runs
242
262
 
@@ -252,11 +272,15 @@ class Controller:
252
272
  env_handler = self._config.get_run_env(env)
253
273
  artifacts = [
254
274
  self._config.artifacts[name].copy() for name in artifact_names]
275
+ kinds, keys = self._split_kinds(artifacts)
255
276
  with self._config.in_project_root():
256
- if remote:
257
- env_handler.remove_remote_artifacts(artifacts, run_ids)
258
- else:
259
- env_handler.remove_local_artifacts(artifacts, run_ids)
277
+ for kind, artifact_list in kinds.items():
278
+ if remote:
279
+ env_handler.remove_remote_artifacts(
280
+ kind, artifact_list, run_ids)
281
+ else:
282
+ env_handler.remove_local_artifacts(
283
+ kind, artifact_list, run_ids)
260
284
 
261
285
  def pull_artifacts(
262
286
  self,
@@ -267,9 +291,11 @@ class Controller:
267
291
  env_handler = self._config.get_run_env(env)
268
292
  artifacts = [
269
293
  self._config.artifacts[name].copy() for name in artifact_names]
294
+ kinds, keys = self._split_kinds(artifacts)
270
295
  with self._config.in_project_root():
271
- env_handler.remove_local_artifacts(artifacts, run_ids)
272
- env_handler.pull_artifacts(artifacts, run_ids)
296
+ for kind, artifact_list in kinds.items():
297
+ env_handler.remove_local_artifacts(kind, artifact_list, run_ids)
298
+ env_handler.pull_artifacts(kind, artifact_list, run_ids)
273
299
 
274
300
  def push_artifacts(
275
301
  self,
@@ -280,9 +306,12 @@ class Controller:
280
306
  env_handler = self._config.get_run_env(env)
281
307
  artifacts = [
282
308
  self._config.artifacts[name].copy() for name in artifact_names]
309
+ kinds, keys = self._split_kinds(artifacts)
283
310
  with self._config.in_project_root():
284
- env_handler.remove_remote_artifacts(artifacts, run_ids)
285
- env_handler.push_artifacts(artifacts, run_ids)
311
+ for kind, artifact_list in kinds.items():
312
+ env_handler.remove_remote_artifacts(
313
+ kind, artifact_list, run_ids)
314
+ env_handler.push_artifacts(kind, artifact_list, run_ids)
286
315
 
287
316
  def copy_artifacts(
288
317
  self,
@@ -300,17 +329,19 @@ class Controller:
300
329
  self._config.artifacts[name].copy() for name in artifact_names
301
330
  if self._config.artifacts[name].scope == 'run'
302
331
  ]
332
+ kinds, keys = self._split_kinds(artifacts)
303
333
  with self._config.in_project_root():
304
- if remote:
305
- env_handler.remove_remote_artifacts(
306
- artifacts, [[to_run_id]]*len(artifacts))
307
- env_handler.copy_remote_artifacts(
308
- artifacts, from_run_id, to_run_id)
309
- else:
310
- env_handler.remove_local_artifacts(
311
- artifacts, [[to_run_id]]*len(artifacts))
312
- env_handler.copy_local_artifacts(
313
- artifacts, from_run_id, to_run_id)
334
+ for kind, artifact_list in kinds.items():
335
+ if remote:
336
+ env_handler.remove_remote_artifacts(
337
+ kind, artifact_list, [[to_run_id]]*len(artifact_list))
338
+ env_handler.copy_remote_artifacts(
339
+ kind, artifact_list, from_run_id, to_run_id)
340
+ else:
341
+ env_handler.remove_local_artifacts(
342
+ kind, artifact_list, [[to_run_id]]*len(artifact_list))
343
+ env_handler.copy_local_artifacts(
344
+ kind, artifact_list, from_run_id, to_run_id)
314
345
 
315
346
  def move_artifacts(
316
347
  self,
@@ -328,17 +359,19 @@ class Controller:
328
359
  self._config.artifacts[name].copy() for name in artifact_names
329
360
  if self._config.artifacts[name].scope == 'run'
330
361
  ]
362
+ kinds, keys = self._split_kinds(artifacts)
331
363
  with self._config.in_project_root():
332
- if remote:
333
- env_handler.remove_remote_artifacts(
334
- artifacts, [[to_run_id]]*len(artifacts))
335
- env_handler.move_remote_artifacts(
336
- artifacts, from_run_id, to_run_id)
337
- else:
338
- env_handler.remove_local_artifacts(
339
- artifacts, [[to_run_id]]*len(artifacts))
340
- env_handler.move_local_artifacts(
341
- artifacts, from_run_id, to_run_id)
364
+ for kind, artifact_list in kinds.items():
365
+ if remote:
366
+ env_handler.remove_remote_artifacts(
367
+ kind, artifact_list, [[to_run_id]]*len(artifact_list))
368
+ env_handler.move_remote_artifacts(
369
+ kind, artifact_list, from_run_id, to_run_id)
370
+ else:
371
+ env_handler.remove_local_artifacts(
372
+ kind, artifact_list, [[to_run_id]]*len(artifact_list))
373
+ env_handler.move_local_artifacts(
374
+ kind, artifact_list, from_run_id, to_run_id)
342
375
 
343
376
  def run_pipeline(
344
377
  self,
@@ -386,11 +419,14 @@ class Controller:
386
419
  refs[step.name] = ':'.join([repo, run_id])
387
420
 
388
421
  for loc in step.outputs.values():
389
- overwrite_artifacts.append(
390
- self._config.artifacts[loc.artifact_name].copy())
422
+ artifact = self._config.artifacts[loc.artifact_name].copy()
423
+ if artifact.scope == 'run':
424
+ overwrite_artifacts.append(artifact)
391
425
  for loc in step.temporaries.values():
392
- overwrite_artifacts.append(
393
- self._config.artifacts[loc.artifact_name].copy())
426
+ artifact = self._config.artifacts[loc.artifact_name].copy()
427
+ overwrite_artifacts.append(artifact)
428
+
429
+ overwrite_artifacts, _ = self._split_kinds(overwrite_artifacts)
394
430
 
395
431
  temp_path = 'temp/run'
396
432
  ensure_path(temp_path, clear=True)
@@ -253,9 +253,8 @@ class EnvironmentHandler:
253
253
  def get_local_artifact_ref(self, artifact, run_id=None):
254
254
  if artifact.kind != 'file':
255
255
  raise CloudSubmitError(
256
- f'Cannot get path for local artifact {artifact.name}. '
257
- "Only artifacts of kind 'file' are supported and this one "
258
- f'is of kind {repr(artifact.kind)}.'
256
+ f'Cannot get path for local artifact {artifact.name} '
257
+ f'of kind {repr(artifact.kind)}.'
259
258
  )
260
259
  if artifact.scope == 'project':
261
260
  return os.path.join('artifacts', 'shared', artifact.name)
@@ -277,18 +276,18 @@ class EnvironmentHandler:
277
276
  def get_remote_artifact_ref(self, artifact, run_id=None):
278
277
  raise NotImplementedError
279
278
 
280
- def list_local_artifacts(self, artifacts, run_ids=None):
279
+ def list_local_artifacts(self, kind, artifacts, run_ids=None):
280
+ if kind != 'file':
281
+ raise CloudSubmitError(
282
+ 'Cannot list run IDs for local artifacts of kind '
283
+ f'{repr(kind)}. '
284
+ )
285
+
281
286
  if run_ids is not None:
282
287
  run_ids = set(run_ids)
283
288
 
284
289
  results = []
285
290
  for artifact in artifacts:
286
- if artifact.kind != 'file':
287
- raise CloudSubmitError(
288
- f'Cannot list run IDs for local artifact {artifact.name}. '
289
- "Only artifacts of kind 'file' are supported and this one "
290
- f'is of kind {repr(artifact.kind)}.'
291
- )
292
291
  if artifact.scope == 'run':
293
292
  pattern = os.path.join(
294
293
  'artifacts', 'users', self._user,
@@ -317,27 +316,33 @@ class EnvironmentHandler:
317
316
  results.append(ids)
318
317
  return results
319
318
 
320
- def list_remote_artifacts(self, artifacts, run_ids=None):
319
+ def list_remote_artifacts(self, kind, artifacts, run_ids=None):
321
320
  raise NotImplementedError
322
321
 
323
- def push_artifacts(self, artifacts, run_ids):
322
+ def push_artifacts(self, kind, artifacts, run_ids):
324
323
  raise NotImplementedError
325
324
 
326
- def pull_artifacts(self, artifacts, run_ids):
325
+ def pull_artifacts(self, kind, artifacts, run_ids):
327
326
  raise NotImplementedError
328
327
 
329
- def remove_local_artifacts(self, artifacts, run_ids):
328
+ def remove_local_artifacts(self, kind, artifacts, run_ids):
329
+ if kind != 'file':
330
+ raise CloudSubmitError(
331
+ f'Cannot remove local artifacts of kind {repr(kind)}. '
332
+ )
330
333
  for artifact, runs in zip(artifacts, run_ids):
331
- if artifact.kind != 'file':
332
- continue
333
334
  for run_id in runs:
334
335
  path = self.get_local_artifact_ref(artifact, run_id)
335
336
  clear_path(path)
336
337
 
337
- def remove_remote_artifacts(self, artifacts, run_ids):
338
+ def remove_remote_artifacts(self, kind, artifacts, run_ids):
338
339
  raise NotImplementedError
339
340
 
340
- def copy_local_artifacts(self, artifacts, from_run_id, to_run_id):
341
+ def copy_local_artifacts(self, kind, artifacts, from_run_id, to_run_id):
342
+ if kind != 'file':
343
+ raise CloudSubmitError(
344
+ f'Cannot copy local artifacts of kind {repr(kind)}. '
345
+ )
341
346
  for artifact in artifacts:
342
347
  src_path = self.get_local_artifact_ref(artifact, from_run_id)
343
348
  dst_path = self.get_local_artifact_ref(artifact, to_run_id)
@@ -351,10 +356,14 @@ class EnvironmentHandler:
351
356
  else:
352
357
  raise
353
358
 
354
- def copy_remote_artifacts(self, artifacts, from_run_id, to_run_id):
359
+ def copy_remote_artifacts(self, kind, artifacts, from_run_id, to_run_id):
355
360
  raise NotImplementedError
356
361
 
357
- def move_local_artifacts(self, artifacts, from_run_id, to_run_id):
362
+ def move_local_artifacts(self, kind, artifacts, from_run_id, to_run_id):
363
+ if kind != 'file':
364
+ raise CloudSubmitError(
365
+ f'Cannot move local artifacts of kind {repr(kind)}. '
366
+ )
358
367
  for artifact in artifacts:
359
368
  src_path = self.get_local_artifact_ref(artifact, from_run_id)
360
369
  dst_path = self.get_local_artifact_ref(artifact, to_run_id)
@@ -244,7 +244,13 @@ class LocalAWSEnv(LocalEnv):
244
244
  lines = lines.split('\n')
245
245
  return [l.split()[-1] for l in lines]
246
246
 
247
- def list_remote_artifacts(self, artifacts, run_ids=None):
247
+ def list_remote_artifacts(self, kind, artifacts, run_ids=None):
248
+ if kind != 'file':
249
+ raise CloudSubmitError(
250
+ 'Cannot list run IDs for remote artifacts of kind '
251
+ f'{repr(kind)}. '
252
+ )
253
+
248
254
  if run_ids is not None:
249
255
  run_ids = set(run_ids)
250
256
 
@@ -252,8 +258,6 @@ class LocalAWSEnv(LocalEnv):
252
258
  user_artifacts = set()
253
259
  run_artifacts = set()
254
260
  for a in artifacts:
255
- if a.kind != 'file':
256
- continue
257
261
  if a.scope == 'project':
258
262
  project_artifacts.add(a.name)
259
263
  elif a.scope == 'user':
@@ -303,7 +307,12 @@ class LocalAWSEnv(LocalEnv):
303
307
  results = [sorted(results[a.name]) for a in artifacts]
304
308
  return results
305
309
 
306
- def remove_remote_artifacts(self, artifacts, run_ids):
310
+ def remove_remote_artifacts(self, kind, artifacts, run_ids):
311
+ if kind != 'file':
312
+ raise CloudSubmitError(
313
+ f'Cannot remove remote artifacts of kind {repr(kind)}.'
314
+ )
315
+
307
316
  path = '/'.join(
308
317
  ['s3:/', self._s3_bucket, self._s3_prefix, self._project])
309
318
  command = [
@@ -354,7 +363,12 @@ class LocalAWSEnv(LocalEnv):
354
363
 
355
364
  run_command(command)
356
365
 
357
- def push_artifacts(self, artifacts, run_ids):
366
+ def push_artifacts(self, kind, artifacts, run_ids):
367
+ if kind != 'file':
368
+ raise CloudSubmitError(
369
+ f'Cannot push artifacts of kind {repr(kind)}.'
370
+ )
371
+
358
372
  remote_path = '/'.join(
359
373
  ['s3:/', self._s3_bucket, self._s3_prefix, self._project])
360
374
  local_path = 'artifacts'
@@ -404,7 +418,12 @@ class LocalAWSEnv(LocalEnv):
404
418
 
405
419
  run_command(command)
406
420
 
407
- def pull_artifacts(self, artifacts, run_ids):
421
+ def pull_artifacts(self, kind, artifacts, run_ids):
422
+ if kind != 'file':
423
+ raise CloudSubmitError(
424
+ f'Cannot pull artifacts of kind {repr(kind)}.'
425
+ )
426
+
408
427
  remote_path = '/'.join(
409
428
  ['s3:/', self._s3_bucket, self._s3_prefix, self._project])
410
429
  local_path = 'artifacts'
@@ -454,7 +473,12 @@ class LocalAWSEnv(LocalEnv):
454
473
 
455
474
  run_command(command)
456
475
 
457
- def copy_remote_artifacts(self, artifacts, from_run_id, to_run_id):
476
+ def copy_remote_artifacts(self, kind, artifacts, from_run_id, to_run_id):
477
+ if kind != 'file':
478
+ raise CloudSubmitError(
479
+ f'Cannot copy remote artifacts of kind {repr(kind)}.'
480
+ )
481
+
458
482
  base_path = '/'.join([
459
483
  's3:/', self._s3_bucket, self._s3_prefix, self._project,
460
484
  'users', self._user, 'runs',
@@ -480,7 +504,12 @@ class LocalAWSEnv(LocalEnv):
480
504
  ])
481
505
  run_command(command)
482
506
 
483
- def move_remote_artifacts(self, artifacts, from_run_id, to_run_id):
507
+ def move_remote_artifacts(self, kind, artifacts, from_run_id, to_run_id):
508
+ if kind != 'file':
509
+ raise CloudSubmitError(
510
+ f'Cannot move remote artifacts of kind {repr(kind)}.'
511
+ )
512
+
484
513
  base_path = '/'.join([
485
514
  's3:/', self._s3_bucket, self._s3_prefix, self._project,
486
515
  'users', self._user, 'runs',
@@ -79,16 +79,16 @@ class LocalEnv(EnvironmentHandler):
79
79
  def get_remote_artifact_ref(self, artifact, run_id=None):
80
80
  return ''
81
81
 
82
- def list_remote_artifacts(self, artifacts, run_ids=None):
82
+ def list_remote_artifacts(self, kind, artifacts, run_ids=None):
83
83
  return [[] for a in artifacts]
84
84
 
85
- def remove_remote_artifacts(self, artifacts, run_ids):
85
+ def remove_remote_artifacts(self, kind, artifacts, run_ids):
86
86
  pass
87
87
 
88
- def copy_remote_artifacts(self, artifacts, from_run_id, to_run_id):
88
+ def copy_remote_artifacts(self, kind, artifacts, from_run_id, to_run_id):
89
89
  pass
90
90
 
91
- def move_remote_artifacts(self, artifacts, from_run_id, to_run_id):
91
+ def move_remote_artifacts(self, kind, artifacts, from_run_id, to_run_id):
92
92
  pass
93
93
 
94
94
  def run_pipeline(
@@ -109,10 +109,11 @@ class LocalEnv(EnvironmentHandler):
109
109
  ensure_path(artifacts_user_path)
110
110
  ensure_path(artifacts_run_path)
111
111
 
112
- self.remove_local_artifacts(
113
- overwrite_artifacts, [[run_id]]*len(overwrite_artifacts))
114
- self.remove_remote_artifacts(
115
- overwrite_artifacts, [[run_id]]*len(overwrite_artifacts))
112
+ for kind, artifacts in overwrite_artifacts.items():
113
+ self.remove_local_artifacts(
114
+ kind, artifacts, [[run_id]]*len(artifacts))
115
+ self.remove_remote_artifacts(
116
+ kind, artifacts, [[run_id]]*len(artifacts))
116
117
 
117
118
  for step in pipeline.steps:
118
119
  if step.name not in image_refs:
File without changes
File without changes
File without changes