cloud-submit 0.3__tar.gz → 0.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. {cloud_submit-0.3 → cloud_submit-0.4}/PKG-INFO +2 -2
  2. {cloud_submit-0.3 → cloud_submit-0.4}/pyproject.toml +1 -1
  3. {cloud_submit-0.3 → cloud_submit-0.4}/src/cloud_submit/cli.py +24 -4
  4. {cloud_submit-0.3 → cloud_submit-0.4}/src/cloud_submit/controller.py +6 -4
  5. {cloud_submit-0.3 → cloud_submit-0.4}/src/cloud_submit/environment_handler.py +2 -1
  6. {cloud_submit-0.3 → cloud_submit-0.4}/src/cloud_submit/envs/aws/local_environment_handler.py +5 -3
  7. {cloud_submit-0.3 → cloud_submit-0.4}/src/cloud_submit/envs/aws/local_execution_handler.py +8 -4
  8. {cloud_submit-0.3 → cloud_submit-0.4}/.gitignore +0 -0
  9. {cloud_submit-0.3 → cloud_submit-0.4}/LICENSE +0 -0
  10. {cloud_submit-0.3 → cloud_submit-0.4}/README.md +0 -0
  11. {cloud_submit-0.3 → cloud_submit-0.4}/docs/examples/aws/src/csub/build_config.py +0 -0
  12. {cloud_submit-0.3 → cloud_submit-0.4}/docs/examples/aws/src/fit.py +0 -0
  13. {cloud_submit-0.3 → cloud_submit-0.4}/docs/examples/aws/src/generate.py +0 -0
  14. {cloud_submit-0.3 → cloud_submit-0.4}/docs/examples/aws/src/parameters.py +0 -0
  15. {cloud_submit-0.3 → cloud_submit-0.4}/docs/examples/aws/userconfig/template.yaml +0 -0
  16. {cloud_submit-0.3 → cloud_submit-0.4}/docs/examples/basic/src/csub/build_config.py +0 -0
  17. {cloud_submit-0.3 → cloud_submit-0.4}/docs/examples/basic/src/fit.py +0 -0
  18. {cloud_submit-0.3 → cloud_submit-0.4}/docs/examples/basic/src/generate.py +0 -0
  19. {cloud_submit-0.3 → cloud_submit-0.4}/docs/examples/basic/src/parameters.py +0 -0
  20. {cloud_submit-0.3 → cloud_submit-0.4}/docs/examples/basic/userconfig/template.yaml +0 -0
  21. {cloud_submit-0.3 → cloud_submit-0.4}/docs/examples/multi-worker/src/csub/build_config.py +0 -0
  22. {cloud_submit-0.3 → cloud_submit-0.4}/docs/examples/multi-worker/src/fit.py +0 -0
  23. {cloud_submit-0.3 → cloud_submit-0.4}/docs/examples/multi-worker/src/generate.py +0 -0
  24. {cloud_submit-0.3 → cloud_submit-0.4}/docs/examples/multi-worker/src/parameters.py +0 -0
  25. {cloud_submit-0.3 → cloud_submit-0.4}/docs/examples/multi-worker/userconfig/template.yaml +0 -0
  26. {cloud_submit-0.3 → cloud_submit-0.4}/src/cloud_submit/__init__.py +0 -0
  27. {cloud_submit-0.3 → cloud_submit-0.4}/src/cloud_submit/config.py +0 -0
  28. {cloud_submit-0.3 → cloud_submit-0.4}/src/cloud_submit/envs/aws/__init__.py +0 -0
  29. {cloud_submit-0.3 → cloud_submit-0.4}/src/cloud_submit/envs/aws/remote_environment_handler.py +0 -0
  30. {cloud_submit-0.3 → cloud_submit-0.4}/src/cloud_submit/envs/aws/remote_execution_handler.py +0 -0
  31. {cloud_submit-0.3 → cloud_submit-0.4}/src/cloud_submit/envs/aws/s3_tools.py +0 -0
  32. {cloud_submit-0.3 → cloud_submit-0.4}/src/cloud_submit/envs/local/environment_handler.py +0 -0
  33. {cloud_submit-0.3 → cloud_submit-0.4}/src/cloud_submit/envs/local/execution_handler.py +0 -0
  34. {cloud_submit-0.3 → cloud_submit-0.4}/src/cloud_submit/execution/base_execution_handler.py +0 -0
  35. {cloud_submit-0.3 → cloud_submit-0.4}/src/cloud_submit/execution/config.py +0 -0
  36. {cloud_submit-0.3 → cloud_submit-0.4}/src/cloud_submit/execution/execute.py +0 -0
  37. {cloud_submit-0.3 → cloud_submit-0.4}/src/cloud_submit/execution/utils.py +0 -0
  38. {cloud_submit-0.3 → cloud_submit-0.4}/src/cloud_submit/images.py +0 -0
  39. {cloud_submit-0.3 → cloud_submit-0.4}/src/cloud_submit/utils.py +0 -0
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.4
1
+ Metadata-Version: 2.5
2
2
  Name: cloud-submit
3
- Version: 0.3
3
+ Version: 0.4
4
4
  Summary: A tool for building processing pipelines in Python and executing them locally or in the cloud.
5
5
  Project-URL: Homepage, https://github.com/mwiebusch78/cloud-submit
6
6
  Project-URL: Bug Tracker, https://github.com/mwiebusch78/cloud-submit/issues
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "cloud-submit"
7
- version = "0.3"
7
+ version = "0.4"
8
8
  authors = [
9
9
  { name="Martin Wiebusch" },
10
10
  ]
@@ -82,7 +82,7 @@ def get_steps(pipeline, steps_arg):
82
82
  tokens = steps_arg.split(',')
83
83
  all_steps = [step.name for step in pipeline.steps]
84
84
  indices = {step: i for i, step in enumerate(all_steps)}
85
-
85
+
86
86
  selected = set()
87
87
  for token in tokens:
88
88
  token = token.split(':')
@@ -99,7 +99,7 @@ def get_steps(pipeline, steps_arg):
99
99
  end = indices[token[1]]
100
100
  except KeyError:
101
101
  raise CloudSubmitError(f'Invalid step name: {token[1]}')
102
- for step in all_steps[beg:end+1]:
102
+ for step in all_steps[begin:end+1]:
103
103
  selected.add(step)
104
104
  else:
105
105
  raise CloudSubmitError(f'Invalid step range: {":".join(token)}')
@@ -306,9 +306,19 @@ def images():
306
306
  is_flag=True,
307
307
  help='Ignore IMAGES arguments and build all base images.',
308
308
  )
309
+ @click.option(
310
+ '--no-cache',
311
+ is_flag=True,
312
+ help='Do not use docker cache when building images.',
313
+ )
314
+ @click.option(
315
+ '--pull',
316
+ is_flag=True,
317
+ help='Instruct docker to always attempt to pull all referenced images.',
318
+ )
309
319
  @click.argument('images', type=str, nargs=-1)
310
320
  @click.pass_context
311
- def build_images(ctx, env, build_id, build_all, images):
321
+ def build_images(ctx, env, build_id, build_all, no_cache, pull, images):
312
322
  init(ctx)
313
323
  config = ctx.obj['config']
314
324
  controller = ctx.obj['controller']
@@ -321,8 +331,18 @@ def build_images(ctx, env, build_id, build_all, images):
321
331
  name for name, image in config.images.items()
322
332
  if isinstance(image, BaseImage)
323
333
  ]
334
+ docker_flags = []
335
+ if no_cache:
336
+ docker_flags.append('--no-cache')
337
+ if pull:
338
+ docker_flags.append('--pull')
324
339
  try:
325
- controller.build_images(images, build_id=build_id, env=env)
340
+ controller.build_images(
341
+ images,
342
+ build_id=build_id,
343
+ env=env,
344
+ docker_flags=docker_flags,
345
+ )
326
346
  except CloudSubmitError as e:
327
347
  abort(str(e))
328
348
 
@@ -61,7 +61,7 @@ class Controller:
61
61
  except FileNotFoundError:
62
62
  pass
63
63
 
64
- def _build_image(self, image, build_id, env, rebuild=False):
64
+ def _build_image(self, image, build_id, env, docker_flags, rebuild=False):
65
65
  is_execution_image = isinstance(image, ExecutionImage)
66
66
 
67
67
  if not rebuild and not is_execution_image:
@@ -96,7 +96,7 @@ class Controller:
96
96
  image.setup_builddir(path, parent_ref)
97
97
 
98
98
  print(f'Building image {image.name}.')
99
- ref = env.build_image(path, image, build_id)
99
+ ref = env.build_image(path, image, build_id, docker_flags)
100
100
  if ref is None:
101
101
  raise CloudSubmitError(
102
102
  'build_image method of environment handler did not '
@@ -105,10 +105,12 @@ class Controller:
105
105
  if not is_execution_image:
106
106
  self._save_image_ref(image, ref)
107
107
 
108
- def build_images(self, images, build_id=None, env=None):
108
+ def build_images(self, images, build_id=None, env=None, docker_flags=None):
109
109
  now = dt.datetime.now(tz=dt.UTC)
110
110
  env_handler = self._config.get_build_env(env)
111
111
  build_id = env_handler.generate_build_id(now, build_id)
112
+ if docker_flags is None:
113
+ docker_flags = []
112
114
 
113
115
  images = set(images)
114
116
  for image in images:
@@ -127,7 +129,7 @@ class Controller:
127
129
  continue
128
130
  rebuild = image_name in images
129
131
  self._build_image(
130
- image, build_id, env_handler, rebuild=rebuild)
132
+ image, build_id, env_handler, docker_flags, rebuild=rebuild)
131
133
 
132
134
  def list_image_refs(self, images=None, ids=None, env=None, remote=False):
133
135
  env_handler = self._config.get_build_env(env)
@@ -235,13 +235,14 @@ class EnvironmentHandler:
235
235
  def remove_remote_image_refs(self, refs):
236
236
  raise NotImplementedError
237
237
 
238
- def build_image(self, path, image, build_id):
238
+ def build_image(self, path, image, build_id, docker_flags):
239
239
  repo = self.get_image_repo(image)
240
240
  ref = ':'.join([repo, build_id])
241
241
  command = [
242
242
  self._docker_command,
243
243
  'buildx',
244
244
  'build',
245
+ *docker_flags,
245
246
  *(image.args),
246
247
  '-t', ref,
247
248
  path,
@@ -129,9 +129,9 @@ class LocalAWSEnv(LocalEnv):
129
129
  os.path.join(path, 's3_tools.py'),
130
130
  )
131
131
 
132
- def _build_image(self, path, image, build_id, push):
132
+ def _build_image(self, path, image, build_id, docker_flags, push):
133
133
  if not push:
134
- ref = super().build_image(path, image, build_id)
134
+ ref = super().build_image(path, image, build_id, docker_flags)
135
135
  else:
136
136
  self._docker_login()
137
137
  repo = self.get_image_repo(image)
@@ -140,6 +140,7 @@ class LocalAWSEnv(LocalEnv):
140
140
  self._docker_command,
141
141
  'buildx',
142
142
  'build',
143
+ *docker_flags,
143
144
  *(image.args),
144
145
  ]
145
146
  if self._docker_platforms is not None:
@@ -148,11 +149,12 @@ class LocalAWSEnv(LocalEnv):
148
149
  run_command(command)
149
150
  return ref
150
151
 
151
- def build_image(self, path, image, build_id):
152
+ def build_image(self, path, image, build_id, docker_flags):
152
153
  return self._build_image(
153
154
  path=path,
154
155
  image=image,
155
156
  build_id=build_id,
157
+ docker_flags=docker_flags,
156
158
  push=isinstance(image, BaseImage),
157
159
  )
158
160
 
@@ -37,9 +37,13 @@ class ExecutionHandler(BaseExecutionHandler):
37
37
  'environment.'
38
38
  )
39
39
  remote_path = self.get_artifact_reference('uri', artifact)
40
- remote_path = '/'.join(remote_path.split('/')[:-1])
40
+ remote_dir = '/'.join(remote_path.split('/')[:-1])
41
41
  local_path = self.get_artifact_reference('path', artifact)
42
- local_path = os.path.dirname(local_path)
42
+ local_dir = os.path.dirname(local_path)
43
+
44
+ if os.path.exists(local_path):
45
+ print(f'Using local data for artifact {artifact.name}.')
46
+ return
43
47
 
44
48
  print(f'Downloading artifact {artifact.name}.')
45
49
  command = [
@@ -50,8 +54,8 @@ class ExecutionHandler(BaseExecutionHandler):
50
54
  '--region', self._aws_region,
51
55
  '--recursive',
52
56
  '--no-progress',
53
- remote_path,
54
- local_path,
57
+ remote_dir,
58
+ local_dir,
55
59
  '--exclude', '*',
56
60
  '--include', artifact.name,
57
61
  '--include', '/'.join([artifact.name, '*'])
File without changes
File without changes
File without changes