cloud-submit 0.1.2__tar.gz → 0.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. {cloud_submit-0.1.2 → cloud_submit-0.3}/PKG-INFO +1 -1
  2. {cloud_submit-0.1.2 → cloud_submit-0.3}/docs/examples/aws/src/csub/build_config.py +14 -7
  3. {cloud_submit-0.1.2 → cloud_submit-0.3}/docs/examples/aws/userconfig/template.yaml +5 -0
  4. {cloud_submit-0.1.2 → cloud_submit-0.3}/docs/examples/basic/src/csub/build_config.py +4 -4
  5. {cloud_submit-0.1.2 → cloud_submit-0.3}/docs/examples/multi-worker/src/csub/build_config.py +4 -4
  6. {cloud_submit-0.1.2 → cloud_submit-0.3}/pyproject.toml +1 -1
  7. {cloud_submit-0.1.2 → cloud_submit-0.3}/src/cloud_submit/__init__.py +2 -3
  8. {cloud_submit-0.1.2 → cloud_submit-0.3}/src/cloud_submit/cli.py +18 -8
  9. {cloud_submit-0.1.2 → cloud_submit-0.3}/src/cloud_submit/controller.py +69 -24
  10. {cloud_submit-0.1.2 → cloud_submit-0.3}/src/cloud_submit/environment_handler.py +47 -66
  11. {cloud_submit-0.1.2 → cloud_submit-0.3}/src/cloud_submit/envs/aws/local_environment_handler.py +60 -33
  12. cloud_submit-0.3/src/cloud_submit/envs/aws/local_execution_handler.py +63 -0
  13. {cloud_submit-0.1.2 → cloud_submit-0.3}/src/cloud_submit/envs/aws/remote_environment_handler.py +36 -19
  14. {cloud_submit-0.1.2 → cloud_submit-0.3}/src/cloud_submit/envs/aws/remote_execution_handler.py +24 -23
  15. {cloud_submit-0.1.2 → cloud_submit-0.3}/src/cloud_submit/envs/aws/s3_tools.py +1 -2
  16. {cloud_submit-0.1.2 → cloud_submit-0.3}/src/cloud_submit/envs/local/environment_handler.py +53 -3
  17. {cloud_submit-0.1.2 → cloud_submit-0.3}/src/cloud_submit/execution/base_execution_handler.py +7 -7
  18. {cloud_submit-0.1.2 → cloud_submit-0.3}/src/cloud_submit/execution/config.py +11 -15
  19. {cloud_submit-0.1.2 → cloud_submit-0.3}/src/cloud_submit/execution/execute.py +22 -22
  20. {cloud_submit-0.1.2 → cloud_submit-0.3}/src/cloud_submit/execution/utils.py +18 -3
  21. {cloud_submit-0.1.2 → cloud_submit-0.3}/src/cloud_submit/images.py +4 -0
  22. {cloud_submit-0.1.2 → cloud_submit-0.3}/src/cloud_submit/utils.py +0 -19
  23. cloud_submit-0.1.2/src/cloud_submit/envs/aws/local_execution_handler.py +0 -5
  24. {cloud_submit-0.1.2 → cloud_submit-0.3}/.gitignore +0 -0
  25. {cloud_submit-0.1.2 → cloud_submit-0.3}/LICENSE +0 -0
  26. {cloud_submit-0.1.2 → cloud_submit-0.3}/README.md +0 -0
  27. {cloud_submit-0.1.2 → cloud_submit-0.3}/docs/examples/aws/src/fit.py +0 -0
  28. {cloud_submit-0.1.2 → cloud_submit-0.3}/docs/examples/aws/src/generate.py +0 -0
  29. {cloud_submit-0.1.2 → cloud_submit-0.3}/docs/examples/aws/src/parameters.py +0 -0
  30. {cloud_submit-0.1.2 → cloud_submit-0.3}/docs/examples/basic/src/fit.py +0 -0
  31. {cloud_submit-0.1.2 → cloud_submit-0.3}/docs/examples/basic/src/generate.py +0 -0
  32. {cloud_submit-0.1.2 → cloud_submit-0.3}/docs/examples/basic/src/parameters.py +0 -0
  33. {cloud_submit-0.1.2 → cloud_submit-0.3}/docs/examples/basic/userconfig/template.yaml +0 -0
  34. {cloud_submit-0.1.2 → cloud_submit-0.3}/docs/examples/multi-worker/src/fit.py +0 -0
  35. {cloud_submit-0.1.2 → cloud_submit-0.3}/docs/examples/multi-worker/src/generate.py +0 -0
  36. {cloud_submit-0.1.2 → cloud_submit-0.3}/docs/examples/multi-worker/src/parameters.py +0 -0
  37. {cloud_submit-0.1.2 → cloud_submit-0.3}/docs/examples/multi-worker/userconfig/template.yaml +0 -0
  38. {cloud_submit-0.1.2 → cloud_submit-0.3}/src/cloud_submit/config.py +0 -0
  39. {cloud_submit-0.1.2 → cloud_submit-0.3}/src/cloud_submit/envs/aws/__init__.py +0 -0
  40. {cloud_submit-0.1.2 → cloud_submit-0.3}/src/cloud_submit/envs/local/execution_handler.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: cloud-submit
3
- Version: 0.1.2
3
+ Version: 0.3
4
4
  Summary: A tool for building processing pipelines in Python and executing them locally or in the cloud.
5
5
  Project-URL: Homepage, https://github.com/mwiebusch78/cloud-submit
6
6
  Project-URL: Bug Tracker, https://github.com/mwiebusch78/cloud-submit/issues
@@ -39,11 +39,16 @@ def build_config(project_root, userconfig):
39
39
  cs.BaseImage(
40
40
  name='base',
41
41
  instructions=r"""
42
- FROM python
42
+ FROM python:3.12.3
43
+ ARG TARGETARCH
43
44
  RUN \
45
+ case "$TARGETARCH" in \
46
+ amd64) url="https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" ;; \
47
+ arm64) url="https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" ;; \
48
+ *) echo "unsupported architecture: $TARGETARCH" >&2; exit 1 ;; \
49
+ esac && \
44
50
  cd /opt && \
45
- curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" \
46
- -o "awscliv2.zip" && \
51
+ curl "$url" -o "awscliv2.zip" && \
47
52
  unzip awscliv2.zip && \
48
53
  ./aws/install && \
49
54
  rm -rf awscliv2.zip aws
@@ -86,7 +91,7 @@ def build_config(project_root, userconfig):
86
91
  'sigma': PARAMS['sigma'],
87
92
  },
88
93
  outputs={
89
- 'train_data_path': cs.local('train_data.parquet'),
94
+ 'train_data_path': cs.aref('train_data.parquet', 'path'),
90
95
  },
91
96
  ),
92
97
  cs.Step(
@@ -99,11 +104,11 @@ def build_config(project_root, userconfig):
99
104
  function='fit:fit_step',
100
105
  image='exec',
101
106
  inputs={
102
- 'train_data_path': cs.local('train_data.parquet'),
107
+ 'train_data_path': cs.aref('train_data.parquet', 'path'),
103
108
  },
104
109
  outputs={
105
- 'coefficients_path': cs.local('coefficients.json'),
106
- 'predictions_path': cs.local('predictions.parquet'),
110
+ 'coefficients_path': cs.aref('coefficients.json', 'path'),
111
+ 'predictions_path': cs.aref('predictions.parquet', 'path'),
107
112
  },
108
113
  ),
109
114
  ],
@@ -125,6 +130,7 @@ def build_config(project_root, userconfig):
125
130
  aws_region=userconfig['aws_region'],
126
131
  aws_profile=userconfig['aws_profile'],
127
132
  docker_namespace=userconfig['docker_namespace'],
133
+ docker_platforms=userconfig['docker_platforms'],
128
134
  s3_bucket=userconfig['s3_bucket'],
129
135
  s3_prefix=userconfig['s3_prefix'],
130
136
  ),
@@ -136,6 +142,7 @@ def build_config(project_root, userconfig):
136
142
  aws_region=userconfig['aws_region'],
137
143
  aws_profile=userconfig['aws_profile'],
138
144
  docker_namespace=userconfig['docker_namespace'],
145
+ docker_platforms=userconfig['docker_platforms'],
139
146
  s3_bucket=userconfig['s3_bucket'],
140
147
  s3_prefix=userconfig['s3_prefix'],
141
148
  ecs_cluster_arn=userconfig['ecs_cluster_arn'],
@@ -24,6 +24,11 @@ aws_profile: default
24
24
  # registry.
25
25
  docker_namespace: csub
26
26
 
27
+ # This is the value of the --platform option used when building docker images.
28
+ # To build images that work on both x86_64 and ARM architecture set this to
29
+ # "linux/amd64,linux/arm64"
30
+ docker_platforms: null
31
+
27
32
  # This is the name of the S3 bucket where your artifacts will be stored.
28
33
  s3_bucket: # TODO: add the name of your S3 bucket.
29
34
 
@@ -70,7 +70,7 @@ def build_config(project_root, userconfig):
70
70
  'sigma': PARAMS['sigma'],
71
71
  },
72
72
  outputs={
73
- 'train_data_path': cs.local('train_data.parquet'),
73
+ 'train_data_path': cs.aref('train_data.parquet', 'path'),
74
74
  },
75
75
  ),
76
76
  cs.Step(
@@ -78,11 +78,11 @@ def build_config(project_root, userconfig):
78
78
  function='fit:fit_step',
79
79
  image='exec',
80
80
  inputs={
81
- 'train_data_path': cs.local('train_data.parquet'),
81
+ 'train_data_path': cs.aref('train_data.parquet', 'path'),
82
82
  },
83
83
  outputs={
84
- 'coefficients_path': cs.local('coefficients.json'),
85
- 'predictions_path': cs.local('predictions.parquet'),
84
+ 'coefficients_path': cs.aref('coefficients.json', 'path'),
85
+ 'predictions_path': cs.aref('predictions.parquet', 'path'),
86
86
  },
87
87
  ),
88
88
  ],
@@ -69,7 +69,7 @@ def build_config(project_root, userconfig):
69
69
  'sigma': PARAMS['sigma'],
70
70
  },
71
71
  outputs={
72
- 'train_data_path': cs.local('train_data.parquet'),
72
+ 'train_data_path': cs.aref('train_data.parquet', 'path'),
73
73
  },
74
74
  pass_submit_timestamp_as='submit_timestamp',
75
75
  ),
@@ -78,11 +78,11 @@ def build_config(project_root, userconfig):
78
78
  function='fit:fit_step',
79
79
  image='exec',
80
80
  inputs={
81
- 'train_data_path': cs.local('train_data.parquet'),
81
+ 'train_data_path': cs.aref('train_data.parquet', 'path'),
82
82
  },
83
83
  outputs={
84
- 'coefficients_path': cs.local('coefficients.json'),
85
- 'predictions_path': cs.local('predictions.parquet'),
84
+ 'coefficients_path': cs.aref('coefficients.json', 'path'),
85
+ 'predictions_path': cs.aref('predictions.parquet', 'path'),
86
86
  },
87
87
  num_workers=PARAMS['num_folds'],
88
88
  pass_num_workers_as='num_folds',
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "cloud-submit"
7
- version = "0.1.2"
7
+ version = "0.3"
8
8
  authors = [
9
9
  { name="Martin Wiebusch" },
10
10
  ]
@@ -6,12 +6,11 @@ from .config import Config
6
6
  from .execution.config import (
7
7
  ConfigError,
8
8
  Artifact,
9
- ArtifactLocation,
9
+ ArtifactReference,
10
10
  Spec,
11
11
  Step,
12
12
  Pipeline,
13
- local,
14
- remote,
13
+ aref,
15
14
  )
16
15
  from .execution.utils import (
17
16
  CloudSubmitError,
@@ -7,6 +7,7 @@ import yaml
7
7
 
8
8
  from .execution.utils import CloudSubmitError
9
9
  from .controller import Controller
10
+ from .images import BaseImage
10
11
 
11
12
 
12
13
  def abort(msg):
@@ -303,11 +304,11 @@ def images():
303
304
  @click.option(
304
305
  '--all', '-a', 'build_all',
305
306
  is_flag=True,
306
- help='Build all images.',
307
+ help='Ignore IMAGES arguments and build all base images.',
307
308
  )
308
309
  @click.argument('images', type=str, nargs=-1)
309
310
  @click.pass_context
310
- def build(ctx, env, build_id, build_all, images):
311
+ def build_images(ctx, env, build_id, build_all, images):
311
312
  init(ctx)
312
313
  config = ctx.obj['config']
313
314
  controller = ctx.obj['controller']
@@ -316,9 +317,12 @@ def build(ctx, env, build_id, build_all, images):
316
317
  if image not in config.images:
317
318
  abort(f'No definition found for image {image}.')
318
319
  if build_all:
319
- images = list(config.images.keys())
320
+ images = [
321
+ name for name, image in config.images.items()
322
+ if isinstance(image, BaseImage)
323
+ ]
320
324
  try:
321
- controller.build(images, build_id=build_id, env=env)
325
+ controller.build_images(images, build_id=build_id, env=env)
322
326
  except CloudSubmitError as e:
323
327
  abort(str(e))
324
328
 
@@ -495,7 +499,10 @@ def remove_images(ctx, env, images, local, remote, build_ids):
495
499
  def set_image(ctx, env, image, ref):
496
500
  init(ctx)
497
501
  controller = ctx.obj['controller']
498
- controller.set_image(image, ref, env=env)
502
+ try:
503
+ controller.set_image(image, ref, env=env)
504
+ except CloudSubmitError as e:
505
+ abort(str(e))
499
506
 
500
507
 
501
508
  @images.command(
@@ -522,7 +529,10 @@ def unset_image(ctx, env, unset_all, images):
522
529
  if unset_all:
523
530
  images = list(config.images.keys())
524
531
  for image in images:
525
- controller.unset_image(image, env=env)
532
+ try:
533
+ controller.unset_image(image, env=env)
534
+ except CloudSubmitError as e:
535
+ abort(str(e))
526
536
 
527
537
 
528
538
  # artifacts subgroup
@@ -605,10 +615,10 @@ def list_artifacts(ctx, env, artifacts, local, remote, run_ids):
605
615
  artifact = config.artifacts[name]
606
616
  for run_id in runs:
607
617
  if local:
608
- path = env_handler.get_local_artifact_path(
618
+ path = env_handler.get_local_artifact_ref(
609
619
  artifact, run_id)
610
620
  else:
611
- path = env_handler.get_remote_artifact_path(
621
+ path = env_handler.get_remote_artifact_ref(
612
622
  artifact, run_id)
613
623
  data.append((
614
624
  name,
@@ -32,14 +32,44 @@ class Controller:
32
32
  self._config.export_pipelines(os.path.join(path, 'pipelines.json'))
33
33
  self._config.export_artifacts(os.path.join(path, 'artifacts.json'))
34
34
 
35
+ def _get_image_ref_path(self, image):
36
+ if not isinstance(image, BaseImage):
37
+ raise ValueError(f'Cannot handle image of type {type(image)}')
38
+ return os.path.join('images', self._config.user_name, image.name)
39
+
40
+ def _get_image_ref(self, image):
41
+ ensure_path(os.path.join('images', self._config.user_name))
42
+ path = self._get_image_ref_path(image)
43
+ try:
44
+ with open(path, 'r') as stream:
45
+ ref = stream.read().strip()
46
+ except FileNotFoundError:
47
+ return None
48
+ return ref
49
+
50
+ def _save_image_ref(self, image, ref):
51
+ ensure_path(os.path.join('images', self._config.user_name))
52
+ path = self._get_image_ref_path(image)
53
+ with open(path, 'w') as stream:
54
+ stream.write(ref)
55
+
56
+ def _clear_image_ref(self, image):
57
+ ensure_path(os.path.join('images', self._config.user_name))
58
+ path = self._get_image_ref_path(image)
59
+ try:
60
+ os.remove(path)
61
+ except FileNotFoundError:
62
+ pass
63
+
35
64
  def _build_image(self, image, build_id, env, rebuild=False):
36
- if not rebuild:
37
- ref = env.get_image_ref(image)
65
+ is_execution_image = isinstance(image, ExecutionImage)
66
+
67
+ if not rebuild and not is_execution_image:
68
+ ref = self._get_image_ref(image)
38
69
  if ref is not None:
39
70
  print(f'Using existing build of {image.name}: {ref}')
40
71
  return ref
41
72
 
42
- is_execution_image = isinstance(image, ExecutionImage)
43
73
  path = os.path.join('temp', 'build', image.name)
44
74
  if is_execution_image:
45
75
  path = os.path.join(path, env.name)
@@ -58,7 +88,7 @@ class Controller:
58
88
  if parent_image is None:
59
89
  raise CloudSubmitError(
60
90
  f'Could not find declaration for image {image.parent}.')
61
- parent_ref = env.get_image_ref(parent_image)
91
+ parent_ref = self._get_image_ref(parent_image)
62
92
  if parent_ref is None:
63
93
  raise CloudSubmitError(
64
94
  f'Could not find reference for image {image.parent}.')
@@ -72,9 +102,10 @@ class Controller:
72
102
  'build_image method of environment handler did not '
73
103
  'return a valid image reference.'
74
104
  )
75
- env.save_image_ref(image, ref)
105
+ if not is_execution_image:
106
+ self._save_image_ref(image, ref)
76
107
 
77
- def build(self, images, build_id=None, env=None):
108
+ def build_images(self, images, build_id=None, env=None):
78
109
  now = dt.datetime.now(tz=dt.UTC)
79
110
  env_handler = self._config.get_build_env(env)
80
111
  build_id = env_handler.generate_build_id(now, build_id)
@@ -114,19 +145,18 @@ class Controller:
114
145
  results = []
115
146
  for image in images:
116
147
  image = self._config.images[image]
117
- repo_name = env_handler.get_image_repo_name(image)
148
+ repo_name = env_handler.get_image_repo(image)
118
149
  if remote:
119
- tags = env_handler.list_remote_image_tags(repo_name)
150
+ tags = env_handler.list_remote_image_tags(image)
120
151
  else:
121
- tags = env_handler.list_local_image_tags(repo_name)
152
+ tags = env_handler.list_local_image_tags(image)
122
153
  if ids is not None:
123
154
  tags = [t for t in tags if t in ids]
124
155
  for tag in tags:
125
156
  results.append(':'.join([repo_name, tag]))
126
157
  return results
127
158
 
128
- def list_images(self, images=None, env=None):
129
- env_handler = self._config.get_build_env(env)
159
+ def list_images(self, images=None):
130
160
  if images is not None:
131
161
  images = set(images)
132
162
  result = []
@@ -136,15 +166,18 @@ class Controller:
136
166
  continue
137
167
  if isinstance(i, BaseImage):
138
168
  tpe = 'base'
169
+ ref = self._get_image_ref(i)
139
170
  elif isinstance(i, ExecutionImage):
140
171
  tpe = 'execution'
172
+ ref = None
141
173
  else:
142
174
  raise ValueError('Unknown image type.')
143
- ref = env_handler.get_image_ref(i)
144
175
  result.append((i.name, tpe, ref))
145
176
  return result
146
177
 
147
178
  def remove_image_refs(self, refs, remote=False, env=None):
179
+ if not refs:
180
+ return
148
181
  env_handler = self._config.get_build_env(env)
149
182
  if remote:
150
183
  env_handler.remove_remote_image_refs(refs)
@@ -156,16 +189,26 @@ class Controller:
156
189
  image = self._config.images.get(image_name)
157
190
  if image is None:
158
191
  raise CloudSubmitError(f'Unknown image name {image_name}.')
192
+ if not isinstance(image, BaseImage):
193
+ raise CloudSubmitError(
194
+ f'Cannot set ref for image {image_name} '
195
+ 'since it is not a base image.'
196
+ )
159
197
  with self._config.in_project_root():
160
- env_handler.save_image_ref(image, ref)
198
+ self._save_image_ref(image, ref)
161
199
 
162
200
  def unset_image(self, image_name, env=None):
163
201
  env_handler = self._config.get_build_env(env)
164
202
  image = self._config.images.get(image_name)
165
203
  if image is None:
166
204
  raise CloudSubmitError(f'Unknown image name {image_name}.')
205
+ if not isinstance(image, BaseImage):
206
+ raise CloudSubmitError(
207
+ f'Cannot unset ref for image {image_name} '
208
+ 'since it is not a base image.'
209
+ )
167
210
  with self._config.in_project_root():
168
- env_handler.clear_image_ref(image)
211
+ self._clear_image_ref(image)
169
212
 
170
213
  def list_artifacts(
171
214
  self,
@@ -202,6 +245,8 @@ class Controller:
202
245
  remote=False,
203
246
  env=None,
204
247
  ):
248
+ if not artifact_names:
249
+ return
205
250
  env_handler = self._config.get_run_env(env)
206
251
  artifacts = [
207
252
  self._config.artifacts[name].copy() for name in artifact_names]
@@ -327,7 +372,7 @@ class Controller:
327
372
 
328
373
  with self._config.in_project_root():
329
374
  images = [step.image for step in steps]
330
- self.build(images, build_id=run_id, env=build_env)
375
+ self.build_images(images, build_id=run_id, env=build_env)
331
376
  refs = {}
332
377
  overwrite_artifacts = []
333
378
  for step in steps:
@@ -335,11 +380,8 @@ class Controller:
335
380
  if image is None:
336
381
  raise CloudSubmitError(
337
382
  f'Cannot find declaration for image {step.image}.')
338
- ref = env_handler.get_image_ref(image)
339
- if ref is None:
340
- raise CloudSubmitError(
341
- f'Could not find image ref for image {step.image}')
342
- refs[step.name] = ref
383
+ repo = env_handler.get_image_repo(image)
384
+ refs[step.name] = ':'.join([repo, run_id])
343
385
 
344
386
  for loc in step.outputs.values():
345
387
  overwrite_artifacts.append(
@@ -351,10 +393,14 @@ class Controller:
351
393
  temp_path = 'temp/run'
352
394
  ensure_path(temp_path, clear=True)
353
395
  env_handler.run_pipeline(
354
- pipeline, refs, overwrite_artifacts,
355
- timestamp, run_id, temp_path,
396
+ pipeline=pipeline,
397
+ image_refs=refs,
398
+ overwrite_artifacts=overwrite_artifacts,
399
+ timestamp=timestamp,
400
+ run_id=run_id,
401
+ temp_path=temp_path,
356
402
  )
357
-
403
+
358
404
  return {
359
405
  'run_id': run_id,
360
406
  'start_timestamp': now,
@@ -371,5 +417,4 @@ class Controller:
371
417
  since = dt.datetime.now(tz=dt.UTC) - since
372
418
  env_handler = self._config.get_run_env(env)
373
419
  env_handler.print_logs(run_id, since, stream=stream)
374
-
375
420
 
@@ -19,20 +19,32 @@ from .execution.config import to_utc
19
19
 
20
20
  class EnvironmentHandler:
21
21
  def __init__(
22
- self,
23
- name,
24
- project,
25
- user,
26
- docker_command='docker',
27
- docker_registry=None,
28
- docker_namespace='csub',
22
+ self,
23
+ name,
24
+ project,
25
+ user,
26
+ envvars=None,
27
+ secrets=None,
28
+ docker_command='docker',
29
+ docker_registry=None,
30
+ docker_namespace='csub',
29
31
  ):
30
- self.name = name
31
- self._project = project
32
- self._user = user
32
+ self.name = str(name)
33
+ self._project = str(project)
34
+ self._user = str(user)
33
35
  self._docker_command = str(docker_command)
34
36
  self._docker_registry = docker_registry
35
- self._docker_namespace = docker_namespace
37
+ self._docker_namespace = str(docker_namespace)
38
+
39
+ self._envvars = {
40
+ 'CSUB_PROJECT': self._project,
41
+ 'CSUB_USER': self._user,
42
+ }
43
+ if envvars is not None:
44
+ self._envvars.update(envvars)
45
+ self._secrets = {}
46
+ if secrets is not None:
47
+ self._secrets.update(secrets)
36
48
 
37
49
  def _generate_id(self, timestamp, id):
38
50
  if id is not None:
@@ -126,41 +138,7 @@ class EnvironmentHandler:
126
138
  """
127
139
  raise NotImplementedError
128
140
 
129
- def _get_image_ref_path(self, image):
130
- if isinstance(image, BaseImage):
131
- image_name = image.name
132
- elif isinstance(image, ExecutionImage):
133
- image_name = '.'.join([image.name, self.name])
134
- else:
135
- raise ValueError(f'Cannot handle image of type {type(image)}')
136
-
137
- return os.path.join('images', self._user, image_name)
138
-
139
- def get_image_ref(self, image):
140
- ensure_path(os.path.join('images', self._user))
141
- path = self._get_image_ref_path(image)
142
- try:
143
- with open(path, 'r') as stream:
144
- ref = stream.read().strip()
145
- except FileNotFoundError:
146
- return None
147
- return ref
148
-
149
- def save_image_ref(self, image, ref):
150
- ensure_path(os.path.join('images', self._user))
151
- path = self._get_image_ref_path(image)
152
- with open(path, 'w') as stream:
153
- stream.write(ref)
154
-
155
- def clear_image_ref(self, image):
156
- ensure_path(os.path.join('images', self._user))
157
- path = self._get_image_ref_path(image)
158
- try:
159
- os.remove(path)
160
- except FileNotFoundError:
161
- pass
162
-
163
- def get_image_repo_name(self, image):
141
+ def get_image_repo(self, image):
164
142
  prefix = []
165
143
  if self._docker_registry is not None:
166
144
  prefix.append(self._docker_registry)
@@ -188,7 +166,7 @@ class EnvironmentHandler:
188
166
 
189
167
  def pull_base_image(self, ref):
190
168
  """Pull an image from the remote registry.
191
-
169
+
192
170
  Note:
193
171
  Derived classes usually need to overload this to handle authentication
194
172
  with the remote registry. In a local environment where there is no
@@ -199,21 +177,20 @@ class EnvironmentHandler:
199
177
  Args:
200
178
  ref (str): The full docker image reference for the requested image.
201
179
  """
202
- run_command([
203
- self._docker_command,
204
- 'pull',
205
- ref,
206
- ])
180
+ command = [self._docker_command, 'pull', ref]
181
+ run_command(command)
207
182
 
208
- def list_local_image_tags(self, repo_name):
183
+ def list_local_image_tags(self, image):
209
184
  """List locally available tags for a docker image repo.
210
185
 
211
186
  Args:
212
- repo_name (str): The fully qualified name of the docker image repo.
187
+ image (cloud_submit.Image): The object describing the docker image
188
+ whose tags should be listed.
213
189
 
214
190
  Returns:
215
191
  tags (list of str): The list of tags that are available locally.
216
192
  """
193
+ repo_name = self.get_image_repo(image)
217
194
  result = run_command(
218
195
  [
219
196
  self._docker_command,
@@ -230,7 +207,7 @@ class EnvironmentHandler:
230
207
  return []
231
208
  return result.split('\n')
232
209
 
233
- def list_remote_image_tags(self, repo_name):
210
+ def list_remote_image_tags(self, image):
234
211
  """List remotely available tags for a docker image repo.
235
212
 
236
213
  Note:
@@ -238,7 +215,8 @@ class EnvironmentHandler:
238
215
  remote registry.
239
216
 
240
217
  Args:
241
- repo_name (str): The fully qualified name of the docker image repo.
218
+ image (cloud_submit.Image): The object describing the docker image
219
+ whose tags should be listed.
242
220
 
243
221
  Returns:
244
222
  tags (list of str): The list of the tags that are available in the
@@ -258,17 +236,20 @@ class EnvironmentHandler:
258
236
  raise NotImplementedError
259
237
 
260
238
  def build_image(self, path, image, build_id):
261
- repo = self.get_image_repo_name(image)
239
+ repo = self.get_image_repo(image)
262
240
  ref = ':'.join([repo, build_id])
263
- run_command([
241
+ command = [
264
242
  self._docker_command,
243
+ 'buildx',
265
244
  'build',
245
+ *(image.args),
266
246
  '-t', ref,
267
247
  path,
268
- ])
248
+ ]
249
+ run_command(command)
269
250
  return ref
270
251
 
271
- def get_local_artifact_path(self, artifact, run_id=None):
252
+ def get_local_artifact_ref(self, artifact, run_id=None):
272
253
  if artifact.kind != 'file':
273
254
  raise CloudSubmitError(
274
255
  f'Cannot get path for local artifact {artifact.name}. '
@@ -292,7 +273,7 @@ class EnvironmentHandler:
292
273
  raise CloudSubmitError(
293
274
  f'Unknown scope {artifact.scope} for artifact {artifact.name}.')
294
275
 
295
- def get_remote_artifact_path(self, artifact, run_id=None):
276
+ def get_remote_artifact_ref(self, artifact, run_id=None):
296
277
  raise NotImplementedError
297
278
 
298
279
  def list_local_artifacts(self, artifacts, run_ids=None):
@@ -349,7 +330,7 @@ class EnvironmentHandler:
349
330
  if artifact.kind != 'file':
350
331
  continue
351
332
  for run_id in runs:
352
- path = self.get_local_artifact_path(artifact, run_id)
333
+ path = self.get_local_artifact_ref(artifact, run_id)
353
334
  clear_path(path)
354
335
 
355
336
  def remove_remote_artifacts(self, artifacts, run_ids):
@@ -357,8 +338,8 @@ class EnvironmentHandler:
357
338
 
358
339
  def copy_local_artifacts(self, artifacts, from_run_id, to_run_id):
359
340
  for artifact in artifacts:
360
- src_path = self.get_local_artifact_path(artifact, from_run_id)
361
- dst_path = self.get_local_artifact_path(artifact, to_run_id)
341
+ src_path = self.get_local_artifact_ref(artifact, from_run_id)
342
+ dst_path = self.get_local_artifact_ref(artifact, to_run_id)
362
343
  if os.path.exists(src_path):
363
344
  ensure_path(os.path.dirname(dst_path))
364
345
  try:
@@ -374,8 +355,8 @@ class EnvironmentHandler:
374
355
 
375
356
  def move_local_artifacts(self, artifacts, from_run_id, to_run_id):
376
357
  for artifact in artifacts:
377
- src_path = self.get_local_artifact_path(artifact, from_run_id)
378
- dst_path = self.get_local_artifact_path(artifact, to_run_id)
358
+ src_path = self.get_local_artifact_ref(artifact, from_run_id)
359
+ dst_path = self.get_local_artifact_ref(artifact, to_run_id)
379
360
  if os.path.exists(src_path):
380
361
  ensure_path(os.path.dirname(dst_path))
381
362
  shutil.move(src_path, dst_path)