cloud-submit 0.2__tar.gz → 0.3.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. {cloud_submit-0.2 → cloud_submit-0.3.1}/PKG-INFO +2 -2
  2. {cloud_submit-0.2 → cloud_submit-0.3.1}/docs/examples/aws/src/csub/build_config.py +11 -6
  3. {cloud_submit-0.2 → cloud_submit-0.3.1}/docs/examples/basic/src/csub/build_config.py +4 -4
  4. {cloud_submit-0.2 → cloud_submit-0.3.1}/docs/examples/multi-worker/src/csub/build_config.py +4 -4
  5. {cloud_submit-0.2 → cloud_submit-0.3.1}/pyproject.toml +1 -1
  6. {cloud_submit-0.2 → cloud_submit-0.3.1}/src/cloud_submit/__init__.py +2 -3
  7. {cloud_submit-0.2 → cloud_submit-0.3.1}/src/cloud_submit/cli.py +18 -8
  8. {cloud_submit-0.2 → cloud_submit-0.3.1}/src/cloud_submit/controller.py +67 -22
  9. {cloud_submit-0.2 → cloud_submit-0.3.1}/src/cloud_submit/environment_handler.py +28 -49
  10. {cloud_submit-0.2 → cloud_submit-0.3.1}/src/cloud_submit/envs/aws/local_environment_handler.py +32 -8
  11. cloud_submit-0.3.1/src/cloud_submit/envs/aws/local_execution_handler.py +67 -0
  12. {cloud_submit-0.2 → cloud_submit-0.3.1}/src/cloud_submit/envs/aws/remote_environment_handler.py +33 -22
  13. {cloud_submit-0.2 → cloud_submit-0.3.1}/src/cloud_submit/envs/aws/remote_execution_handler.py +24 -23
  14. {cloud_submit-0.2 → cloud_submit-0.3.1}/src/cloud_submit/envs/aws/s3_tools.py +1 -2
  15. {cloud_submit-0.2 → cloud_submit-0.3.1}/src/cloud_submit/envs/local/environment_handler.py +51 -2
  16. {cloud_submit-0.2 → cloud_submit-0.3.1}/src/cloud_submit/execution/base_execution_handler.py +7 -7
  17. {cloud_submit-0.2 → cloud_submit-0.3.1}/src/cloud_submit/execution/config.py +11 -15
  18. {cloud_submit-0.2 → cloud_submit-0.3.1}/src/cloud_submit/execution/execute.py +22 -22
  19. {cloud_submit-0.2 → cloud_submit-0.3.1}/src/cloud_submit/execution/utils.py +18 -3
  20. {cloud_submit-0.2 → cloud_submit-0.3.1}/src/cloud_submit/images.py +4 -0
  21. {cloud_submit-0.2 → cloud_submit-0.3.1}/src/cloud_submit/utils.py +0 -19
  22. cloud_submit-0.2/docs/examples/aws/userconfig/.ipynb_checkpoints/default-checkpoint.yaml +0 -13
  23. cloud_submit-0.2/src/cloud_submit/envs/aws/local_execution_handler.py +0 -5
  24. {cloud_submit-0.2 → cloud_submit-0.3.1}/.gitignore +0 -0
  25. {cloud_submit-0.2 → cloud_submit-0.3.1}/LICENSE +0 -0
  26. {cloud_submit-0.2 → cloud_submit-0.3.1}/README.md +0 -0
  27. {cloud_submit-0.2 → cloud_submit-0.3.1}/docs/examples/aws/src/fit.py +0 -0
  28. {cloud_submit-0.2 → cloud_submit-0.3.1}/docs/examples/aws/src/generate.py +0 -0
  29. {cloud_submit-0.2 → cloud_submit-0.3.1}/docs/examples/aws/src/parameters.py +0 -0
  30. {cloud_submit-0.2 → cloud_submit-0.3.1}/docs/examples/aws/userconfig/template.yaml +0 -0
  31. {cloud_submit-0.2 → cloud_submit-0.3.1}/docs/examples/basic/src/fit.py +0 -0
  32. {cloud_submit-0.2 → cloud_submit-0.3.1}/docs/examples/basic/src/generate.py +0 -0
  33. {cloud_submit-0.2 → cloud_submit-0.3.1}/docs/examples/basic/src/parameters.py +0 -0
  34. {cloud_submit-0.2 → cloud_submit-0.3.1}/docs/examples/basic/userconfig/template.yaml +0 -0
  35. {cloud_submit-0.2 → cloud_submit-0.3.1}/docs/examples/multi-worker/src/fit.py +0 -0
  36. {cloud_submit-0.2 → cloud_submit-0.3.1}/docs/examples/multi-worker/src/generate.py +0 -0
  37. {cloud_submit-0.2 → cloud_submit-0.3.1}/docs/examples/multi-worker/src/parameters.py +0 -0
  38. {cloud_submit-0.2 → cloud_submit-0.3.1}/docs/examples/multi-worker/userconfig/template.yaml +0 -0
  39. {cloud_submit-0.2 → cloud_submit-0.3.1}/src/cloud_submit/config.py +0 -0
  40. {cloud_submit-0.2 → cloud_submit-0.3.1}/src/cloud_submit/envs/aws/__init__.py +0 -0
  41. {cloud_submit-0.2 → cloud_submit-0.3.1}/src/cloud_submit/envs/local/execution_handler.py +0 -0
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.4
1
+ Metadata-Version: 2.5
2
2
  Name: cloud-submit
3
- Version: 0.2
3
+ Version: 0.3.1
4
4
  Summary: A tool for building processing pipelines in Python and executing them locally or in the cloud.
5
5
  Project-URL: Homepage, https://github.com/mwiebusch78/cloud-submit
6
6
  Project-URL: Bug Tracker, https://github.com/mwiebusch78/cloud-submit/issues
@@ -40,10 +40,15 @@ def build_config(project_root, userconfig):
40
40
  name='base',
41
41
  instructions=r"""
42
42
  FROM python:3.12.3
43
+ ARG TARGETARCH
43
44
  RUN \
45
+ case "$TARGETARCH" in \
46
+ amd64) url="https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" ;; \
47
+ arm64) url="https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" ;; \
48
+ *) echo "unsupported architecture: $TARGETARCH" >&2; exit 1 ;; \
49
+ esac && \
44
50
  cd /opt && \
45
- curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" \
46
- -o "awscliv2.zip" && \
51
+ curl "$url" -o "awscliv2.zip" && \
47
52
  unzip awscliv2.zip && \
48
53
  ./aws/install && \
49
54
  rm -rf awscliv2.zip aws
@@ -86,7 +91,7 @@ def build_config(project_root, userconfig):
86
91
  'sigma': PARAMS['sigma'],
87
92
  },
88
93
  outputs={
89
- 'train_data_path': cs.local('train_data.parquet'),
94
+ 'train_data_path': cs.aref('train_data.parquet', 'path'),
90
95
  },
91
96
  ),
92
97
  cs.Step(
@@ -99,11 +104,11 @@ def build_config(project_root, userconfig):
99
104
  function='fit:fit_step',
100
105
  image='exec',
101
106
  inputs={
102
- 'train_data_path': cs.local('train_data.parquet'),
107
+ 'train_data_path': cs.aref('train_data.parquet', 'path'),
103
108
  },
104
109
  outputs={
105
- 'coefficients_path': cs.local('coefficients.json'),
106
- 'predictions_path': cs.local('predictions.parquet'),
110
+ 'coefficients_path': cs.aref('coefficients.json', 'path'),
111
+ 'predictions_path': cs.aref('predictions.parquet', 'path'),
107
112
  },
108
113
  ),
109
114
  ],
@@ -70,7 +70,7 @@ def build_config(project_root, userconfig):
70
70
  'sigma': PARAMS['sigma'],
71
71
  },
72
72
  outputs={
73
- 'train_data_path': cs.local('train_data.parquet'),
73
+ 'train_data_path': cs.aref('train_data.parquet', 'path'),
74
74
  },
75
75
  ),
76
76
  cs.Step(
@@ -78,11 +78,11 @@ def build_config(project_root, userconfig):
78
78
  function='fit:fit_step',
79
79
  image='exec',
80
80
  inputs={
81
- 'train_data_path': cs.local('train_data.parquet'),
81
+ 'train_data_path': cs.aref('train_data.parquet', 'path'),
82
82
  },
83
83
  outputs={
84
- 'coefficients_path': cs.local('coefficients.json'),
85
- 'predictions_path': cs.local('predictions.parquet'),
84
+ 'coefficients_path': cs.aref('coefficients.json', 'path'),
85
+ 'predictions_path': cs.aref('predictions.parquet', 'path'),
86
86
  },
87
87
  ),
88
88
  ],
@@ -69,7 +69,7 @@ def build_config(project_root, userconfig):
69
69
  'sigma': PARAMS['sigma'],
70
70
  },
71
71
  outputs={
72
- 'train_data_path': cs.local('train_data.parquet'),
72
+ 'train_data_path': cs.aref('train_data.parquet', 'path'),
73
73
  },
74
74
  pass_submit_timestamp_as='submit_timestamp',
75
75
  ),
@@ -78,11 +78,11 @@ def build_config(project_root, userconfig):
78
78
  function='fit:fit_step',
79
79
  image='exec',
80
80
  inputs={
81
- 'train_data_path': cs.local('train_data.parquet'),
81
+ 'train_data_path': cs.aref('train_data.parquet', 'path'),
82
82
  },
83
83
  outputs={
84
- 'coefficients_path': cs.local('coefficients.json'),
85
- 'predictions_path': cs.local('predictions.parquet'),
84
+ 'coefficients_path': cs.aref('coefficients.json', 'path'),
85
+ 'predictions_path': cs.aref('predictions.parquet', 'path'),
86
86
  },
87
87
  num_workers=PARAMS['num_folds'],
88
88
  pass_num_workers_as='num_folds',
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "cloud-submit"
7
- version = "0.2"
7
+ version = "0.3.1"
8
8
  authors = [
9
9
  { name="Martin Wiebusch" },
10
10
  ]
@@ -6,12 +6,11 @@ from .config import Config
6
6
  from .execution.config import (
7
7
  ConfigError,
8
8
  Artifact,
9
- ArtifactLocation,
9
+ ArtifactReference,
10
10
  Spec,
11
11
  Step,
12
12
  Pipeline,
13
- local,
14
- remote,
13
+ aref,
15
14
  )
16
15
  from .execution.utils import (
17
16
  CloudSubmitError,
@@ -7,6 +7,7 @@ import yaml
7
7
 
8
8
  from .execution.utils import CloudSubmitError
9
9
  from .controller import Controller
10
+ from .images import BaseImage
10
11
 
11
12
 
12
13
  def abort(msg):
@@ -303,11 +304,11 @@ def images():
303
304
  @click.option(
304
305
  '--all', '-a', 'build_all',
305
306
  is_flag=True,
306
- help='Build all images.',
307
+ help='Ignore IMAGES arguments and build all base images.',
307
308
  )
308
309
  @click.argument('images', type=str, nargs=-1)
309
310
  @click.pass_context
310
- def build(ctx, env, build_id, build_all, images):
311
+ def build_images(ctx, env, build_id, build_all, images):
311
312
  init(ctx)
312
313
  config = ctx.obj['config']
313
314
  controller = ctx.obj['controller']
@@ -316,9 +317,12 @@ def build(ctx, env, build_id, build_all, images):
316
317
  if image not in config.images:
317
318
  abort(f'No definition found for image {image}.')
318
319
  if build_all:
319
- images = list(config.images.keys())
320
+ images = [
321
+ name for name, image in config.images.items()
322
+ if isinstance(image, BaseImage)
323
+ ]
320
324
  try:
321
- controller.build(images, build_id=build_id, env=env)
325
+ controller.build_images(images, build_id=build_id, env=env)
322
326
  except CloudSubmitError as e:
323
327
  abort(str(e))
324
328
 
@@ -495,7 +499,10 @@ def remove_images(ctx, env, images, local, remote, build_ids):
495
499
  def set_image(ctx, env, image, ref):
496
500
  init(ctx)
497
501
  controller = ctx.obj['controller']
498
- controller.set_image(image, ref, env=env)
502
+ try:
503
+ controller.set_image(image, ref, env=env)
504
+ except CloudSubmitError as e:
505
+ abort(str(e))
499
506
 
500
507
 
501
508
  @images.command(
@@ -522,7 +529,10 @@ def unset_image(ctx, env, unset_all, images):
522
529
  if unset_all:
523
530
  images = list(config.images.keys())
524
531
  for image in images:
525
- controller.unset_image(image, env=env)
532
+ try:
533
+ controller.unset_image(image, env=env)
534
+ except CloudSubmitError as e:
535
+ abort(str(e))
526
536
 
527
537
 
528
538
  # artifacts subgroup
@@ -605,10 +615,10 @@ def list_artifacts(ctx, env, artifacts, local, remote, run_ids):
605
615
  artifact = config.artifacts[name]
606
616
  for run_id in runs:
607
617
  if local:
608
- path = env_handler.get_local_artifact_path(
618
+ path = env_handler.get_local_artifact_ref(
609
619
  artifact, run_id)
610
620
  else:
611
- path = env_handler.get_remote_artifact_path(
621
+ path = env_handler.get_remote_artifact_ref(
612
622
  artifact, run_id)
613
623
  data.append((
614
624
  name,
@@ -32,14 +32,44 @@ class Controller:
32
32
  self._config.export_pipelines(os.path.join(path, 'pipelines.json'))
33
33
  self._config.export_artifacts(os.path.join(path, 'artifacts.json'))
34
34
 
35
+ def _get_image_ref_path(self, image):
36
+ if not isinstance(image, BaseImage):
37
+ raise ValueError(f'Cannot handle image of type {type(image)}')
38
+ return os.path.join('images', self._config.user_name, image.name)
39
+
40
+ def _get_image_ref(self, image):
41
+ ensure_path(os.path.join('images', self._config.user_name))
42
+ path = self._get_image_ref_path(image)
43
+ try:
44
+ with open(path, 'r') as stream:
45
+ ref = stream.read().strip()
46
+ except FileNotFoundError:
47
+ return None
48
+ return ref
49
+
50
+ def _save_image_ref(self, image, ref):
51
+ ensure_path(os.path.join('images', self._config.user_name))
52
+ path = self._get_image_ref_path(image)
53
+ with open(path, 'w') as stream:
54
+ stream.write(ref)
55
+
56
+ def _clear_image_ref(self, image):
57
+ ensure_path(os.path.join('images', self._config.user_name))
58
+ path = self._get_image_ref_path(image)
59
+ try:
60
+ os.remove(path)
61
+ except FileNotFoundError:
62
+ pass
63
+
35
64
  def _build_image(self, image, build_id, env, rebuild=False):
36
- if not rebuild:
37
- ref = env.get_image_ref(image)
65
+ is_execution_image = isinstance(image, ExecutionImage)
66
+
67
+ if not rebuild and not is_execution_image:
68
+ ref = self._get_image_ref(image)
38
69
  if ref is not None:
39
70
  print(f'Using existing build of {image.name}: {ref}')
40
71
  return ref
41
72
 
42
- is_execution_image = isinstance(image, ExecutionImage)
43
73
  path = os.path.join('temp', 'build', image.name)
44
74
  if is_execution_image:
45
75
  path = os.path.join(path, env.name)
@@ -58,7 +88,7 @@ class Controller:
58
88
  if parent_image is None:
59
89
  raise CloudSubmitError(
60
90
  f'Could not find declaration for image {image.parent}.')
61
- parent_ref = env.get_image_ref(parent_image)
91
+ parent_ref = self._get_image_ref(parent_image)
62
92
  if parent_ref is None:
63
93
  raise CloudSubmitError(
64
94
  f'Could not find reference for image {image.parent}.')
@@ -72,9 +102,10 @@ class Controller:
72
102
  'build_image method of environment handler did not '
73
103
  'return a valid image reference.'
74
104
  )
75
- env.save_image_ref(image, ref)
105
+ if not is_execution_image:
106
+ self._save_image_ref(image, ref)
76
107
 
77
- def build(self, images, build_id=None, env=None):
108
+ def build_images(self, images, build_id=None, env=None):
78
109
  now = dt.datetime.now(tz=dt.UTC)
79
110
  env_handler = self._config.get_build_env(env)
80
111
  build_id = env_handler.generate_build_id(now, build_id)
@@ -114,7 +145,7 @@ class Controller:
114
145
  results = []
115
146
  for image in images:
116
147
  image = self._config.images[image]
117
- repo_name = env_handler.get_image_repo_name(image)
148
+ repo_name = env_handler.get_image_repo(image)
118
149
  if remote:
119
150
  tags = env_handler.list_remote_image_tags(image)
120
151
  else:
@@ -125,8 +156,7 @@ class Controller:
125
156
  results.append(':'.join([repo_name, tag]))
126
157
  return results
127
158
 
128
- def list_images(self, images=None, env=None):
129
- env_handler = self._config.get_build_env(env)
159
+ def list_images(self, images=None):
130
160
  if images is not None:
131
161
  images = set(images)
132
162
  result = []
@@ -136,15 +166,18 @@ class Controller:
136
166
  continue
137
167
  if isinstance(i, BaseImage):
138
168
  tpe = 'base'
169
+ ref = self._get_image_ref(i)
139
170
  elif isinstance(i, ExecutionImage):
140
171
  tpe = 'execution'
172
+ ref = None
141
173
  else:
142
174
  raise ValueError('Unknown image type.')
143
- ref = env_handler.get_image_ref(i)
144
175
  result.append((i.name, tpe, ref))
145
176
  return result
146
177
 
147
178
  def remove_image_refs(self, refs, remote=False, env=None):
179
+ if not refs:
180
+ return
148
181
  env_handler = self._config.get_build_env(env)
149
182
  if remote:
150
183
  env_handler.remove_remote_image_refs(refs)
@@ -156,16 +189,26 @@ class Controller:
156
189
  image = self._config.images.get(image_name)
157
190
  if image is None:
158
191
  raise CloudSubmitError(f'Unknown image name {image_name}.')
192
+ if not isinstance(image, BaseImage):
193
+ raise CloudSubmitError(
194
+ f'Cannot set ref for image {image_name} '
195
+ 'since it is not a base image.'
196
+ )
159
197
  with self._config.in_project_root():
160
- env_handler.save_image_ref(image, ref)
198
+ self._save_image_ref(image, ref)
161
199
 
162
200
  def unset_image(self, image_name, env=None):
163
201
  env_handler = self._config.get_build_env(env)
164
202
  image = self._config.images.get(image_name)
165
203
  if image is None:
166
204
  raise CloudSubmitError(f'Unknown image name {image_name}.')
205
+ if not isinstance(image, BaseImage):
206
+ raise CloudSubmitError(
207
+ f'Cannot unset ref for image {image_name} '
208
+ 'since it is not a base image.'
209
+ )
167
210
  with self._config.in_project_root():
168
- env_handler.clear_image_ref(image)
211
+ self._clear_image_ref(image)
169
212
 
170
213
  def list_artifacts(
171
214
  self,
@@ -202,6 +245,8 @@ class Controller:
202
245
  remote=False,
203
246
  env=None,
204
247
  ):
248
+ if not artifact_names:
249
+ return
205
250
  env_handler = self._config.get_run_env(env)
206
251
  artifacts = [
207
252
  self._config.artifacts[name].copy() for name in artifact_names]
@@ -327,7 +372,7 @@ class Controller:
327
372
 
328
373
  with self._config.in_project_root():
329
374
  images = [step.image for step in steps]
330
- self.build(images, build_id=run_id, env=build_env)
375
+ self.build_images(images, build_id=run_id, env=build_env)
331
376
  refs = {}
332
377
  overwrite_artifacts = []
333
378
  for step in steps:
@@ -335,11 +380,8 @@ class Controller:
335
380
  if image is None:
336
381
  raise CloudSubmitError(
337
382
  f'Cannot find declaration for image {step.image}.')
338
- ref = env_handler.get_image_ref(image)
339
- if ref is None:
340
- raise CloudSubmitError(
341
- f'Could not find image ref for image {step.image}')
342
- refs[step.name] = ref
383
+ repo = env_handler.get_image_repo(image)
384
+ refs[step.name] = ':'.join([repo, run_id])
343
385
 
344
386
  for loc in step.outputs.values():
345
387
  overwrite_artifacts.append(
@@ -351,10 +393,14 @@ class Controller:
351
393
  temp_path = 'temp/run'
352
394
  ensure_path(temp_path, clear=True)
353
395
  env_handler.run_pipeline(
354
- pipeline, refs, overwrite_artifacts,
355
- timestamp, run_id, temp_path,
396
+ pipeline=pipeline,
397
+ image_refs=refs,
398
+ overwrite_artifacts=overwrite_artifacts,
399
+ timestamp=timestamp,
400
+ run_id=run_id,
401
+ temp_path=temp_path,
356
402
  )
357
-
403
+
358
404
  return {
359
405
  'run_id': run_id,
360
406
  'start_timestamp': now,
@@ -371,5 +417,4 @@ class Controller:
371
417
  since = dt.datetime.now(tz=dt.UTC) - since
372
418
  env_handler = self._config.get_run_env(env)
373
419
  env_handler.print_logs(run_id, since, stream=stream)
374
-
375
420
 
@@ -23,16 +23,28 @@ class EnvironmentHandler:
23
23
  name,
24
24
  project,
25
25
  user,
26
+ envvars=None,
27
+ secrets=None,
26
28
  docker_command='docker',
27
29
  docker_registry=None,
28
30
  docker_namespace='csub',
29
31
  ):
30
- self.name = name
31
- self._project = project
32
- self._user = user
32
+ self.name = str(name)
33
+ self._project = str(project)
34
+ self._user = str(user)
33
35
  self._docker_command = str(docker_command)
34
36
  self._docker_registry = docker_registry
35
- self._docker_namespace = docker_namespace
37
+ self._docker_namespace = str(docker_namespace)
38
+
39
+ self._envvars = {
40
+ 'CSUB_PROJECT': self._project,
41
+ 'CSUB_USER': self._user,
42
+ }
43
+ if envvars is not None:
44
+ self._envvars.update(envvars)
45
+ self._secrets = {}
46
+ if secrets is not None:
47
+ self._secrets.update(secrets)
36
48
 
37
49
  def _generate_id(self, timestamp, id):
38
50
  if id is not None:
@@ -126,41 +138,7 @@ class EnvironmentHandler:
126
138
  """
127
139
  raise NotImplementedError
128
140
 
129
- def _get_image_ref_path(self, image):
130
- if isinstance(image, BaseImage):
131
- image_name = image.name
132
- elif isinstance(image, ExecutionImage):
133
- image_name = '.'.join([image.name, self.name])
134
- else:
135
- raise ValueError(f'Cannot handle image of type {type(image)}')
136
-
137
- return os.path.join('images', self._user, image_name)
138
-
139
- def get_image_ref(self, image):
140
- ensure_path(os.path.join('images', self._user))
141
- path = self._get_image_ref_path(image)
142
- try:
143
- with open(path, 'r') as stream:
144
- ref = stream.read().strip()
145
- except FileNotFoundError:
146
- return None
147
- return ref
148
-
149
- def save_image_ref(self, image, ref):
150
- ensure_path(os.path.join('images', self._user))
151
- path = self._get_image_ref_path(image)
152
- with open(path, 'w') as stream:
153
- stream.write(ref)
154
-
155
- def clear_image_ref(self, image):
156
- ensure_path(os.path.join('images', self._user))
157
- path = self._get_image_ref_path(image)
158
- try:
159
- os.remove(path)
160
- except FileNotFoundError:
161
- pass
162
-
163
- def get_image_repo_name(self, image):
141
+ def get_image_repo(self, image):
164
142
  prefix = []
165
143
  if self._docker_registry is not None:
166
144
  prefix.append(self._docker_registry)
@@ -188,7 +166,7 @@ class EnvironmentHandler:
188
166
 
189
167
  def pull_base_image(self, ref):
190
168
  """Pull an image from the remote registry.
191
-
169
+
192
170
  Note:
193
171
  Derived classes usually need to overload this to handle authentication
194
172
  with the remote registry. In a local environment where there is no
@@ -212,7 +190,7 @@ class EnvironmentHandler:
212
190
  Returns:
213
191
  tags (list of str): The list of tags that are available locally.
214
192
  """
215
- repo_name = self.get_image_repo_name(image)
193
+ repo_name = self.get_image_repo(image)
216
194
  result = run_command(
217
195
  [
218
196
  self._docker_command,
@@ -258,19 +236,20 @@ class EnvironmentHandler:
258
236
  raise NotImplementedError
259
237
 
260
238
  def build_image(self, path, image, build_id):
261
- repo = self.get_image_repo_name(image)
239
+ repo = self.get_image_repo(image)
262
240
  ref = ':'.join([repo, build_id])
263
241
  command = [
264
242
  self._docker_command,
265
243
  'buildx',
266
244
  'build',
245
+ *(image.args),
267
246
  '-t', ref,
268
247
  path,
269
248
  ]
270
249
  run_command(command)
271
250
  return ref
272
251
 
273
- def get_local_artifact_path(self, artifact, run_id=None):
252
+ def get_local_artifact_ref(self, artifact, run_id=None):
274
253
  if artifact.kind != 'file':
275
254
  raise CloudSubmitError(
276
255
  f'Cannot get path for local artifact {artifact.name}. '
@@ -294,7 +273,7 @@ class EnvironmentHandler:
294
273
  raise CloudSubmitError(
295
274
  f'Unknown scope {artifact.scope} for artifact {artifact.name}.')
296
275
 
297
- def get_remote_artifact_path(self, artifact, run_id=None):
276
+ def get_remote_artifact_ref(self, artifact, run_id=None):
298
277
  raise NotImplementedError
299
278
 
300
279
  def list_local_artifacts(self, artifacts, run_ids=None):
@@ -351,7 +330,7 @@ class EnvironmentHandler:
351
330
  if artifact.kind != 'file':
352
331
  continue
353
332
  for run_id in runs:
354
- path = self.get_local_artifact_path(artifact, run_id)
333
+ path = self.get_local_artifact_ref(artifact, run_id)
355
334
  clear_path(path)
356
335
 
357
336
  def remove_remote_artifacts(self, artifacts, run_ids):
@@ -359,8 +338,8 @@ class EnvironmentHandler:
359
338
 
360
339
  def copy_local_artifacts(self, artifacts, from_run_id, to_run_id):
361
340
  for artifact in artifacts:
362
- src_path = self.get_local_artifact_path(artifact, from_run_id)
363
- dst_path = self.get_local_artifact_path(artifact, to_run_id)
341
+ src_path = self.get_local_artifact_ref(artifact, from_run_id)
342
+ dst_path = self.get_local_artifact_ref(artifact, to_run_id)
364
343
  if os.path.exists(src_path):
365
344
  ensure_path(os.path.dirname(dst_path))
366
345
  try:
@@ -376,8 +355,8 @@ class EnvironmentHandler:
376
355
 
377
356
  def move_local_artifacts(self, artifacts, from_run_id, to_run_id):
378
357
  for artifact in artifacts:
379
- src_path = self.get_local_artifact_path(artifact, from_run_id)
380
- dst_path = self.get_local_artifact_path(artifact, to_run_id)
358
+ src_path = self.get_local_artifact_ref(artifact, from_run_id)
359
+ dst_path = self.get_local_artifact_ref(artifact, to_run_id)
381
360
  if os.path.exists(src_path):
382
361
  ensure_path(os.path.dirname(dst_path))
383
362
  shutil.move(src_path, dst_path)
@@ -17,7 +17,7 @@ from cloud_submit import (
17
17
  BaseImage,
18
18
  )
19
19
 
20
- from .s3_tools import get_remote_artifact_path
20
+ from .s3_tools import get_remote_artifact_ref
21
21
 
22
22
 
23
23
 
@@ -36,20 +36,29 @@ class LocalAWSEnv(LocalEnv):
36
36
  aws_profile,
37
37
  s3_bucket,
38
38
  s3_prefix,
39
+ envvars=None,
40
+ secrets=None,
39
41
  docker_command='docker',
40
42
  docker_namespace='csub',
41
43
  docker_platforms=None,
42
44
  docker_login_refresh_hours=6,
43
45
  aws_command='aws',
46
+ container_aws_command='aws',
44
47
  ):
45
48
  super().__init__(
46
49
  name=name,
47
50
  project=project,
48
51
  user=user,
52
+ envvars=None,
53
+ secrets=None,
49
54
  docker_command=docker_command,
50
55
  docker_registry=\
51
56
  f'{aws_account_id}.dkr.ecr.{aws_region}.amazonaws.com',
52
57
  docker_namespace=docker_namespace,
58
+ docker_mounts=[(
59
+ os.path.join(os.environ['HOME'], '.aws'),
60
+ '/root/.aws',
61
+ )],
53
62
  )
54
63
  self._aws_account_id = str(aws_account_id)
55
64
  self._aws_region = str(aws_region)
@@ -57,11 +66,21 @@ class LocalAWSEnv(LocalEnv):
57
66
  self._s3_bucket = str(s3_bucket)
58
67
  self._s3_prefix = str(s3_prefix)
59
68
  self._aws_command = str(aws_command)
69
+ self._container_aws_command = str(container_aws_command)
60
70
  self._docker_login_refresh_hours = float(docker_login_refresh_hours)
61
71
  self._docker_platforms = docker_platforms
62
72
  if self._docker_platforms is not None:
63
73
  self._docker_platforms = str(docker_platforms)
64
74
 
75
+ self._envvars['CSUB_AWS_REGION'] = self._aws_region
76
+ self._envvars['CSUB_AWS_PROFILE'] = self._aws_profile
77
+ self._envvars['CSUB_AWS_COMMAND'] = self._container_aws_command
78
+ self._envvars['CSUB_S3_BUCKET'] = self._s3_bucket
79
+ self._envvars['CSUB_S3_PREFIX'] = self._s3_prefix
80
+ if envvars is not None:
81
+ self._envvars.update(envvars)
82
+ if secrets is not None:
83
+ self._secrets.update(secrets)
65
84
  self._last_docker_login_ts = None
66
85
 
67
86
  def _docker_login(self):
@@ -71,7 +90,7 @@ class LocalAWSEnv(LocalEnv):
71
90
  (now - self._last_docker_login_ts).total_seconds()
72
91
  if seconds_since_last_login < self._docker_login_refresh_hours*3600:
73
92
  return
74
-
93
+
75
94
  result = run_command(
76
95
  [
77
96
  self._aws_command,
@@ -105,18 +124,23 @@ class LocalAWSEnv(LocalEnv):
105
124
  os.path.join(sourcedir, 'local_execution_handler.py'),
106
125
  os.path.join(path, 'execution_handler.py'),
107
126
  )
127
+ shutil.copyfile(
128
+ os.path.join(sourcedir, 's3_tools.py'),
129
+ os.path.join(path, 's3_tools.py'),
130
+ )
108
131
 
109
132
  def _build_image(self, path, image, build_id, push):
110
133
  if not push:
111
134
  ref = super().build_image(path, image, build_id)
112
135
  else:
113
136
  self._docker_login()
114
- repo = self.get_image_repo_name(image)
137
+ repo = self.get_image_repo(image)
115
138
  ref = ':'.join([repo, build_id])
116
139
  command = [
117
140
  self._docker_command,
118
141
  'buildx',
119
142
  'build',
143
+ *(image.args),
120
144
  ]
121
145
  if self._docker_platforms is not None:
122
146
  command.extend(['--platform', self._docker_platforms])
@@ -128,7 +152,7 @@ class LocalAWSEnv(LocalEnv):
128
152
  return self._build_image(
129
153
  path=path,
130
154
  image=image,
131
- build_id=build_id,
155
+ build_id=build_id,
132
156
  push=isinstance(image, BaseImage),
133
157
  )
134
158
 
@@ -160,7 +184,7 @@ class LocalAWSEnv(LocalEnv):
160
184
  def list_remote_image_tags(self, image):
161
185
  if isinstance(image, ExecutionImage):
162
186
  return []
163
- repo_name = self.get_image_repo_name(image)
187
+ repo_name = self.get_image_repo(image)
164
188
  return self._list_remote_image_tags(repo_name)
165
189
 
166
190
  def remove_remote_image_refs(self, refs):
@@ -185,8 +209,8 @@ class LocalAWSEnv(LocalEnv):
185
209
  *[f'imageTag={tag}' for tag in tag_list],
186
210
  ])
187
211
 
188
- def get_remote_artifact_path(self, artifact, run_id=None):
189
- return get_remote_artifact_path(
212
+ def get_remote_artifact_ref(self, artifact, run_id=None):
213
+ return get_remote_artifact_ref(
190
214
  artifact,
191
215
  self._project,
192
216
  self._user,
@@ -273,7 +297,7 @@ class LocalAWSEnv(LocalEnv):
273
297
  if len(key) > 1 and key[1] in run_artifacts:
274
298
  artifact = key[1]
275
299
  results[artifact].add(run_id)
276
-
300
+
277
301
  results = [sorted(results[a.name]) for a in artifacts]
278
302
  return results
279
303