cloud-submit 0.2__tar.gz → 0.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cloud_submit-0.2 → cloud_submit-0.3}/PKG-INFO +1 -1
- {cloud_submit-0.2 → cloud_submit-0.3}/docs/examples/aws/src/csub/build_config.py +11 -6
- {cloud_submit-0.2 → cloud_submit-0.3}/docs/examples/basic/src/csub/build_config.py +4 -4
- {cloud_submit-0.2 → cloud_submit-0.3}/docs/examples/multi-worker/src/csub/build_config.py +4 -4
- {cloud_submit-0.2 → cloud_submit-0.3}/pyproject.toml +1 -1
- {cloud_submit-0.2 → cloud_submit-0.3}/src/cloud_submit/__init__.py +2 -3
- {cloud_submit-0.2 → cloud_submit-0.3}/src/cloud_submit/cli.py +18 -8
- {cloud_submit-0.2 → cloud_submit-0.3}/src/cloud_submit/controller.py +67 -22
- {cloud_submit-0.2 → cloud_submit-0.3}/src/cloud_submit/environment_handler.py +28 -49
- {cloud_submit-0.2 → cloud_submit-0.3}/src/cloud_submit/envs/aws/local_environment_handler.py +32 -8
- cloud_submit-0.3/src/cloud_submit/envs/aws/local_execution_handler.py +63 -0
- {cloud_submit-0.2 → cloud_submit-0.3}/src/cloud_submit/envs/aws/remote_environment_handler.py +33 -22
- {cloud_submit-0.2 → cloud_submit-0.3}/src/cloud_submit/envs/aws/remote_execution_handler.py +24 -23
- {cloud_submit-0.2 → cloud_submit-0.3}/src/cloud_submit/envs/aws/s3_tools.py +1 -2
- {cloud_submit-0.2 → cloud_submit-0.3}/src/cloud_submit/envs/local/environment_handler.py +51 -2
- {cloud_submit-0.2 → cloud_submit-0.3}/src/cloud_submit/execution/base_execution_handler.py +7 -7
- {cloud_submit-0.2 → cloud_submit-0.3}/src/cloud_submit/execution/config.py +11 -15
- {cloud_submit-0.2 → cloud_submit-0.3}/src/cloud_submit/execution/execute.py +22 -22
- {cloud_submit-0.2 → cloud_submit-0.3}/src/cloud_submit/execution/utils.py +18 -3
- {cloud_submit-0.2 → cloud_submit-0.3}/src/cloud_submit/images.py +4 -0
- {cloud_submit-0.2 → cloud_submit-0.3}/src/cloud_submit/utils.py +0 -19
- cloud_submit-0.2/docs/examples/aws/userconfig/.ipynb_checkpoints/default-checkpoint.yaml +0 -13
- cloud_submit-0.2/src/cloud_submit/envs/aws/local_execution_handler.py +0 -5
- {cloud_submit-0.2 → cloud_submit-0.3}/.gitignore +0 -0
- {cloud_submit-0.2 → cloud_submit-0.3}/LICENSE +0 -0
- {cloud_submit-0.2 → cloud_submit-0.3}/README.md +0 -0
- {cloud_submit-0.2 → cloud_submit-0.3}/docs/examples/aws/src/fit.py +0 -0
- {cloud_submit-0.2 → cloud_submit-0.3}/docs/examples/aws/src/generate.py +0 -0
- {cloud_submit-0.2 → cloud_submit-0.3}/docs/examples/aws/src/parameters.py +0 -0
- {cloud_submit-0.2 → cloud_submit-0.3}/docs/examples/aws/userconfig/template.yaml +0 -0
- {cloud_submit-0.2 → cloud_submit-0.3}/docs/examples/basic/src/fit.py +0 -0
- {cloud_submit-0.2 → cloud_submit-0.3}/docs/examples/basic/src/generate.py +0 -0
- {cloud_submit-0.2 → cloud_submit-0.3}/docs/examples/basic/src/parameters.py +0 -0
- {cloud_submit-0.2 → cloud_submit-0.3}/docs/examples/basic/userconfig/template.yaml +0 -0
- {cloud_submit-0.2 → cloud_submit-0.3}/docs/examples/multi-worker/src/fit.py +0 -0
- {cloud_submit-0.2 → cloud_submit-0.3}/docs/examples/multi-worker/src/generate.py +0 -0
- {cloud_submit-0.2 → cloud_submit-0.3}/docs/examples/multi-worker/src/parameters.py +0 -0
- {cloud_submit-0.2 → cloud_submit-0.3}/docs/examples/multi-worker/userconfig/template.yaml +0 -0
- {cloud_submit-0.2 → cloud_submit-0.3}/src/cloud_submit/config.py +0 -0
- {cloud_submit-0.2 → cloud_submit-0.3}/src/cloud_submit/envs/aws/__init__.py +0 -0
- {cloud_submit-0.2 → cloud_submit-0.3}/src/cloud_submit/envs/local/execution_handler.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cloud-submit
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3
|
|
4
4
|
Summary: A tool for building processing pipelines in Python and executing them locally or in the cloud.
|
|
5
5
|
Project-URL: Homepage, https://github.com/mwiebusch78/cloud-submit
|
|
6
6
|
Project-URL: Bug Tracker, https://github.com/mwiebusch78/cloud-submit/issues
|
|
@@ -40,10 +40,15 @@ def build_config(project_root, userconfig):
|
|
|
40
40
|
name='base',
|
|
41
41
|
instructions=r"""
|
|
42
42
|
FROM python:3.12.3
|
|
43
|
+
ARG TARGETARCH
|
|
43
44
|
RUN \
|
|
45
|
+
case "$TARGETARCH" in \
|
|
46
|
+
amd64) url="https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" ;; \
|
|
47
|
+
arm64) url="https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" ;; \
|
|
48
|
+
*) echo "unsupported architecture: $TARGETARCH" >&2; exit 1 ;; \
|
|
49
|
+
esac && \
|
|
44
50
|
cd /opt && \
|
|
45
|
-
curl "
|
|
46
|
-
-o "awscliv2.zip" && \
|
|
51
|
+
curl "$url" -o "awscliv2.zip" && \
|
|
47
52
|
unzip awscliv2.zip && \
|
|
48
53
|
./aws/install && \
|
|
49
54
|
rm -rf awscliv2.zip aws
|
|
@@ -86,7 +91,7 @@ def build_config(project_root, userconfig):
|
|
|
86
91
|
'sigma': PARAMS['sigma'],
|
|
87
92
|
},
|
|
88
93
|
outputs={
|
|
89
|
-
'train_data_path': cs.
|
|
94
|
+
'train_data_path': cs.aref('train_data.parquet', 'path'),
|
|
90
95
|
},
|
|
91
96
|
),
|
|
92
97
|
cs.Step(
|
|
@@ -99,11 +104,11 @@ def build_config(project_root, userconfig):
|
|
|
99
104
|
function='fit:fit_step',
|
|
100
105
|
image='exec',
|
|
101
106
|
inputs={
|
|
102
|
-
'train_data_path': cs.
|
|
107
|
+
'train_data_path': cs.aref('train_data.parquet', 'path'),
|
|
103
108
|
},
|
|
104
109
|
outputs={
|
|
105
|
-
'coefficients_path': cs.
|
|
106
|
-
'predictions_path': cs.
|
|
110
|
+
'coefficients_path': cs.aref('coefficients.json', 'path'),
|
|
111
|
+
'predictions_path': cs.aref('predictions.parquet', 'path'),
|
|
107
112
|
},
|
|
108
113
|
),
|
|
109
114
|
],
|
|
@@ -70,7 +70,7 @@ def build_config(project_root, userconfig):
|
|
|
70
70
|
'sigma': PARAMS['sigma'],
|
|
71
71
|
},
|
|
72
72
|
outputs={
|
|
73
|
-
'train_data_path': cs.
|
|
73
|
+
'train_data_path': cs.aref('train_data.parquet', 'path'),
|
|
74
74
|
},
|
|
75
75
|
),
|
|
76
76
|
cs.Step(
|
|
@@ -78,11 +78,11 @@ def build_config(project_root, userconfig):
|
|
|
78
78
|
function='fit:fit_step',
|
|
79
79
|
image='exec',
|
|
80
80
|
inputs={
|
|
81
|
-
'train_data_path': cs.
|
|
81
|
+
'train_data_path': cs.aref('train_data.parquet', 'path'),
|
|
82
82
|
},
|
|
83
83
|
outputs={
|
|
84
|
-
'coefficients_path': cs.
|
|
85
|
-
'predictions_path': cs.
|
|
84
|
+
'coefficients_path': cs.aref('coefficients.json', 'path'),
|
|
85
|
+
'predictions_path': cs.aref('predictions.parquet', 'path'),
|
|
86
86
|
},
|
|
87
87
|
),
|
|
88
88
|
],
|
|
@@ -69,7 +69,7 @@ def build_config(project_root, userconfig):
|
|
|
69
69
|
'sigma': PARAMS['sigma'],
|
|
70
70
|
},
|
|
71
71
|
outputs={
|
|
72
|
-
'train_data_path': cs.
|
|
72
|
+
'train_data_path': cs.aref('train_data.parquet', 'path'),
|
|
73
73
|
},
|
|
74
74
|
pass_submit_timestamp_as='submit_timestamp',
|
|
75
75
|
),
|
|
@@ -78,11 +78,11 @@ def build_config(project_root, userconfig):
|
|
|
78
78
|
function='fit:fit_step',
|
|
79
79
|
image='exec',
|
|
80
80
|
inputs={
|
|
81
|
-
'train_data_path': cs.
|
|
81
|
+
'train_data_path': cs.aref('train_data.parquet', 'path'),
|
|
82
82
|
},
|
|
83
83
|
outputs={
|
|
84
|
-
'coefficients_path': cs.
|
|
85
|
-
'predictions_path': cs.
|
|
84
|
+
'coefficients_path': cs.aref('coefficients.json', 'path'),
|
|
85
|
+
'predictions_path': cs.aref('predictions.parquet', 'path'),
|
|
86
86
|
},
|
|
87
87
|
num_workers=PARAMS['num_folds'],
|
|
88
88
|
pass_num_workers_as='num_folds',
|
|
@@ -6,12 +6,11 @@ from .config import Config
|
|
|
6
6
|
from .execution.config import (
|
|
7
7
|
ConfigError,
|
|
8
8
|
Artifact,
|
|
9
|
-
|
|
9
|
+
ArtifactReference,
|
|
10
10
|
Spec,
|
|
11
11
|
Step,
|
|
12
12
|
Pipeline,
|
|
13
|
-
|
|
14
|
-
remote,
|
|
13
|
+
aref,
|
|
15
14
|
)
|
|
16
15
|
from .execution.utils import (
|
|
17
16
|
CloudSubmitError,
|
|
@@ -7,6 +7,7 @@ import yaml
|
|
|
7
7
|
|
|
8
8
|
from .execution.utils import CloudSubmitError
|
|
9
9
|
from .controller import Controller
|
|
10
|
+
from .images import BaseImage
|
|
10
11
|
|
|
11
12
|
|
|
12
13
|
def abort(msg):
|
|
@@ -303,11 +304,11 @@ def images():
|
|
|
303
304
|
@click.option(
|
|
304
305
|
'--all', '-a', 'build_all',
|
|
305
306
|
is_flag=True,
|
|
306
|
-
help='
|
|
307
|
+
help='Ignore IMAGES arguments and build all base images.',
|
|
307
308
|
)
|
|
308
309
|
@click.argument('images', type=str, nargs=-1)
|
|
309
310
|
@click.pass_context
|
|
310
|
-
def
|
|
311
|
+
def build_images(ctx, env, build_id, build_all, images):
|
|
311
312
|
init(ctx)
|
|
312
313
|
config = ctx.obj['config']
|
|
313
314
|
controller = ctx.obj['controller']
|
|
@@ -316,9 +317,12 @@ def build(ctx, env, build_id, build_all, images):
|
|
|
316
317
|
if image not in config.images:
|
|
317
318
|
abort(f'No definition found for image {image}.')
|
|
318
319
|
if build_all:
|
|
319
|
-
images =
|
|
320
|
+
images = [
|
|
321
|
+
name for name, image in config.images.items()
|
|
322
|
+
if isinstance(image, BaseImage)
|
|
323
|
+
]
|
|
320
324
|
try:
|
|
321
|
-
controller.
|
|
325
|
+
controller.build_images(images, build_id=build_id, env=env)
|
|
322
326
|
except CloudSubmitError as e:
|
|
323
327
|
abort(str(e))
|
|
324
328
|
|
|
@@ -495,7 +499,10 @@ def remove_images(ctx, env, images, local, remote, build_ids):
|
|
|
495
499
|
def set_image(ctx, env, image, ref):
|
|
496
500
|
init(ctx)
|
|
497
501
|
controller = ctx.obj['controller']
|
|
498
|
-
|
|
502
|
+
try:
|
|
503
|
+
controller.set_image(image, ref, env=env)
|
|
504
|
+
except CloudSubmitError as e:
|
|
505
|
+
abort(str(e))
|
|
499
506
|
|
|
500
507
|
|
|
501
508
|
@images.command(
|
|
@@ -522,7 +529,10 @@ def unset_image(ctx, env, unset_all, images):
|
|
|
522
529
|
if unset_all:
|
|
523
530
|
images = list(config.images.keys())
|
|
524
531
|
for image in images:
|
|
525
|
-
|
|
532
|
+
try:
|
|
533
|
+
controller.unset_image(image, env=env)
|
|
534
|
+
except CloudSubmitError as e:
|
|
535
|
+
abort(str(e))
|
|
526
536
|
|
|
527
537
|
|
|
528
538
|
# artifacts subgroup
|
|
@@ -605,10 +615,10 @@ def list_artifacts(ctx, env, artifacts, local, remote, run_ids):
|
|
|
605
615
|
artifact = config.artifacts[name]
|
|
606
616
|
for run_id in runs:
|
|
607
617
|
if local:
|
|
608
|
-
path = env_handler.
|
|
618
|
+
path = env_handler.get_local_artifact_ref(
|
|
609
619
|
artifact, run_id)
|
|
610
620
|
else:
|
|
611
|
-
path = env_handler.
|
|
621
|
+
path = env_handler.get_remote_artifact_ref(
|
|
612
622
|
artifact, run_id)
|
|
613
623
|
data.append((
|
|
614
624
|
name,
|
|
@@ -32,14 +32,44 @@ class Controller:
|
|
|
32
32
|
self._config.export_pipelines(os.path.join(path, 'pipelines.json'))
|
|
33
33
|
self._config.export_artifacts(os.path.join(path, 'artifacts.json'))
|
|
34
34
|
|
|
35
|
+
def _get_image_ref_path(self, image):
|
|
36
|
+
if not isinstance(image, BaseImage):
|
|
37
|
+
raise ValueError(f'Cannot handle image of type {type(image)}')
|
|
38
|
+
return os.path.join('images', self._config.user_name, image.name)
|
|
39
|
+
|
|
40
|
+
def _get_image_ref(self, image):
|
|
41
|
+
ensure_path(os.path.join('images', self._config.user_name))
|
|
42
|
+
path = self._get_image_ref_path(image)
|
|
43
|
+
try:
|
|
44
|
+
with open(path, 'r') as stream:
|
|
45
|
+
ref = stream.read().strip()
|
|
46
|
+
except FileNotFoundError:
|
|
47
|
+
return None
|
|
48
|
+
return ref
|
|
49
|
+
|
|
50
|
+
def _save_image_ref(self, image, ref):
|
|
51
|
+
ensure_path(os.path.join('images', self._config.user_name))
|
|
52
|
+
path = self._get_image_ref_path(image)
|
|
53
|
+
with open(path, 'w') as stream:
|
|
54
|
+
stream.write(ref)
|
|
55
|
+
|
|
56
|
+
def _clear_image_ref(self, image):
|
|
57
|
+
ensure_path(os.path.join('images', self._config.user_name))
|
|
58
|
+
path = self._get_image_ref_path(image)
|
|
59
|
+
try:
|
|
60
|
+
os.remove(path)
|
|
61
|
+
except FileNotFoundError:
|
|
62
|
+
pass
|
|
63
|
+
|
|
35
64
|
def _build_image(self, image, build_id, env, rebuild=False):
|
|
36
|
-
|
|
37
|
-
|
|
65
|
+
is_execution_image = isinstance(image, ExecutionImage)
|
|
66
|
+
|
|
67
|
+
if not rebuild and not is_execution_image:
|
|
68
|
+
ref = self._get_image_ref(image)
|
|
38
69
|
if ref is not None:
|
|
39
70
|
print(f'Using existing build of {image.name}: {ref}')
|
|
40
71
|
return ref
|
|
41
72
|
|
|
42
|
-
is_execution_image = isinstance(image, ExecutionImage)
|
|
43
73
|
path = os.path.join('temp', 'build', image.name)
|
|
44
74
|
if is_execution_image:
|
|
45
75
|
path = os.path.join(path, env.name)
|
|
@@ -58,7 +88,7 @@ class Controller:
|
|
|
58
88
|
if parent_image is None:
|
|
59
89
|
raise CloudSubmitError(
|
|
60
90
|
f'Could not find declaration for image {image.parent}.')
|
|
61
|
-
parent_ref =
|
|
91
|
+
parent_ref = self._get_image_ref(parent_image)
|
|
62
92
|
if parent_ref is None:
|
|
63
93
|
raise CloudSubmitError(
|
|
64
94
|
f'Could not find reference for image {image.parent}.')
|
|
@@ -72,9 +102,10 @@ class Controller:
|
|
|
72
102
|
'build_image method of environment handler did not '
|
|
73
103
|
'return a valid image reference.'
|
|
74
104
|
)
|
|
75
|
-
|
|
105
|
+
if not is_execution_image:
|
|
106
|
+
self._save_image_ref(image, ref)
|
|
76
107
|
|
|
77
|
-
def
|
|
108
|
+
def build_images(self, images, build_id=None, env=None):
|
|
78
109
|
now = dt.datetime.now(tz=dt.UTC)
|
|
79
110
|
env_handler = self._config.get_build_env(env)
|
|
80
111
|
build_id = env_handler.generate_build_id(now, build_id)
|
|
@@ -114,7 +145,7 @@ class Controller:
|
|
|
114
145
|
results = []
|
|
115
146
|
for image in images:
|
|
116
147
|
image = self._config.images[image]
|
|
117
|
-
repo_name = env_handler.
|
|
148
|
+
repo_name = env_handler.get_image_repo(image)
|
|
118
149
|
if remote:
|
|
119
150
|
tags = env_handler.list_remote_image_tags(image)
|
|
120
151
|
else:
|
|
@@ -125,8 +156,7 @@ class Controller:
|
|
|
125
156
|
results.append(':'.join([repo_name, tag]))
|
|
126
157
|
return results
|
|
127
158
|
|
|
128
|
-
def list_images(self, images=None
|
|
129
|
-
env_handler = self._config.get_build_env(env)
|
|
159
|
+
def list_images(self, images=None):
|
|
130
160
|
if images is not None:
|
|
131
161
|
images = set(images)
|
|
132
162
|
result = []
|
|
@@ -136,15 +166,18 @@ class Controller:
|
|
|
136
166
|
continue
|
|
137
167
|
if isinstance(i, BaseImage):
|
|
138
168
|
tpe = 'base'
|
|
169
|
+
ref = self._get_image_ref(i)
|
|
139
170
|
elif isinstance(i, ExecutionImage):
|
|
140
171
|
tpe = 'execution'
|
|
172
|
+
ref = None
|
|
141
173
|
else:
|
|
142
174
|
raise ValueError('Unknown image type.')
|
|
143
|
-
ref = env_handler.get_image_ref(i)
|
|
144
175
|
result.append((i.name, tpe, ref))
|
|
145
176
|
return result
|
|
146
177
|
|
|
147
178
|
def remove_image_refs(self, refs, remote=False, env=None):
|
|
179
|
+
if not refs:
|
|
180
|
+
return
|
|
148
181
|
env_handler = self._config.get_build_env(env)
|
|
149
182
|
if remote:
|
|
150
183
|
env_handler.remove_remote_image_refs(refs)
|
|
@@ -156,16 +189,26 @@ class Controller:
|
|
|
156
189
|
image = self._config.images.get(image_name)
|
|
157
190
|
if image is None:
|
|
158
191
|
raise CloudSubmitError(f'Unknown image name {image_name}.')
|
|
192
|
+
if not isinstance(image, BaseImage):
|
|
193
|
+
raise CloudSubmitError(
|
|
194
|
+
f'Cannot set ref for image {image_name} '
|
|
195
|
+
'since it is not a base image.'
|
|
196
|
+
)
|
|
159
197
|
with self._config.in_project_root():
|
|
160
|
-
|
|
198
|
+
self._save_image_ref(image, ref)
|
|
161
199
|
|
|
162
200
|
def unset_image(self, image_name, env=None):
|
|
163
201
|
env_handler = self._config.get_build_env(env)
|
|
164
202
|
image = self._config.images.get(image_name)
|
|
165
203
|
if image is None:
|
|
166
204
|
raise CloudSubmitError(f'Unknown image name {image_name}.')
|
|
205
|
+
if not isinstance(image, BaseImage):
|
|
206
|
+
raise CloudSubmitError(
|
|
207
|
+
f'Cannot unset ref for image {image_name} '
|
|
208
|
+
'since it is not a base image.'
|
|
209
|
+
)
|
|
167
210
|
with self._config.in_project_root():
|
|
168
|
-
|
|
211
|
+
self._clear_image_ref(image)
|
|
169
212
|
|
|
170
213
|
def list_artifacts(
|
|
171
214
|
self,
|
|
@@ -202,6 +245,8 @@ class Controller:
|
|
|
202
245
|
remote=False,
|
|
203
246
|
env=None,
|
|
204
247
|
):
|
|
248
|
+
if not artifact_names:
|
|
249
|
+
return
|
|
205
250
|
env_handler = self._config.get_run_env(env)
|
|
206
251
|
artifacts = [
|
|
207
252
|
self._config.artifacts[name].copy() for name in artifact_names]
|
|
@@ -327,7 +372,7 @@ class Controller:
|
|
|
327
372
|
|
|
328
373
|
with self._config.in_project_root():
|
|
329
374
|
images = [step.image for step in steps]
|
|
330
|
-
self.
|
|
375
|
+
self.build_images(images, build_id=run_id, env=build_env)
|
|
331
376
|
refs = {}
|
|
332
377
|
overwrite_artifacts = []
|
|
333
378
|
for step in steps:
|
|
@@ -335,11 +380,8 @@ class Controller:
|
|
|
335
380
|
if image is None:
|
|
336
381
|
raise CloudSubmitError(
|
|
337
382
|
f'Cannot find declaration for image {step.image}.')
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
raise CloudSubmitError(
|
|
341
|
-
f'Could not find image ref for image {step.image}')
|
|
342
|
-
refs[step.name] = ref
|
|
383
|
+
repo = env_handler.get_image_repo(image)
|
|
384
|
+
refs[step.name] = ':'.join([repo, run_id])
|
|
343
385
|
|
|
344
386
|
for loc in step.outputs.values():
|
|
345
387
|
overwrite_artifacts.append(
|
|
@@ -351,10 +393,14 @@ class Controller:
|
|
|
351
393
|
temp_path = 'temp/run'
|
|
352
394
|
ensure_path(temp_path, clear=True)
|
|
353
395
|
env_handler.run_pipeline(
|
|
354
|
-
pipeline,
|
|
355
|
-
|
|
396
|
+
pipeline=pipeline,
|
|
397
|
+
image_refs=refs,
|
|
398
|
+
overwrite_artifacts=overwrite_artifacts,
|
|
399
|
+
timestamp=timestamp,
|
|
400
|
+
run_id=run_id,
|
|
401
|
+
temp_path=temp_path,
|
|
356
402
|
)
|
|
357
|
-
|
|
403
|
+
|
|
358
404
|
return {
|
|
359
405
|
'run_id': run_id,
|
|
360
406
|
'start_timestamp': now,
|
|
@@ -371,5 +417,4 @@ class Controller:
|
|
|
371
417
|
since = dt.datetime.now(tz=dt.UTC) - since
|
|
372
418
|
env_handler = self._config.get_run_env(env)
|
|
373
419
|
env_handler.print_logs(run_id, since, stream=stream)
|
|
374
|
-
|
|
375
420
|
|
|
@@ -23,16 +23,28 @@ class EnvironmentHandler:
|
|
|
23
23
|
name,
|
|
24
24
|
project,
|
|
25
25
|
user,
|
|
26
|
+
envvars=None,
|
|
27
|
+
secrets=None,
|
|
26
28
|
docker_command='docker',
|
|
27
29
|
docker_registry=None,
|
|
28
30
|
docker_namespace='csub',
|
|
29
31
|
):
|
|
30
|
-
self.name = name
|
|
31
|
-
self._project = project
|
|
32
|
-
self._user = user
|
|
32
|
+
self.name = str(name)
|
|
33
|
+
self._project = str(project)
|
|
34
|
+
self._user = str(user)
|
|
33
35
|
self._docker_command = str(docker_command)
|
|
34
36
|
self._docker_registry = docker_registry
|
|
35
|
-
self._docker_namespace = docker_namespace
|
|
37
|
+
self._docker_namespace = str(docker_namespace)
|
|
38
|
+
|
|
39
|
+
self._envvars = {
|
|
40
|
+
'CSUB_PROJECT': self._project,
|
|
41
|
+
'CSUB_USER': self._user,
|
|
42
|
+
}
|
|
43
|
+
if envvars is not None:
|
|
44
|
+
self._envvars.update(envvars)
|
|
45
|
+
self._secrets = {}
|
|
46
|
+
if secrets is not None:
|
|
47
|
+
self._secrets.update(secrets)
|
|
36
48
|
|
|
37
49
|
def _generate_id(self, timestamp, id):
|
|
38
50
|
if id is not None:
|
|
@@ -126,41 +138,7 @@ class EnvironmentHandler:
|
|
|
126
138
|
"""
|
|
127
139
|
raise NotImplementedError
|
|
128
140
|
|
|
129
|
-
def
|
|
130
|
-
if isinstance(image, BaseImage):
|
|
131
|
-
image_name = image.name
|
|
132
|
-
elif isinstance(image, ExecutionImage):
|
|
133
|
-
image_name = '.'.join([image.name, self.name])
|
|
134
|
-
else:
|
|
135
|
-
raise ValueError(f'Cannot handle image of type {type(image)}')
|
|
136
|
-
|
|
137
|
-
return os.path.join('images', self._user, image_name)
|
|
138
|
-
|
|
139
|
-
def get_image_ref(self, image):
|
|
140
|
-
ensure_path(os.path.join('images', self._user))
|
|
141
|
-
path = self._get_image_ref_path(image)
|
|
142
|
-
try:
|
|
143
|
-
with open(path, 'r') as stream:
|
|
144
|
-
ref = stream.read().strip()
|
|
145
|
-
except FileNotFoundError:
|
|
146
|
-
return None
|
|
147
|
-
return ref
|
|
148
|
-
|
|
149
|
-
def save_image_ref(self, image, ref):
|
|
150
|
-
ensure_path(os.path.join('images', self._user))
|
|
151
|
-
path = self._get_image_ref_path(image)
|
|
152
|
-
with open(path, 'w') as stream:
|
|
153
|
-
stream.write(ref)
|
|
154
|
-
|
|
155
|
-
def clear_image_ref(self, image):
|
|
156
|
-
ensure_path(os.path.join('images', self._user))
|
|
157
|
-
path = self._get_image_ref_path(image)
|
|
158
|
-
try:
|
|
159
|
-
os.remove(path)
|
|
160
|
-
except FileNotFoundError:
|
|
161
|
-
pass
|
|
162
|
-
|
|
163
|
-
def get_image_repo_name(self, image):
|
|
141
|
+
def get_image_repo(self, image):
|
|
164
142
|
prefix = []
|
|
165
143
|
if self._docker_registry is not None:
|
|
166
144
|
prefix.append(self._docker_registry)
|
|
@@ -188,7 +166,7 @@ class EnvironmentHandler:
|
|
|
188
166
|
|
|
189
167
|
def pull_base_image(self, ref):
|
|
190
168
|
"""Pull an image from the remote registry.
|
|
191
|
-
|
|
169
|
+
|
|
192
170
|
Note:
|
|
193
171
|
Derived classes usually need to overload this to handle authentication
|
|
194
172
|
with the remote registry. In a local environment where there is no
|
|
@@ -212,7 +190,7 @@ class EnvironmentHandler:
|
|
|
212
190
|
Returns:
|
|
213
191
|
tags (list of str): The list of tags that are available locally.
|
|
214
192
|
"""
|
|
215
|
-
repo_name = self.
|
|
193
|
+
repo_name = self.get_image_repo(image)
|
|
216
194
|
result = run_command(
|
|
217
195
|
[
|
|
218
196
|
self._docker_command,
|
|
@@ -258,19 +236,20 @@ class EnvironmentHandler:
|
|
|
258
236
|
raise NotImplementedError
|
|
259
237
|
|
|
260
238
|
def build_image(self, path, image, build_id):
|
|
261
|
-
repo = self.
|
|
239
|
+
repo = self.get_image_repo(image)
|
|
262
240
|
ref = ':'.join([repo, build_id])
|
|
263
241
|
command = [
|
|
264
242
|
self._docker_command,
|
|
265
243
|
'buildx',
|
|
266
244
|
'build',
|
|
245
|
+
*(image.args),
|
|
267
246
|
'-t', ref,
|
|
268
247
|
path,
|
|
269
248
|
]
|
|
270
249
|
run_command(command)
|
|
271
250
|
return ref
|
|
272
251
|
|
|
273
|
-
def
|
|
252
|
+
def get_local_artifact_ref(self, artifact, run_id=None):
|
|
274
253
|
if artifact.kind != 'file':
|
|
275
254
|
raise CloudSubmitError(
|
|
276
255
|
f'Cannot get path for local artifact {artifact.name}. '
|
|
@@ -294,7 +273,7 @@ class EnvironmentHandler:
|
|
|
294
273
|
raise CloudSubmitError(
|
|
295
274
|
f'Unknown scope {artifact.scope} for artifact {artifact.name}.')
|
|
296
275
|
|
|
297
|
-
def
|
|
276
|
+
def get_remote_artifact_ref(self, artifact, run_id=None):
|
|
298
277
|
raise NotImplementedError
|
|
299
278
|
|
|
300
279
|
def list_local_artifacts(self, artifacts, run_ids=None):
|
|
@@ -351,7 +330,7 @@ class EnvironmentHandler:
|
|
|
351
330
|
if artifact.kind != 'file':
|
|
352
331
|
continue
|
|
353
332
|
for run_id in runs:
|
|
354
|
-
path = self.
|
|
333
|
+
path = self.get_local_artifact_ref(artifact, run_id)
|
|
355
334
|
clear_path(path)
|
|
356
335
|
|
|
357
336
|
def remove_remote_artifacts(self, artifacts, run_ids):
|
|
@@ -359,8 +338,8 @@ class EnvironmentHandler:
|
|
|
359
338
|
|
|
360
339
|
def copy_local_artifacts(self, artifacts, from_run_id, to_run_id):
|
|
361
340
|
for artifact in artifacts:
|
|
362
|
-
src_path = self.
|
|
363
|
-
dst_path = self.
|
|
341
|
+
src_path = self.get_local_artifact_ref(artifact, from_run_id)
|
|
342
|
+
dst_path = self.get_local_artifact_ref(artifact, to_run_id)
|
|
364
343
|
if os.path.exists(src_path):
|
|
365
344
|
ensure_path(os.path.dirname(dst_path))
|
|
366
345
|
try:
|
|
@@ -376,8 +355,8 @@ class EnvironmentHandler:
|
|
|
376
355
|
|
|
377
356
|
def move_local_artifacts(self, artifacts, from_run_id, to_run_id):
|
|
378
357
|
for artifact in artifacts:
|
|
379
|
-
src_path = self.
|
|
380
|
-
dst_path = self.
|
|
358
|
+
src_path = self.get_local_artifact_ref(artifact, from_run_id)
|
|
359
|
+
dst_path = self.get_local_artifact_ref(artifact, to_run_id)
|
|
381
360
|
if os.path.exists(src_path):
|
|
382
361
|
ensure_path(os.path.dirname(dst_path))
|
|
383
362
|
shutil.move(src_path, dst_path)
|
{cloud_submit-0.2 → cloud_submit-0.3}/src/cloud_submit/envs/aws/local_environment_handler.py
RENAMED
|
@@ -17,7 +17,7 @@ from cloud_submit import (
|
|
|
17
17
|
BaseImage,
|
|
18
18
|
)
|
|
19
19
|
|
|
20
|
-
from .s3_tools import
|
|
20
|
+
from .s3_tools import get_remote_artifact_ref
|
|
21
21
|
|
|
22
22
|
|
|
23
23
|
|
|
@@ -36,20 +36,29 @@ class LocalAWSEnv(LocalEnv):
|
|
|
36
36
|
aws_profile,
|
|
37
37
|
s3_bucket,
|
|
38
38
|
s3_prefix,
|
|
39
|
+
envvars=None,
|
|
40
|
+
secrets=None,
|
|
39
41
|
docker_command='docker',
|
|
40
42
|
docker_namespace='csub',
|
|
41
43
|
docker_platforms=None,
|
|
42
44
|
docker_login_refresh_hours=6,
|
|
43
45
|
aws_command='aws',
|
|
46
|
+
container_aws_command='aws',
|
|
44
47
|
):
|
|
45
48
|
super().__init__(
|
|
46
49
|
name=name,
|
|
47
50
|
project=project,
|
|
48
51
|
user=user,
|
|
52
|
+
envvars=None,
|
|
53
|
+
secrets=None,
|
|
49
54
|
docker_command=docker_command,
|
|
50
55
|
docker_registry=\
|
|
51
56
|
f'{aws_account_id}.dkr.ecr.{aws_region}.amazonaws.com',
|
|
52
57
|
docker_namespace=docker_namespace,
|
|
58
|
+
docker_mounts=[(
|
|
59
|
+
os.path.join(os.environ['HOME'], '.aws'),
|
|
60
|
+
'/root/.aws',
|
|
61
|
+
)],
|
|
53
62
|
)
|
|
54
63
|
self._aws_account_id = str(aws_account_id)
|
|
55
64
|
self._aws_region = str(aws_region)
|
|
@@ -57,11 +66,21 @@ class LocalAWSEnv(LocalEnv):
|
|
|
57
66
|
self._s3_bucket = str(s3_bucket)
|
|
58
67
|
self._s3_prefix = str(s3_prefix)
|
|
59
68
|
self._aws_command = str(aws_command)
|
|
69
|
+
self._container_aws_command = str(container_aws_command)
|
|
60
70
|
self._docker_login_refresh_hours = float(docker_login_refresh_hours)
|
|
61
71
|
self._docker_platforms = docker_platforms
|
|
62
72
|
if self._docker_platforms is not None:
|
|
63
73
|
self._docker_platforms = str(docker_platforms)
|
|
64
74
|
|
|
75
|
+
self._envvars['CSUB_AWS_REGION'] = self._aws_region
|
|
76
|
+
self._envvars['CSUB_AWS_PROFILE'] = self._aws_profile
|
|
77
|
+
self._envvars['CSUB_AWS_COMMAND'] = self._container_aws_command
|
|
78
|
+
self._envvars['CSUB_S3_BUCKET'] = self._s3_bucket
|
|
79
|
+
self._envvars['CSUB_S3_PREFIX'] = self._s3_prefix
|
|
80
|
+
if envvars is not None:
|
|
81
|
+
self._envvars.update(envvars)
|
|
82
|
+
if secrets is not None:
|
|
83
|
+
self._secrets.update(secrets)
|
|
65
84
|
self._last_docker_login_ts = None
|
|
66
85
|
|
|
67
86
|
def _docker_login(self):
|
|
@@ -71,7 +90,7 @@ class LocalAWSEnv(LocalEnv):
|
|
|
71
90
|
(now - self._last_docker_login_ts).total_seconds()
|
|
72
91
|
if seconds_since_last_login < self._docker_login_refresh_hours*3600:
|
|
73
92
|
return
|
|
74
|
-
|
|
93
|
+
|
|
75
94
|
result = run_command(
|
|
76
95
|
[
|
|
77
96
|
self._aws_command,
|
|
@@ -105,18 +124,23 @@ class LocalAWSEnv(LocalEnv):
|
|
|
105
124
|
os.path.join(sourcedir, 'local_execution_handler.py'),
|
|
106
125
|
os.path.join(path, 'execution_handler.py'),
|
|
107
126
|
)
|
|
127
|
+
shutil.copyfile(
|
|
128
|
+
os.path.join(sourcedir, 's3_tools.py'),
|
|
129
|
+
os.path.join(path, 's3_tools.py'),
|
|
130
|
+
)
|
|
108
131
|
|
|
109
132
|
def _build_image(self, path, image, build_id, push):
|
|
110
133
|
if not push:
|
|
111
134
|
ref = super().build_image(path, image, build_id)
|
|
112
135
|
else:
|
|
113
136
|
self._docker_login()
|
|
114
|
-
repo = self.
|
|
137
|
+
repo = self.get_image_repo(image)
|
|
115
138
|
ref = ':'.join([repo, build_id])
|
|
116
139
|
command = [
|
|
117
140
|
self._docker_command,
|
|
118
141
|
'buildx',
|
|
119
142
|
'build',
|
|
143
|
+
*(image.args),
|
|
120
144
|
]
|
|
121
145
|
if self._docker_platforms is not None:
|
|
122
146
|
command.extend(['--platform', self._docker_platforms])
|
|
@@ -128,7 +152,7 @@ class LocalAWSEnv(LocalEnv):
|
|
|
128
152
|
return self._build_image(
|
|
129
153
|
path=path,
|
|
130
154
|
image=image,
|
|
131
|
-
build_id=build_id,
|
|
155
|
+
build_id=build_id,
|
|
132
156
|
push=isinstance(image, BaseImage),
|
|
133
157
|
)
|
|
134
158
|
|
|
@@ -160,7 +184,7 @@ class LocalAWSEnv(LocalEnv):
|
|
|
160
184
|
def list_remote_image_tags(self, image):
|
|
161
185
|
if isinstance(image, ExecutionImage):
|
|
162
186
|
return []
|
|
163
|
-
repo_name = self.
|
|
187
|
+
repo_name = self.get_image_repo(image)
|
|
164
188
|
return self._list_remote_image_tags(repo_name)
|
|
165
189
|
|
|
166
190
|
def remove_remote_image_refs(self, refs):
|
|
@@ -185,8 +209,8 @@ class LocalAWSEnv(LocalEnv):
|
|
|
185
209
|
*[f'imageTag={tag}' for tag in tag_list],
|
|
186
210
|
])
|
|
187
211
|
|
|
188
|
-
def
|
|
189
|
-
return
|
|
212
|
+
def get_remote_artifact_ref(self, artifact, run_id=None):
|
|
213
|
+
return get_remote_artifact_ref(
|
|
190
214
|
artifact,
|
|
191
215
|
self._project,
|
|
192
216
|
self._user,
|
|
@@ -273,7 +297,7 @@ class LocalAWSEnv(LocalEnv):
|
|
|
273
297
|
if len(key) > 1 and key[1] in run_artifacts:
|
|
274
298
|
artifact = key[1]
|
|
275
299
|
results[artifact].add(run_id)
|
|
276
|
-
|
|
300
|
+
|
|
277
301
|
results = [sorted(results[a.name]) for a in artifacts]
|
|
278
302
|
return results
|
|
279
303
|
|