cloud-submit 0.1.2__tar.gz → 0.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cloud_submit-0.1.2 → cloud_submit-0.3}/PKG-INFO +1 -1
- {cloud_submit-0.1.2 → cloud_submit-0.3}/docs/examples/aws/src/csub/build_config.py +14 -7
- {cloud_submit-0.1.2 → cloud_submit-0.3}/docs/examples/aws/userconfig/template.yaml +5 -0
- {cloud_submit-0.1.2 → cloud_submit-0.3}/docs/examples/basic/src/csub/build_config.py +4 -4
- {cloud_submit-0.1.2 → cloud_submit-0.3}/docs/examples/multi-worker/src/csub/build_config.py +4 -4
- {cloud_submit-0.1.2 → cloud_submit-0.3}/pyproject.toml +1 -1
- {cloud_submit-0.1.2 → cloud_submit-0.3}/src/cloud_submit/__init__.py +2 -3
- {cloud_submit-0.1.2 → cloud_submit-0.3}/src/cloud_submit/cli.py +18 -8
- {cloud_submit-0.1.2 → cloud_submit-0.3}/src/cloud_submit/controller.py +69 -24
- {cloud_submit-0.1.2 → cloud_submit-0.3}/src/cloud_submit/environment_handler.py +47 -66
- {cloud_submit-0.1.2 → cloud_submit-0.3}/src/cloud_submit/envs/aws/local_environment_handler.py +60 -33
- cloud_submit-0.3/src/cloud_submit/envs/aws/local_execution_handler.py +63 -0
- {cloud_submit-0.1.2 → cloud_submit-0.3}/src/cloud_submit/envs/aws/remote_environment_handler.py +36 -19
- {cloud_submit-0.1.2 → cloud_submit-0.3}/src/cloud_submit/envs/aws/remote_execution_handler.py +24 -23
- {cloud_submit-0.1.2 → cloud_submit-0.3}/src/cloud_submit/envs/aws/s3_tools.py +1 -2
- {cloud_submit-0.1.2 → cloud_submit-0.3}/src/cloud_submit/envs/local/environment_handler.py +53 -3
- {cloud_submit-0.1.2 → cloud_submit-0.3}/src/cloud_submit/execution/base_execution_handler.py +7 -7
- {cloud_submit-0.1.2 → cloud_submit-0.3}/src/cloud_submit/execution/config.py +11 -15
- {cloud_submit-0.1.2 → cloud_submit-0.3}/src/cloud_submit/execution/execute.py +22 -22
- {cloud_submit-0.1.2 → cloud_submit-0.3}/src/cloud_submit/execution/utils.py +18 -3
- {cloud_submit-0.1.2 → cloud_submit-0.3}/src/cloud_submit/images.py +4 -0
- {cloud_submit-0.1.2 → cloud_submit-0.3}/src/cloud_submit/utils.py +0 -19
- cloud_submit-0.1.2/src/cloud_submit/envs/aws/local_execution_handler.py +0 -5
- {cloud_submit-0.1.2 → cloud_submit-0.3}/.gitignore +0 -0
- {cloud_submit-0.1.2 → cloud_submit-0.3}/LICENSE +0 -0
- {cloud_submit-0.1.2 → cloud_submit-0.3}/README.md +0 -0
- {cloud_submit-0.1.2 → cloud_submit-0.3}/docs/examples/aws/src/fit.py +0 -0
- {cloud_submit-0.1.2 → cloud_submit-0.3}/docs/examples/aws/src/generate.py +0 -0
- {cloud_submit-0.1.2 → cloud_submit-0.3}/docs/examples/aws/src/parameters.py +0 -0
- {cloud_submit-0.1.2 → cloud_submit-0.3}/docs/examples/basic/src/fit.py +0 -0
- {cloud_submit-0.1.2 → cloud_submit-0.3}/docs/examples/basic/src/generate.py +0 -0
- {cloud_submit-0.1.2 → cloud_submit-0.3}/docs/examples/basic/src/parameters.py +0 -0
- {cloud_submit-0.1.2 → cloud_submit-0.3}/docs/examples/basic/userconfig/template.yaml +0 -0
- {cloud_submit-0.1.2 → cloud_submit-0.3}/docs/examples/multi-worker/src/fit.py +0 -0
- {cloud_submit-0.1.2 → cloud_submit-0.3}/docs/examples/multi-worker/src/generate.py +0 -0
- {cloud_submit-0.1.2 → cloud_submit-0.3}/docs/examples/multi-worker/src/parameters.py +0 -0
- {cloud_submit-0.1.2 → cloud_submit-0.3}/docs/examples/multi-worker/userconfig/template.yaml +0 -0
- {cloud_submit-0.1.2 → cloud_submit-0.3}/src/cloud_submit/config.py +0 -0
- {cloud_submit-0.1.2 → cloud_submit-0.3}/src/cloud_submit/envs/aws/__init__.py +0 -0
- {cloud_submit-0.1.2 → cloud_submit-0.3}/src/cloud_submit/envs/local/execution_handler.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cloud-submit
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3
|
|
4
4
|
Summary: A tool for building processing pipelines in Python and executing them locally or in the cloud.
|
|
5
5
|
Project-URL: Homepage, https://github.com/mwiebusch78/cloud-submit
|
|
6
6
|
Project-URL: Bug Tracker, https://github.com/mwiebusch78/cloud-submit/issues
|
|
@@ -39,11 +39,16 @@ def build_config(project_root, userconfig):
|
|
|
39
39
|
cs.BaseImage(
|
|
40
40
|
name='base',
|
|
41
41
|
instructions=r"""
|
|
42
|
-
FROM python
|
|
42
|
+
FROM python:3.12.3
|
|
43
|
+
ARG TARGETARCH
|
|
43
44
|
RUN \
|
|
45
|
+
case "$TARGETARCH" in \
|
|
46
|
+
amd64) url="https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" ;; \
|
|
47
|
+
arm64) url="https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" ;; \
|
|
48
|
+
*) echo "unsupported architecture: $TARGETARCH" >&2; exit 1 ;; \
|
|
49
|
+
esac && \
|
|
44
50
|
cd /opt && \
|
|
45
|
-
curl "
|
|
46
|
-
-o "awscliv2.zip" && \
|
|
51
|
+
curl "$url" -o "awscliv2.zip" && \
|
|
47
52
|
unzip awscliv2.zip && \
|
|
48
53
|
./aws/install && \
|
|
49
54
|
rm -rf awscliv2.zip aws
|
|
@@ -86,7 +91,7 @@ def build_config(project_root, userconfig):
|
|
|
86
91
|
'sigma': PARAMS['sigma'],
|
|
87
92
|
},
|
|
88
93
|
outputs={
|
|
89
|
-
'train_data_path': cs.
|
|
94
|
+
'train_data_path': cs.aref('train_data.parquet', 'path'),
|
|
90
95
|
},
|
|
91
96
|
),
|
|
92
97
|
cs.Step(
|
|
@@ -99,11 +104,11 @@ def build_config(project_root, userconfig):
|
|
|
99
104
|
function='fit:fit_step',
|
|
100
105
|
image='exec',
|
|
101
106
|
inputs={
|
|
102
|
-
'train_data_path': cs.
|
|
107
|
+
'train_data_path': cs.aref('train_data.parquet', 'path'),
|
|
103
108
|
},
|
|
104
109
|
outputs={
|
|
105
|
-
'coefficients_path': cs.
|
|
106
|
-
'predictions_path': cs.
|
|
110
|
+
'coefficients_path': cs.aref('coefficients.json', 'path'),
|
|
111
|
+
'predictions_path': cs.aref('predictions.parquet', 'path'),
|
|
107
112
|
},
|
|
108
113
|
),
|
|
109
114
|
],
|
|
@@ -125,6 +130,7 @@ def build_config(project_root, userconfig):
|
|
|
125
130
|
aws_region=userconfig['aws_region'],
|
|
126
131
|
aws_profile=userconfig['aws_profile'],
|
|
127
132
|
docker_namespace=userconfig['docker_namespace'],
|
|
133
|
+
docker_platforms=userconfig['docker_platforms'],
|
|
128
134
|
s3_bucket=userconfig['s3_bucket'],
|
|
129
135
|
s3_prefix=userconfig['s3_prefix'],
|
|
130
136
|
),
|
|
@@ -136,6 +142,7 @@ def build_config(project_root, userconfig):
|
|
|
136
142
|
aws_region=userconfig['aws_region'],
|
|
137
143
|
aws_profile=userconfig['aws_profile'],
|
|
138
144
|
docker_namespace=userconfig['docker_namespace'],
|
|
145
|
+
docker_platforms=userconfig['docker_platforms'],
|
|
139
146
|
s3_bucket=userconfig['s3_bucket'],
|
|
140
147
|
s3_prefix=userconfig['s3_prefix'],
|
|
141
148
|
ecs_cluster_arn=userconfig['ecs_cluster_arn'],
|
|
@@ -24,6 +24,11 @@ aws_profile: default
|
|
|
24
24
|
# registry.
|
|
25
25
|
docker_namespace: csub
|
|
26
26
|
|
|
27
|
+
# This is the value of the --platform option used when building docker images.
|
|
28
|
+
# To build images that work on both x86_64 and ARM architecture set this to
|
|
29
|
+
# "linux/amd64,linux/arm64"
|
|
30
|
+
docker_platforms: null
|
|
31
|
+
|
|
27
32
|
# This is the name of the S3 bucket where your artifacts will be stored.
|
|
28
33
|
s3_bucket: # TODO: add the name of your S3 bucket.
|
|
29
34
|
|
|
@@ -70,7 +70,7 @@ def build_config(project_root, userconfig):
|
|
|
70
70
|
'sigma': PARAMS['sigma'],
|
|
71
71
|
},
|
|
72
72
|
outputs={
|
|
73
|
-
'train_data_path': cs.
|
|
73
|
+
'train_data_path': cs.aref('train_data.parquet', 'path'),
|
|
74
74
|
},
|
|
75
75
|
),
|
|
76
76
|
cs.Step(
|
|
@@ -78,11 +78,11 @@ def build_config(project_root, userconfig):
|
|
|
78
78
|
function='fit:fit_step',
|
|
79
79
|
image='exec',
|
|
80
80
|
inputs={
|
|
81
|
-
'train_data_path': cs.
|
|
81
|
+
'train_data_path': cs.aref('train_data.parquet', 'path'),
|
|
82
82
|
},
|
|
83
83
|
outputs={
|
|
84
|
-
'coefficients_path': cs.
|
|
85
|
-
'predictions_path': cs.
|
|
84
|
+
'coefficients_path': cs.aref('coefficients.json', 'path'),
|
|
85
|
+
'predictions_path': cs.aref('predictions.parquet', 'path'),
|
|
86
86
|
},
|
|
87
87
|
),
|
|
88
88
|
],
|
|
@@ -69,7 +69,7 @@ def build_config(project_root, userconfig):
|
|
|
69
69
|
'sigma': PARAMS['sigma'],
|
|
70
70
|
},
|
|
71
71
|
outputs={
|
|
72
|
-
'train_data_path': cs.
|
|
72
|
+
'train_data_path': cs.aref('train_data.parquet', 'path'),
|
|
73
73
|
},
|
|
74
74
|
pass_submit_timestamp_as='submit_timestamp',
|
|
75
75
|
),
|
|
@@ -78,11 +78,11 @@ def build_config(project_root, userconfig):
|
|
|
78
78
|
function='fit:fit_step',
|
|
79
79
|
image='exec',
|
|
80
80
|
inputs={
|
|
81
|
-
'train_data_path': cs.
|
|
81
|
+
'train_data_path': cs.aref('train_data.parquet', 'path'),
|
|
82
82
|
},
|
|
83
83
|
outputs={
|
|
84
|
-
'coefficients_path': cs.
|
|
85
|
-
'predictions_path': cs.
|
|
84
|
+
'coefficients_path': cs.aref('coefficients.json', 'path'),
|
|
85
|
+
'predictions_path': cs.aref('predictions.parquet', 'path'),
|
|
86
86
|
},
|
|
87
87
|
num_workers=PARAMS['num_folds'],
|
|
88
88
|
pass_num_workers_as='num_folds',
|
|
@@ -6,12 +6,11 @@ from .config import Config
|
|
|
6
6
|
from .execution.config import (
|
|
7
7
|
ConfigError,
|
|
8
8
|
Artifact,
|
|
9
|
-
|
|
9
|
+
ArtifactReference,
|
|
10
10
|
Spec,
|
|
11
11
|
Step,
|
|
12
12
|
Pipeline,
|
|
13
|
-
|
|
14
|
-
remote,
|
|
13
|
+
aref,
|
|
15
14
|
)
|
|
16
15
|
from .execution.utils import (
|
|
17
16
|
CloudSubmitError,
|
|
@@ -7,6 +7,7 @@ import yaml
|
|
|
7
7
|
|
|
8
8
|
from .execution.utils import CloudSubmitError
|
|
9
9
|
from .controller import Controller
|
|
10
|
+
from .images import BaseImage
|
|
10
11
|
|
|
11
12
|
|
|
12
13
|
def abort(msg):
|
|
@@ -303,11 +304,11 @@ def images():
|
|
|
303
304
|
@click.option(
|
|
304
305
|
'--all', '-a', 'build_all',
|
|
305
306
|
is_flag=True,
|
|
306
|
-
help='
|
|
307
|
+
help='Ignore IMAGES arguments and build all base images.',
|
|
307
308
|
)
|
|
308
309
|
@click.argument('images', type=str, nargs=-1)
|
|
309
310
|
@click.pass_context
|
|
310
|
-
def
|
|
311
|
+
def build_images(ctx, env, build_id, build_all, images):
|
|
311
312
|
init(ctx)
|
|
312
313
|
config = ctx.obj['config']
|
|
313
314
|
controller = ctx.obj['controller']
|
|
@@ -316,9 +317,12 @@ def build(ctx, env, build_id, build_all, images):
|
|
|
316
317
|
if image not in config.images:
|
|
317
318
|
abort(f'No definition found for image {image}.')
|
|
318
319
|
if build_all:
|
|
319
|
-
images =
|
|
320
|
+
images = [
|
|
321
|
+
name for name, image in config.images.items()
|
|
322
|
+
if isinstance(image, BaseImage)
|
|
323
|
+
]
|
|
320
324
|
try:
|
|
321
|
-
controller.
|
|
325
|
+
controller.build_images(images, build_id=build_id, env=env)
|
|
322
326
|
except CloudSubmitError as e:
|
|
323
327
|
abort(str(e))
|
|
324
328
|
|
|
@@ -495,7 +499,10 @@ def remove_images(ctx, env, images, local, remote, build_ids):
|
|
|
495
499
|
def set_image(ctx, env, image, ref):
|
|
496
500
|
init(ctx)
|
|
497
501
|
controller = ctx.obj['controller']
|
|
498
|
-
|
|
502
|
+
try:
|
|
503
|
+
controller.set_image(image, ref, env=env)
|
|
504
|
+
except CloudSubmitError as e:
|
|
505
|
+
abort(str(e))
|
|
499
506
|
|
|
500
507
|
|
|
501
508
|
@images.command(
|
|
@@ -522,7 +529,10 @@ def unset_image(ctx, env, unset_all, images):
|
|
|
522
529
|
if unset_all:
|
|
523
530
|
images = list(config.images.keys())
|
|
524
531
|
for image in images:
|
|
525
|
-
|
|
532
|
+
try:
|
|
533
|
+
controller.unset_image(image, env=env)
|
|
534
|
+
except CloudSubmitError as e:
|
|
535
|
+
abort(str(e))
|
|
526
536
|
|
|
527
537
|
|
|
528
538
|
# artifacts subgroup
|
|
@@ -605,10 +615,10 @@ def list_artifacts(ctx, env, artifacts, local, remote, run_ids):
|
|
|
605
615
|
artifact = config.artifacts[name]
|
|
606
616
|
for run_id in runs:
|
|
607
617
|
if local:
|
|
608
|
-
path = env_handler.
|
|
618
|
+
path = env_handler.get_local_artifact_ref(
|
|
609
619
|
artifact, run_id)
|
|
610
620
|
else:
|
|
611
|
-
path = env_handler.
|
|
621
|
+
path = env_handler.get_remote_artifact_ref(
|
|
612
622
|
artifact, run_id)
|
|
613
623
|
data.append((
|
|
614
624
|
name,
|
|
@@ -32,14 +32,44 @@ class Controller:
|
|
|
32
32
|
self._config.export_pipelines(os.path.join(path, 'pipelines.json'))
|
|
33
33
|
self._config.export_artifacts(os.path.join(path, 'artifacts.json'))
|
|
34
34
|
|
|
35
|
+
def _get_image_ref_path(self, image):
|
|
36
|
+
if not isinstance(image, BaseImage):
|
|
37
|
+
raise ValueError(f'Cannot handle image of type {type(image)}')
|
|
38
|
+
return os.path.join('images', self._config.user_name, image.name)
|
|
39
|
+
|
|
40
|
+
def _get_image_ref(self, image):
|
|
41
|
+
ensure_path(os.path.join('images', self._config.user_name))
|
|
42
|
+
path = self._get_image_ref_path(image)
|
|
43
|
+
try:
|
|
44
|
+
with open(path, 'r') as stream:
|
|
45
|
+
ref = stream.read().strip()
|
|
46
|
+
except FileNotFoundError:
|
|
47
|
+
return None
|
|
48
|
+
return ref
|
|
49
|
+
|
|
50
|
+
def _save_image_ref(self, image, ref):
|
|
51
|
+
ensure_path(os.path.join('images', self._config.user_name))
|
|
52
|
+
path = self._get_image_ref_path(image)
|
|
53
|
+
with open(path, 'w') as stream:
|
|
54
|
+
stream.write(ref)
|
|
55
|
+
|
|
56
|
+
def _clear_image_ref(self, image):
|
|
57
|
+
ensure_path(os.path.join('images', self._config.user_name))
|
|
58
|
+
path = self._get_image_ref_path(image)
|
|
59
|
+
try:
|
|
60
|
+
os.remove(path)
|
|
61
|
+
except FileNotFoundError:
|
|
62
|
+
pass
|
|
63
|
+
|
|
35
64
|
def _build_image(self, image, build_id, env, rebuild=False):
|
|
36
|
-
|
|
37
|
-
|
|
65
|
+
is_execution_image = isinstance(image, ExecutionImage)
|
|
66
|
+
|
|
67
|
+
if not rebuild and not is_execution_image:
|
|
68
|
+
ref = self._get_image_ref(image)
|
|
38
69
|
if ref is not None:
|
|
39
70
|
print(f'Using existing build of {image.name}: {ref}')
|
|
40
71
|
return ref
|
|
41
72
|
|
|
42
|
-
is_execution_image = isinstance(image, ExecutionImage)
|
|
43
73
|
path = os.path.join('temp', 'build', image.name)
|
|
44
74
|
if is_execution_image:
|
|
45
75
|
path = os.path.join(path, env.name)
|
|
@@ -58,7 +88,7 @@ class Controller:
|
|
|
58
88
|
if parent_image is None:
|
|
59
89
|
raise CloudSubmitError(
|
|
60
90
|
f'Could not find declaration for image {image.parent}.')
|
|
61
|
-
parent_ref =
|
|
91
|
+
parent_ref = self._get_image_ref(parent_image)
|
|
62
92
|
if parent_ref is None:
|
|
63
93
|
raise CloudSubmitError(
|
|
64
94
|
f'Could not find reference for image {image.parent}.')
|
|
@@ -72,9 +102,10 @@ class Controller:
|
|
|
72
102
|
'build_image method of environment handler did not '
|
|
73
103
|
'return a valid image reference.'
|
|
74
104
|
)
|
|
75
|
-
|
|
105
|
+
if not is_execution_image:
|
|
106
|
+
self._save_image_ref(image, ref)
|
|
76
107
|
|
|
77
|
-
def
|
|
108
|
+
def build_images(self, images, build_id=None, env=None):
|
|
78
109
|
now = dt.datetime.now(tz=dt.UTC)
|
|
79
110
|
env_handler = self._config.get_build_env(env)
|
|
80
111
|
build_id = env_handler.generate_build_id(now, build_id)
|
|
@@ -114,19 +145,18 @@ class Controller:
|
|
|
114
145
|
results = []
|
|
115
146
|
for image in images:
|
|
116
147
|
image = self._config.images[image]
|
|
117
|
-
repo_name = env_handler.
|
|
148
|
+
repo_name = env_handler.get_image_repo(image)
|
|
118
149
|
if remote:
|
|
119
|
-
tags = env_handler.list_remote_image_tags(
|
|
150
|
+
tags = env_handler.list_remote_image_tags(image)
|
|
120
151
|
else:
|
|
121
|
-
tags = env_handler.list_local_image_tags(
|
|
152
|
+
tags = env_handler.list_local_image_tags(image)
|
|
122
153
|
if ids is not None:
|
|
123
154
|
tags = [t for t in tags if t in ids]
|
|
124
155
|
for tag in tags:
|
|
125
156
|
results.append(':'.join([repo_name, tag]))
|
|
126
157
|
return results
|
|
127
158
|
|
|
128
|
-
def list_images(self, images=None
|
|
129
|
-
env_handler = self._config.get_build_env(env)
|
|
159
|
+
def list_images(self, images=None):
|
|
130
160
|
if images is not None:
|
|
131
161
|
images = set(images)
|
|
132
162
|
result = []
|
|
@@ -136,15 +166,18 @@ class Controller:
|
|
|
136
166
|
continue
|
|
137
167
|
if isinstance(i, BaseImage):
|
|
138
168
|
tpe = 'base'
|
|
169
|
+
ref = self._get_image_ref(i)
|
|
139
170
|
elif isinstance(i, ExecutionImage):
|
|
140
171
|
tpe = 'execution'
|
|
172
|
+
ref = None
|
|
141
173
|
else:
|
|
142
174
|
raise ValueError('Unknown image type.')
|
|
143
|
-
ref = env_handler.get_image_ref(i)
|
|
144
175
|
result.append((i.name, tpe, ref))
|
|
145
176
|
return result
|
|
146
177
|
|
|
147
178
|
def remove_image_refs(self, refs, remote=False, env=None):
|
|
179
|
+
if not refs:
|
|
180
|
+
return
|
|
148
181
|
env_handler = self._config.get_build_env(env)
|
|
149
182
|
if remote:
|
|
150
183
|
env_handler.remove_remote_image_refs(refs)
|
|
@@ -156,16 +189,26 @@ class Controller:
|
|
|
156
189
|
image = self._config.images.get(image_name)
|
|
157
190
|
if image is None:
|
|
158
191
|
raise CloudSubmitError(f'Unknown image name {image_name}.')
|
|
192
|
+
if not isinstance(image, BaseImage):
|
|
193
|
+
raise CloudSubmitError(
|
|
194
|
+
f'Cannot set ref for image {image_name} '
|
|
195
|
+
'since it is not a base image.'
|
|
196
|
+
)
|
|
159
197
|
with self._config.in_project_root():
|
|
160
|
-
|
|
198
|
+
self._save_image_ref(image, ref)
|
|
161
199
|
|
|
162
200
|
def unset_image(self, image_name, env=None):
|
|
163
201
|
env_handler = self._config.get_build_env(env)
|
|
164
202
|
image = self._config.images.get(image_name)
|
|
165
203
|
if image is None:
|
|
166
204
|
raise CloudSubmitError(f'Unknown image name {image_name}.')
|
|
205
|
+
if not isinstance(image, BaseImage):
|
|
206
|
+
raise CloudSubmitError(
|
|
207
|
+
f'Cannot unset ref for image {image_name} '
|
|
208
|
+
'since it is not a base image.'
|
|
209
|
+
)
|
|
167
210
|
with self._config.in_project_root():
|
|
168
|
-
|
|
211
|
+
self._clear_image_ref(image)
|
|
169
212
|
|
|
170
213
|
def list_artifacts(
|
|
171
214
|
self,
|
|
@@ -202,6 +245,8 @@ class Controller:
|
|
|
202
245
|
remote=False,
|
|
203
246
|
env=None,
|
|
204
247
|
):
|
|
248
|
+
if not artifact_names:
|
|
249
|
+
return
|
|
205
250
|
env_handler = self._config.get_run_env(env)
|
|
206
251
|
artifacts = [
|
|
207
252
|
self._config.artifacts[name].copy() for name in artifact_names]
|
|
@@ -327,7 +372,7 @@ class Controller:
|
|
|
327
372
|
|
|
328
373
|
with self._config.in_project_root():
|
|
329
374
|
images = [step.image for step in steps]
|
|
330
|
-
self.
|
|
375
|
+
self.build_images(images, build_id=run_id, env=build_env)
|
|
331
376
|
refs = {}
|
|
332
377
|
overwrite_artifacts = []
|
|
333
378
|
for step in steps:
|
|
@@ -335,11 +380,8 @@ class Controller:
|
|
|
335
380
|
if image is None:
|
|
336
381
|
raise CloudSubmitError(
|
|
337
382
|
f'Cannot find declaration for image {step.image}.')
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
raise CloudSubmitError(
|
|
341
|
-
f'Could not find image ref for image {step.image}')
|
|
342
|
-
refs[step.name] = ref
|
|
383
|
+
repo = env_handler.get_image_repo(image)
|
|
384
|
+
refs[step.name] = ':'.join([repo, run_id])
|
|
343
385
|
|
|
344
386
|
for loc in step.outputs.values():
|
|
345
387
|
overwrite_artifacts.append(
|
|
@@ -351,10 +393,14 @@ class Controller:
|
|
|
351
393
|
temp_path = 'temp/run'
|
|
352
394
|
ensure_path(temp_path, clear=True)
|
|
353
395
|
env_handler.run_pipeline(
|
|
354
|
-
pipeline,
|
|
355
|
-
|
|
396
|
+
pipeline=pipeline,
|
|
397
|
+
image_refs=refs,
|
|
398
|
+
overwrite_artifacts=overwrite_artifacts,
|
|
399
|
+
timestamp=timestamp,
|
|
400
|
+
run_id=run_id,
|
|
401
|
+
temp_path=temp_path,
|
|
356
402
|
)
|
|
357
|
-
|
|
403
|
+
|
|
358
404
|
return {
|
|
359
405
|
'run_id': run_id,
|
|
360
406
|
'start_timestamp': now,
|
|
@@ -371,5 +417,4 @@ class Controller:
|
|
|
371
417
|
since = dt.datetime.now(tz=dt.UTC) - since
|
|
372
418
|
env_handler = self._config.get_run_env(env)
|
|
373
419
|
env_handler.print_logs(run_id, since, stream=stream)
|
|
374
|
-
|
|
375
420
|
|
|
@@ -19,20 +19,32 @@ from .execution.config import to_utc
|
|
|
19
19
|
|
|
20
20
|
class EnvironmentHandler:
|
|
21
21
|
def __init__(
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
22
|
+
self,
|
|
23
|
+
name,
|
|
24
|
+
project,
|
|
25
|
+
user,
|
|
26
|
+
envvars=None,
|
|
27
|
+
secrets=None,
|
|
28
|
+
docker_command='docker',
|
|
29
|
+
docker_registry=None,
|
|
30
|
+
docker_namespace='csub',
|
|
29
31
|
):
|
|
30
|
-
self.name = name
|
|
31
|
-
self._project = project
|
|
32
|
-
self._user = user
|
|
32
|
+
self.name = str(name)
|
|
33
|
+
self._project = str(project)
|
|
34
|
+
self._user = str(user)
|
|
33
35
|
self._docker_command = str(docker_command)
|
|
34
36
|
self._docker_registry = docker_registry
|
|
35
|
-
self._docker_namespace = docker_namespace
|
|
37
|
+
self._docker_namespace = str(docker_namespace)
|
|
38
|
+
|
|
39
|
+
self._envvars = {
|
|
40
|
+
'CSUB_PROJECT': self._project,
|
|
41
|
+
'CSUB_USER': self._user,
|
|
42
|
+
}
|
|
43
|
+
if envvars is not None:
|
|
44
|
+
self._envvars.update(envvars)
|
|
45
|
+
self._secrets = {}
|
|
46
|
+
if secrets is not None:
|
|
47
|
+
self._secrets.update(secrets)
|
|
36
48
|
|
|
37
49
|
def _generate_id(self, timestamp, id):
|
|
38
50
|
if id is not None:
|
|
@@ -126,41 +138,7 @@ class EnvironmentHandler:
|
|
|
126
138
|
"""
|
|
127
139
|
raise NotImplementedError
|
|
128
140
|
|
|
129
|
-
def
|
|
130
|
-
if isinstance(image, BaseImage):
|
|
131
|
-
image_name = image.name
|
|
132
|
-
elif isinstance(image, ExecutionImage):
|
|
133
|
-
image_name = '.'.join([image.name, self.name])
|
|
134
|
-
else:
|
|
135
|
-
raise ValueError(f'Cannot handle image of type {type(image)}')
|
|
136
|
-
|
|
137
|
-
return os.path.join('images', self._user, image_name)
|
|
138
|
-
|
|
139
|
-
def get_image_ref(self, image):
|
|
140
|
-
ensure_path(os.path.join('images', self._user))
|
|
141
|
-
path = self._get_image_ref_path(image)
|
|
142
|
-
try:
|
|
143
|
-
with open(path, 'r') as stream:
|
|
144
|
-
ref = stream.read().strip()
|
|
145
|
-
except FileNotFoundError:
|
|
146
|
-
return None
|
|
147
|
-
return ref
|
|
148
|
-
|
|
149
|
-
def save_image_ref(self, image, ref):
|
|
150
|
-
ensure_path(os.path.join('images', self._user))
|
|
151
|
-
path = self._get_image_ref_path(image)
|
|
152
|
-
with open(path, 'w') as stream:
|
|
153
|
-
stream.write(ref)
|
|
154
|
-
|
|
155
|
-
def clear_image_ref(self, image):
|
|
156
|
-
ensure_path(os.path.join('images', self._user))
|
|
157
|
-
path = self._get_image_ref_path(image)
|
|
158
|
-
try:
|
|
159
|
-
os.remove(path)
|
|
160
|
-
except FileNotFoundError:
|
|
161
|
-
pass
|
|
162
|
-
|
|
163
|
-
def get_image_repo_name(self, image):
|
|
141
|
+
def get_image_repo(self, image):
|
|
164
142
|
prefix = []
|
|
165
143
|
if self._docker_registry is not None:
|
|
166
144
|
prefix.append(self._docker_registry)
|
|
@@ -188,7 +166,7 @@ class EnvironmentHandler:
|
|
|
188
166
|
|
|
189
167
|
def pull_base_image(self, ref):
|
|
190
168
|
"""Pull an image from the remote registry.
|
|
191
|
-
|
|
169
|
+
|
|
192
170
|
Note:
|
|
193
171
|
Derived classes usually need to overload this to handle authentication
|
|
194
172
|
with the remote registry. In a local environment where there is no
|
|
@@ -199,21 +177,20 @@ class EnvironmentHandler:
|
|
|
199
177
|
Args:
|
|
200
178
|
ref (str): The full docker image reference for the requested image.
|
|
201
179
|
"""
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
'pull',
|
|
205
|
-
ref,
|
|
206
|
-
])
|
|
180
|
+
command = [self._docker_command, 'pull', ref]
|
|
181
|
+
run_command(command)
|
|
207
182
|
|
|
208
|
-
def list_local_image_tags(self,
|
|
183
|
+
def list_local_image_tags(self, image):
|
|
209
184
|
"""List locally available tags for a docker image repo.
|
|
210
185
|
|
|
211
186
|
Args:
|
|
212
|
-
|
|
187
|
+
image (cloud_submit.Image): The object describing the docker image
|
|
188
|
+
whose tags should be listed.
|
|
213
189
|
|
|
214
190
|
Returns:
|
|
215
191
|
tags (list of str): The list of tags that are available locally.
|
|
216
192
|
"""
|
|
193
|
+
repo_name = self.get_image_repo(image)
|
|
217
194
|
result = run_command(
|
|
218
195
|
[
|
|
219
196
|
self._docker_command,
|
|
@@ -230,7 +207,7 @@ class EnvironmentHandler:
|
|
|
230
207
|
return []
|
|
231
208
|
return result.split('\n')
|
|
232
209
|
|
|
233
|
-
def list_remote_image_tags(self,
|
|
210
|
+
def list_remote_image_tags(self, image):
|
|
234
211
|
"""List remotely available tags for a docker image repo.
|
|
235
212
|
|
|
236
213
|
Note:
|
|
@@ -238,7 +215,8 @@ class EnvironmentHandler:
|
|
|
238
215
|
remote registry.
|
|
239
216
|
|
|
240
217
|
Args:
|
|
241
|
-
|
|
218
|
+
image (cloud_submit.Image): The object describing the docker image
|
|
219
|
+
whose tags should be listed.
|
|
242
220
|
|
|
243
221
|
Returns:
|
|
244
222
|
tags (list of str): The list of the tags that are available in the
|
|
@@ -258,17 +236,20 @@ class EnvironmentHandler:
|
|
|
258
236
|
raise NotImplementedError
|
|
259
237
|
|
|
260
238
|
def build_image(self, path, image, build_id):
|
|
261
|
-
repo = self.
|
|
239
|
+
repo = self.get_image_repo(image)
|
|
262
240
|
ref = ':'.join([repo, build_id])
|
|
263
|
-
|
|
241
|
+
command = [
|
|
264
242
|
self._docker_command,
|
|
243
|
+
'buildx',
|
|
265
244
|
'build',
|
|
245
|
+
*(image.args),
|
|
266
246
|
'-t', ref,
|
|
267
247
|
path,
|
|
268
|
-
]
|
|
248
|
+
]
|
|
249
|
+
run_command(command)
|
|
269
250
|
return ref
|
|
270
251
|
|
|
271
|
-
def
|
|
252
|
+
def get_local_artifact_ref(self, artifact, run_id=None):
|
|
272
253
|
if artifact.kind != 'file':
|
|
273
254
|
raise CloudSubmitError(
|
|
274
255
|
f'Cannot get path for local artifact {artifact.name}. '
|
|
@@ -292,7 +273,7 @@ class EnvironmentHandler:
|
|
|
292
273
|
raise CloudSubmitError(
|
|
293
274
|
f'Unknown scope {artifact.scope} for artifact {artifact.name}.')
|
|
294
275
|
|
|
295
|
-
def
|
|
276
|
+
def get_remote_artifact_ref(self, artifact, run_id=None):
|
|
296
277
|
raise NotImplementedError
|
|
297
278
|
|
|
298
279
|
def list_local_artifacts(self, artifacts, run_ids=None):
|
|
@@ -349,7 +330,7 @@ class EnvironmentHandler:
|
|
|
349
330
|
if artifact.kind != 'file':
|
|
350
331
|
continue
|
|
351
332
|
for run_id in runs:
|
|
352
|
-
path = self.
|
|
333
|
+
path = self.get_local_artifact_ref(artifact, run_id)
|
|
353
334
|
clear_path(path)
|
|
354
335
|
|
|
355
336
|
def remove_remote_artifacts(self, artifacts, run_ids):
|
|
@@ -357,8 +338,8 @@ class EnvironmentHandler:
|
|
|
357
338
|
|
|
358
339
|
def copy_local_artifacts(self, artifacts, from_run_id, to_run_id):
|
|
359
340
|
for artifact in artifacts:
|
|
360
|
-
src_path = self.
|
|
361
|
-
dst_path = self.
|
|
341
|
+
src_path = self.get_local_artifact_ref(artifact, from_run_id)
|
|
342
|
+
dst_path = self.get_local_artifact_ref(artifact, to_run_id)
|
|
362
343
|
if os.path.exists(src_path):
|
|
363
344
|
ensure_path(os.path.dirname(dst_path))
|
|
364
345
|
try:
|
|
@@ -374,8 +355,8 @@ class EnvironmentHandler:
|
|
|
374
355
|
|
|
375
356
|
def move_local_artifacts(self, artifacts, from_run_id, to_run_id):
|
|
376
357
|
for artifact in artifacts:
|
|
377
|
-
src_path = self.
|
|
378
|
-
dst_path = self.
|
|
358
|
+
src_path = self.get_local_artifact_ref(artifact, from_run_id)
|
|
359
|
+
dst_path = self.get_local_artifact_ref(artifact, to_run_id)
|
|
379
360
|
if os.path.exists(src_path):
|
|
380
361
|
ensure_path(os.path.dirname(dst_path))
|
|
381
362
|
shutil.move(src_path, dst_path)
|