cloud-submit 0.3.1__tar.gz → 0.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cloud_submit-0.3.1 → cloud_submit-0.4}/PKG-INFO +1 -1
- {cloud_submit-0.3.1 → cloud_submit-0.4}/pyproject.toml +1 -1
- {cloud_submit-0.3.1 → cloud_submit-0.4}/src/cloud_submit/cli.py +24 -4
- {cloud_submit-0.3.1 → cloud_submit-0.4}/src/cloud_submit/controller.py +6 -4
- {cloud_submit-0.3.1 → cloud_submit-0.4}/src/cloud_submit/environment_handler.py +2 -1
- {cloud_submit-0.3.1 → cloud_submit-0.4}/src/cloud_submit/envs/aws/local_environment_handler.py +5 -3
- {cloud_submit-0.3.1 → cloud_submit-0.4}/.gitignore +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.4}/LICENSE +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.4}/README.md +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.4}/docs/examples/aws/src/csub/build_config.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.4}/docs/examples/aws/src/fit.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.4}/docs/examples/aws/src/generate.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.4}/docs/examples/aws/src/parameters.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.4}/docs/examples/aws/userconfig/template.yaml +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.4}/docs/examples/basic/src/csub/build_config.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.4}/docs/examples/basic/src/fit.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.4}/docs/examples/basic/src/generate.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.4}/docs/examples/basic/src/parameters.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.4}/docs/examples/basic/userconfig/template.yaml +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.4}/docs/examples/multi-worker/src/csub/build_config.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.4}/docs/examples/multi-worker/src/fit.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.4}/docs/examples/multi-worker/src/generate.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.4}/docs/examples/multi-worker/src/parameters.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.4}/docs/examples/multi-worker/userconfig/template.yaml +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.4}/src/cloud_submit/__init__.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.4}/src/cloud_submit/config.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.4}/src/cloud_submit/envs/aws/__init__.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.4}/src/cloud_submit/envs/aws/local_execution_handler.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.4}/src/cloud_submit/envs/aws/remote_environment_handler.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.4}/src/cloud_submit/envs/aws/remote_execution_handler.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.4}/src/cloud_submit/envs/aws/s3_tools.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.4}/src/cloud_submit/envs/local/environment_handler.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.4}/src/cloud_submit/envs/local/execution_handler.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.4}/src/cloud_submit/execution/base_execution_handler.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.4}/src/cloud_submit/execution/config.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.4}/src/cloud_submit/execution/execute.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.4}/src/cloud_submit/execution/utils.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.4}/src/cloud_submit/images.py +0 -0
- {cloud_submit-0.3.1 → cloud_submit-0.4}/src/cloud_submit/utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: cloud-submit
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4
|
|
4
4
|
Summary: A tool for building processing pipelines in Python and executing them locally or in the cloud.
|
|
5
5
|
Project-URL: Homepage, https://github.com/mwiebusch78/cloud-submit
|
|
6
6
|
Project-URL: Bug Tracker, https://github.com/mwiebusch78/cloud-submit/issues
|
|
@@ -82,7 +82,7 @@ def get_steps(pipeline, steps_arg):
|
|
|
82
82
|
tokens = steps_arg.split(',')
|
|
83
83
|
all_steps = [step.name for step in pipeline.steps]
|
|
84
84
|
indices = {step: i for i, step in enumerate(all_steps)}
|
|
85
|
-
|
|
85
|
+
|
|
86
86
|
selected = set()
|
|
87
87
|
for token in tokens:
|
|
88
88
|
token = token.split(':')
|
|
@@ -99,7 +99,7 @@ def get_steps(pipeline, steps_arg):
|
|
|
99
99
|
end = indices[token[1]]
|
|
100
100
|
except KeyError:
|
|
101
101
|
raise CloudSubmitError(f'Invalid step name: {token[1]}')
|
|
102
|
-
for step in all_steps[
|
|
102
|
+
for step in all_steps[begin:end+1]:
|
|
103
103
|
selected.add(step)
|
|
104
104
|
else:
|
|
105
105
|
raise CloudSubmitError(f'Invalid step range: {":".join(token)}')
|
|
@@ -306,9 +306,19 @@ def images():
|
|
|
306
306
|
is_flag=True,
|
|
307
307
|
help='Ignore IMAGES arguments and build all base images.',
|
|
308
308
|
)
|
|
309
|
+
@click.option(
|
|
310
|
+
'--no-cache',
|
|
311
|
+
is_flag=True,
|
|
312
|
+
help='Do not use docker cache when building images.',
|
|
313
|
+
)
|
|
314
|
+
@click.option(
|
|
315
|
+
'--pull',
|
|
316
|
+
is_flag=True,
|
|
317
|
+
help='Instruct docker to always attempt to pull all referenced images.',
|
|
318
|
+
)
|
|
309
319
|
@click.argument('images', type=str, nargs=-1)
|
|
310
320
|
@click.pass_context
|
|
311
|
-
def build_images(ctx, env, build_id, build_all, images):
|
|
321
|
+
def build_images(ctx, env, build_id, build_all, no_cache, pull, images):
|
|
312
322
|
init(ctx)
|
|
313
323
|
config = ctx.obj['config']
|
|
314
324
|
controller = ctx.obj['controller']
|
|
@@ -321,8 +331,18 @@ def build_images(ctx, env, build_id, build_all, images):
|
|
|
321
331
|
name for name, image in config.images.items()
|
|
322
332
|
if isinstance(image, BaseImage)
|
|
323
333
|
]
|
|
334
|
+
docker_flags = []
|
|
335
|
+
if no_cache:
|
|
336
|
+
docker_flags.append('--no-cache')
|
|
337
|
+
if pull:
|
|
338
|
+
docker_flags.append('--pull')
|
|
324
339
|
try:
|
|
325
|
-
controller.build_images(
|
|
340
|
+
controller.build_images(
|
|
341
|
+
images,
|
|
342
|
+
build_id=build_id,
|
|
343
|
+
env=env,
|
|
344
|
+
docker_flags=docker_flags,
|
|
345
|
+
)
|
|
326
346
|
except CloudSubmitError as e:
|
|
327
347
|
abort(str(e))
|
|
328
348
|
|
|
@@ -61,7 +61,7 @@ class Controller:
|
|
|
61
61
|
except FileNotFoundError:
|
|
62
62
|
pass
|
|
63
63
|
|
|
64
|
-
def _build_image(self, image, build_id, env, rebuild=False):
|
|
64
|
+
def _build_image(self, image, build_id, env, docker_flags, rebuild=False):
|
|
65
65
|
is_execution_image = isinstance(image, ExecutionImage)
|
|
66
66
|
|
|
67
67
|
if not rebuild and not is_execution_image:
|
|
@@ -96,7 +96,7 @@ class Controller:
|
|
|
96
96
|
image.setup_builddir(path, parent_ref)
|
|
97
97
|
|
|
98
98
|
print(f'Building image {image.name}.')
|
|
99
|
-
ref = env.build_image(path, image, build_id)
|
|
99
|
+
ref = env.build_image(path, image, build_id, docker_flags)
|
|
100
100
|
if ref is None:
|
|
101
101
|
raise CloudSubmitError(
|
|
102
102
|
'build_image method of environment handler did not '
|
|
@@ -105,10 +105,12 @@ class Controller:
|
|
|
105
105
|
if not is_execution_image:
|
|
106
106
|
self._save_image_ref(image, ref)
|
|
107
107
|
|
|
108
|
-
def build_images(self, images, build_id=None, env=None):
|
|
108
|
+
def build_images(self, images, build_id=None, env=None, docker_flags=None):
|
|
109
109
|
now = dt.datetime.now(tz=dt.UTC)
|
|
110
110
|
env_handler = self._config.get_build_env(env)
|
|
111
111
|
build_id = env_handler.generate_build_id(now, build_id)
|
|
112
|
+
if docker_flags is None:
|
|
113
|
+
docker_flags = []
|
|
112
114
|
|
|
113
115
|
images = set(images)
|
|
114
116
|
for image in images:
|
|
@@ -127,7 +129,7 @@ class Controller:
|
|
|
127
129
|
continue
|
|
128
130
|
rebuild = image_name in images
|
|
129
131
|
self._build_image(
|
|
130
|
-
image, build_id, env_handler, rebuild=rebuild)
|
|
132
|
+
image, build_id, env_handler, docker_flags, rebuild=rebuild)
|
|
131
133
|
|
|
132
134
|
def list_image_refs(self, images=None, ids=None, env=None, remote=False):
|
|
133
135
|
env_handler = self._config.get_build_env(env)
|
|
@@ -235,13 +235,14 @@ class EnvironmentHandler:
|
|
|
235
235
|
def remove_remote_image_refs(self, refs):
|
|
236
236
|
raise NotImplementedError
|
|
237
237
|
|
|
238
|
-
def build_image(self, path, image, build_id):
|
|
238
|
+
def build_image(self, path, image, build_id, docker_flags):
|
|
239
239
|
repo = self.get_image_repo(image)
|
|
240
240
|
ref = ':'.join([repo, build_id])
|
|
241
241
|
command = [
|
|
242
242
|
self._docker_command,
|
|
243
243
|
'buildx',
|
|
244
244
|
'build',
|
|
245
|
+
*docker_flags,
|
|
245
246
|
*(image.args),
|
|
246
247
|
'-t', ref,
|
|
247
248
|
path,
|
{cloud_submit-0.3.1 → cloud_submit-0.4}/src/cloud_submit/envs/aws/local_environment_handler.py
RENAMED
|
@@ -129,9 +129,9 @@ class LocalAWSEnv(LocalEnv):
|
|
|
129
129
|
os.path.join(path, 's3_tools.py'),
|
|
130
130
|
)
|
|
131
131
|
|
|
132
|
-
def _build_image(self, path, image, build_id, push):
|
|
132
|
+
def _build_image(self, path, image, build_id, docker_flags, push):
|
|
133
133
|
if not push:
|
|
134
|
-
ref = super().build_image(path, image, build_id)
|
|
134
|
+
ref = super().build_image(path, image, build_id, docker_flags)
|
|
135
135
|
else:
|
|
136
136
|
self._docker_login()
|
|
137
137
|
repo = self.get_image_repo(image)
|
|
@@ -140,6 +140,7 @@ class LocalAWSEnv(LocalEnv):
|
|
|
140
140
|
self._docker_command,
|
|
141
141
|
'buildx',
|
|
142
142
|
'build',
|
|
143
|
+
*docker_flags,
|
|
143
144
|
*(image.args),
|
|
144
145
|
]
|
|
145
146
|
if self._docker_platforms is not None:
|
|
@@ -148,11 +149,12 @@ class LocalAWSEnv(LocalEnv):
|
|
|
148
149
|
run_command(command)
|
|
149
150
|
return ref
|
|
150
151
|
|
|
151
|
-
def build_image(self, path, image, build_id):
|
|
152
|
+
def build_image(self, path, image, build_id, docker_flags):
|
|
152
153
|
return self._build_image(
|
|
153
154
|
path=path,
|
|
154
155
|
image=image,
|
|
155
156
|
build_id=build_id,
|
|
157
|
+
docker_flags=docker_flags,
|
|
156
158
|
push=isinstance(image, BaseImage),
|
|
157
159
|
)
|
|
158
160
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{cloud_submit-0.3.1 → cloud_submit-0.4}/src/cloud_submit/envs/aws/local_execution_handler.py
RENAMED
|
File without changes
|
{cloud_submit-0.3.1 → cloud_submit-0.4}/src/cloud_submit/envs/aws/remote_environment_handler.py
RENAMED
|
File without changes
|
{cloud_submit-0.3.1 → cloud_submit-0.4}/src/cloud_submit/envs/aws/remote_execution_handler.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{cloud_submit-0.3.1 → cloud_submit-0.4}/src/cloud_submit/execution/base_execution_handler.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|