cloud-submit 0.1.2__tar.gz → 0.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. {cloud_submit-0.1.2 → cloud_submit-0.2}/PKG-INFO +1 -1
  2. {cloud_submit-0.1.2 → cloud_submit-0.2}/docs/examples/aws/src/csub/build_config.py +3 -1
  3. cloud_submit-0.2/docs/examples/aws/userconfig/.ipynb_checkpoints/default-checkpoint.yaml +13 -0
  4. {cloud_submit-0.1.2 → cloud_submit-0.2}/docs/examples/aws/userconfig/template.yaml +5 -0
  5. {cloud_submit-0.1.2 → cloud_submit-0.2}/pyproject.toml +1 -1
  6. {cloud_submit-0.1.2 → cloud_submit-0.2}/src/cloud_submit/controller.py +2 -2
  7. {cloud_submit-0.1.2 → cloud_submit-0.2}/src/cloud_submit/environment_handler.py +20 -18
  8. {cloud_submit-0.1.2 → cloud_submit-0.2}/src/cloud_submit/envs/aws/local_environment_handler.py +29 -26
  9. {cloud_submit-0.1.2 → cloud_submit-0.2}/src/cloud_submit/envs/aws/remote_environment_handler.py +7 -1
  10. {cloud_submit-0.1.2 → cloud_submit-0.2}/src/cloud_submit/envs/local/environment_handler.py +4 -3
  11. {cloud_submit-0.1.2 → cloud_submit-0.2}/.gitignore +0 -0
  12. {cloud_submit-0.1.2 → cloud_submit-0.2}/LICENSE +0 -0
  13. {cloud_submit-0.1.2 → cloud_submit-0.2}/README.md +0 -0
  14. {cloud_submit-0.1.2 → cloud_submit-0.2}/docs/examples/aws/src/fit.py +0 -0
  15. {cloud_submit-0.1.2 → cloud_submit-0.2}/docs/examples/aws/src/generate.py +0 -0
  16. {cloud_submit-0.1.2 → cloud_submit-0.2}/docs/examples/aws/src/parameters.py +0 -0
  17. {cloud_submit-0.1.2 → cloud_submit-0.2}/docs/examples/basic/src/csub/build_config.py +0 -0
  18. {cloud_submit-0.1.2 → cloud_submit-0.2}/docs/examples/basic/src/fit.py +0 -0
  19. {cloud_submit-0.1.2 → cloud_submit-0.2}/docs/examples/basic/src/generate.py +0 -0
  20. {cloud_submit-0.1.2 → cloud_submit-0.2}/docs/examples/basic/src/parameters.py +0 -0
  21. {cloud_submit-0.1.2 → cloud_submit-0.2}/docs/examples/basic/userconfig/template.yaml +0 -0
  22. {cloud_submit-0.1.2 → cloud_submit-0.2}/docs/examples/multi-worker/src/csub/build_config.py +0 -0
  23. {cloud_submit-0.1.2 → cloud_submit-0.2}/docs/examples/multi-worker/src/fit.py +0 -0
  24. {cloud_submit-0.1.2 → cloud_submit-0.2}/docs/examples/multi-worker/src/generate.py +0 -0
  25. {cloud_submit-0.1.2 → cloud_submit-0.2}/docs/examples/multi-worker/src/parameters.py +0 -0
  26. {cloud_submit-0.1.2 → cloud_submit-0.2}/docs/examples/multi-worker/userconfig/template.yaml +0 -0
  27. {cloud_submit-0.1.2 → cloud_submit-0.2}/src/cloud_submit/__init__.py +0 -0
  28. {cloud_submit-0.1.2 → cloud_submit-0.2}/src/cloud_submit/cli.py +0 -0
  29. {cloud_submit-0.1.2 → cloud_submit-0.2}/src/cloud_submit/config.py +0 -0
  30. {cloud_submit-0.1.2 → cloud_submit-0.2}/src/cloud_submit/envs/aws/__init__.py +0 -0
  31. {cloud_submit-0.1.2 → cloud_submit-0.2}/src/cloud_submit/envs/aws/local_execution_handler.py +0 -0
  32. {cloud_submit-0.1.2 → cloud_submit-0.2}/src/cloud_submit/envs/aws/remote_execution_handler.py +0 -0
  33. {cloud_submit-0.1.2 → cloud_submit-0.2}/src/cloud_submit/envs/aws/s3_tools.py +0 -0
  34. {cloud_submit-0.1.2 → cloud_submit-0.2}/src/cloud_submit/envs/local/execution_handler.py +0 -0
  35. {cloud_submit-0.1.2 → cloud_submit-0.2}/src/cloud_submit/execution/base_execution_handler.py +0 -0
  36. {cloud_submit-0.1.2 → cloud_submit-0.2}/src/cloud_submit/execution/config.py +0 -0
  37. {cloud_submit-0.1.2 → cloud_submit-0.2}/src/cloud_submit/execution/execute.py +0 -0
  38. {cloud_submit-0.1.2 → cloud_submit-0.2}/src/cloud_submit/execution/utils.py +0 -0
  39. {cloud_submit-0.1.2 → cloud_submit-0.2}/src/cloud_submit/images.py +0 -0
  40. {cloud_submit-0.1.2 → cloud_submit-0.2}/src/cloud_submit/utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: cloud-submit
3
- Version: 0.1.2
3
+ Version: 0.2
4
4
  Summary: A tool for building processing pipelines in Python and executing them locally or in the cloud.
5
5
  Project-URL: Homepage, https://github.com/mwiebusch78/cloud-submit
6
6
  Project-URL: Bug Tracker, https://github.com/mwiebusch78/cloud-submit/issues
@@ -39,7 +39,7 @@ def build_config(project_root, userconfig):
39
39
  cs.BaseImage(
40
40
  name='base',
41
41
  instructions=r"""
42
- FROM python
42
+ FROM python:3.12.3
43
43
  RUN \
44
44
  cd /opt && \
45
45
  curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" \
@@ -125,6 +125,7 @@ def build_config(project_root, userconfig):
125
125
  aws_region=userconfig['aws_region'],
126
126
  aws_profile=userconfig['aws_profile'],
127
127
  docker_namespace=userconfig['docker_namespace'],
128
+ docker_platforms=userconfig['docker_platforms'],
128
129
  s3_bucket=userconfig['s3_bucket'],
129
130
  s3_prefix=userconfig['s3_prefix'],
130
131
  ),
@@ -136,6 +137,7 @@ def build_config(project_root, userconfig):
136
137
  aws_region=userconfig['aws_region'],
137
138
  aws_profile=userconfig['aws_profile'],
138
139
  docker_namespace=userconfig['docker_namespace'],
140
+ docker_platforms=userconfig['docker_platforms'],
139
141
  s3_bucket=userconfig['s3_bucket'],
140
142
  s3_prefix=userconfig['s3_prefix'],
141
143
  ecs_cluster_arn=userconfig['ecs_cluster_arn'],
@@ -0,0 +1,13 @@
1
+ username: martin
2
+ aws_account_id: "505307261052"
3
+ aws_region: eu-north-1
4
+ aws_profile: default
5
+ docker_namespace: csub
6
+ s3_bucket: artifacts-505307261052-eu-north-1-an
7
+ s3_prefix: csub
8
+ ecs_cluster_arn: arn:aws:ecs:eu-north-1:505307261052:cluster/dev-cluster
9
+ ecs_capacity_provider: Infra-ECS-Cluster-dev-cluster-9dedfac2-ManagedInstancesCapacityProvider-0GpOCu8Wkp42
10
+ ecs_infrastructure_role_arn: arn:aws:iam::505307261052:role/ecsInfrastructureRole
11
+ ecs_execution_role_arn: arn:aws:iam::505307261052:role/ecsTaskExecutionRole
12
+ ecs_task_role_arn: arn:aws:iam::505307261052:role/ecsTaskRole
13
+ stepfunctions_role_arn: arn:aws:iam::505307261052:role/stepfunctionsRole
@@ -24,6 +24,11 @@ aws_profile: default
24
24
  # registry.
25
25
  docker_namespace: csub
26
26
 
27
+ # This is the value of the --platform option used when building docker images.
28
+ # To build images that work on both x86_64 and ARM architecture set this to
29
+ # "linux/amd64,linux/arm64"
30
+ docker_platforms: null
31
+
27
32
  # This is the name of the S3 bucket where your artifacts will be stored.
28
33
  s3_bucket: # TODO: add the name of your S3 bucket.
29
34
 
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "cloud-submit"
7
- version = "0.1.2"
7
+ version = "0.2"
8
8
  authors = [
9
9
  { name="Martin Wiebusch" },
10
10
  ]
@@ -116,9 +116,9 @@ class Controller:
116
116
  image = self._config.images[image]
117
117
  repo_name = env_handler.get_image_repo_name(image)
118
118
  if remote:
119
- tags = env_handler.list_remote_image_tags(repo_name)
119
+ tags = env_handler.list_remote_image_tags(image)
120
120
  else:
121
- tags = env_handler.list_local_image_tags(repo_name)
121
+ tags = env_handler.list_local_image_tags(image)
122
122
  if ids is not None:
123
123
  tags = [t for t in tags if t in ids]
124
124
  for tag in tags:
@@ -19,13 +19,13 @@ from .execution.config import to_utc
19
19
 
20
20
  class EnvironmentHandler:
21
21
  def __init__(
22
- self,
23
- name,
24
- project,
25
- user,
26
- docker_command='docker',
27
- docker_registry=None,
28
- docker_namespace='csub',
22
+ self,
23
+ name,
24
+ project,
25
+ user,
26
+ docker_command='docker',
27
+ docker_registry=None,
28
+ docker_namespace='csub',
29
29
  ):
30
30
  self.name = name
31
31
  self._project = project
@@ -199,21 +199,20 @@ class EnvironmentHandler:
199
199
  Args:
200
200
  ref (str): The full docker image reference for the requested image.
201
201
  """
202
- run_command([
203
- self._docker_command,
204
- 'pull',
205
- ref,
206
- ])
202
+ command = [self._docker_command, 'pull', ref]
203
+ run_command(command)
207
204
 
208
- def list_local_image_tags(self, repo_name):
205
+ def list_local_image_tags(self, image):
209
206
  """List locally available tags for a docker image repo.
210
207
 
211
208
  Args:
212
- repo_name (str): The fully qualified name of the docker image repo.
209
+ image (cloud_submit.Image): The object describing the docker image
210
+ whose tags should be listed.
213
211
 
214
212
  Returns:
215
213
  tags (list of str): The list of tags that are available locally.
216
214
  """
215
+ repo_name = self.get_image_repo_name(image)
217
216
  result = run_command(
218
217
  [
219
218
  self._docker_command,
@@ -230,7 +229,7 @@ class EnvironmentHandler:
230
229
  return []
231
230
  return result.split('\n')
232
231
 
233
- def list_remote_image_tags(self, repo_name):
232
+ def list_remote_image_tags(self, image):
234
233
  """List remotely available tags for a docker image repo.
235
234
 
236
235
  Note:
@@ -238,7 +237,8 @@ class EnvironmentHandler:
238
237
  remote registry.
239
238
 
240
239
  Args:
241
- repo_name (str): The fully qualified name of the docker image repo.
240
+ image (cloud_submit.Image): The object describing the docker image
241
+ whose tags should be listed.
242
242
 
243
243
  Returns:
244
244
  tags (list of str): The list of the tags that are available in the
@@ -260,12 +260,14 @@ class EnvironmentHandler:
260
260
  def build_image(self, path, image, build_id):
261
261
  repo = self.get_image_repo_name(image)
262
262
  ref = ':'.join([repo, build_id])
263
- run_command([
263
+ command = [
264
264
  self._docker_command,
265
+ 'buildx',
265
266
  'build',
266
267
  '-t', ref,
267
268
  path,
268
- ])
269
+ ]
270
+ run_command(command)
269
271
  return ref
270
272
 
271
273
  def get_local_artifact_path(self, artifact, run_id=None):
@@ -10,6 +10,7 @@ from cloud_submit import (
10
10
  LocalEnv,
11
11
  ensure_path,
12
12
  CloudSubmitError,
13
+ ExecutionImage,
13
14
  build_docker_mount_option,
14
15
  run_command,
15
16
  parse_image_ref,
@@ -37,6 +38,7 @@ class LocalAWSEnv(LocalEnv):
37
38
  s3_prefix,
38
39
  docker_command='docker',
39
40
  docker_namespace='csub',
41
+ docker_platforms=None,
40
42
  docker_login_refresh_hours=6,
41
43
  aws_command='aws',
42
44
  ):
@@ -56,6 +58,9 @@ class LocalAWSEnv(LocalEnv):
56
58
  self._s3_prefix = str(s3_prefix)
57
59
  self._aws_command = str(aws_command)
58
60
  self._docker_login_refresh_hours = float(docker_login_refresh_hours)
61
+ self._docker_platforms = docker_platforms
62
+ if self._docker_platforms is not None:
63
+ self._docker_platforms = str(docker_platforms)
59
64
 
60
65
  self._last_docker_login_ts = None
61
66
 
@@ -102,26 +107,21 @@ class LocalAWSEnv(LocalEnv):
102
107
  )
103
108
 
104
109
  def _build_image(self, path, image, build_id, push):
105
- ref = super().build_image(path, image, build_id)
106
- self._docker_login()
107
-
108
- if push:
109
- # push image
110
- run_command([self._docker_command, 'push', ref])
111
-
112
- # get image digest
113
- result = run_command(
114
- [
115
- self._docker_command,
116
- 'buildx', 'imagetools', 'inspect',
117
- '--format', '{{json .Manifest.Digest}}',
118
- ref,
119
- ],
120
- stdout=subprocess.PIPE,
121
- text=True,
122
- )
123
- digest = result.stdout.strip('"')
124
- return ref + '@' + digest
110
+ if not push:
111
+ ref = super().build_image(path, image, build_id)
112
+ else:
113
+ self._docker_login()
114
+ repo = self.get_image_repo_name(image)
115
+ ref = ':'.join([repo, build_id])
116
+ command = [
117
+ self._docker_command,
118
+ 'buildx',
119
+ 'build',
120
+ ]
121
+ if self._docker_platforms is not None:
122
+ command.extend(['--platform', self._docker_platforms])
123
+ command.extend(['-t', ref, '--push', path])
124
+ run_command(command)
125
125
  return ref
126
126
 
127
127
  def build_image(self, path, image, build_id):
@@ -134,13 +134,10 @@ class LocalAWSEnv(LocalEnv):
134
134
 
135
135
  def pull_base_image(self, ref):
136
136
  self._docker_login()
137
- run_command([
138
- self._docker_command,
139
- 'pull',
140
- ref,
141
- ])
137
+ command = [self._docker_command, 'pull', ref]
138
+ run_command(command)
142
139
 
143
- def list_remote_image_tags(self, repo_name):
140
+ def _list_remote_image_tags(self, repo_name):
144
141
  self._docker_login()
145
142
  repo_name = '/'.join(repo_name.split('/')[1:])
146
143
  result = run_command(
@@ -160,6 +157,12 @@ class LocalAWSEnv(LocalEnv):
160
157
  tags = [img['imageTag'] for img in result['imageIds']]
161
158
  return tags
162
159
 
160
+ def list_remote_image_tags(self, image):
161
+ if isinstance(image, ExecutionImage):
162
+ return []
163
+ repo_name = self.get_image_repo_name(image)
164
+ return self._list_remote_image_tags(repo_name)
165
+
163
166
  def remove_remote_image_refs(self, refs):
164
167
  tags = {}
165
168
  for ref in refs:
@@ -28,6 +28,7 @@ class RemoteAWSEnv(LocalAWSEnv):
28
28
  stepfunctions_role_arn,
29
29
  docker_command='docker',
30
30
  docker_namespace='csub',
31
+ docker_platforms=None,
31
32
  docker_login_refresh_hours=6,
32
33
  aws_command='aws',
33
34
  log_group_prefix='/ecs/csub/',
@@ -44,6 +45,7 @@ class RemoteAWSEnv(LocalAWSEnv):
44
45
  s3_prefix=s3_prefix,
45
46
  docker_command=docker_command,
46
47
  docker_namespace=docker_namespace,
48
+ docker_platforms=docker_platforms,
47
49
  docker_login_refresh_hours=docker_login_refresh_hours,
48
50
  aws_command=aws_command,
49
51
  )
@@ -64,6 +66,10 @@ class RemoteAWSEnv(LocalAWSEnv):
64
66
  push=True,
65
67
  )
66
68
 
69
+ def list_remote_image_tags(self, image):
70
+ repo_name = self.get_image_repo_name(image)
71
+ return self._list_remote_image_tags(repo_name)
72
+
67
73
  def install_execution_handler(self, path):
68
74
  sourcedir = os.path.dirname(__file__)
69
75
  shutil.copyfile(
@@ -333,8 +339,8 @@ class RemoteAWSEnv(LocalAWSEnv):
333
339
  with open(workflow_path, 'w') as stream:
334
340
  json.dump(workflow, stream)
335
341
 
342
+ workflow_arn = self._make_workflow_arn(workflow_name)
336
343
  if self._check_workflow_exists(workflow_name):
337
- workflow_arn = self._make_workflow_arn(workflow_name)
338
344
  run_command(
339
345
  [
340
346
  self._aws_command,
@@ -44,7 +44,7 @@ class LocalEnv(EnvironmentHandler):
44
44
  f'Could not find image {ref}. You may have to build it again.'
45
45
  )
46
46
 
47
- def list_remote_image_tags(self, repo_name):
47
+ def list_remote_image_tags(self, image):
48
48
  return []
49
49
 
50
50
  def remove_remote_image_refs(self, refs):
@@ -96,7 +96,7 @@ class LocalEnv(EnvironmentHandler):
96
96
  worker_indices = range(step.num_workers)
97
97
 
98
98
  for worker_index in worker_indices:
99
- run_command([
99
+ command = [
100
100
  self._docker_command,
101
101
  'run',
102
102
  '--rm',
@@ -114,7 +114,8 @@ class LocalEnv(EnvironmentHandler):
114
114
  ref,
115
115
  pipeline.name,
116
116
  step.name,
117
- ])
117
+ ]
118
+ run_command(command)
118
119
 
119
120
  def print_logs(self, run_id, start_timestamp, stream=False):
120
121
  raise CloudSubmitError(
File without changes
File without changes
File without changes