cloud-submit 0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cloud_submit/__init__.py +30 -0
- cloud_submit/cli.py +1014 -0
- cloud_submit/config.py +164 -0
- cloud_submit/controller.py +375 -0
- cloud_submit/environment_handler.py +398 -0
- cloud_submit/envs/aws/__init__.py +2 -0
- cloud_submit/envs/aws/local_environment_handler.py +478 -0
- cloud_submit/envs/aws/local_execution_handler.py +5 -0
- cloud_submit/envs/aws/remote_environment_handler.py +401 -0
- cloud_submit/envs/aws/remote_execution_handler.py +87 -0
- cloud_submit/envs/aws/s3_tools.py +46 -0
- cloud_submit/envs/local/environment_handler.py +121 -0
- cloud_submit/envs/local/execution_handler.py +5 -0
- cloud_submit/execution/base_execution_handler.py +49 -0
- cloud_submit/execution/config.py +248 -0
- cloud_submit/execution/execute.py +108 -0
- cloud_submit/execution/utils.py +98 -0
- cloud_submit/images.py +58 -0
- cloud_submit/utils.py +67 -0
- cloud_submit-0.1.dist-info/METADATA +12 -0
- cloud_submit-0.1.dist-info/RECORD +24 -0
- cloud_submit-0.1.dist-info/WHEEL +4 -0
- cloud_submit-0.1.dist-info/entry_points.txt +2 -0
- cloud_submit-0.1.dist-info/licenses/LICENSE +19 -0
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import shutil
|
|
3
|
+
import datetime as dt
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class BaseExecutionHandler:
|
|
7
|
+
def __init__(self):
|
|
8
|
+
pass
|
|
9
|
+
|
|
10
|
+
def get_local_artifact_path(self, artifact):
|
|
11
|
+
if artifact.kind != 'file':
|
|
12
|
+
raise RuntimeError(
|
|
13
|
+
f'Cannot construct local path for artifact {artifact.name} '
|
|
14
|
+
f'of kind {artifact.kind}'
|
|
15
|
+
)
|
|
16
|
+
if artifact.scope not in ['run', 'user', 'project']:
|
|
17
|
+
raise RuntimeError(
|
|
18
|
+
f'Invalid scope {artifact.scope} '
|
|
19
|
+
f'for artifact {artifact.name} '
|
|
20
|
+
)
|
|
21
|
+
return os.path.join(
|
|
22
|
+
'/mnt/artifacts',
|
|
23
|
+
artifact.scope,
|
|
24
|
+
artifact.name,
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
def get_remote_artifact_path(self, artifact):
|
|
28
|
+
raise RuntimeError(
|
|
29
|
+
'Cannot construct remote path for artifact '
|
|
30
|
+
f'{artifact.name}.'
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
def download_artifact(self, artifact):
|
|
34
|
+
pass
|
|
35
|
+
|
|
36
|
+
def upload_artifact(self, artifact):
|
|
37
|
+
pass
|
|
38
|
+
|
|
39
|
+
def get_submit_timestamp(self):
|
|
40
|
+
return dt.datetime.fromisoformat(os.environ['CSUB_TIMESTAMP'])
|
|
41
|
+
|
|
42
|
+
def get_run_id(self):
|
|
43
|
+
return os.environ['CSUB_RUN_ID']
|
|
44
|
+
|
|
45
|
+
def get_worker_index(self):
|
|
46
|
+
return int(os.environ['CSUB_WORKER_INDEX'])
|
|
47
|
+
|
|
48
|
+
def get_run_steps(self):
|
|
49
|
+
return os.environ['CSUB_RUN_STEPS'].split(',')
|
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import datetime as dt
|
|
3
|
+
|
|
4
|
+
from .utils import read_json
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class ConfigError(Exception):
|
|
8
|
+
pass
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def to_utc(timestamp):
|
|
12
|
+
if timestamp is None:
|
|
13
|
+
return None
|
|
14
|
+
if timestamp.tzinfo is None:
|
|
15
|
+
timestamp = timestamp.replace(tzinfo=dt.UTC)
|
|
16
|
+
return timestamp.astimezone(dt.UTC)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class Artifact:
|
|
20
|
+
def __init__(
|
|
21
|
+
self,
|
|
22
|
+
name,
|
|
23
|
+
kind='file',
|
|
24
|
+
scope='run',
|
|
25
|
+
):
|
|
26
|
+
if scope not in ['run', 'user', 'project']:
|
|
27
|
+
raise ValueError(
|
|
28
|
+
f'Invalid value {repr(scope)} for scope. '
|
|
29
|
+
"Must be 'run', 'user' or 'project'."
|
|
30
|
+
)
|
|
31
|
+
self.name = name
|
|
32
|
+
self.kind = kind
|
|
33
|
+
self.scope = scope
|
|
34
|
+
|
|
35
|
+
def __eq__(self, other):
|
|
36
|
+
return (
|
|
37
|
+
isinstance(other, Artifact)
|
|
38
|
+
and other.name == self.name
|
|
39
|
+
and other.kind == self.kind
|
|
40
|
+
and other.scope == self.scope
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
def copy(self):
|
|
44
|
+
return Artifact(self.name, kind=self.kind, scope=self.scope)
|
|
45
|
+
|
|
46
|
+
def to_dict(self):
|
|
47
|
+
return {
|
|
48
|
+
'name': self.name,
|
|
49
|
+
'kind': self.kind,
|
|
50
|
+
'scope': self.scope,
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
@staticmethod
|
|
54
|
+
def from_dict(obj):
|
|
55
|
+
return Artifact(
|
|
56
|
+
obj['name'],
|
|
57
|
+
kind=obj['kind'],
|
|
58
|
+
scope=obj['scope'],
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class ArtifactLocation:
|
|
63
|
+
def __init__(
|
|
64
|
+
self,
|
|
65
|
+
artifact_name,
|
|
66
|
+
is_local,
|
|
67
|
+
sync=True,
|
|
68
|
+
):
|
|
69
|
+
self.artifact_name = str(artifact_name)
|
|
70
|
+
self.is_local = bool(is_local)
|
|
71
|
+
self.sync = bool(sync)
|
|
72
|
+
|
|
73
|
+
def to_dict(self):
|
|
74
|
+
return {
|
|
75
|
+
'artifact_name': self.artifact_name,
|
|
76
|
+
'is_local': self.is_local,
|
|
77
|
+
'sync': self.sync
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
@staticmethod
|
|
81
|
+
def from_dict(obj):
|
|
82
|
+
return ArtifactLocation(
|
|
83
|
+
obj['artifact_name'],
|
|
84
|
+
obj['is_local'],
|
|
85
|
+
sync=obj['sync'],
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def local(name, sync=True):
|
|
90
|
+
return ArtifactLocation(name, is_local=True, sync=sync)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def remote(name, sync=True):
|
|
94
|
+
return ArtifactLocation(name, is_local=False, sync=sync)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
class Spec:
|
|
98
|
+
def __init__(self, **kwargs):
|
|
99
|
+
self._spec = kwargs
|
|
100
|
+
|
|
101
|
+
def get(self, attr, default=None):
|
|
102
|
+
return self._spec.get(attr, default)
|
|
103
|
+
|
|
104
|
+
def __eq__(self, other):
|
|
105
|
+
if other is None:
|
|
106
|
+
return False
|
|
107
|
+
return self._spec == other._spec
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
class Step:
|
|
111
|
+
def __init__(
|
|
112
|
+
self,
|
|
113
|
+
name,
|
|
114
|
+
function,
|
|
115
|
+
image=None,
|
|
116
|
+
spec=None,
|
|
117
|
+
params=None,
|
|
118
|
+
inputs=None,
|
|
119
|
+
outputs=None,
|
|
120
|
+
temporaries=None,
|
|
121
|
+
num_workers=None,
|
|
122
|
+
pass_num_workers_as='num_workers',
|
|
123
|
+
pass_worker_index_as='worker_index',
|
|
124
|
+
pass_submit_timestamp_as=None,
|
|
125
|
+
pass_run_id_as=None,
|
|
126
|
+
):
|
|
127
|
+
self.name = name
|
|
128
|
+
self.function = function
|
|
129
|
+
self.params = params
|
|
130
|
+
if self.params is None:
|
|
131
|
+
self.params = {}
|
|
132
|
+
self.inputs = inputs
|
|
133
|
+
if self.inputs is None:
|
|
134
|
+
self.inputs = {}
|
|
135
|
+
self.outputs = outputs
|
|
136
|
+
if self.outputs is None:
|
|
137
|
+
self.outputs = {}
|
|
138
|
+
self.temporaries = temporaries
|
|
139
|
+
if self.temporaries is None:
|
|
140
|
+
self.temporaries = {}
|
|
141
|
+
self.image = image
|
|
142
|
+
self.spec = spec
|
|
143
|
+
if self.spec is None:
|
|
144
|
+
self.spec = Spec()
|
|
145
|
+
self.num_workers = None
|
|
146
|
+
if num_workers is not None:
|
|
147
|
+
self.num_workers = int(num_workers)
|
|
148
|
+
self.pass_num_workers_as = pass_num_workers_as
|
|
149
|
+
self.pass_worker_index_as = pass_worker_index_as
|
|
150
|
+
self.pass_submit_timestamp_as = pass_submit_timestamp_as
|
|
151
|
+
self.pass_run_id_as = pass_run_id_as
|
|
152
|
+
|
|
153
|
+
def to_dict(self):
|
|
154
|
+
return {
|
|
155
|
+
'name': self.name,
|
|
156
|
+
'function': self.function,
|
|
157
|
+
'params': self.params,
|
|
158
|
+
'inputs': {k: v.to_dict() for k, v in self.inputs.items()},
|
|
159
|
+
'outputs': {k: v.to_dict() for k, v in self.outputs.items()},
|
|
160
|
+
'temporaries':
|
|
161
|
+
{k: v.to_dict() for k, v in self.temporaries.items()},
|
|
162
|
+
'num_workers': self.num_workers,
|
|
163
|
+
'pass_num_workers_as': self.pass_num_workers_as,
|
|
164
|
+
'pass_worker_index_as': self.pass_worker_index_as,
|
|
165
|
+
'pass_submit_timestamp_as': self.pass_submit_timestamp_as,
|
|
166
|
+
'pass_run_id_as': self.pass_run_id_as,
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
@staticmethod
|
|
170
|
+
def from_dict(obj):
|
|
171
|
+
return Step(
|
|
172
|
+
obj['name'],
|
|
173
|
+
obj['function'],
|
|
174
|
+
params=obj['params'],
|
|
175
|
+
inputs={
|
|
176
|
+
k: ArtifactLocation.from_dict(v)
|
|
177
|
+
for k, v in obj['inputs'].items()
|
|
178
|
+
},
|
|
179
|
+
outputs={
|
|
180
|
+
k: ArtifactLocation.from_dict(v)
|
|
181
|
+
for k, v in obj['outputs'].items()
|
|
182
|
+
},
|
|
183
|
+
temporaries={
|
|
184
|
+
k: ArtifactLocation.from_dict(v)
|
|
185
|
+
for k, v in obj['temporaries'].items()
|
|
186
|
+
},
|
|
187
|
+
num_workers=obj['num_workers'],
|
|
188
|
+
pass_num_workers_as=obj['pass_num_workers_as'],
|
|
189
|
+
pass_worker_index_as=obj['pass_worker_index_as'],
|
|
190
|
+
pass_submit_timestamp_as=obj['pass_submit_timestamp_as'],
|
|
191
|
+
pass_run_id_as=obj['pass_run_id_as'],
|
|
192
|
+
)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
class Pipeline:
|
|
196
|
+
def __init__(
|
|
197
|
+
self,
|
|
198
|
+
name,
|
|
199
|
+
steps,
|
|
200
|
+
default_submit_timestamp=None,
|
|
201
|
+
):
|
|
202
|
+
self.name = name
|
|
203
|
+
self.steps = list(steps)
|
|
204
|
+
self.default_submit_timestamp = to_utc(default_submit_timestamp)
|
|
205
|
+
|
|
206
|
+
step_names = set()
|
|
207
|
+
for step in self.steps:
|
|
208
|
+
if step.name in step_names:
|
|
209
|
+
raise ConfigError(
|
|
210
|
+
f'Duplicate step name {step.name} '
|
|
211
|
+
f'in pipeline {self.name}.'
|
|
212
|
+
)
|
|
213
|
+
step_names.add(step.name)
|
|
214
|
+
|
|
215
|
+
def to_dict(self):
|
|
216
|
+
return {
|
|
217
|
+
'name': self.name,
|
|
218
|
+
'steps': [s.to_dict() for s in self.steps],
|
|
219
|
+
'default_submit_timestamp': self.default_submit_timestamp
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
@staticmethod
|
|
223
|
+
def from_dict(obj):
|
|
224
|
+
return Pipeline(
|
|
225
|
+
name=obj['name'],
|
|
226
|
+
steps=[Step.from_dict(s) for s in obj['steps']],
|
|
227
|
+
default_submit_timestamp=obj['default_submit_timestamp']
|
|
228
|
+
)
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def read_pipelines():
|
|
232
|
+
path = os.path.join(os.path.dirname(__file__), 'pipelines.json')
|
|
233
|
+
obj = read_json(path)
|
|
234
|
+
pipelines = {}
|
|
235
|
+
for name, steps in obj.items():
|
|
236
|
+
pipelines[name] = [Step.from_dict(s) for s in steps]
|
|
237
|
+
return pipelines
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def read_artifacts():
|
|
241
|
+
path = os.path.join(os.path.dirname(__file__), 'artifacts.json')
|
|
242
|
+
obj = read_json(path)
|
|
243
|
+
artifacts = {}
|
|
244
|
+
for artifact in obj['artifacts']:
|
|
245
|
+
artifact = Artifact.from_dict(artifact)
|
|
246
|
+
artifacts[artifact.name] = artifact
|
|
247
|
+
return artifacts
|
|
248
|
+
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import sys
|
|
3
|
+
import importlib
|
|
4
|
+
import copy
|
|
5
|
+
import datetime as dt
|
|
6
|
+
|
|
7
|
+
from .config import read_pipelines, read_artifacts
|
|
8
|
+
from .execution_handler import create_execution_handler
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
if __name__ == '__main__':
|
|
12
|
+
if len(sys.argv) != 3:
|
|
13
|
+
raise SystemExit(
|
|
14
|
+
'Invalid number of command line arguments. '
|
|
15
|
+
'Expected PIPELINE and STEPS.'
|
|
16
|
+
)
|
|
17
|
+
pipeline_name = sys.argv[1]
|
|
18
|
+
steps = set(sys.argv[2].split(','))
|
|
19
|
+
|
|
20
|
+
os.makedirs('/mnt/artifacts/project', exist_ok=True)
|
|
21
|
+
os.makedirs('/mnt/artifacts/user', exist_ok=True)
|
|
22
|
+
os.makedirs('/mnt/artifacts/run', exist_ok=True)
|
|
23
|
+
|
|
24
|
+
pipelines = read_pipelines()
|
|
25
|
+
artifacts = read_artifacts()
|
|
26
|
+
try:
|
|
27
|
+
pipeline_steps = pipelines[pipeline_name]
|
|
28
|
+
except KeyError:
|
|
29
|
+
raise SystemExit(
|
|
30
|
+
f'Invalid pipeline name: {pipeline_name}')
|
|
31
|
+
|
|
32
|
+
def get_artifact(name):
|
|
33
|
+
try:
|
|
34
|
+
return artifacts[name]
|
|
35
|
+
except KeyError:
|
|
36
|
+
raise SystemExit(f'Invalid artifact name {name}')
|
|
37
|
+
|
|
38
|
+
def get_artifact_path(eh, loc):
|
|
39
|
+
artifact = get_artifact(loc.artifact_name)
|
|
40
|
+
if loc.is_local:
|
|
41
|
+
return eh.get_local_artifact_path(artifact)
|
|
42
|
+
return eh.get_remote_artifact_path(artifact)
|
|
43
|
+
|
|
44
|
+
eh = create_execution_handler()
|
|
45
|
+
timestamp = eh.get_submit_timestamp()
|
|
46
|
+
run_id = eh.get_run_id()
|
|
47
|
+
worker_index = eh.get_worker_index()
|
|
48
|
+
run_steps = set(eh.get_run_steps())
|
|
49
|
+
run_steps = [step.name for step in pipeline_steps if step.name in run_steps]
|
|
50
|
+
|
|
51
|
+
synced_artifacts = set()
|
|
52
|
+
|
|
53
|
+
for step in pipeline_steps:
|
|
54
|
+
if step.name not in steps:
|
|
55
|
+
continue
|
|
56
|
+
if step.name == run_steps[0] and step.num_workers is None:
|
|
57
|
+
print('STARTING EXECUTION.')
|
|
58
|
+
|
|
59
|
+
print(f'Executing step: {step.name}')
|
|
60
|
+
for loc in step.inputs.values():
|
|
61
|
+
if loc.is_local and loc.artifact_name not in synced_artifacts:
|
|
62
|
+
artifact = get_artifact(loc.artifact_name)
|
|
63
|
+
eh.download_artifact(artifact)
|
|
64
|
+
synced_artifacts.add(loc.artifact_name)
|
|
65
|
+
|
|
66
|
+
module_name, function_name = step.function.split(':')
|
|
67
|
+
module = importlib.import_module(module_name)
|
|
68
|
+
function = getattr(module, function_name)
|
|
69
|
+
|
|
70
|
+
kwargs = copy.deepcopy(step.params)
|
|
71
|
+
for kw, loc in step.inputs.items():
|
|
72
|
+
kwargs[kw] = get_artifact_path(eh, loc)
|
|
73
|
+
for kw, loc in step.outputs.items():
|
|
74
|
+
kwargs[kw] = get_artifact_path(eh, loc)
|
|
75
|
+
for kw, loc in step.temporaries.items():
|
|
76
|
+
kwargs[kw] = get_artifact_path(eh, loc)
|
|
77
|
+
if step.num_workers is not None:
|
|
78
|
+
if worker_index < 0:
|
|
79
|
+
raise SystemExit(
|
|
80
|
+
f'Invalid worker index {worker_index} in step {step.name}.'
|
|
81
|
+
)
|
|
82
|
+
kwargs[step.pass_num_workers_as] = step.num_workers
|
|
83
|
+
kwargs[step.pass_worker_index_as] = worker_index
|
|
84
|
+
if step.pass_submit_timestamp_as is not None:
|
|
85
|
+
kwargs[step.pass_submit_timestamp_as] = timestamp
|
|
86
|
+
if step.pass_run_id_as is not None:
|
|
87
|
+
kwargs[step.pass_run_id_as] = run_id
|
|
88
|
+
|
|
89
|
+
success = True
|
|
90
|
+
try:
|
|
91
|
+
function(**kwargs)
|
|
92
|
+
except Exception:
|
|
93
|
+
success = False
|
|
94
|
+
raise
|
|
95
|
+
finally:
|
|
96
|
+
remote_outputs = set(
|
|
97
|
+
l.artifact_name for l in step.outputs.values()
|
|
98
|
+
if not l.is_local
|
|
99
|
+
)
|
|
100
|
+
for loc in step.outputs.values():
|
|
101
|
+
artifact = get_artifact(loc.artifact_name)
|
|
102
|
+
if loc.is_local:
|
|
103
|
+
synced_artifacts.add(loc.artifact_name)
|
|
104
|
+
if loc.artifact_name not in remote_outputs:
|
|
105
|
+
eh.upload_artifact(artifact)
|
|
106
|
+
if success and step.name == run_steps[-1]:
|
|
107
|
+
print('EXECUTION FINISHED SUCCESSFULLY.')
|
|
108
|
+
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import sys
|
|
3
|
+
import shutil
|
|
4
|
+
import shlex
|
|
5
|
+
import json
|
|
6
|
+
import subprocess
|
|
7
|
+
import datetime as dt
|
|
8
|
+
import re
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
_timedelta_regex = re.compile(r'^(-?[\d]+)d([\d]+)s([\d]+)u$')
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class DecodingError(Exception):
|
|
15
|
+
pass
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class CloudSubmitError(Exception):
|
|
19
|
+
pass
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def clear_path(path):
|
|
23
|
+
shutil.rmtree(path, ignore_errors=True)
|
|
24
|
+
try:
|
|
25
|
+
os.remove(path)
|
|
26
|
+
except FileNotFoundError:
|
|
27
|
+
pass
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def ensure_path(path, clear=False):
|
|
31
|
+
if clear:
|
|
32
|
+
clear_path(path)
|
|
33
|
+
os.makedirs(path, exist_ok=True)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _encode(obj):
|
|
37
|
+
if isinstance(obj, dt.datetime):
|
|
38
|
+
return {'$datetime': obj.isoformat()}
|
|
39
|
+
elif isinstance(obj, dt.date):
|
|
40
|
+
return {'$date': obj.isoformat()}
|
|
41
|
+
elif isinstance(obj, dt.timedelta):
|
|
42
|
+
return {'$timedelta': f'{obj.days}d{obj.seconds}s{obj.microseconds}u'}
|
|
43
|
+
else:
|
|
44
|
+
raise TypeError(f'Object of type {type(obj)} is not JSON serializable')
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _decode(obj):
|
|
48
|
+
if len(obj) == 1:
|
|
49
|
+
key, value = next(iter(obj.items()))
|
|
50
|
+
if key == '$datetime':
|
|
51
|
+
return dt.datetime.fromisoformat(value)
|
|
52
|
+
elif key == '$date':
|
|
53
|
+
return dt.date.fromisoformat(value)
|
|
54
|
+
elif key == '$timedelta':
|
|
55
|
+
match = _timedelta_regex.match(value)
|
|
56
|
+
if not match:
|
|
57
|
+
raise DecodingError(
|
|
58
|
+
f'Expecting timedelta expression but got {repr(value)}')
|
|
59
|
+
return dt.timedelta(
|
|
60
|
+
days=int(match.group(1)),
|
|
61
|
+
seconds=int(match.group(2)),
|
|
62
|
+
microseconds=int(match.group(3)),
|
|
63
|
+
)
|
|
64
|
+
return obj
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def read_json(path):
|
|
68
|
+
with open(path, 'r') as stream:
|
|
69
|
+
obj = json.load(stream, object_hook=_decode)
|
|
70
|
+
return obj
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def write_json(obj, path):
|
|
74
|
+
with open(path, 'w') as stream:
|
|
75
|
+
json.dump(obj, stream, default=_encode, indent=4)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def run_command(command, check=True, hide_stderr=False, **kwargs):
|
|
79
|
+
if check and hide_stderr:
|
|
80
|
+
kwargs['stderr'] = subprocess.PIPE
|
|
81
|
+
try:
|
|
82
|
+
result = subprocess.run(command, **kwargs)
|
|
83
|
+
except FileNotFoundError:
|
|
84
|
+
raise CloudSubmitError(
|
|
85
|
+
'Error. Command not found when trying to execute:\n'
|
|
86
|
+
+ ' '.join([shlex.quote(part) for part in command])
|
|
87
|
+
)
|
|
88
|
+
except KeyboardInterrupt:
|
|
89
|
+
raise CloudSubmitError('Aborted on user request.')
|
|
90
|
+
if check and result.returncode != 0:
|
|
91
|
+
if hide_stderr:
|
|
92
|
+
sys.stderr.write(result.stderr)
|
|
93
|
+
msg = (
|
|
94
|
+
f'Command exited with status code {result.returncode}:\n'
|
|
95
|
+
+ ' '.join([shlex.quote(part) for part in command])
|
|
96
|
+
)
|
|
97
|
+
raise CloudSubmitError(msg)
|
|
98
|
+
return result
|
cloud_submit/images.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import os
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class Image:
|
|
5
|
+
def __init__(
|
|
6
|
+
self,
|
|
7
|
+
name,
|
|
8
|
+
parent=None,
|
|
9
|
+
instructions=None,
|
|
10
|
+
setup_builddir_callback=None,
|
|
11
|
+
):
|
|
12
|
+
self.name = name
|
|
13
|
+
self.parent = parent
|
|
14
|
+
self._instructions = instructions if instructions is not None else ''
|
|
15
|
+
self._setup_builddir_callback = setup_builddir_callback
|
|
16
|
+
|
|
17
|
+
def setup_builddir(self, path, base_image):
|
|
18
|
+
if self._setup_builddir_callback is not None:
|
|
19
|
+
self._setup_builddir_callback(path, mode)
|
|
20
|
+
with open(os.path.join(path, 'Dockerfile'), 'w') as dockerfile:
|
|
21
|
+
if self.parent is not None:
|
|
22
|
+
dockerfile.write(f'FROM {base_image}\n\n')
|
|
23
|
+
dockerfile.write(self._instructions)
|
|
24
|
+
dockerfile.write('\n')
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class BaseImage(Image):
|
|
28
|
+
pass
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class ExecutionImage(Image):
|
|
32
|
+
def __init__(
|
|
33
|
+
self,
|
|
34
|
+
name,
|
|
35
|
+
parent=None,
|
|
36
|
+
instructions=None,
|
|
37
|
+
setup_builddir_callback=None,
|
|
38
|
+
python_cmd='python',
|
|
39
|
+
):
|
|
40
|
+
super().__init__(
|
|
41
|
+
name=name,
|
|
42
|
+
parent=parent,
|
|
43
|
+
instructions=instructions,
|
|
44
|
+
setup_builddir_callback=setup_builddir_callback,
|
|
45
|
+
)
|
|
46
|
+
self._python_cmd = python_cmd
|
|
47
|
+
|
|
48
|
+
def setup_builddir(self, path, base_image):
|
|
49
|
+
super().setup_builddir(path, base_image)
|
|
50
|
+
with open(os.path.join(path, 'Dockerfile'), 'a') as dockerfile:
|
|
51
|
+
dockerfile.write(
|
|
52
|
+
'\nCOPY --chown=root:root src /root/src/\n'
|
|
53
|
+
'ENV PYTHONPATH="/root/src:$PYTHONPATH"\n'
|
|
54
|
+
'WORKDIR /root\n'
|
|
55
|
+
f'ENTRYPOINT ["/usr/bin/env", "{self._python_cmd}", "-u", '
|
|
56
|
+
'"-m", "csub.execute"]\n'
|
|
57
|
+
)
|
|
58
|
+
|
cloud_submit/utils.py
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import re
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def build_docker_mount_option(source, dest):
|
|
6
|
+
source = os.path.abspath(source)
|
|
7
|
+
volume = os.environ.get('CSUB_DOD_VOLUME', None)
|
|
8
|
+
if not volume:
|
|
9
|
+
return f'type=bind,src={source},dst={dest}'
|
|
10
|
+
dod_mount = os.environ.get('CSUB_DOD_MOUNT_POINT', None)
|
|
11
|
+
if not dod_mount:
|
|
12
|
+
raise CloudSubmitError(
|
|
13
|
+
'CSUB_DOD_MOUNT_POINT variable must be set to use cloud-submit '
|
|
14
|
+
'in a docker-outside-docker setup. If do not want to use '
|
|
15
|
+
'cloud-submit in docker-outside-docker mode make sure that the '
|
|
16
|
+
'variable CSUB_DOD_VOLUME is *not* set.'
|
|
17
|
+
)
|
|
18
|
+
dod_mount = os.path.abspath(dod_mount)
|
|
19
|
+
if os.path.commonpath([source, dod_mount]) != dod_mount:
|
|
20
|
+
raise CloudSubmitError(
|
|
21
|
+
f'Artifact directory {path} is not a subpath of {dod_mount}. '
|
|
22
|
+
'Change the CSUB_DOD_MOUNT_POINT variable or move your '
|
|
23
|
+
'project directory.'
|
|
24
|
+
)
|
|
25
|
+
subpath = os.path.relpath(source, start=dod_mount)
|
|
26
|
+
return (
|
|
27
|
+
'type=volume,'
|
|
28
|
+
f'src={volume},'
|
|
29
|
+
f'dst={dest},'
|
|
30
|
+
f'volume-subpath={subpath}'
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def run_command(command, check=True, **kwargs):
|
|
35
|
+
try:
|
|
36
|
+
result = subprocess.run(command, **kwargs)
|
|
37
|
+
except FileNotFoundError:
|
|
38
|
+
raise CloudSubmitError(
|
|
39
|
+
'Error. Command not found when trying to execute:\n'
|
|
40
|
+
+ ' '.join([shlex.quote(part) for part in command])
|
|
41
|
+
)
|
|
42
|
+
except KeyboardInterrupt:
|
|
43
|
+
raise CloudSubmitError('Aborted on user request.')
|
|
44
|
+
if check and result.returncode != 0:
|
|
45
|
+
msg = (
|
|
46
|
+
f'Command exited with status code {result.returncode}:\n'
|
|
47
|
+
+ ' '.join([shlex.quote(part) for part in command])
|
|
48
|
+
)
|
|
49
|
+
raise CloudSubmitError(msg)
|
|
50
|
+
return result
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
_IMAGE_REF_REGEX = re.compile(
|
|
54
|
+
'^([a-zA-Z0-9-.:]+)/([a-z0-9-_./]+):([a-zA-Z0-9-_.]+)(@([a-zA-Z0-9:]+))?$'
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def parse_image_ref(ref):
|
|
59
|
+
match = _IMAGE_REF_REGEX.match(ref)
|
|
60
|
+
if match is None:
|
|
61
|
+
raise CloudSubmitError(f'Could not parse image ref {ref}')
|
|
62
|
+
return (
|
|
63
|
+
match.group(1),
|
|
64
|
+
match.group(2),
|
|
65
|
+
match.group(3),
|
|
66
|
+
match.group(5),
|
|
67
|
+
)
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: cloud-submit
|
|
3
|
+
Version: 0.1
|
|
4
|
+
Summary: A tool for building processing pipelines in Python and executing them locally or in the cloud.
|
|
5
|
+
Author: Martin Wiebusch
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
8
|
+
Classifier: Operating System :: OS Independent
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Requires-Python: >=3.7
|
|
11
|
+
Requires-Dist: click
|
|
12
|
+
Requires-Dist: pyyaml
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
cloud_submit/__init__.py,sha256=Oy7My15-bLvmL6G_Y53e_OBG4xk7TC69G60GvZUrgHY,659
|
|
2
|
+
cloud_submit/cli.py,sha256=POr0dPo8pN1xyBcqvAtIyampdIp0QW_XSY7RXl3p_8I,26805
|
|
3
|
+
cloud_submit/config.py,sha256=nNZheN9r0vZ9jisolifrnb5SRe4GdSiAz5SjrfJ_lrQ,6226
|
|
4
|
+
cloud_submit/controller.py,sha256=peLTcrkgQ-LHrFUbiCvO_gTkJ1vqAd6ADOIBHtEIFLw,13626
|
|
5
|
+
cloud_submit/environment_handler.py,sha256=W1xp5DNa9MAGqz42034sF38BfWjKKsLKaUu9lKGVpss,14245
|
|
6
|
+
cloud_submit/images.py,sha256=6zo0XP8qaZ17J4bpZp4pzMS6kDmDWoF6UeU8fgW1Fig,1716
|
|
7
|
+
cloud_submit/utils.py,sha256=vG4sDrxfs-Irem6zncjk38K4ALBR350N_DclhWDd2e0,2176
|
|
8
|
+
cloud_submit/envs/aws/__init__.py,sha256=9YzB6gf91zqKVy_jrRc8AvNW1vdIdIowRf79s1LK5QE,104
|
|
9
|
+
cloud_submit/envs/aws/local_environment_handler.py,sha256=xTRrJ5MKU7tjqKAtnVBl-4vxbjq8Lg1cVocShfx2d2U,15800
|
|
10
|
+
cloud_submit/envs/aws/local_execution_handler.py,sha256=hpJJFjYYw3u2SPK53yBd_k6AVGOiv4DyERWSfgI06YU,125
|
|
11
|
+
cloud_submit/envs/aws/remote_environment_handler.py,sha256=idO2Ug3MYJbxN8Sv_g6_jW0GkBbvWA8Oq3cLRJlVt7I,14693
|
|
12
|
+
cloud_submit/envs/aws/remote_execution_handler.py,sha256=S29bTVVgbPIieUt-RDCpbY4Hd6G9ZrRze2kdXUGdcnw,2903
|
|
13
|
+
cloud_submit/envs/aws/s3_tools.py,sha256=a9ooG3tk6LNuXLT73dFZtGJAkYXjbPK76R7JFek7dBw,1316
|
|
14
|
+
cloud_submit/envs/local/environment_handler.py,sha256=m15bnJiU8YM-vWJQbJui6mgjh2oCeQLZXFKV3HVcINY,3891
|
|
15
|
+
cloud_submit/envs/local/execution_handler.py,sha256=hpJJFjYYw3u2SPK53yBd_k6AVGOiv4DyERWSfgI06YU,125
|
|
16
|
+
cloud_submit/execution/base_execution_handler.py,sha256=YgfjbKxgigoha3cBJCGbtXnoVnZrr-QLqg_o2BgVRYc,1332
|
|
17
|
+
cloud_submit/execution/config.py,sha256=JDNzsuj5GjbRFqS6ibWhfE29UQ4089iZC7QgvMTf6qE,6763
|
|
18
|
+
cloud_submit/execution/execute.py,sha256=pPTSZ0bj67nZt_-81p-gpzi-b4S5WuHwc22hUy1COgs,3822
|
|
19
|
+
cloud_submit/execution/utils.py,sha256=7SMTIWHqyGoM0pSjqAEOXuYnuLmpzyCpFES3DQw7o8s,2646
|
|
20
|
+
cloud_submit-0.1.dist-info/METADATA,sha256=L2l8Bh4Y0Gmpcttdd82X9U12Zx2bZjUD9CIfrC7pOAU,416
|
|
21
|
+
cloud_submit-0.1.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
|
|
22
|
+
cloud_submit-0.1.dist-info/entry_points.txt,sha256=O20VZUbR139AENr5zt_wUFSbAoiPj6klDEpEXruCrQE,47
|
|
23
|
+
cloud_submit-0.1.dist-info/licenses/LICENSE,sha256=7EI8xVBu6h_7_JlVw-yPhhOZlpY9hP8wal7kHtqKT_E,1074
|
|
24
|
+
cloud_submit-0.1.dist-info/RECORD,,
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
Copyright (c) 2018 The Python Packaging Authority
|
|
2
|
+
|
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
4
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
5
|
+
in the Software without restriction, including without limitation the rights
|
|
6
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
7
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
8
|
+
furnished to do so, subject to the following conditions:
|
|
9
|
+
|
|
10
|
+
The above copyright notice and this permission notice shall be included in all
|
|
11
|
+
copies or substantial portions of the Software.
|
|
12
|
+
|
|
13
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
14
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
15
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
16
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
17
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
18
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
19
|
+
SOFTWARE.
|